dashboards/tests/test_llm_source.py

"""LLM engine parsing: real captures, provenance on every number, absent never 0.

The engines do not export the names people expect, so these tests are pinned to captures of
the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentation.
"""
from __future__ import annotations

import re

import pytest

from dashboards.rates import RateEstimator
from dashboards.sources.llm import (
    engine_result,
    fleet_totals,
    parse_gufo_metrics,
    parse_vllm_metrics,
    rate_key,
    series_names,
)
from slots_contract import load_slots_capture


def bump_metric(text: str, name: str, delta: float) -> str:
    """Return the exposition with one counter increased — a scrape `dt` later."""
    pattern = re.compile(rf"^({re.escape(name)}(?:\{{[^}}]*\}})?) (\S+)$", re.M)

    def replace(match: re.Match[str]) -> str:
        return f"{match.group(1)} {float(match.group(2)) + delta}"

    bumped, count = pattern.subn(replace, text)
    assert count, f"{name} not present in the capture"
    return bumped


# ---- gufo / llama.cpp ----------------------------------------------------- #

def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
    metrics = parse_gufo_metrics(metrics_text("gufo"))
    assert metrics["tokens_generated_total"]["value"] == 877_336.0
    assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0
    assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0
    assert metrics["requests_processing"]["value"] == 0.0
    assert metrics["device_lost_total"]["value"] == 0.0
    assert metrics["kv_cache_usage_ratio"]["value"] == 0.0


def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None:
    metrics = parse_gufo_metrics(metrics_text("gufo"))
    accept = metrics["spec_accept_rate"]["value"]
    assert accept == pytest.approx(578_514 / 789_028)
    assert str(metrics["spec_accept_rate"]["source"]).startswith("computed")


def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None:
    """gufo's prompt counter excludes cache hits; the lifetime total must not."""
    metrics = parse_gufo_metrics(metrics_text("gufo"))
    total = metrics["tokens_total"]["value"]
    assert total == 1_350_666 + 124_529_105 + 877_336


def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None:
    """llamacpp:predicted_tokens_seconds is ONE request's last pass — never the headline."""
    text = metrics_text("gufo")
    rates = RateEstimator(span=10)
    now = 1_000.0
    first = parse_gufo_metrics(text, rates, now)
    assert "decode_tokens_per_second" not in first or first["decode_tokens_per_second"]["value"] is None

    text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0)
    metrics = parse_gufo_metrics(text, rates, now + 10.0)
    decode = metrics["decode_tokens_per_second"]
    assert decode["value"] == pytest.approx(40.0)
    assert decode["window"] == pytest.approx(10.0)
    assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"]
    # the native gauge is still there, labelled, because it answers a different question
    assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)


def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None:
    text = metrics_text("gufo")
    rates = RateEstimator(span=10)
    parse_gufo_metrics(text, rates, 1_000.0)
    text = bump_metric(text, "llamacpp:prompt_tokens_total", 1_000.0)
    metrics = parse_gufo_metrics(text, rates, 1_010.0)
    assert metrics["prompt_tokens_per_second"]["value"] == pytest.approx(100.0)


def test_gufo_counter_reset_produces_n_a_not_a_spike(metrics_text) -> None:
    text = metrics_text("gufo")
    rates = RateEstimator(span=60)
    parse_gufo_metrics(text, rates, 1_000.0)
    restarted = bump_metric(text, "llamacpp:tokens_predicted_total", -800_000.0)
    metrics = parse_gufo_metrics(restarted, rates, 1_010.0)
    assert metrics.get("decode_tokens_per_second", {}).get("value") is None


def test_gufo_a_stalled_poll_then_the_catch_up_is_not_a_spike(metrics_text) -> None:
    """The exact stored incident, through the production parse path.

    2026-10-18 09:16:31 on deskwork: a 15 s gap in every source, a first result carrying 114
    tokens (its body was generated ~2 s into the stall), then the poll loop's 0.2 s catch-up
    printing **3284.59 tok/s** for an engine doing 53. `test_rates.py` holds the estimator-level
    pair; this is the same defect where the operator sees it.
    """
    text = metrics_text("gufo")
    rates = RateEstimator(span=10.0)
    parse_gufo_metrics(text, rates, 1_000.0)
    text = bump_metric(text, "llamacpp:tokens_predicted_total", 114.0)
    parse_gufo_metrics(text, rates, 1_015.25)
    text = bump_metric(text, "llamacpp:tokens_predicted_total", 713.0)
    metrics = parse_gufo_metrics(text, rates, 1_015.47)
    cell = metrics["decode_tokens_per_second"]
    assert cell["window"] == pytest.approx(0.22)
    assert cell["value"] is None or cell["value"] < 100.0, (
        f"53 tok/s of real work printed as {cell['value']:.0f} tok/s")


def test_gufo_a_stale_scrape_body_never_becomes_a_rate_above_the_engine(metrics_text) -> None:
    """The other half of the stall: the reply is stamped when the poll *finished*, not when the
    engine rendered it, so its counter is the stall's length older than its own timestamp.

    That sample then anchors every window for the rest of the span and each poll re-diffs the
    same skewed delta over a longer one — the live 2026-10-19 12:42 incident printed 232 tok/s
    for one request running at ~50, then a decaying 146 / 114 / 95 tail. The estimator's jump
    gate holds the first of those and now refuses to call a window anchored on the same sample a
    confirmation, so nothing above the engine's real rate reaches the payload.
    """
    text = metrics_text("gufo")
    rates = RateEstimator(span=10.0)
    parse_gufo_metrics(text, rates, 1_000.0)
    # (stamp, tokens the engine had really produced by then): the 1011 reply is the stalled one —
    # its body was rendered ~9 s before the poll stamped it — and from there the counter advances
    # at a steady 45 tok/s, so 45 is the only rate that is true to print.
    timeline = [(1_011.0, 90.0), (1_012.0, 540.0), (1_014.0, 630.0), (1_016.0, 720.0),
                (1_018.0, 810.0), (1_022.0, 990.0), (1_026.0, 1_170.0)]
    produced = 0.0
    cells = []
    for stamp, total in timeline:
        text = bump_metric(text, "llamacpp:tokens_predicted_total", total - produced)
        produced = total
        cells.append(parse_gufo_metrics(text, rates, stamp)["decode_tokens_per_second"])
    for cell in cells:
        assert cell["value"] is None or cell["value"] <= 100.0, \
            f"a 45 tok/s engine printed {cell['value']:.0f} tok/s ({cells})"
    assert cells[-1]["value"] == pytest.approx(45.0), "the stall ends in the truth, not a tail"
    assert cells[-1]["window"] == pytest.approx(10.0)


def test_absent_gufo_metrics_are_none_with_no_source() -> None:
    metrics = parse_gufo_metrics("# nothing here\n")
    assert metrics["requests_processing"]["value"] is None
    assert metrics["requests_processing"]["source"] is None
    assert metrics.get("spec_accept_rate", {}).get("value") is None


# ---- severity: KV pressure arrives with a verdict ------------------------- #

def test_gufo_kv_cache_cell_keeps_provenance_and_gains_severity(metrics_text) -> None:
    metrics = parse_gufo_metrics(metrics_text("gufo"))
    cell = metrics["kv_cache_usage_ratio"]
    assert cell["value"] == 0.0
    assert cell["source"] == "llamacpp:kv_cache_usage_ratio"
    assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
    assert cell["severity"] == "ok"


@pytest.mark.parametrize("ratio,expected", [
    ("0.74", "ok"), ("0.75", "warn"), ("0.89", "warn"), ("0.9", "fail"), ("1.0", "fail"),
])
def test_kv_ratio_is_classified_on_a_percent_scale(metrics_text, ratio, expected) -> None:
    """0.9 of a KV cache is 90 %, not 0.9 % — classifying the raw ratio is a permanent green."""
    text = re.sub(r"^(llamacpp:kv_cache_usage_ratio) \S+$", rf"\g<1> {ratio}",
                  metrics_text("gufo"), count=1, flags=re.M)
    metrics = parse_gufo_metrics(text)
    assert metrics["kv_cache_usage_ratio"]["severity"] == expected
    assert metrics["kv_cache_usage_ratio"]["value"] == pytest.approx(float(ratio)), \
        "the scale is for the verdict only; the published value is untouched"


def test_absent_kv_cell_gets_no_verdict() -> None:
    metrics = parse_gufo_metrics("# nothing here\n")
    assert metrics["kv_cache_usage_ratio"]["severity"] is None


# ---- vLLM ----------------------------------------------------------------- #

def test_vllm_counters_and_gauges(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    assert metrics["prompt_tokens_total"]["value"] == pytest.approx(1.8389909e7)
    assert metrics["generation_tokens_total"]["value"] == pytest.approx(611_163.0)
    usage = metrics["gpu_cache_usage"]["value"]
    assert usage is None or 0.0 <= usage <= 1.0, "a ratio summed over label sets would exceed 1"
    assert metrics["num_requests_running"]["value"] is not None
    assert metrics["num_preemptions_total"]["value"] is not None


def test_vllm_kv_cache_cell_carries_a_severity(metrics_text) -> None:
    assert parse_vllm_metrics(metrics_text("vllm"))["gpu_cache_usage"]["severity"] == "ok"
    pressed = bump_metric(metrics_text("vllm"), "vllm:kv_cache_usage_perc", 0.95)
    cell = parse_vllm_metrics(pressed)["gpu_cache_usage"]
    assert cell["value"] == pytest.approx(0.95)
    assert cell["severity"] == "fail"
    assert cell["source"] == "vllm:kv_cache_usage_perc"


def test_vllm_absent_kv_cell_gets_no_verdict(metrics_text) -> None:
    stripped = "\n".join(line for line in metrics_text("vllm").splitlines()
                         if "kv_cache_usage" not in line and "gpu_cache_usage" not in line)
    metrics = parse_vllm_metrics(stripped)
    assert metrics["gpu_cache_usage"]["value"] is None
    assert metrics["gpu_cache_usage"]["severity"] is None


def test_vllm_latency_histograms(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    ttft = metrics["time_to_first_token_seconds"]
    assert ttft and ttft["count"] > 0 and ttft["mean"] > 0
    assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"]
    assert metrics["time_per_output_token_seconds"]["mean"] > 0


def test_vllm_per_request_speed_is_the_reciprocal_of_tpot(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    tpot = metrics["time_per_output_token_seconds"]["mean"]
    per_request = metrics["per_request_decode_tokens_per_second"]
    assert per_request["value"] == pytest.approx(1.0 / tpot)
    assert per_request["derived"] is True


def test_vllm_prefix_cache_hit_ratio(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    ratio = metrics["prefix_cache_hit_ratio"]["value"]
    assert ratio is not None and 0.0 <= ratio <= 1.0
    assert "hits" in metrics["prefix_cache_hit_ratio"]["source"]


def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) -> None:
    text = metrics_text("vllm")
    rates = RateEstimator(span=10)
    parse_vllm_metrics(text, rates, 2_000.0)
    text = bump_metric(text, "vllm:generation_tokens_total", 250.0)
    metrics = parse_vllm_metrics(text, rates, 2_010.0)
    assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0)


def test_vllm_a_joining_label_set_does_not_lend_its_carried_counter(metrics_text) -> None:
    """A second `engine=`/`model_name=` series appearing mid-window must not be a rate.

    vLLM's counters are labelled, so `scalar(agg="sum")` added whatever label sets exist right
    now. A rank joining (or a reloaded model whose old series is still exported) carries its own
    cumulative counter, and `value < samples[-1]` never fires for an *increase*, so the carried
    value was diffed. Confirmed as a defect, not as this fleet's trigger: `generation_tokens_total`
    has exactly one label set live (`{engine="0",model_name="juggernaut"}`). Fixed in task 03 by
    diffing each label set under its own estimator key — the joining series has no baseline and
    contributes 0 tokens until it has one.
    """
    text = metrics_text("vllm")
    rates = RateEstimator(span=10.0)
    parse_vllm_metrics(text, rates, 2_000.0)
    gained = text + 'vllm:generation_tokens_total{engine="1",model_name="juggernaut"} 500000.0\n'
    metrics = parse_vllm_metrics(gained, rates, 2_010.0)
    cell = metrics["decode_tokens_per_second"]
    assert cell["value"] is None or cell["value"] < 100.0, (
        f"a joining label set printed {cell['value']:.0f} tok/s")


def test_vllm_a_label_set_that_survives_starts_contributing_its_own_delta(metrics_text) -> None:
    """Fresh series contribute nothing on their first poll and their real Δ on the second."""
    text = metrics_text("vllm")
    rates = RateEstimator(span=10.0)
    parse_vllm_metrics(text, rates, 2_000.0)
    gained = text + 'vllm:generation_tokens_total{engine="1",model_name="juggernaut"} 500000.0\n'
    metrics = parse_vllm_metrics(gained, rates, 2_010.0)
    assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(0.0), \
        "the joining series is baseline-less: 500 000 carried, 0 tokens diffed"
    grown = bump_metric(gained, "vllm:generation_tokens_total", 100.0)  # bumps every label set
    metrics = parse_vllm_metrics(grown, rates, 2_020.0)
    assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(20.0), \
        "two series x 100 tokens / 10 s: the rate is the sum of the per-series rates"
    assert metrics["decode_tokens_per_second"]["window"] == pytest.approx(10.0)


def test_gufo_measured_cell_keeps_the_provenance_shape(metrics_text) -> None:
    """Rule 2 after the per-series rewrite: value, `computed:Δ<series>/Δt`, `Δ<window>`."""
    text = metrics_text("gufo")
    rates = RateEstimator(span=10)
    parse_gufo_metrics(text, rates, 1_000.0)
    text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0)
    decode = parse_gufo_metrics(text, rates, 1_010.0)["decode_tokens_per_second"]
    assert decode["value"] == pytest.approx(40.0)
    assert decode["derived"] is True
    assert decode["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt"
    assert decode["window"] == pytest.approx(10.0), "the widest window a contributing used"


def test_gufo_a_refused_window_is_n_a_and_names_the_window_it_refused(metrics_text) -> None:
    """The stall refusal is an answer: value None, but the series and Δ still travel."""
    text = metrics_text("gufo")
    rates = RateEstimator(span=10)
    parse_gufo_metrics(text, rates, 1_000.0)
    text = bump_metric(text, "llamacpp:tokens_predicted_total", 50.0)
    parse_gufo_metrics(text, rates, 1_010.0)
    text = bump_metric(text, "llamacpp:tokens_predicted_total", 60.0)
    decode = parse_gufo_metrics(text, rates, 1_010.3)["decode_tokens_per_second"]
    assert decode["value"] is None, "0.3 s is the loop's catch-up gap, not a rate"
    assert decode["window"] == pytest.approx(0.3)
    assert decode["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt"


def test_rate_key_puts_the_label_set_in_the_estimator_key() -> None:
    """The key is the resolved name plus its labels, label order included-stable."""
    assert rate_key("llamacpp:tokens_predicted_total", {}) == "llamacpp:tokens_predicted_total"
    assert rate_key("vllm:generation_tokens_total", {"model_name": "m", "engine": "0"}) == \
        'vllm:generation_tokens_total{engine="0",model_name="m"}'
    assert rate_key("m", {"weird": 'a"b\\c'}) == 'm{weird="a\\"b\\\\c"}', \
        "a quote in a label value must not splice two keys into one"


def test_vllm_waiting_requests_are_broken_down_by_reason(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    reasons = metrics["waiting_by_reason"]["value"]
    assert isinstance(reasons, dict)


def test_vllm_empty_exposition_is_all_none() -> None:
    metrics = parse_vllm_metrics("")
    assert metrics["num_requests_running"]["value"] is None
    assert metrics["time_to_first_token_seconds"] is None
    assert metrics["per_request_decode_tokens_per_second"]["value"] is None


# ---- Result assembly ------------------------------------------------------ #

def test_engine_result_payload_and_samples(metrics_text) -> None:
    rates = RateEstimator(span=10)
    result = engine_result("gufo", "http://gufo.internal:8731", metrics_text("gufo"), rates,
                           now=3_000.0)
    payload = result.payload
    assert payload["engine"] == "gufo"
    assert payload["upstream"] == "http://gufo.internal:8731"
    assert payload["series_count"] > 10
    assert payload["unparsed"] == 0
    assert payload["sessions"]["detail"], "a missing /slots must be explained, not blank"
    for sample in result.samples:
        assert sample.source == "llm" and sample.host == "gufo"
        assert sample.metric.startswith("llm.")
        assert sample.value == sample.value and sample.value is not None  # no NaN


def test_engine_result_charts_a_measured_rate_after_the_second_poll(metrics_text) -> None:
    rates = RateEstimator(span=10)
    engine_result("gufo", "http://g:1", metrics_text("gufo"), rates, now=3_000.0)
    bumped = bump_metric(metrics_text("gufo"), "llamacpp:tokens_predicted_total", 100.0)
    result = engine_result("gufo", "http://g:1", bumped, rates, now=3_010.0)
    metrics = result.payload["metrics"]
    assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(10.0)
    assert any(s.metric == "llm.decode_tokens_per_second" for s in result.samples)


def test_vllm_stores_the_tpot_series_the_cell_sparkline_draws(metrics_text) -> None:
    """Phase 04 added `llm.tpot_seconds` so the vLLM TPOT row has a shape, not a dead slot.

    Hermetic twin of the live check: the captured exposition must resolve the histogram the
    page charts, or `/llm` shows a permanent "no history" next to a real mean.
    """
    result = engine_result("vllm", "http://v:1", metrics_text("vllm"), RateEstimator(), now=1.0)
    mean = result.payload["metrics"]["time_per_output_token_seconds__mean"]["value"]
    assert mean is not None, "the vLLM capture lost its TPOT histogram — re-capture fixtures"
    stored = {s.metric: s.value for s in result.samples}
    assert stored["llm.tpot_seconds"] == pytest.approx(mean)
    # gufo publishes no output-token histogram, so it stores no TPOT series (never a 0)
    gufo = engine_result("gufo", "http://g:1", metrics_text("gufo"), RateEstimator(), now=1.0)
    assert "llm.tpot_seconds" not in {s.metric for s in gufo.samples}


def test_gufo_stores_the_spec_accept_series_the_cell_sparkline_draws(metrics_text) -> None:
    """`llm.spec_accept_rate` is the gufo-only half of the phase-04 history addition."""
    result = engine_result("gufo", "http://g:1", metrics_text("gufo"), RateEstimator(), now=1.0)
    ratio = result.payload["metrics"]["spec_accept_rate"]["value"]
    assert ratio is not None and 0.0 < ratio <= 1.0
    stored = {s.metric: s.value for s in result.samples}
    assert stored["llm.spec_accept_rate"] == pytest.approx(ratio)
    vllm = engine_result("vllm", "http://v:1", metrics_text("vllm"), RateEstimator(), now=1.0)
    assert "llm.spec_accept_rate" not in {s.metric for s in vllm.samples}


def test_sessions_pass_through_slots_for_gufo() -> None:
    """`sessions.list` is the raw gufo rows — the grid, not the source, reads slot fields.

    The rows are phase 07's live capture (`tests/fixtures/slots_gufo.json`), not a guessed
    shape: the old table's `idle`/`n_prompt`/`n_predict` names were invented and the server
    never answered them. `_sessions()` must pass the rows through untouched — the shape is
    contracted separately in `tests/slots_contract.py`.
    """
    slots = load_slots_capture()
    result = engine_result("gufo", "http://g:1", "", RateEstimator(), slots=slots, now=1.0)
    sessions = result.payload["sessions"]
    assert sessions["list"] == slots
    assert sessions["list"][0]["is_processing"] is False, "the passthrough re-shaped the rows"
    assert sessions["source"] == "gufo /slots"


def test_sessions_say_vllm_has_no_per_session_view() -> None:
    result = engine_result("vllm", "http://v:1", "", RateEstimator(), now=1.0)
    sessions = result.payload["sessions"]
    assert sessions["list"] == []
    assert "vLLM" in sessions["detail"]


# ---- fleet totals --------------------------------------------------------- #

def _engine(name: str, **values) -> dict:
    metrics = {key: {"value": value} for key, value in values.items()}
    return {"engine": name, "metrics": metrics}


def test_fleet_totals_sum_only_engines_that_answered() -> None:
    engines = [
        _engine("gufo", decode_tokens_per_second=40.0, prompt_tokens_per_second=10.0,
                kv_cache_usage_ratio=0.25, requests_processing=1.0, requests_deferred=0.0,
                tokens_total=100.0),
        _engine("vllm", decode_tokens_per_second=10.0, prompt_tokens_per_second=5.0,
                gpu_cache_usage=0.75, num_requests_running=2.0, num_requests_waiting=3.0,
                tokens_total=200.0),
        {},  # an engine whose poll failed
    ]
    totals = fleet_totals(engines)
    assert totals["engines_total"] == 3
    assert totals["engines_live"] == 2
    assert totals["decode_tokens_per_second"] == 50.0
    assert totals["prompt_tokens_per_second"] == 15.0
    assert totals["requests_active"] == 3.0
    assert totals["requests_waiting"] == 3.0
    assert totals["kv_cache_usage_max"] == 0.75, "cache pressure is a max, never a sum"
    assert totals["tokens_total"] == 300.0


def test_fleet_totals_report_the_gufo_vllm_seam() -> None:
    totals = fleet_totals([_engine("gufo"), _engine("vllm")])
    assert totals["seams"], "the two engines count prompt tokens differently; say so"


def test_fleet_totals_of_nothing_is_none_not_zero() -> None:
    totals = fleet_totals([])
    assert totals["decode_tokens_per_second"] is None
    assert totals["tokens_total"] is None
    assert totals["engines_live"] == 0


def test_series_names_are_the_parse_catalog() -> None:
    assert "tokens_generated_total" in series_names("gufo")
    assert "generation_tokens_total" in series_names("vllm")
    assert "generation_tokens_total" not in series_names("gufo")


def test_gufo_a_measured_zero_is_not_replaced_by_the_gauge(metrics_text) -> None:
    """`measured or gauge` made an idle engine advertise its last request's speed."""
    text = metrics_text("gufo")
    rates = RateEstimator(span=10)
    parse_gufo_metrics(text, rates, 1_000.0)
    metrics = parse_gufo_metrics(text, rates, 1_010.0)   # counters unchanged: truly idle
    decode = metrics["decode_tokens_per_second"]
    assert decode["value"] == 0.0
    assert "predicted_tokens_seconds" not in str(decode["source"]), "native gauge must not leak in"


def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None:
    text = metrics_text("vllm")
    rates = RateEstimator(span=10)
    parse_vllm_metrics(text, rates, 2_000.0)
    metrics = parse_vllm_metrics(text, rates, 2_010.0)
    decode = metrics["decode_tokens_per_second"]
    assert decode["value"] == 0.0
    assert "running/tpot" not in str(decode["source"])