dashboards/tests/test_llm_source.py

"""LLM engine parsing: real captures, provenance on every number, absent never 0.

The engines do not export the names people expect, so these tests are pinned to captures of
the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentation.
"""
from __future__ import annotations

import re

import pytest

from dashboards.rates import RateEstimator
from dashboards.sources.llm import (
    engine_result,
    fleet_totals,
    parse_gufo_metrics,
    parse_vllm_metrics,
    series_names,
)


def bump_metric(text: str, name: str, delta: float) -> str:
    """Return the exposition with one counter increased — a scrape `dt` later."""
    pattern = re.compile(rf"^({re.escape(name)}(?:\{{[^}}]*\}})?) (\S+)$", re.M)

    def replace(match: re.Match[str]) -> str:
        return f"{match.group(1)} {float(match.group(2)) + delta}"

    bumped, count = pattern.subn(replace, text)
    assert count, f"{name} not present in the capture"
    return bumped


# ---- gufo / llama.cpp ----------------------------------------------------- #

def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
    metrics = parse_gufo_metrics(metrics_text("gufo"))
    assert metrics["tokens_generated_total"]["value"] == 877_336.0
    assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0
    assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0
    assert metrics["requests_processing"]["value"] == 0.0
    assert metrics["device_lost_total"]["value"] == 0.0
    assert metrics["kv_cache_usage_ratio"]["value"] == 0.0


def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None:
    metrics = parse_gufo_metrics(metrics_text("gufo"))
    accept = metrics["spec_accept_rate"]["value"]
    assert accept == pytest.approx(578_514 / 789_028)
    assert str(metrics["spec_accept_rate"]["source"]).startswith("computed")


def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None:
    """gufo's prompt counter excludes cache hits; the lifetime total must not."""
    metrics = parse_gufo_metrics(metrics_text("gufo"))
    total = metrics["tokens_total"]["value"]
    assert total == 1_350_666 + 124_529_105 + 877_336


def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None:
    """llamacpp:predicted_tokens_seconds is ONE request's last pass — never the headline."""
    text = metrics_text("gufo")
    rates = RateEstimator(span=10)
    now = 1_000.0
    first = parse_gufo_metrics(text, rates, now)
    assert "decode_tokens_per_second" not in first or first["decode_tokens_per_second"]["value"] is None

    text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0)
    metrics = parse_gufo_metrics(text, rates, now + 10.0)
    decode = metrics["decode_tokens_per_second"]
    assert decode["value"] == pytest.approx(40.0)
    assert decode["window"] == pytest.approx(10.0)
    assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"]
    # the native gauge is still there, labelled, because it answers a different question
    assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)


def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None:
    text = metrics_text("gufo")
    rates = RateEstimator(span=10)
    parse_gufo_metrics(text, rates, 1_000.0)
    text = bump_metric(text, "llamacpp:prompt_tokens_total", 1_000.0)
    metrics = parse_gufo_metrics(text, rates, 1_010.0)
    assert metrics["prompt_tokens_per_second"]["value"] == pytest.approx(100.0)


def test_gufo_counter_reset_produces_n_a_not_a_spike(metrics_text) -> None:
    text = metrics_text("gufo")
    rates = RateEstimator(span=60)
    parse_gufo_metrics(text, rates, 1_000.0)
    restarted = bump_metric(text, "llamacpp:tokens_predicted_total", -800_000.0)
    metrics = parse_gufo_metrics(restarted, rates, 1_010.0)
    assert metrics.get("decode_tokens_per_second", {}).get("value") is None


def test_absent_gufo_metrics_are_none_with_no_source() -> None:
    metrics = parse_gufo_metrics("# nothing here\n")
    assert metrics["requests_processing"]["value"] is None
    assert metrics["requests_processing"]["source"] is None
    assert metrics.get("spec_accept_rate", {}).get("value") is None


# ---- severity: KV pressure arrives with a verdict ------------------------- #

def test_gufo_kv_cache_cell_keeps_provenance_and_gains_severity(metrics_text) -> None:
    metrics = parse_gufo_metrics(metrics_text("gufo"))
    cell = metrics["kv_cache_usage_ratio"]
    assert cell["value"] == 0.0
    assert cell["source"] == "llamacpp:kv_cache_usage_ratio"
    assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
    assert cell["severity"] == "ok"


@pytest.mark.parametrize("ratio,expected", [
    ("0.74", "ok"), ("0.75", "warn"), ("0.89", "warn"), ("0.9", "fail"), ("1.0", "fail"),
])
def test_kv_ratio_is_classified_on_a_percent_scale(metrics_text, ratio, expected) -> None:
    """0.9 of a KV cache is 90 %, not 0.9 % — classifying the raw ratio is a permanent green."""
    text = re.sub(r"^(llamacpp:kv_cache_usage_ratio) \S+$", rf"\g<1> {ratio}",
                  metrics_text("gufo"), count=1, flags=re.M)
    metrics = parse_gufo_metrics(text)
    assert metrics["kv_cache_usage_ratio"]["severity"] == expected
    assert metrics["kv_cache_usage_ratio"]["value"] == pytest.approx(float(ratio)), \
        "the scale is for the verdict only; the published value is untouched"


def test_absent_kv_cell_gets_no_verdict() -> None:
    metrics = parse_gufo_metrics("# nothing here\n")
    assert metrics["kv_cache_usage_ratio"]["severity"] is None


# ---- vLLM ----------------------------------------------------------------- #

def test_vllm_counters_and_gauges(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    assert metrics["prompt_tokens_total"]["value"] == pytest.approx(1.8389909e7)
    assert metrics["generation_tokens_total"]["value"] == pytest.approx(611_163.0)
    usage = metrics["gpu_cache_usage"]["value"]
    assert usage is None or 0.0 <= usage <= 1.0, "a ratio summed over label sets would exceed 1"
    assert metrics["num_requests_running"]["value"] is not None
    assert metrics["num_preemptions_total"]["value"] is not None


def test_vllm_kv_cache_cell_carries_a_severity(metrics_text) -> None:
    assert parse_vllm_metrics(metrics_text("vllm"))["gpu_cache_usage"]["severity"] == "ok"
    pressed = bump_metric(metrics_text("vllm"), "vllm:kv_cache_usage_perc", 0.95)
    cell = parse_vllm_metrics(pressed)["gpu_cache_usage"]
    assert cell["value"] == pytest.approx(0.95)
    assert cell["severity"] == "fail"
    assert cell["source"] == "vllm:kv_cache_usage_perc"


def test_vllm_absent_kv_cell_gets_no_verdict(metrics_text) -> None:
    stripped = "\n".join(line for line in metrics_text("vllm").splitlines()
                         if "kv_cache_usage" not in line and "gpu_cache_usage" not in line)
    metrics = parse_vllm_metrics(stripped)
    assert metrics["gpu_cache_usage"]["value"] is None
    assert metrics["gpu_cache_usage"]["severity"] is None


def test_vllm_latency_histograms(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    ttft = metrics["time_to_first_token_seconds"]
    assert ttft and ttft["count"] > 0 and ttft["mean"] > 0
    assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"]
    assert metrics["time_per_output_token_seconds"]["mean"] > 0


def test_vllm_per_request_speed_is_the_reciprocal_of_tpot(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    tpot = metrics["time_per_output_token_seconds"]["mean"]
    per_request = metrics["per_request_decode_tokens_per_second"]
    assert per_request["value"] == pytest.approx(1.0 / tpot)
    assert per_request["derived"] is True


def test_vllm_prefix_cache_hit_ratio(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    ratio = metrics["prefix_cache_hit_ratio"]["value"]
    assert ratio is not None and 0.0 <= ratio <= 1.0
    assert "hits" in metrics["prefix_cache_hit_ratio"]["source"]


def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) -> None:
    text = metrics_text("vllm")
    rates = RateEstimator(span=10)
    parse_vllm_metrics(text, rates, 2_000.0)
    text = bump_metric(text, "vllm:generation_tokens_total", 250.0)
    metrics = parse_vllm_metrics(text, rates, 2_010.0)
    assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0)


def test_vllm_waiting_requests_are_broken_down_by_reason(metrics_text) -> None:
    metrics = parse_vllm_metrics(metrics_text("vllm"))
    reasons = metrics["waiting_by_reason"]["value"]
    assert isinstance(reasons, dict)


def test_vllm_empty_exposition_is_all_none() -> None:
    metrics = parse_vllm_metrics("")
    assert metrics["num_requests_running"]["value"] is None
    assert metrics["time_to_first_token_seconds"] is None
    assert metrics["per_request_decode_tokens_per_second"]["value"] is None


# ---- Result assembly ------------------------------------------------------ #

def test_engine_result_payload_and_samples(metrics_text) -> None:
    rates = RateEstimator(span=10)
    result = engine_result("gufo", "http://gufo.internal:8731", metrics_text("gufo"), rates,
                           now=3_000.0)
    payload = result.payload
    assert payload["engine"] == "gufo"
    assert payload["upstream"] == "http://gufo.internal:8731"
    assert payload["series_count"] > 10
    assert payload["unparsed"] == 0
    assert payload["sessions"]["detail"], "a missing /slots must be explained, not blank"
    for sample in result.samples:
        assert sample.source == "llm" and sample.host == "gufo"
        assert sample.metric.startswith("llm.")
        assert sample.value == sample.value and sample.value is not None  # no NaN


def test_engine_result_charts_a_measured_rate_after_the_second_poll(metrics_text) -> None:
    rates = RateEstimator(span=10)
    engine_result("gufo", "http://g:1", metrics_text("gufo"), rates, now=3_000.0)
    bumped = bump_metric(metrics_text("gufo"), "llamacpp:tokens_predicted_total", 100.0)
    result = engine_result("gufo", "http://g:1", bumped, rates, now=3_010.0)
    metrics = result.payload["metrics"]
    assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(10.0)
    assert any(s.metric == "llm.decode_tokens_per_second" for s in result.samples)


def test_sessions_pass_through_slots_for_gufo() -> None:
    slots = [{"id": 0, "idle": False, "n_prompt": 12, "n_predict": 40}]
    result = engine_result("gufo", "http://g:1", "", RateEstimator(), slots=slots, now=1.0)
    sessions = result.payload["sessions"]
    assert sessions["list"] == slots
    assert sessions["source"] == "gufo /slots"


def test_sessions_say_vllm_has_no_per_session_view() -> None:
    result = engine_result("vllm", "http://v:1", "", RateEstimator(), now=1.0)
    sessions = result.payload["sessions"]
    assert sessions["list"] == []
    assert "vLLM" in sessions["detail"]


# ---- fleet totals --------------------------------------------------------- #

def _engine(name: str, **values) -> dict:
    metrics = {key: {"value": value} for key, value in values.items()}
    return {"engine": name, "metrics": metrics}


def test_fleet_totals_sum_only_engines_that_answered() -> None:
    engines = [
        _engine("gufo", decode_tokens_per_second=40.0, prompt_tokens_per_second=10.0,
                kv_cache_usage_ratio=0.25, requests_processing=1.0, requests_deferred=0.0,
                tokens_total=100.0),
        _engine("vllm", decode_tokens_per_second=10.0, prompt_tokens_per_second=5.0,
                gpu_cache_usage=0.75, num_requests_running=2.0, num_requests_waiting=3.0,
                tokens_total=200.0),
        {},  # an engine whose poll failed
    ]
    totals = fleet_totals(engines)
    assert totals["engines_total"] == 3
    assert totals["engines_live"] == 2
    assert totals["decode_tokens_per_second"] == 50.0
    assert totals["prompt_tokens_per_second"] == 15.0
    assert totals["requests_active"] == 3.0
    assert totals["requests_waiting"] == 3.0
    assert totals["kv_cache_usage_max"] == 0.75, "cache pressure is a max, never a sum"
    assert totals["tokens_total"] == 300.0


def test_fleet_totals_report_the_gufo_vllm_seam() -> None:
    totals = fleet_totals([_engine("gufo"), _engine("vllm")])
    assert totals["seams"], "the two engines count prompt tokens differently; say so"


def test_fleet_totals_of_nothing_is_none_not_zero() -> None:
    totals = fleet_totals([])
    assert totals["decode_tokens_per_second"] is None
    assert totals["tokens_total"] is None
    assert totals["engines_live"] == 0


def test_series_names_are_the_parse_catalog() -> None:
    assert "tokens_generated_total" in series_names("gufo")
    assert "generation_tokens_total" in series_names("vllm")
    assert "generation_tokens_total" not in series_names("gufo")


def test_gufo_a_measured_zero_is_not_replaced_by_the_gauge(metrics_text) -> None:
    """`measured or gauge` made an idle engine advertise its last request's speed."""
    text = metrics_text("gufo")
    rates = RateEstimator(span=10)
    parse_gufo_metrics(text, rates, 1_000.0)
    metrics = parse_gufo_metrics(text, rates, 1_010.0)   # counters unchanged: truly idle
    decode = metrics["decode_tokens_per_second"]
    assert decode["value"] == 0.0
    assert "predicted_tokens_seconds" not in str(decode["source"]), "native gauge must not leak in"


def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None:
    text = metrics_text("vllm")
    rates = RateEstimator(span=10)
    parse_vllm_metrics(text, rates, 2_000.0)
    metrics = parse_vllm_metrics(text, rates, 2_010.0)
    decode = metrics["decode_tokens_per_second"]
    assert decode["value"] == 0.0
    assert "running/tpot" not in str(decode["source"])