"""Ground truth: the payloads captured from the DEPLOYED engines. Neither engine exports the series names people expect, and gufo's `/slots` is a bare JSON array. These fixtures are verbatim captures (gufo on deskwork-ai:8731, vLLM on driveripper-ai:8000); if a field stops mapping, the engine renamed something and this test tells you which one instead of the dashboard quietly showing `n/a` in production. """ from __future__ import annotations import json import re from pathlib import Path import pytest from app import RateEstimator, parse_gufo_metrics, parse_vllm_metrics FIXTURES = Path(__file__).resolve().parent / "fixtures" @pytest.fixture(scope="module") def gufo_text() -> str: return (FIXTURES / "gufo_metrics_real.txt").read_text() @pytest.fixture(scope="module") def vllm_text() -> str: return (FIXTURES / "vllm_metrics_real.txt").read_text() # --------------------------------------------------------------------------- # # gufo # --------------------------------------------------------------------------- # def test_gufo_real_names_are_the_primary_aliases(gufo_text: str) -> None: m = parse_gufo_metrics(gufo_text) assert m["tokens_prompt_total"]["source"] == "llamacpp:prompt_tokens_total" assert m["tokens_generated_total"]["source"] == "llamacpp:tokens_predicted_total" # gufo names its speed gauges *_seconds while the unit is tokens/second assert m["prompt_tokens_per_second_native"]["source"] == "llamacpp:prompt_tokens_seconds" assert m["decode_tokens_per_second_native"]["source"] == "llamacpp:predicted_tokens_seconds" # parsed without an estimator, both throughput cells degrade to those one-request gauges assert m["prompt_tokens_per_second"]["source"] == "llamacpp:prompt_tokens_seconds" # parsed without an estimator, the throughput cell degrades to that one-request gauge assert m["decode_tokens_per_second"]["source"] == "llamacpp:predicted_tokens_seconds" def test_gufo_aggregate_decode_rate_resolves_on_the_real_counter(gufo_text: str) -> None: """The Δ-rates key off the RESOLVED counter names, so they must work on gufo's real names (`tokens_predicted_total`, `prompt_tokens_total`) and land in the same ballpark as gufo's own busy-time counter (`tokens_predicted_seconds_total`). """ rates = RateEstimator() parse_gufo_metrics(gufo_text, rates, now=0.0) bumped = gufo_text.replace( "llamacpp:tokens_predicted_total 173091", "llamacpp:tokens_predicted_total 173191" ).replace( # +1000 generated and +1000 prompt tokens over 2s -> 500 tok/s of each "llamacpp:prompt_tokens_total 374263", "llamacpp:prompt_tokens_total 375263" ) assert bumped != gufo_text # the capture moved the counter; re-check this test m = parse_gufo_metrics(bumped, rates, now=2.0) d = m["decode_tokens_per_second"] assert d["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt" assert d["value"] == pytest.approx(50.0) # sanity: the lifetime mean from the two cumulative counters is the same order of # magnitude, so the windowed aggregate is not quietly summing something else busy = float(re.search(r"tokens_predicted_seconds_total ([0-9.]+)", gufo_text).group(1)) assert 5.0 < (173091.0 / busy) < 100.0 # prompt throughput from the same two scrapes: gufo's prompt counter EXCLUDES cache hits, # so this is newly-computed prefill work across all slots, not prompt tokens accepted p = m["prompt_tokens_per_second"] assert p["source"] == "computed:Δllamacpp:prompt_tokens_total/Δt" assert p["value"] == pytest.approx(500.0) assert m["prompt_tokens_per_second_native"]["value"] == pytest.approx(820.2841172907755) def test_gufo_every_banner_and_strip_field_resolves(gufo_text: str) -> None: m = parse_gufo_metrics(gufo_text) for key in ( "requests_processing", "requests_deferred", "kv_cache_usage_ratio", "tokens_prompt_total", "tokens_generated_total", "prompt_tokens_per_second", "decode_tokens_per_second", "tokens_total", "n_tokens_max", "device_lost_total", ): assert m[key]["value"] is not None, f"gufo {key} stopped resolving" assert 0.0 <= m["kv_cache_usage_ratio"]["value"] <= 1.0 def test_gufo_totals_include_cache_hits_which_are_a_separate_counter(gufo_text: str) -> None: m = parse_gufo_metrics(gufo_text) prompt = m["tokens_prompt_total"]["value"] cached = m["tokens_prompt_cached_total"]["value"] generated = m["tokens_generated_total"]["value"] assert m["tokens_total"]["value"] == pytest.approx(prompt + cached + generated) # 29.6M cached vs 0.37M computed on the live box — cache hits dominate assert m["prompt_cache_hit_ratio"]["value"] == pytest.approx(cached / (prompt + cached)) assert 0.0 < m["prompt_cache_hit_ratio"]["value"] < 1.0 def test_gufo_speculative_decoding_accept_rate(gufo_text: str) -> None: m = parse_gufo_metrics(gufo_text) accept = m["spec_accept_rate"] assert accept["derived"] is True assert 0.0 <= accept["value"] <= 1.0 def test_gufo_device_lost_is_zero_on_a_healthy_box(gufo_text: str) -> None: assert parse_gufo_metrics(gufo_text)["device_lost_total"]["value"] == 0.0 # --------------------------------------------------------------------------- # # gufo /slots # --------------------------------------------------------------------------- # def test_gufo_slots_is_a_bare_array_not_an_object() -> None: """The UI must not assume {"slots": [...]} — the deployed engine returns a list.""" data = json.loads((FIXTURES / "gufo_slots_real.json").read_text()) assert isinstance(data, list) assert len(data) >= 1 assert all(isinstance(slot, dict) and "id" in slot for slot in data) def test_gufo_slot_fields_the_cards_depend_on() -> None: data = json.loads((FIXTURES / "gufo_slots_real.json").read_text()) slot = data[0] for key in ( "id", "state", "is_processing", "id_task", "n_prompt_tokens", "n_prompt_tokens_cache", "n_ctx", "next_token", "model", ): assert key in slot, f"gufo slot no longer reports {key}" assert slot["next_token"], "next_token[] should still be present, with n_decoded/n_remain" assert "n_decoded" in slot["next_token"][0] # an idle slot reports -1 for "nothing pending", which is not a count if not slot["is_processing"]: assert slot["id_task"] == -1 # --------------------------------------------------------------------------- # # vLLM # --------------------------------------------------------------------------- # def test_vllm_real_names_are_the_primary_aliases(vllm_text: str) -> None: m = parse_vllm_metrics(vllm_text) assert m["gpu_cache_usage"]["source"] == "vllm:kv_cache_usage_perc" # not gpu_cache_usage assert m["generation_tokens_total"]["source"] == "vllm:generation_tokens_total" assert m["time_per_output_token_seconds"]["source"] == "vllm:inter_token_latency_seconds" def test_vllm_every_banner_and_card_field_resolves(vllm_text: str) -> None: m = parse_vllm_metrics(vllm_text) for key in ( "num_requests_running", "num_requests_waiting", "gpu_cache_usage", "prompt_tokens_total", "generation_tokens_total", "tokens_total", "waiting_capacity", "prefix_cache_hit_ratio", "num_preemptions_total", ): assert m[key]["value"] is not None, f"vLLM {key} stopped resolving" for key in ( "time_to_first_token_seconds__mean", "time_per_output_token_seconds__mean", "request_queue_time_seconds__mean", ): assert m[key]["value"] is not None, f"vLLM {key} stopped resolving" assert 0.0 <= m["gpu_cache_usage"]["value"] <= 1.0 assert 0.0 <= m["prefix_cache_hit_ratio"]["value"] <= 1.0 def test_vllm_waiting_reasons_from_the_live_labels(vllm_text: str) -> None: # the deployed build tags reasons as capacity/deferred, not defragmentation m = parse_vllm_metrics(vllm_text) assert "capacity" in m["waiting_by_reason"]["value"] def test_vllm_histograms_survive_engine_and_model_labels(vllm_text: str) -> None: m = parse_vllm_metrics(vllm_text) ttft = m["time_to_first_token_seconds"] assert ttft["count"] > 0 and ttft["mean"] > 0 # single engine label set here, so an interpolated p95 is legitimate assert ttft["p95"] is None or ttft["p95"] >= ttft["mean"] def test_vllm_prompt_tok_s_is_answered_even_without_a_rate_gauge(vllm_text: str) -> None: # this build exports no throughput series at all, so prompt tok/s comes from the # prefill histograms; it must not render n/a on a healthy, busy engine m = parse_vllm_metrics(vllm_text) p = m["prompt_tokens_per_second"] assert p["value"] is not None, "prompt tok/s regressed to n/a" assert p["derived"] is True assert p["source"] == "computed:mean(request_prompt_tokens)/mean(request_prefill_time_seconds)" assert p["value"] > 1000 # live box sits around 4.7k tok/s; single digits mean a bug assert m["prefill_tokens_per_second_lifetime"]["value"] == pytest.approx(p["value"]) def test_vllm_prompt_tok_s_switches_to_measured_rate_once_scraping(vllm_text: str) -> None: from app import RateEstimator rates = RateEstimator() parse_vllm_metrics(vllm_text, rates, now=1000.0) m = parse_vllm_metrics(vllm_text, rates, now=1001.0) # same counters -> zero throughput p = m["prompt_tokens_per_second"] assert p["source"] == "computed:Δvllm:prompt_tokens_total/Δt" assert p["window"] == pytest.approx(1.0) # unchanged capture -> genuinely zero throughput in that window, not a missing gauge assert p["value"] == 0.0 def test_vllm_decode_throughput_is_derived_on_this_build(vllm_text: str) -> None: # no vllm:*throughput gauge exists on the deployed version -> computed from TPOT m = parse_vllm_metrics(vllm_text) assert m["decode_tokens_per_second"]["derived"] is True assert m["decode_tokens_per_second"]["source"] == "computed:running/tpot"