ducoterra
3d5ccd7a39
**Phase 07 final verification pass — all tasks already in `complete/`; every criterion re-verified live.**
- **Old slot table gone**: no `#slots-card`/`#slots-table`/`renderSlots` in HTML/JS/CSS (only the new grid's `.slots-empty` *class*); pinned by `test_the_broken_slot_table_is_gone_from_html_js_and_css`.
- **`tests/fixtures/slots_gufo.json`**: present; live `GET ${GUFO_BASE_URL}/slots` this pass matches it key-for-key (row + `next_token[0]`), so the capture is reproducible; drift proven to fail (mutation tests + live `test_gufo_slots_answer_the_pinned_shape`, 9 live tests pass).
- **Aggregate block**: live page shows `2 of 2 engines`, running/waiting/prompt/decode/total-tokens straight from `fleet_totals()`, dot `ok`, no verdict colour; on a throwaway single-engine instance (:8091, private temp DB) `#fleet` is `hidden`, one full-width strip card + one panel, zero banners → "one clean half".
- **Strip + slot grid on live data**: 2 `.card.stat` (KV bars `bar ok` 13 %/0 %), gufo grid rendered real cards incl. a busy slot (`#2189`, 67849 prompt tok, 99 % cache share, `processing`), vLLM note-only with no grid; no `NaN`/`undefined` anywhere. Screenshots: `data/logs/llm-phase07-final-live-{two-engines,single-engine}.png`, `ups-charts-aria-labels.png`.
- **Defect found in earlier work and fixed**: `/ups`' four `LineChart`s shipped no `ariaLabel`, violating AGENTS.md's "every chart names itself for assistive tech" (`/system`+`/llm` were pinned, `/ups` wasn't) → added labels to `page-ups.js` and `test_every_ups_chart_names_itself_for_assistive_tech`; browser-verified `role="img"` + label on all four.
- **Gate**: `bash .agents/validate.sh` → **exit 0**; `uv run pytest --cov --cov-fail-under=94` → **604 passed, 9 skipped, coverage 97.91 %**; `uv run ruff check src tests agent` → All checks passed; `uv run pyright` → **0 errors**; `node --check` every page script ok; `uv run pytest -m live --live` → **9 passed**; `uv pip list --outdated` empty.
- **Docs**: README + AGENTS.md reviewed end-to-end; fixed test count (582→604), `/llm` source column + both architecture diagrams now name gufo `/slots`, AGENTS.md agent line gained `GET /schema` + status page, ariaLabel convention now covers all three chart pages, README Testing block now lists `bash .agents/validate.sh`. Env-var table, agent/install flags, Containerfile facts, `/api/*` shapes, range chips, remote URL all verified accurate — no other changes needed.
- **Live-infra**: no fleet/host/config change; the temp :8091 instance, its temp DB and the fake processes are removed; dev server restarted and `/api/health` `ok` (all 6 sources ok). One thing I deliberately did **not** touch: the gitignored local `.env.dev` still carries a comment claiming the NUT `[dashboards]` account has only battery-test `instcmds`, whereas AGENTS.md/README record `instcmds = all` (verified on hardware 2026-10-17) — operator-owned file, flag for the operator to reword.
- **Next pending phase**: `08_events_paging_and_copy`.
350 lines
16 KiB
Python
350 lines
16 KiB
Python
"""LLM engine parsing: real captures, provenance on every number, absent never 0.
|
|
|
|
The engines do not export the names people expect, so these tests are pinned to captures of
|
|
the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentation.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
import pytest
|
|
|
|
from dashboards.rates import RateEstimator
|
|
from dashboards.sources.llm import (
|
|
engine_result,
|
|
fleet_totals,
|
|
parse_gufo_metrics,
|
|
parse_vllm_metrics,
|
|
series_names,
|
|
)
|
|
from slots_contract import load_slots_capture
|
|
|
|
|
|
def bump_metric(text: str, name: str, delta: float) -> str:
|
|
"""Return the exposition with one counter increased — a scrape `dt` later."""
|
|
pattern = re.compile(rf"^({re.escape(name)}(?:\{{[^}}]*\}})?) (\S+)$", re.M)
|
|
|
|
def replace(match: re.Match[str]) -> str:
|
|
return f"{match.group(1)} {float(match.group(2)) + delta}"
|
|
|
|
bumped, count = pattern.subn(replace, text)
|
|
assert count, f"{name} not present in the capture"
|
|
return bumped
|
|
|
|
|
|
# ---- gufo / llama.cpp ----------------------------------------------------- #
|
|
|
|
def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
assert metrics["tokens_generated_total"]["value"] == 877_336.0
|
|
assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0
|
|
assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0
|
|
assert metrics["requests_processing"]["value"] == 0.0
|
|
assert metrics["device_lost_total"]["value"] == 0.0
|
|
assert metrics["kv_cache_usage_ratio"]["value"] == 0.0
|
|
|
|
|
|
def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None:
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
accept = metrics["spec_accept_rate"]["value"]
|
|
assert accept == pytest.approx(578_514 / 789_028)
|
|
assert str(metrics["spec_accept_rate"]["source"]).startswith("computed")
|
|
|
|
|
|
def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None:
|
|
"""gufo's prompt counter excludes cache hits; the lifetime total must not."""
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
total = metrics["tokens_total"]["value"]
|
|
assert total == 1_350_666 + 124_529_105 + 877_336
|
|
|
|
|
|
def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None:
|
|
"""llamacpp:predicted_tokens_seconds is ONE request's last pass — never the headline."""
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
now = 1_000.0
|
|
first = parse_gufo_metrics(text, rates, now)
|
|
assert "decode_tokens_per_second" not in first or first["decode_tokens_per_second"]["value"] is None
|
|
|
|
text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0)
|
|
metrics = parse_gufo_metrics(text, rates, now + 10.0)
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == pytest.approx(40.0)
|
|
assert decode["window"] == pytest.approx(10.0)
|
|
assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"]
|
|
# the native gauge is still there, labelled, because it answers a different question
|
|
assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)
|
|
|
|
|
|
def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None:
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
text = bump_metric(text, "llamacpp:prompt_tokens_total", 1_000.0)
|
|
metrics = parse_gufo_metrics(text, rates, 1_010.0)
|
|
assert metrics["prompt_tokens_per_second"]["value"] == pytest.approx(100.0)
|
|
|
|
|
|
def test_gufo_counter_reset_produces_n_a_not_a_spike(metrics_text) -> None:
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=60)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
restarted = bump_metric(text, "llamacpp:tokens_predicted_total", -800_000.0)
|
|
metrics = parse_gufo_metrics(restarted, rates, 1_010.0)
|
|
assert metrics.get("decode_tokens_per_second", {}).get("value") is None
|
|
|
|
|
|
def test_absent_gufo_metrics_are_none_with_no_source() -> None:
|
|
metrics = parse_gufo_metrics("# nothing here\n")
|
|
assert metrics["requests_processing"]["value"] is None
|
|
assert metrics["requests_processing"]["source"] is None
|
|
assert metrics.get("spec_accept_rate", {}).get("value") is None
|
|
|
|
|
|
# ---- severity: KV pressure arrives with a verdict ------------------------- #
|
|
|
|
def test_gufo_kv_cache_cell_keeps_provenance_and_gains_severity(metrics_text) -> None:
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
cell = metrics["kv_cache_usage_ratio"]
|
|
assert cell["value"] == 0.0
|
|
assert cell["source"] == "llamacpp:kv_cache_usage_ratio"
|
|
assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
|
|
assert cell["severity"] == "ok"
|
|
|
|
|
|
@pytest.mark.parametrize("ratio,expected", [
|
|
("0.74", "ok"), ("0.75", "warn"), ("0.89", "warn"), ("0.9", "fail"), ("1.0", "fail"),
|
|
])
|
|
def test_kv_ratio_is_classified_on_a_percent_scale(metrics_text, ratio, expected) -> None:
|
|
"""0.9 of a KV cache is 90 %, not 0.9 % — classifying the raw ratio is a permanent green."""
|
|
text = re.sub(r"^(llamacpp:kv_cache_usage_ratio) \S+$", rf"\g<1> {ratio}",
|
|
metrics_text("gufo"), count=1, flags=re.M)
|
|
metrics = parse_gufo_metrics(text)
|
|
assert metrics["kv_cache_usage_ratio"]["severity"] == expected
|
|
assert metrics["kv_cache_usage_ratio"]["value"] == pytest.approx(float(ratio)), \
|
|
"the scale is for the verdict only; the published value is untouched"
|
|
|
|
|
|
def test_absent_kv_cell_gets_no_verdict() -> None:
|
|
metrics = parse_gufo_metrics("# nothing here\n")
|
|
assert metrics["kv_cache_usage_ratio"]["severity"] is None
|
|
|
|
|
|
# ---- vLLM ----------------------------------------------------------------- #
|
|
|
|
def test_vllm_counters_and_gauges(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
assert metrics["prompt_tokens_total"]["value"] == pytest.approx(1.8389909e7)
|
|
assert metrics["generation_tokens_total"]["value"] == pytest.approx(611_163.0)
|
|
usage = metrics["gpu_cache_usage"]["value"]
|
|
assert usage is None or 0.0 <= usage <= 1.0, "a ratio summed over label sets would exceed 1"
|
|
assert metrics["num_requests_running"]["value"] is not None
|
|
assert metrics["num_preemptions_total"]["value"] is not None
|
|
|
|
|
|
def test_vllm_kv_cache_cell_carries_a_severity(metrics_text) -> None:
|
|
assert parse_vllm_metrics(metrics_text("vllm"))["gpu_cache_usage"]["severity"] == "ok"
|
|
pressed = bump_metric(metrics_text("vllm"), "vllm:kv_cache_usage_perc", 0.95)
|
|
cell = parse_vllm_metrics(pressed)["gpu_cache_usage"]
|
|
assert cell["value"] == pytest.approx(0.95)
|
|
assert cell["severity"] == "fail"
|
|
assert cell["source"] == "vllm:kv_cache_usage_perc"
|
|
|
|
|
|
def test_vllm_absent_kv_cell_gets_no_verdict(metrics_text) -> None:
|
|
stripped = "\n".join(line for line in metrics_text("vllm").splitlines()
|
|
if "kv_cache_usage" not in line and "gpu_cache_usage" not in line)
|
|
metrics = parse_vllm_metrics(stripped)
|
|
assert metrics["gpu_cache_usage"]["value"] is None
|
|
assert metrics["gpu_cache_usage"]["severity"] is None
|
|
|
|
|
|
def test_vllm_latency_histograms(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
ttft = metrics["time_to_first_token_seconds"]
|
|
assert ttft and ttft["count"] > 0 and ttft["mean"] > 0
|
|
assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"]
|
|
assert metrics["time_per_output_token_seconds"]["mean"] > 0
|
|
|
|
|
|
def test_vllm_per_request_speed_is_the_reciprocal_of_tpot(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
tpot = metrics["time_per_output_token_seconds"]["mean"]
|
|
per_request = metrics["per_request_decode_tokens_per_second"]
|
|
assert per_request["value"] == pytest.approx(1.0 / tpot)
|
|
assert per_request["derived"] is True
|
|
|
|
|
|
def test_vllm_prefix_cache_hit_ratio(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
ratio = metrics["prefix_cache_hit_ratio"]["value"]
|
|
assert ratio is not None and 0.0 <= ratio <= 1.0
|
|
assert "hits" in metrics["prefix_cache_hit_ratio"]["source"]
|
|
|
|
|
|
def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) -> None:
|
|
text = metrics_text("vllm")
|
|
rates = RateEstimator(span=10)
|
|
parse_vllm_metrics(text, rates, 2_000.0)
|
|
text = bump_metric(text, "vllm:generation_tokens_total", 250.0)
|
|
metrics = parse_vllm_metrics(text, rates, 2_010.0)
|
|
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0)
|
|
|
|
|
|
def test_vllm_waiting_requests_are_broken_down_by_reason(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
reasons = metrics["waiting_by_reason"]["value"]
|
|
assert isinstance(reasons, dict)
|
|
|
|
|
|
def test_vllm_empty_exposition_is_all_none() -> None:
|
|
metrics = parse_vllm_metrics("")
|
|
assert metrics["num_requests_running"]["value"] is None
|
|
assert metrics["time_to_first_token_seconds"] is None
|
|
assert metrics["per_request_decode_tokens_per_second"]["value"] is None
|
|
|
|
|
|
# ---- Result assembly ------------------------------------------------------ #
|
|
|
|
def test_engine_result_payload_and_samples(metrics_text) -> None:
|
|
rates = RateEstimator(span=10)
|
|
result = engine_result("gufo", "http://gufo.internal:8731", metrics_text("gufo"), rates,
|
|
now=3_000.0)
|
|
payload = result.payload
|
|
assert payload["engine"] == "gufo"
|
|
assert payload["upstream"] == "http://gufo.internal:8731"
|
|
assert payload["series_count"] > 10
|
|
assert payload["unparsed"] == 0
|
|
assert payload["sessions"]["detail"], "a missing /slots must be explained, not blank"
|
|
for sample in result.samples:
|
|
assert sample.source == "llm" and sample.host == "gufo"
|
|
assert sample.metric.startswith("llm.")
|
|
assert sample.value == sample.value and sample.value is not None # no NaN
|
|
|
|
|
|
def test_engine_result_charts_a_measured_rate_after_the_second_poll(metrics_text) -> None:
|
|
rates = RateEstimator(span=10)
|
|
engine_result("gufo", "http://g:1", metrics_text("gufo"), rates, now=3_000.0)
|
|
bumped = bump_metric(metrics_text("gufo"), "llamacpp:tokens_predicted_total", 100.0)
|
|
result = engine_result("gufo", "http://g:1", bumped, rates, now=3_010.0)
|
|
metrics = result.payload["metrics"]
|
|
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(10.0)
|
|
assert any(s.metric == "llm.decode_tokens_per_second" for s in result.samples)
|
|
|
|
|
|
def test_vllm_stores_the_tpot_series_the_cell_sparkline_draws(metrics_text) -> None:
|
|
"""Phase 04 added `llm.tpot_seconds` so the vLLM TPOT row has a shape, not a dead slot.
|
|
|
|
Hermetic twin of the live check: the captured exposition must resolve the histogram the
|
|
page charts, or `/llm` shows a permanent "no history" next to a real mean.
|
|
"""
|
|
result = engine_result("vllm", "http://v:1", metrics_text("vllm"), RateEstimator(), now=1.0)
|
|
mean = result.payload["metrics"]["time_per_output_token_seconds__mean"]["value"]
|
|
assert mean is not None, "the vLLM capture lost its TPOT histogram — re-capture fixtures"
|
|
stored = {s.metric: s.value for s in result.samples}
|
|
assert stored["llm.tpot_seconds"] == pytest.approx(mean)
|
|
# gufo publishes no output-token histogram, so it stores no TPOT series (never a 0)
|
|
gufo = engine_result("gufo", "http://g:1", metrics_text("gufo"), RateEstimator(), now=1.0)
|
|
assert "llm.tpot_seconds" not in {s.metric for s in gufo.samples}
|
|
|
|
|
|
def test_gufo_stores_the_spec_accept_series_the_cell_sparkline_draws(metrics_text) -> None:
|
|
"""`llm.spec_accept_rate` is the gufo-only half of the phase-04 history addition."""
|
|
result = engine_result("gufo", "http://g:1", metrics_text("gufo"), RateEstimator(), now=1.0)
|
|
ratio = result.payload["metrics"]["spec_accept_rate"]["value"]
|
|
assert ratio is not None and 0.0 < ratio <= 1.0
|
|
stored = {s.metric: s.value for s in result.samples}
|
|
assert stored["llm.spec_accept_rate"] == pytest.approx(ratio)
|
|
vllm = engine_result("vllm", "http://v:1", metrics_text("vllm"), RateEstimator(), now=1.0)
|
|
assert "llm.spec_accept_rate" not in {s.metric for s in vllm.samples}
|
|
|
|
|
|
def test_sessions_pass_through_slots_for_gufo() -> None:
|
|
"""`sessions.list` is the raw gufo rows — the grid, not the source, reads slot fields.
|
|
|
|
The rows are phase 07's live capture (`tests/fixtures/slots_gufo.json`), not a guessed
|
|
shape: the old table's `idle`/`n_prompt`/`n_predict` names were invented and the server
|
|
never answered them. `_sessions()` must pass the rows through untouched — the shape is
|
|
contracted separately in `tests/slots_contract.py`.
|
|
"""
|
|
slots = load_slots_capture()
|
|
result = engine_result("gufo", "http://g:1", "", RateEstimator(), slots=slots, now=1.0)
|
|
sessions = result.payload["sessions"]
|
|
assert sessions["list"] == slots
|
|
assert sessions["list"][0]["is_processing"] is False, "the passthrough re-shaped the rows"
|
|
assert sessions["source"] == "gufo /slots"
|
|
|
|
|
|
def test_sessions_say_vllm_has_no_per_session_view() -> None:
|
|
result = engine_result("vllm", "http://v:1", "", RateEstimator(), now=1.0)
|
|
sessions = result.payload["sessions"]
|
|
assert sessions["list"] == []
|
|
assert "vLLM" in sessions["detail"]
|
|
|
|
|
|
# ---- fleet totals --------------------------------------------------------- #
|
|
|
|
def _engine(name: str, **values) -> dict:
|
|
metrics = {key: {"value": value} for key, value in values.items()}
|
|
return {"engine": name, "metrics": metrics}
|
|
|
|
|
|
def test_fleet_totals_sum_only_engines_that_answered() -> None:
|
|
engines = [
|
|
_engine("gufo", decode_tokens_per_second=40.0, prompt_tokens_per_second=10.0,
|
|
kv_cache_usage_ratio=0.25, requests_processing=1.0, requests_deferred=0.0,
|
|
tokens_total=100.0),
|
|
_engine("vllm", decode_tokens_per_second=10.0, prompt_tokens_per_second=5.0,
|
|
gpu_cache_usage=0.75, num_requests_running=2.0, num_requests_waiting=3.0,
|
|
tokens_total=200.0),
|
|
{}, # an engine whose poll failed
|
|
]
|
|
totals = fleet_totals(engines)
|
|
assert totals["engines_total"] == 3
|
|
assert totals["engines_live"] == 2
|
|
assert totals["decode_tokens_per_second"] == 50.0
|
|
assert totals["prompt_tokens_per_second"] == 15.0
|
|
assert totals["requests_active"] == 3.0
|
|
assert totals["requests_waiting"] == 3.0
|
|
assert totals["kv_cache_usage_max"] == 0.75, "cache pressure is a max, never a sum"
|
|
assert totals["tokens_total"] == 300.0
|
|
|
|
|
|
def test_fleet_totals_report_the_gufo_vllm_seam() -> None:
|
|
totals = fleet_totals([_engine("gufo"), _engine("vllm")])
|
|
assert totals["seams"], "the two engines count prompt tokens differently; say so"
|
|
|
|
|
|
def test_fleet_totals_of_nothing_is_none_not_zero() -> None:
|
|
totals = fleet_totals([])
|
|
assert totals["decode_tokens_per_second"] is None
|
|
assert totals["tokens_total"] is None
|
|
assert totals["engines_live"] == 0
|
|
|
|
|
|
def test_series_names_are_the_parse_catalog() -> None:
|
|
assert "tokens_generated_total" in series_names("gufo")
|
|
assert "generation_tokens_total" in series_names("vllm")
|
|
assert "generation_tokens_total" not in series_names("gufo")
|
|
|
|
|
|
def test_gufo_a_measured_zero_is_not_replaced_by_the_gauge(metrics_text) -> None:
|
|
"""`measured or gauge` made an idle engine advertise its last request's speed."""
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
metrics = parse_gufo_metrics(text, rates, 1_010.0) # counters unchanged: truly idle
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == 0.0
|
|
assert "predicted_tokens_seconds" not in str(decode["source"]), "native gauge must not leak in"
|
|
|
|
|
|
def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None:
|
|
text = metrics_text("vllm")
|
|
rates = RateEstimator(span=10)
|
|
parse_vllm_metrics(text, rates, 2_000.0)
|
|
metrics = parse_vllm_metrics(text, rates, 2_010.0)
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == 0.0
|
|
assert "running/tpot" not in str(decode["source"])
|