ducoterra
f44a617ef9
**Phase 04 final verification pass — all tasks already in `complete/`; verified + fixed 3 defects found in verification.**
- Verified `/llm` live (both engines, gufo-only instance on :8099, `llm_monitor` on :8095) in a real browser; dev server left running/healthy.
- Completed task-04's truncated layout fix: `#engines` now carries `.items-start` (the CSS rule existed but nothing used it) — gufo 544 px vs vLLM 715 px, no blank card; pinned by `test_llm_engine_panels_are_not_stretched_to_their_tallest_sibling`.
- Fixed: event rows said the subject twice ("gufo gufo is answering again"), and `/ups` mangled mid-message names ("sent to apc: OK" → "sent to OK") → shared `Dash.eventText()` (strips `UpstreamError: ` + a *leading* name only, mirroring `alerts._clean_error`) used by `/system`, `/llm`, `/ups`; new test.
- Fixed: count sparklines autoscaled rollup averages (queue max 0.1379) into full-height spikes → `sparkline({minSpan})`, `minSpan: 1` on sessions/queued (spike 30 px → 4 px); assertions added.
- Tests: `bash .agents/validate.sh` → **EXIT=0** (`uv run pytest --cov --cov-fail-under=94`: **485 passed, 7 skipped**, coverage **96.87%**; `ruff check src tests agent`: All checks passed; `pyright`: 0 errors; `node --check` all static JS clean; repo hygiene clean; `uv pip list --outdated` empty). `uv run pytest -m live --live` → **7 passed**.
- Completion criteria: heroes largest per engine ✔ (38.4 px vs 16.8 px row values); sparkline per cell + selected window ✔ (5/5 gufo, 7/7 vLLM; 6 h chip moved KV max 53 %→58 %); mean+p95 without hover ✔ (243 / 48.8 / 285 ms feet); provenance tag ✔ (`Δ10.0s`/`Δ8.0s` + `title`=exact source); single-engine ✔ (1 panel, totals hidden, zero red); screenshots ✔ `data/logs/llm-verify-{both-engines,charts-and-slots,single-engine,monitor-reference}.png` + `NOTES-chart-set.md`; pytest/cov/ruff/pyright ✔; earlier phases unchanged-behaviour ✔ (all 01–03 tests pass).
- Deviation (documented, not code): throughput heroes stay **neutral ink** — `severity.py` has no throughput row and 0 tok/s idle is healthy; colour comes from rated cells (KV) + reachability. Kept per `NOTES-chart-set`.
- Deviation: could not append the task-04 written verdict to `NOTES-chart-set.md` (it lives in `todo/`, which I must not edit). Verdict: ours reads better than `:8095` — bigger throughput numbers, per-cell shape over the selected window, percentiles inline; `llm_monitor` only sparklines decode+KV and has no window control.
- Docs: AGENTS.md updated (485 test count, `sparkline` `minSpan` rule, `Dash.eventText` event-row rule, `/llm` panel `items-start`); README reviewed in full — commands, config table (all `config.py` vars present), architecture bullets, page descriptions verified accurate; no other inconsistencies found.
- Note for the harness: `.agents/phases/todo/03…/NOTES-verify.md` and `todo/04…/NOTES-chart-set.md` are untracked notes left behind by their phases (task files moved, notes did not).
- Next pending phase: `05_ups_hero_battery_glyph`.
341 lines
15 KiB
Python
341 lines
15 KiB
Python
"""LLM engine parsing: real captures, provenance on every number, absent never 0.
|
|
|
|
The engines do not export the names people expect, so these tests are pinned to captures of
|
|
the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentation.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
import pytest
|
|
|
|
from dashboards.rates import RateEstimator
|
|
from dashboards.sources.llm import (
|
|
engine_result,
|
|
fleet_totals,
|
|
parse_gufo_metrics,
|
|
parse_vllm_metrics,
|
|
series_names,
|
|
)
|
|
|
|
|
|
def bump_metric(text: str, name: str, delta: float) -> str:
|
|
"""Return the exposition with one counter increased — a scrape `dt` later."""
|
|
pattern = re.compile(rf"^({re.escape(name)}(?:\{{[^}}]*\}})?) (\S+)$", re.M)
|
|
|
|
def replace(match: re.Match[str]) -> str:
|
|
return f"{match.group(1)} {float(match.group(2)) + delta}"
|
|
|
|
bumped, count = pattern.subn(replace, text)
|
|
assert count, f"{name} not present in the capture"
|
|
return bumped
|
|
|
|
|
|
# ---- gufo / llama.cpp ----------------------------------------------------- #
|
|
|
|
def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
assert metrics["tokens_generated_total"]["value"] == 877_336.0
|
|
assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0
|
|
assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0
|
|
assert metrics["requests_processing"]["value"] == 0.0
|
|
assert metrics["device_lost_total"]["value"] == 0.0
|
|
assert metrics["kv_cache_usage_ratio"]["value"] == 0.0
|
|
|
|
|
|
def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None:
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
accept = metrics["spec_accept_rate"]["value"]
|
|
assert accept == pytest.approx(578_514 / 789_028)
|
|
assert str(metrics["spec_accept_rate"]["source"]).startswith("computed")
|
|
|
|
|
|
def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None:
|
|
"""gufo's prompt counter excludes cache hits; the lifetime total must not."""
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
total = metrics["tokens_total"]["value"]
|
|
assert total == 1_350_666 + 124_529_105 + 877_336
|
|
|
|
|
|
def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None:
|
|
"""llamacpp:predicted_tokens_seconds is ONE request's last pass — never the headline."""
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
now = 1_000.0
|
|
first = parse_gufo_metrics(text, rates, now)
|
|
assert "decode_tokens_per_second" not in first or first["decode_tokens_per_second"]["value"] is None
|
|
|
|
text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0)
|
|
metrics = parse_gufo_metrics(text, rates, now + 10.0)
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == pytest.approx(40.0)
|
|
assert decode["window"] == pytest.approx(10.0)
|
|
assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"]
|
|
# the native gauge is still there, labelled, because it answers a different question
|
|
assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)
|
|
|
|
|
|
def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None:
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
text = bump_metric(text, "llamacpp:prompt_tokens_total", 1_000.0)
|
|
metrics = parse_gufo_metrics(text, rates, 1_010.0)
|
|
assert metrics["prompt_tokens_per_second"]["value"] == pytest.approx(100.0)
|
|
|
|
|
|
def test_gufo_counter_reset_produces_n_a_not_a_spike(metrics_text) -> None:
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=60)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
restarted = bump_metric(text, "llamacpp:tokens_predicted_total", -800_000.0)
|
|
metrics = parse_gufo_metrics(restarted, rates, 1_010.0)
|
|
assert metrics.get("decode_tokens_per_second", {}).get("value") is None
|
|
|
|
|
|
def test_absent_gufo_metrics_are_none_with_no_source() -> None:
|
|
metrics = parse_gufo_metrics("# nothing here\n")
|
|
assert metrics["requests_processing"]["value"] is None
|
|
assert metrics["requests_processing"]["source"] is None
|
|
assert metrics.get("spec_accept_rate", {}).get("value") is None
|
|
|
|
|
|
# ---- severity: KV pressure arrives with a verdict ------------------------- #
|
|
|
|
def test_gufo_kv_cache_cell_keeps_provenance_and_gains_severity(metrics_text) -> None:
|
|
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
|
cell = metrics["kv_cache_usage_ratio"]
|
|
assert cell["value"] == 0.0
|
|
assert cell["source"] == "llamacpp:kv_cache_usage_ratio"
|
|
assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
|
|
assert cell["severity"] == "ok"
|
|
|
|
|
|
@pytest.mark.parametrize("ratio,expected", [
|
|
("0.74", "ok"), ("0.75", "warn"), ("0.89", "warn"), ("0.9", "fail"), ("1.0", "fail"),
|
|
])
|
|
def test_kv_ratio_is_classified_on_a_percent_scale(metrics_text, ratio, expected) -> None:
|
|
"""0.9 of a KV cache is 90 %, not 0.9 % — classifying the raw ratio is a permanent green."""
|
|
text = re.sub(r"^(llamacpp:kv_cache_usage_ratio) \S+$", rf"\g<1> {ratio}",
|
|
metrics_text("gufo"), count=1, flags=re.M)
|
|
metrics = parse_gufo_metrics(text)
|
|
assert metrics["kv_cache_usage_ratio"]["severity"] == expected
|
|
assert metrics["kv_cache_usage_ratio"]["value"] == pytest.approx(float(ratio)), \
|
|
"the scale is for the verdict only; the published value is untouched"
|
|
|
|
|
|
def test_absent_kv_cell_gets_no_verdict() -> None:
|
|
metrics = parse_gufo_metrics("# nothing here\n")
|
|
assert metrics["kv_cache_usage_ratio"]["severity"] is None
|
|
|
|
|
|
# ---- vLLM ----------------------------------------------------------------- #
|
|
|
|
def test_vllm_counters_and_gauges(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
assert metrics["prompt_tokens_total"]["value"] == pytest.approx(1.8389909e7)
|
|
assert metrics["generation_tokens_total"]["value"] == pytest.approx(611_163.0)
|
|
usage = metrics["gpu_cache_usage"]["value"]
|
|
assert usage is None or 0.0 <= usage <= 1.0, "a ratio summed over label sets would exceed 1"
|
|
assert metrics["num_requests_running"]["value"] is not None
|
|
assert metrics["num_preemptions_total"]["value"] is not None
|
|
|
|
|
|
def test_vllm_kv_cache_cell_carries_a_severity(metrics_text) -> None:
|
|
assert parse_vllm_metrics(metrics_text("vllm"))["gpu_cache_usage"]["severity"] == "ok"
|
|
pressed = bump_metric(metrics_text("vllm"), "vllm:kv_cache_usage_perc", 0.95)
|
|
cell = parse_vllm_metrics(pressed)["gpu_cache_usage"]
|
|
assert cell["value"] == pytest.approx(0.95)
|
|
assert cell["severity"] == "fail"
|
|
assert cell["source"] == "vllm:kv_cache_usage_perc"
|
|
|
|
|
|
def test_vllm_absent_kv_cell_gets_no_verdict(metrics_text) -> None:
|
|
stripped = "\n".join(line for line in metrics_text("vllm").splitlines()
|
|
if "kv_cache_usage" not in line and "gpu_cache_usage" not in line)
|
|
metrics = parse_vllm_metrics(stripped)
|
|
assert metrics["gpu_cache_usage"]["value"] is None
|
|
assert metrics["gpu_cache_usage"]["severity"] is None
|
|
|
|
|
|
def test_vllm_latency_histograms(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
ttft = metrics["time_to_first_token_seconds"]
|
|
assert ttft and ttft["count"] > 0 and ttft["mean"] > 0
|
|
assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"]
|
|
assert metrics["time_per_output_token_seconds"]["mean"] > 0
|
|
|
|
|
|
def test_vllm_per_request_speed_is_the_reciprocal_of_tpot(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
tpot = metrics["time_per_output_token_seconds"]["mean"]
|
|
per_request = metrics["per_request_decode_tokens_per_second"]
|
|
assert per_request["value"] == pytest.approx(1.0 / tpot)
|
|
assert per_request["derived"] is True
|
|
|
|
|
|
def test_vllm_prefix_cache_hit_ratio(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
ratio = metrics["prefix_cache_hit_ratio"]["value"]
|
|
assert ratio is not None and 0.0 <= ratio <= 1.0
|
|
assert "hits" in metrics["prefix_cache_hit_ratio"]["source"]
|
|
|
|
|
|
def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) -> None:
|
|
text = metrics_text("vllm")
|
|
rates = RateEstimator(span=10)
|
|
parse_vllm_metrics(text, rates, 2_000.0)
|
|
text = bump_metric(text, "vllm:generation_tokens_total", 250.0)
|
|
metrics = parse_vllm_metrics(text, rates, 2_010.0)
|
|
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0)
|
|
|
|
|
|
def test_vllm_waiting_requests_are_broken_down_by_reason(metrics_text) -> None:
|
|
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
|
reasons = metrics["waiting_by_reason"]["value"]
|
|
assert isinstance(reasons, dict)
|
|
|
|
|
|
def test_vllm_empty_exposition_is_all_none() -> None:
|
|
metrics = parse_vllm_metrics("")
|
|
assert metrics["num_requests_running"]["value"] is None
|
|
assert metrics["time_to_first_token_seconds"] is None
|
|
assert metrics["per_request_decode_tokens_per_second"]["value"] is None
|
|
|
|
|
|
# ---- Result assembly ------------------------------------------------------ #
|
|
|
|
def test_engine_result_payload_and_samples(metrics_text) -> None:
|
|
rates = RateEstimator(span=10)
|
|
result = engine_result("gufo", "http://gufo.internal:8731", metrics_text("gufo"), rates,
|
|
now=3_000.0)
|
|
payload = result.payload
|
|
assert payload["engine"] == "gufo"
|
|
assert payload["upstream"] == "http://gufo.internal:8731"
|
|
assert payload["series_count"] > 10
|
|
assert payload["unparsed"] == 0
|
|
assert payload["sessions"]["detail"], "a missing /slots must be explained, not blank"
|
|
for sample in result.samples:
|
|
assert sample.source == "llm" and sample.host == "gufo"
|
|
assert sample.metric.startswith("llm.")
|
|
assert sample.value == sample.value and sample.value is not None # no NaN
|
|
|
|
|
|
def test_engine_result_charts_a_measured_rate_after_the_second_poll(metrics_text) -> None:
|
|
rates = RateEstimator(span=10)
|
|
engine_result("gufo", "http://g:1", metrics_text("gufo"), rates, now=3_000.0)
|
|
bumped = bump_metric(metrics_text("gufo"), "llamacpp:tokens_predicted_total", 100.0)
|
|
result = engine_result("gufo", "http://g:1", bumped, rates, now=3_010.0)
|
|
metrics = result.payload["metrics"]
|
|
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(10.0)
|
|
assert any(s.metric == "llm.decode_tokens_per_second" for s in result.samples)
|
|
|
|
|
|
def test_vllm_stores_the_tpot_series_the_cell_sparkline_draws(metrics_text) -> None:
|
|
"""Phase 04 added `llm.tpot_seconds` so the vLLM TPOT row has a shape, not a dead slot.
|
|
|
|
Hermetic twin of the live check: the captured exposition must resolve the histogram the
|
|
page charts, or `/llm` shows a permanent "no history" next to a real mean.
|
|
"""
|
|
result = engine_result("vllm", "http://v:1", metrics_text("vllm"), RateEstimator(), now=1.0)
|
|
mean = result.payload["metrics"]["time_per_output_token_seconds__mean"]["value"]
|
|
assert mean is not None, "the vLLM capture lost its TPOT histogram — re-capture fixtures"
|
|
stored = {s.metric: s.value for s in result.samples}
|
|
assert stored["llm.tpot_seconds"] == pytest.approx(mean)
|
|
# gufo publishes no output-token histogram, so it stores no TPOT series (never a 0)
|
|
gufo = engine_result("gufo", "http://g:1", metrics_text("gufo"), RateEstimator(), now=1.0)
|
|
assert "llm.tpot_seconds" not in {s.metric for s in gufo.samples}
|
|
|
|
|
|
def test_gufo_stores_the_spec_accept_series_the_cell_sparkline_draws(metrics_text) -> None:
|
|
"""`llm.spec_accept_rate` is the gufo-only half of the phase-04 history addition."""
|
|
result = engine_result("gufo", "http://g:1", metrics_text("gufo"), RateEstimator(), now=1.0)
|
|
ratio = result.payload["metrics"]["spec_accept_rate"]["value"]
|
|
assert ratio is not None and 0.0 < ratio <= 1.0
|
|
stored = {s.metric: s.value for s in result.samples}
|
|
assert stored["llm.spec_accept_rate"] == pytest.approx(ratio)
|
|
vllm = engine_result("vllm", "http://v:1", metrics_text("vllm"), RateEstimator(), now=1.0)
|
|
assert "llm.spec_accept_rate" not in {s.metric for s in vllm.samples}
|
|
|
|
|
|
def test_sessions_pass_through_slots_for_gufo() -> None:
|
|
slots = [{"id": 0, "idle": False, "n_prompt": 12, "n_predict": 40}]
|
|
result = engine_result("gufo", "http://g:1", "", RateEstimator(), slots=slots, now=1.0)
|
|
sessions = result.payload["sessions"]
|
|
assert sessions["list"] == slots
|
|
assert sessions["source"] == "gufo /slots"
|
|
|
|
|
|
def test_sessions_say_vllm_has_no_per_session_view() -> None:
|
|
result = engine_result("vllm", "http://v:1", "", RateEstimator(), now=1.0)
|
|
sessions = result.payload["sessions"]
|
|
assert sessions["list"] == []
|
|
assert "vLLM" in sessions["detail"]
|
|
|
|
|
|
# ---- fleet totals --------------------------------------------------------- #
|
|
|
|
def _engine(name: str, **values) -> dict:
|
|
metrics = {key: {"value": value} for key, value in values.items()}
|
|
return {"engine": name, "metrics": metrics}
|
|
|
|
|
|
def test_fleet_totals_sum_only_engines_that_answered() -> None:
|
|
engines = [
|
|
_engine("gufo", decode_tokens_per_second=40.0, prompt_tokens_per_second=10.0,
|
|
kv_cache_usage_ratio=0.25, requests_processing=1.0, requests_deferred=0.0,
|
|
tokens_total=100.0),
|
|
_engine("vllm", decode_tokens_per_second=10.0, prompt_tokens_per_second=5.0,
|
|
gpu_cache_usage=0.75, num_requests_running=2.0, num_requests_waiting=3.0,
|
|
tokens_total=200.0),
|
|
{}, # an engine whose poll failed
|
|
]
|
|
totals = fleet_totals(engines)
|
|
assert totals["engines_total"] == 3
|
|
assert totals["engines_live"] == 2
|
|
assert totals["decode_tokens_per_second"] == 50.0
|
|
assert totals["prompt_tokens_per_second"] == 15.0
|
|
assert totals["requests_active"] == 3.0
|
|
assert totals["requests_waiting"] == 3.0
|
|
assert totals["kv_cache_usage_max"] == 0.75, "cache pressure is a max, never a sum"
|
|
assert totals["tokens_total"] == 300.0
|
|
|
|
|
|
def test_fleet_totals_report_the_gufo_vllm_seam() -> None:
|
|
totals = fleet_totals([_engine("gufo"), _engine("vllm")])
|
|
assert totals["seams"], "the two engines count prompt tokens differently; say so"
|
|
|
|
|
|
def test_fleet_totals_of_nothing_is_none_not_zero() -> None:
|
|
totals = fleet_totals([])
|
|
assert totals["decode_tokens_per_second"] is None
|
|
assert totals["tokens_total"] is None
|
|
assert totals["engines_live"] == 0
|
|
|
|
|
|
def test_series_names_are_the_parse_catalog() -> None:
|
|
assert "tokens_generated_total" in series_names("gufo")
|
|
assert "generation_tokens_total" in series_names("vllm")
|
|
assert "generation_tokens_total" not in series_names("gufo")
|
|
|
|
|
|
def test_gufo_a_measured_zero_is_not_replaced_by_the_gauge(metrics_text) -> None:
|
|
"""`measured or gauge` made an idle engine advertise its last request's speed."""
|
|
text = metrics_text("gufo")
|
|
rates = RateEstimator(span=10)
|
|
parse_gufo_metrics(text, rates, 1_000.0)
|
|
metrics = parse_gufo_metrics(text, rates, 1_010.0) # counters unchanged: truly idle
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == 0.0
|
|
assert "predicted_tokens_seconds" not in str(decode["source"]), "native gauge must not leak in"
|
|
|
|
|
|
def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None:
|
|
text = metrics_text("vllm")
|
|
rates = RateEstimator(span=10)
|
|
parse_vllm_metrics(text, rates, 2_000.0)
|
|
metrics = parse_vllm_metrics(text, rates, 2_010.0)
|
|
decode = metrics["decode_tokens_per_second"]
|
|
assert decode["value"] == 0.0
|
|
assert "running/tpot" not in str(decode["source"])
|