phase: 34_radiance_llm_engine
Everything is verified and green. Final report: **Phase 34 final verification pass — complete** - Verified all 6 completed tasks' work against the live fleet and hermetic suite: `llm:radiance` job registered, `llm_engines()` registry drives 404 gate / history defaults / `/api/config` (additive, byte-identical gufo/vllm keys), radiance field map + `/stats` sidecar + 7-position spec curve, `/llm` strip/curve/rows. - **Defect found & fixed (pre-existing, task 05):** the vLLM-only sentence "vLLM exposes no per-slot JSON…" rendered on the **radiance** panel (`slotSection`'s non-gufo branch). Fix: vLLM line gated to `engine === 'vllm'`; radiance's panel slot section renders nothing (its session home is the strip) — `page-llm.js` + the served-byte pin in `tests/test_api.py` (now asserts the radiance branch is empty and the sentence can't leak). Browser-verified; re-shot `data/logs/34-llm-radiance-rest.png` (the 4.5-min wait found no in-flight request, so the busy shot remains the earlier pass's — its strip rows still stand; only the panel text predates the fix). - Tests: `uv run pytest` → **2031 passed, 19 skipped** (100%); `uv run pytest --cov` → **98.73%** (floor 94%; config 100%, llm 99%, collectors 98%, api 99%); `uv run ruff check src tests agent && uv run pyright` → clean; `node --check` OK; `uv pip list --outdated` → empty. - `uv run pytest -m live --live` → **13 passed, 2 skipped** (vLLM unset — correct; live-write opt-in — correct); the read-only radiance test passed. - Completion criteria: (1) engines `gufo`+`radiance`, `fleet.engines_live` 2, `config.llm.radiance.enabled` true — **pass (live curl)**; (2) unset → 404 no-dial + no job — **pass (spy-pinned, green)**; (3) fixtures present, both families (79 `radiance:` / 198 `vllm:` lines), drift pin — **pass**; (4) panel/curve/rows — **pass (re-shot rest screenshot)**; (5) live radiance test — **pass**; (6) suite/coverage/lint/`bash .agents/validate.sh` (exit 0) — **pass**; (7) docs — below; (8) gufo/vLLM behaviour unchanged — **pass (existing pins untouched)**. - Documentation: **README.md** full review — env table, LLM-engines section, architecture diagram, capture-script and `/llm` section all name radiance correctly; no inconsistencies found, no changes needed. **AGENTS.md** — the three task-06 fixes (fleet line, route row, module bullets) are in place and all other statements match reality; no changes needed. - Dev server healthy (`./scripts/dev-server.sh status`: all sources ok, `llm:radiance` answering); all changes left in the working tree for the harness. - Next pending phase: none — 34 was the last phase in `todo/`.
This commit is contained in:
+646
-13
@@ -5,17 +5,25 @@ the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentati
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards.prom import Prom
|
||||
from dashboards.rates import MIN_WINDOW_SECONDS, RateEstimator
|
||||
from dashboards.severity import classify
|
||||
from dashboards.sources.llm import (
|
||||
RADIANCE_HISTOGRAMS,
|
||||
RADIANCE_OPTIONAL,
|
||||
RADIANCE_SERIES,
|
||||
engine_result,
|
||||
fleet_totals,
|
||||
parse_gufo_metrics,
|
||||
parse_radiance_metrics,
|
||||
parse_vllm_metrics,
|
||||
rate_key,
|
||||
series_names,
|
||||
@@ -62,19 +70,31 @@ def poll_engine(metrics_text: Any, engine: str, span: float, stamps: Sequence[fl
|
||||
# ---- gufo / llama.cpp ----------------------------------------------------- #
|
||||
|
||||
def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
|
||||
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
||||
assert metrics["tokens_generated_total"]["value"] == 877_336.0
|
||||
assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0
|
||||
assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0
|
||||
assert metrics["requests_processing"]["value"] == 0.0
|
||||
assert metrics["device_lost_total"]["value"] == 0.0
|
||||
assert metrics["kv_cache_usage_ratio"]["value"] == 0.0
|
||||
"""The cell's value is the number the exposition declares, verbatim: no
|
||||
conversion, no scaling, and the series' own name is the source. Compared against
|
||||
the raw `Prom` parse of the same capture — never a value frozen at capture time
|
||||
(the counters reset on every engine restart)."""
|
||||
text = metrics_text("gufo")
|
||||
prom = Prom(text)
|
||||
metrics = parse_gufo_metrics(text)
|
||||
for key, name in (
|
||||
("tokens_generated_total", "llamacpp:tokens_predicted_total"),
|
||||
("tokens_prompt_total", "llamacpp:prompt_tokens_total"),
|
||||
("tokens_prompt_cached_total", "llamacpp:prompt_tokens_cached_total"),
|
||||
("requests_processing", "llamacpp:requests_processing"),
|
||||
("device_lost_total", "gufo_device_lost_total"),
|
||||
("kv_cache_usage_ratio", "llamacpp:kv_cache_usage_ratio"),
|
||||
):
|
||||
assert metrics[key]["value"] == prom.scalar([name]), key
|
||||
assert metrics[key]["source"] == name, "the counter's own name is the source"
|
||||
|
||||
|
||||
def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None:
|
||||
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
||||
accept = metrics["spec_accept_rate"]["value"]
|
||||
assert accept == pytest.approx(578_514 / 789_028)
|
||||
accepted = metrics["spec_accepted_tokens_total"]["value"]
|
||||
draft_tokens = metrics["spec_draft_tokens_total"]["value"]
|
||||
assert accept == pytest.approx(accepted / draft_tokens)
|
||||
assert str(metrics["spec_accept_rate"]["source"]).startswith("computed")
|
||||
|
||||
|
||||
@@ -82,7 +102,10 @@ def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None:
|
||||
"""gufo's prompt counter excludes cache hits; the lifetime total must not."""
|
||||
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
||||
total = metrics["tokens_total"]["value"]
|
||||
assert total == 1_350_666 + 124_529_105 + 877_336
|
||||
prompt = metrics["tokens_prompt_total"]["value"]
|
||||
cached = metrics["tokens_prompt_cached_total"]["value"] or 0.0
|
||||
generated = metrics["tokens_generated_total"]["value"]
|
||||
assert total == pytest.approx(prompt + cached + generated)
|
||||
|
||||
|
||||
def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None:
|
||||
@@ -99,8 +122,11 @@ def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_te
|
||||
assert decode["value"] == pytest.approx(40.0)
|
||||
assert decode["window"] == pytest.approx(10.0)
|
||||
assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"]
|
||||
# the native gauge is still there, labelled, because it answers a different question
|
||||
assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)
|
||||
# the native gauge is still there, labelled, because it answers a different question —
|
||||
# verbatim the exposition's own number (a volatile gauge: no frozen value)
|
||||
assert metrics["decode_tokens_per_second_native"]["value"] == \
|
||||
Prom(text).scalar(["llamacpp:predicted_tokens_seconds"])
|
||||
assert metrics["decode_tokens_per_second_native"]["source"] == "llamacpp:predicted_tokens_seconds"
|
||||
|
||||
|
||||
def test_gufo_the_window_the_llm_jobs_are_spanned_by_is_one_that_fills(metrics_text) -> None:
|
||||
@@ -323,10 +349,12 @@ def test_absent_gufo_metrics_are_none_with_no_source() -> None:
|
||||
def test_gufo_kv_cache_cell_keeps_provenance_and_gains_severity(metrics_text) -> None:
|
||||
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
||||
cell = metrics["kv_cache_usage_ratio"]
|
||||
assert cell["value"] == 0.0
|
||||
assert cell["value"] == Prom(metrics_text("gufo")).scalar(["llamacpp:kv_cache_usage_ratio"]), \
|
||||
"verbatim the exposition's own ratio — the cell stays what it was"
|
||||
assert cell["source"] == "llamacpp:kv_cache_usage_ratio"
|
||||
assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
|
||||
assert cell["severity"] == "ok"
|
||||
assert cell["severity"] == classify("llm.kv_cache_usage_pct", cell["value"] * 100.0), \
|
||||
"the verdict is the table's for this value (percent scale)"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ratio,expected", [
|
||||
@@ -667,3 +695,608 @@ def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None:
|
||||
decode = metrics["decode_tokens_per_second"]
|
||||
assert decode["value"] == 0.0
|
||||
assert "running/tpot" not in str(decode["source"])
|
||||
|
||||
|
||||
# ---- radiance: the ground-truth fixtures (phase 34, task 01) ---------------- #
|
||||
#
|
||||
# Radiance is a vLLM derivative: its /metrics carries the `radiance:` native series plus
|
||||
# a `vllm:` compatibility family, and the capture records /stats and /v1/models in the
|
||||
# same pass. These tests pin the *shape and the name set* of those captures, never the
|
||||
# numbers (the nut_*.vars precedent): a metric rename or a new build's series fails here
|
||||
# before a dashboard does.
|
||||
|
||||
|
||||
def _load_radiance_json(name: str) -> Any:
|
||||
"""Load a radiance JSON fixture, skipping with the capture command like the others."""
|
||||
path = Path(__file__).resolve().parent / "fixtures" / name
|
||||
if not path.exists():
|
||||
pytest.skip(f"missing fixture {path.name} — run scripts/capture-fixtures.sh --with-llm")
|
||||
# the capture script writes its verbatim header as `#` comment lines; the document
|
||||
# is what follows them
|
||||
body = "\n".join(line for line in path.read_text().splitlines() if not line.startswith("#"))
|
||||
return json.loads(body)
|
||||
|
||||
|
||||
def _radiance_map_names() -> set[str]:
|
||||
return ({name for names, _ in RADIANCE_SERIES.values() for name in names}
|
||||
| {name for names in RADIANCE_HISTOGRAMS.values() for name in names})
|
||||
|
||||
|
||||
def test_radiance_fixture_name_set_is_pinned_by_the_map_and_optionals(metrics_text) -> None:
|
||||
"""The drift pin: every name in the capture's radiance:/vllm: families is either a
|
||||
payload cell (the map) or a declared non-cell (RADIANCE_OPTIONAL), and vice versa —
|
||||
so a rename, a dropped series, or a typo in a map name fails the suite before a
|
||||
dashboard does."""
|
||||
names = set()
|
||||
for line in metrics_text("radiance").splitlines():
|
||||
if line and not line.startswith("#"):
|
||||
names.add(line.split("{")[0].split()[0])
|
||||
contract = {name for name in names if name.startswith(("radiance:", "vllm:"))}
|
||||
assert contract, "the capture lost its series — re-run the capture with the engine up"
|
||||
map_names = _radiance_map_names()
|
||||
unexplained = contract - map_names - RADIANCE_OPTIONAL
|
||||
assert not unexplained, (
|
||||
f"names in the capture that are neither a payload cell nor declared optional: "
|
||||
f"{sorted(unexplained)}")
|
||||
absent_everywhere = map_names - contract - RADIANCE_OPTIONAL
|
||||
assert not absent_everywhere, (
|
||||
f"map names in neither the capture nor RADIANCE_OPTIONAL (a rename or a typo): "
|
||||
f"{sorted(absent_everywhere)}")
|
||||
stale = RADIANCE_OPTIONAL - contract - map_names
|
||||
assert not stale, f"RADIANCE_OPTIONAL entries that are neither real names: {sorted(stale)}"
|
||||
|
||||
|
||||
def test_radiance_fixture_carries_both_families(metrics_text) -> None:
|
||||
"""The vLLM-derived shape is part of the contract: the capture must hold both the
|
||||
native series and the compatibility family, or the name set has drifted."""
|
||||
text = metrics_text("radiance")
|
||||
native = sum(1 for line in text.splitlines() if line.startswith("radiance:"))
|
||||
compat = sum(1 for line in text.splitlines() if line.startswith("vllm:"))
|
||||
assert native > 0, "the capture lost its radiance: native series"
|
||||
assert compat > 0, "the capture lost its vllm: compatibility family"
|
||||
|
||||
|
||||
def test_radiance_per_position_series_has_the_seven_positions_the_phase_pins(
|
||||
metrics_text,
|
||||
) -> None:
|
||||
"""The per-position spec-acceptance series is the 7-bar curve: its label set is
|
||||
position="0".."6", each position a series of its own."""
|
||||
positions = set()
|
||||
for line in metrics_text("radiance").splitlines():
|
||||
if line.startswith("radiance:draft_position_acceptance_ratio{"):
|
||||
m = re.search(r'position="(\d+)"', line)
|
||||
assert m, f"a per-position line without a position label: {line}"
|
||||
positions.add(m.group(1))
|
||||
assert positions == {str(i) for i in range(7)}, (
|
||||
f"the phase pins seven draft positions; the capture has {sorted(positions)}")
|
||||
|
||||
|
||||
STATS_TOP_LEVEL_KEYS = (
|
||||
"cards", "checkpoints_written", "ckpt_evictions", "ckpt_refused", "decode_tps",
|
||||
"draft_accept", "draft_positions", "hit_cut_tokens", "hit_cuts", "in_flight",
|
||||
"kv_bytes_carved", "kv_bytes_total", "kv_bytes_used", "kv_tokens_cached",
|
||||
"kv_tokens_max", "kv_tokens_total", "kv_tokens_used", "kv_util",
|
||||
"linear_cached_tokens", "linear_hit_rate", "link", "model", "now_ms", "passes",
|
||||
"preempted", "preemptions", "prefill_chunks_capped", "prefill_tps",
|
||||
"prefix_evictions", "prefix_hit_rate", "queue_cap", "released", "requests",
|
||||
"rows_unspec", "running", "started_at", "startup_s", "step_ns", "step_ns_decode",
|
||||
"step_ns_decode_best", "step_ns_decode_recent", "steps", "steps_decode", "steps_held",
|
||||
"steps_mixed", "steps_prefill", "total_requests", "uptime_s", "version", "waiting",
|
||||
)
|
||||
|
||||
|
||||
def test_radiance_stats_fixture_shape() -> None:
|
||||
"""The /stats document: the top-level key set, the per-card VRAM rows, the requests
|
||||
list and the per-position spec-acceptance list — shape, not numbers. A capture at
|
||||
rest records `requests: []` (the header comment says so); a row shape is recorded
|
||||
by re-running the capture with a request in flight."""
|
||||
stats = _load_radiance_json("stats_radiance.json")
|
||||
assert isinstance(stats, dict)
|
||||
assert tuple(sorted(stats.keys())) == STATS_TOP_LEVEL_KEYS, (
|
||||
"the /stats top-level key set changed — the shape the page reads drifted; "
|
||||
"re-capture and re-decide the payload set")
|
||||
cards = stats["cards"]
|
||||
assert isinstance(cards, list) and cards, "cards[] is the per-card VRAM panel's source"
|
||||
for card in cards:
|
||||
assert isinstance(card, dict)
|
||||
assert isinstance(card["index"], int)
|
||||
assert isinstance(card["name"], str) and card["name"]
|
||||
assert isinstance(card["vram_used"], int)
|
||||
assert isinstance(card["vram_total"], int)
|
||||
assert 0 < card["vram_used"] <= card["vram_total"]
|
||||
assert isinstance(stats["requests"], list)
|
||||
assert isinstance(stats["queue_cap"], int)
|
||||
assert isinstance(stats["in_flight"], int)
|
||||
positions = stats["draft_positions"]
|
||||
assert isinstance(positions, list)
|
||||
assert len(positions) == 7, (
|
||||
"the phase pins seven draft positions; the capture has "
|
||||
f"{len(positions)} — the 7-bar curve needs re-deciding")
|
||||
for pos in positions:
|
||||
assert isinstance(pos, dict)
|
||||
assert isinstance(pos["reached"], int)
|
||||
assert isinstance(pos["accepted"], int)
|
||||
assert isinstance(pos["rate"], float)
|
||||
|
||||
|
||||
def test_radiance_models_fixture_shape() -> None:
|
||||
"""The /v1/models document: one model, and the capabilities / max_model_len it
|
||||
publishes — the only place the engine states what it can do."""
|
||||
models = _load_radiance_json("models_radiance.json")
|
||||
assert isinstance(models, dict)
|
||||
assert models["object"] == "list"
|
||||
data = models["data"]
|
||||
assert isinstance(data, list) and data
|
||||
model = data[0]
|
||||
assert model["owned_by"] == "radiance", "the engine's own word for its family"
|
||||
assert isinstance(model["id"], str) and model["id"]
|
||||
assert isinstance(model["max_model_len"], int)
|
||||
assert isinstance(model["capabilities"], dict)
|
||||
|
||||
|
||||
# ---- radiance: the field map (phase 34, task 03) ----------------------- #
|
||||
#
|
||||
# The map is built from the task-01 fixture (rule 7); the tests below pin the three
|
||||
# throughput arms (gauge wins / measured fills / n/a on a refused window), the severity
|
||||
# pass-through on the existing KV row, the /stats session arms, the whitelisted stats
|
||||
# block and the additive history series. As always: shape and name set, never the
|
||||
# numbers of a live engine.
|
||||
|
||||
|
||||
def _radiance_stats() -> dict:
|
||||
return _load_radiance_json("stats_radiance.json")
|
||||
|
||||
|
||||
# ---- the fixture parse -------------------------------------------------- #
|
||||
|
||||
def test_radiance_fixture_parse_every_mapped_key_resolves(metrics_text) -> None:
|
||||
"""Every mapped key resolves on the verbatim capture, with its pinned source string."""
|
||||
metrics = parse_radiance_metrics(metrics_text("radiance"))
|
||||
expected_source = {
|
||||
"decode_tokens_per_second_native": "radiance:decode_tokens_per_second",
|
||||
"prompt_tokens_per_second_native": "radiance:prefill_tokens_per_second",
|
||||
"num_requests_running": "vllm:num_requests_running",
|
||||
"num_requests_waiting": "vllm:num_requests_waiting",
|
||||
"gpu_cache_usage": "vllm:kv_cache_usage_perc",
|
||||
"kv_cache_max_tokens": "radiance:kv_cache_max_tokens",
|
||||
"kv_cache_max_bytes": "radiance:kv_cache_max_bytes",
|
||||
"queue_depth": "radiance:http_queue_depth",
|
||||
"admission_rejected_total": "radiance:http_admission_rejected_total",
|
||||
"num_preemptions_total": "vllm:num_preemptions_total",
|
||||
"prompt_tokens_total": "vllm:prompt_tokens_total",
|
||||
"generation_tokens_total": "vllm:generation_tokens_total",
|
||||
"gpu_prefix_cache_hit_rate": "vllm:gpu_prefix_cache_hit_rate",
|
||||
"spec_accept_rate": "radiance:draft_acceptance_ratio",
|
||||
"decode_step_seconds_recent": "radiance:decode_step_seconds_recent",
|
||||
"decode_step_seconds_best": "radiance:decode_step_seconds_best",
|
||||
}
|
||||
for key, source in expected_source.items():
|
||||
cell = metrics[key]
|
||||
assert set(cell) >= {"value", "source", "derived"}, f"{key} lost its cell shape"
|
||||
assert cell["source"] == source, f"{key} resolved to {cell['source']!r}, not {source!r}"
|
||||
assert cell["value"] is not None, f"{key} should resolve on the capture"
|
||||
# a map name this build does not carry: n/a, never 0
|
||||
assert metrics["num_requests_swapped"]["value"] is None
|
||||
assert metrics["num_requests_swapped"]["source"] is None
|
||||
|
||||
|
||||
def test_radiance_a_series_removed_from_the_capture_is_n_a_never_zero(metrics_text) -> None:
|
||||
"""The completion-criteria arm: a line removed from a fixture copy answers
|
||||
`{\"value\": null}` — 0 would claim \"the engine published zero\", which it did not."""
|
||||
by_key = {
|
||||
"radiance:http_queue_depth": "queue_depth",
|
||||
"radiance:kv_cache_max_bytes": "kv_cache_max_bytes",
|
||||
"radiance:decode_step_seconds_best": "decode_step_seconds_best",
|
||||
"vllm:num_preemptions_total": "num_preemptions_total",
|
||||
"vllm:gpu_prefix_cache_hit_rate": "gpu_prefix_cache_hit_rate",
|
||||
"radiance:draft_acceptance_ratio": "spec_accept_rate",
|
||||
}
|
||||
for name, key in by_key.items():
|
||||
text = "\n".join(line for line in metrics_text("radiance").splitlines()
|
||||
if not line.startswith(name))
|
||||
cell = parse_radiance_metrics(text)[key]
|
||||
assert cell["value"] is None, f"{key} answered a value for a removed series: {cell}"
|
||||
assert cell["source"] is None
|
||||
|
||||
|
||||
def test_radiance_spec_positions_are_the_ordered_seven(metrics_text) -> None:
|
||||
metrics = parse_radiance_metrics(metrics_text("radiance"))
|
||||
positions = metrics["spec_positions"]
|
||||
assert [p["position"] for p in positions] == list(range(7)), \
|
||||
"the phase pins seven draft positions, in the order the engine publishes them"
|
||||
for p in positions:
|
||||
assert 0.0 <= p["ratio"] <= 1.0, "a per-position ratio is a 0-1 gauge"
|
||||
# the ratios are a running-window aggregate: volatile between captures, so the
|
||||
# pin is the shape (seven, ordered, in-range) — the name set is pinned above
|
||||
prom = Prom(metrics_text("radiance"))
|
||||
by_position = prom.by_label(["radiance:draft_position_acceptance_ratio"], "position")
|
||||
for p in positions:
|
||||
assert p["ratio"] == by_position[str(p["position"])], \
|
||||
f"position {p['position']} is not the exposition's own number"
|
||||
|
||||
|
||||
def test_radiance_an_absent_position_stays_absent_not_zeroed(metrics_text) -> None:
|
||||
text = "\n".join(
|
||||
line for line in metrics_text("radiance").splitlines()
|
||||
if not ('radiance:draft_position_acceptance_ratio{' in line
|
||||
and 'position="6"' in line)
|
||||
)
|
||||
positions = parse_radiance_metrics(text)["spec_positions"]
|
||||
assert [p["position"] for p in positions] == [0, 1, 2, 3, 4, 5], \
|
||||
"position 6 is absent from this capture: the list holds what the engine published"
|
||||
assert all(p["ratio"] is not None and p["ratio"] > 0.0 for p in positions), \
|
||||
"an absent position must not be filled with a 0"
|
||||
|
||||
|
||||
def test_radiance_tokens_total_is_the_two_vllm_counters(metrics_text) -> None:
|
||||
metrics = parse_radiance_metrics(metrics_text("radiance"))
|
||||
prompt = metrics["prompt_tokens_total"]["value"]
|
||||
generated = metrics["generation_tokens_total"]["value"]
|
||||
assert metrics["tokens_total"]["value"] == pytest.approx(prompt + generated)
|
||||
assert metrics["tokens_total"]["source"] == "computed:prompt+generated"
|
||||
# both counters removed: n/a, never 0
|
||||
text = metrics_text("radiance")
|
||||
for name in ("vllm:prompt_tokens_total", "vllm:generation_tokens_total"):
|
||||
text = "\n".join(line for line in text.splitlines()
|
||||
if not line.startswith(name))
|
||||
assert parse_radiance_metrics(text)["tokens_total"]["value"] is None
|
||||
|
||||
|
||||
def test_radiance_prefix_hit_ratio_is_the_gauge_source_named_not_derived(metrics_text) -> None:
|
||||
"""This build publishes no prefix counters, so a hits/queries ratio would be an
|
||||
invented number — the gauge is the answer, under the shared UI key."""
|
||||
text = metrics_text("radiance")
|
||||
metrics = parse_radiance_metrics(text)
|
||||
gauge = Prom(text).scalar(["vllm:gpu_prefix_cache_hit_rate"])
|
||||
cell = metrics["prefix_cache_hit_ratio"]
|
||||
assert cell["value"] == pytest.approx(gauge)
|
||||
assert cell["source"] == "vllm:gpu_prefix_cache_hit_rate"
|
||||
assert cell["derived"] is False, "the gauge is not a computed ratio"
|
||||
# the gauge cell itself is the payload cell too (the map entry)
|
||||
assert metrics["gpu_prefix_cache_hit_rate"]["value"] == pytest.approx(gauge)
|
||||
|
||||
|
||||
def test_radiance_latency_histograms_are_the_vllm_shaped_family(metrics_text) -> None:
|
||||
metrics = parse_radiance_metrics(metrics_text("radiance"))
|
||||
ttft = metrics["time_to_first_token_seconds"]
|
||||
assert ttft and ttft["count"] > 0 and ttft["mean"] > 0
|
||||
assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"]
|
||||
assert metrics["time_per_output_token_seconds"]["mean"] > 0
|
||||
assert metrics["e2e_request_latency_seconds"]["count"] > 0
|
||||
# no request_queue_time on this build: the cell is absent, never back-filled
|
||||
assert "request_queue_time_seconds" not in metrics
|
||||
assert "request_queue_time_seconds__mean" not in metrics
|
||||
|
||||
|
||||
# ---- rule 2, the three throughput arms ----------------------------------- #
|
||||
|
||||
def test_radiance_throughput_arm_gauge_wins(metrics_text) -> None:
|
||||
"""The native gauge is the scheduler's recent-window aggregate, so it wins."""
|
||||
text = metrics_text("radiance")
|
||||
metrics = parse_radiance_metrics(text, RateEstimator(span=10), 1_000.0)
|
||||
decode = metrics["decode_tokens_per_second"]
|
||||
# the gauge's value is the exposition's own number (volatile — the pin is the
|
||||
# passthrough, not a frozen value), and it wins the cell
|
||||
assert decode["value"] == metrics["decode_tokens_per_second_native"]["value"]
|
||||
assert decode["value"] == Prom(text).scalar(["radiance:decode_tokens_per_second"], agg="avg")
|
||||
assert decode["source"] == "radiance:decode_tokens_per_second"
|
||||
assert decode["derived"] is False, "a scraped gauge is not a computed Δ"
|
||||
assert "window" not in decode, "a scraped gauge names no Δ window"
|
||||
prompt = metrics["prompt_tokens_per_second"]
|
||||
assert prompt["value"] == metrics["prompt_tokens_per_second_native"]["value"]
|
||||
assert prompt["source"] == "radiance:prefill_tokens_per_second"
|
||||
|
||||
|
||||
def test_radiance_throughput_arm_measured_delta_when_the_gauge_is_absent(metrics_text) -> None:
|
||||
"""A gauge this build does not publish falls back to the measured Δ, per label set."""
|
||||
text = metrics_text("radiance")
|
||||
text = "\n".join(line for line in text.splitlines()
|
||||
if not line.startswith("radiance:decode_tokens_per_second"))
|
||||
rates = RateEstimator(span=10)
|
||||
parse_radiance_metrics(text, rates, 1_000.0)
|
||||
text = bump_metric(text, "vllm:generation_tokens_total", 250.0)
|
||||
metrics = parse_radiance_metrics(text, rates, 1_010.0)
|
||||
decode = metrics["decode_tokens_per_second"]
|
||||
assert decode["value"] == pytest.approx(25.0)
|
||||
assert decode["derived"] is True
|
||||
assert decode["source"] == "computed:Δvllm:generation_tokens_total/Δt"
|
||||
assert decode["window"] == pytest.approx(10.0), \
|
||||
"the label must name the window the value used"
|
||||
# the arms are independent: the prompt gauge still wins its own cell
|
||||
prompt = metrics["prompt_tokens_per_second"]
|
||||
assert prompt["value"] == metrics["prompt_tokens_per_second_native"]["value"]
|
||||
assert prompt["derived"] is False
|
||||
|
||||
|
||||
def test_radiance_throughput_arm_a_refused_window_is_n_a_with_its_label(metrics_text) -> None:
|
||||
"""A refused window is still an answer: n/a beside the honest Δ<window> it refused."""
|
||||
text = metrics_text("radiance")
|
||||
text = "\n".join(line for line in text.splitlines()
|
||||
if not line.startswith("radiance:decode_tokens_per_second"))
|
||||
rates = RateEstimator(span=10)
|
||||
parse_radiance_metrics(text, rates, 1_000.0)
|
||||
text = bump_metric(text, "vllm:generation_tokens_total", 50.0)
|
||||
parse_radiance_metrics(text, rates, 1_010.0)
|
||||
text = bump_metric(text, "vllm:generation_tokens_total", 60.0)
|
||||
decode = parse_radiance_metrics(text, rates, 1_010.3)["decode_tokens_per_second"]
|
||||
assert decode["value"] is None, "0.3 s is the loop's catch-up gap, not a rate"
|
||||
assert decode["window"] == pytest.approx(0.3)
|
||||
assert decode["window"] < MIN_WINDOW_SECONDS
|
||||
assert decode["source"] == "computed:Δvllm:generation_tokens_total/Δt"
|
||||
|
||||
|
||||
def test_radiance_throughput_arm_both_absent_is_n_a_never_zero(metrics_text) -> None:
|
||||
text = metrics_text("radiance")
|
||||
for name in ("radiance:decode_tokens_per_second", "radiance:prefill_tokens_per_second",
|
||||
"vllm:generation_tokens_total", "vllm:prompt_tokens_total"):
|
||||
text = "\n".join(line for line in text.splitlines() if not line.startswith(name))
|
||||
metrics = parse_radiance_metrics(text) # no estimator: no measured arm either
|
||||
for key in ("decode_tokens_per_second", "prompt_tokens_per_second"):
|
||||
cell = metrics[key]
|
||||
assert cell["value"] is None, f"{key} printed a value with no source: {cell}"
|
||||
assert cell["source"] is None
|
||||
|
||||
|
||||
# ---- severity: the KV cell is rated, the spec cell is not --------------- #
|
||||
|
||||
def test_radiance_kv_cell_carries_the_severity_verdict(metrics_text) -> None:
|
||||
"""Radiance publishes the same 0-1 ratio the vLLM map feeds the table: the
|
||||
existing `llm.kv_cache_usage_pct` row rates it, no new threshold anywhere."""
|
||||
text = metrics_text("radiance")
|
||||
base = Prom(text).scalar(["vllm:kv_cache_usage_perc"], agg="max")
|
||||
assert base is not None
|
||||
cell = parse_radiance_metrics(text)["gpu_cache_usage"]
|
||||
assert cell["value"] == pytest.approx(base)
|
||||
assert cell["source"] == "vllm:kv_cache_usage_perc"
|
||||
assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
|
||||
assert cell["severity"] == classify("llm.kv_cache_usage_pct", base * 100.0), \
|
||||
"the table's verdict for this value (percent scale)"
|
||||
# push it above the table's fail bound (90 %): the verdict follows the value
|
||||
target = 0.99
|
||||
pressed = bump_metric(text, "vllm:kv_cache_usage_perc", target - base)
|
||||
cell = parse_radiance_metrics(pressed)["gpu_cache_usage"]
|
||||
assert cell["value"] == pytest.approx(target)
|
||||
assert cell["severity"] == "fail"
|
||||
|
||||
|
||||
def test_radiance_spec_accept_carries_no_severity(metrics_text) -> None:
|
||||
"""Decision 4 pinned both ways: the KV cell is rated, the spec cell is not — an
|
||||
efficiency number is not a verdict."""
|
||||
text = metrics_text("radiance")
|
||||
metrics = parse_radiance_metrics(text)
|
||||
assert metrics["spec_accept_rate"]["value"] == \
|
||||
Prom(text).scalar(["radiance:draft_acceptance_ratio"], agg="avg")
|
||||
assert "severity" not in metrics["spec_accept_rate"]
|
||||
assert metrics["gpu_cache_usage"]["severity"] is not None, \
|
||||
"the other direction: the KV cell is rated"
|
||||
|
||||
|
||||
# ---- /stats as the session source ---------------------------------------- #
|
||||
|
||||
def test_radiance_sessions_stats_arm_at_rest_counts_and_cap(metrics_text) -> None:
|
||||
"""The at-rest capture: requests[] empty → the counts-otherwise sentence, the cap
|
||||
from /stats, `active` from the /metrics gauge (one definition in every arm).
|
||||
The numbers come from the capture itself — volatile between re-captures."""
|
||||
stats = _radiance_stats()
|
||||
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
|
||||
RateEstimator(), stats=stats, now=1.0)
|
||||
sessions = result.payload["sessions"]
|
||||
active_expected = Prom(metrics_text("radiance")).scalar(["vllm:num_requests_running"])
|
||||
assert sessions["active"] == active_expected, "active stays the /metrics gauge in every arm"
|
||||
assert sessions["running"] == stats["running"]
|
||||
assert sessions["waiting"] == stats["waiting"]
|
||||
assert sessions["queue_cap"] == stats["queue_cap"]
|
||||
assert sessions["list"] == stats["requests"]
|
||||
assert sessions["source"] == "radiance /stats"
|
||||
if not stats["requests"]:
|
||||
assert sessions["detail"] == (
|
||||
f"radiance reports counts, not sessions: {stats['running']:g} running"
|
||||
f" of cap {stats['queue_cap']:g}. Per-request detail appears here while "
|
||||
f"requests are in flight.")
|
||||
else:
|
||||
assert sessions["detail"] is None, "rows in flight: no at-rest sentence"
|
||||
|
||||
|
||||
def test_radiance_sessions_stats_arm_non_empty_rows_verbatim(metrics_text) -> None:
|
||||
"""A request in flight: the rows are the engine's own shape, verbatim."""
|
||||
stats = _radiance_stats()
|
||||
rows = [{"id": 42, "model": "juggernaut", "state": "running",
|
||||
"prompt_tokens": 1_200, "completed_tokens": 250}]
|
||||
stats = {**stats, "requests": rows, "running": 1, "waiting": 2}
|
||||
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
|
||||
RateEstimator(), stats=stats, now=1.0)
|
||||
sessions = result.payload["sessions"]
|
||||
assert sessions["list"] == rows, "the rows pass through untouched"
|
||||
assert sessions["source"] == "radiance /stats"
|
||||
assert sessions["detail"] is None
|
||||
assert sessions["running"] == 1
|
||||
assert sessions["waiting"] == 2
|
||||
assert sessions["queue_cap"] == stats["queue_cap"] # the capture's cap, untouched
|
||||
|
||||
|
||||
def test_radiance_sessions_missing_stats_fall_back_to_the_gauges(metrics_text) -> None:
|
||||
"""The sidecar failed: the poll is still healthy, the counts come from the
|
||||
/metrics gauges (a different provenance, said so), and the cap is null — never 0."""
|
||||
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
|
||||
RateEstimator(), now=1.0) # stats=None: /stats did not answer
|
||||
sessions = result.payload["sessions"]
|
||||
assert sessions["running"] == 0.0, "the /metrics gauge's value"
|
||||
assert sessions["waiting"] == 0.0
|
||||
assert sessions["queue_cap"] is None, "null, not 0 — rule 2"
|
||||
assert sessions["list"] == []
|
||||
assert "did not answer" in sessions["detail"]
|
||||
assert "gauges" in sessions["detail"]
|
||||
assert "stats" not in result.payload, "no stats block: the sidecar never answered"
|
||||
# the metrics passthroughs degrade with it
|
||||
assert result.payload["metrics"]["queue_cap"]["value"] is None
|
||||
assert result.payload["metrics"]["vram_used_pct"]["value"] is None
|
||||
|
||||
|
||||
# ---- the whitelisted stats block ------------------------------------------ #
|
||||
|
||||
def test_radiance_payload_stats_block_is_whitelisted(metrics_text) -> None:
|
||||
stats = _radiance_stats()
|
||||
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
|
||||
RateEstimator(), stats=stats, now=1.0)
|
||||
block = result.payload["stats"]
|
||||
assert set(block) == {"model", "version", "uptime_s", "startup_s",
|
||||
"total_requests", "cards"}, \
|
||||
"the engine internals (link / passes / steps* / …) stay out of the payload"
|
||||
assert block["model"] == "juggernaut"
|
||||
assert block["version"] == "1.15.1"
|
||||
assert isinstance(block["uptime_s"], int) and block["uptime_s"] >= 0, \
|
||||
"uptime is the engine's own seconds — volatile between captures"
|
||||
card = block["cards"][0]
|
||||
src = stats["cards"][0]
|
||||
assert set(card) == {"index", "name", "vram_used", "vram_total", "vram_used_pct"}
|
||||
assert card["vram_used"] == src["vram_used"], \
|
||||
"the raw bytes travel as the engine published them"
|
||||
assert card["vram_total"] == src["vram_total"]
|
||||
pct = card["vram_used_pct"]
|
||||
assert pct["derived"] is True, "the one computed number in the block"
|
||||
assert pct["source"] == "computed:vram_used/vram_total*100"
|
||||
assert pct["value"] == pytest.approx(src["vram_used"] / src["vram_total"] * 100.0)
|
||||
|
||||
|
||||
def test_radiance_a_renamed_stats_key_degrades_to_absent(metrics_text) -> None:
|
||||
"""A radiance upgrade that renames a /stats key degrades it to absent, not a
|
||||
crash — the poll still returns its /metrics payload."""
|
||||
stats = _radiance_stats()
|
||||
stats["engine_version"] = stats.pop("version")
|
||||
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
|
||||
RateEstimator(), stats=stats, now=1.0)
|
||||
block = result.payload["stats"]
|
||||
assert "version" not in block, "renamed away: absent, never a guessed value"
|
||||
assert "engine_version" not in block, "and not carried under the new name either"
|
||||
assert block["model"] == "juggernaut", "the rest of the block is intact"
|
||||
assert result.payload["metrics"]["spec_accept_rate"]["value"] is not None, \
|
||||
"the poll still returns"
|
||||
# and a renamed card key degrades the computed pct, not the whole block
|
||||
stats2 = _radiance_stats()
|
||||
card = {**stats2["cards"][0], "vram_in_use": stats2["cards"][0]["vram_used"]}
|
||||
del card["vram_used"]
|
||||
stats2["cards"] = [card]
|
||||
result2 = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
|
||||
RateEstimator(), stats=stats2, now=1.0)
|
||||
row = result2.payload["stats"]["cards"][0]
|
||||
assert "vram_used" not in row and "vram_in_use" not in row
|
||||
assert row["vram_used_pct"]["value"] is None, "the computed number degrades to n/a"
|
||||
|
||||
|
||||
# ---- the /stats passthroughs in the metrics dict ------------------------- #
|
||||
|
||||
def test_radiance_queue_cap_is_the_stats_passthrough(metrics_text) -> None:
|
||||
stats = _radiance_stats()
|
||||
metrics = parse_radiance_metrics(metrics_text("radiance"), stats=stats)
|
||||
cell = metrics["queue_cap"]
|
||||
assert cell["value"] == stats["queue_cap"], "the /stats number, verbatim"
|
||||
assert cell["source"] == "radiance /stats"
|
||||
assert cell["derived"] is False
|
||||
absent = parse_radiance_metrics(metrics_text("radiance"))
|
||||
assert absent["queue_cap"]["value"] is None, "null, not 0 — rule 2"
|
||||
assert absent["queue_cap"]["source"] is None
|
||||
|
||||
|
||||
# ---- the history samples ------------------------------------------------- #
|
||||
|
||||
def test_radiance_history_samples_include_the_new_series(metrics_text) -> None:
|
||||
text = metrics_text("radiance")
|
||||
stats = _radiance_stats()
|
||||
result = engine_result("radiance", "http://r:8000", text,
|
||||
RateEstimator(), stats=stats, now=1.0)
|
||||
names = {s.metric for s in result.samples}
|
||||
for expected in ("llm.decode_tokens_per_second", "llm.prompt_tokens_per_second",
|
||||
"llm.kv_cache_usage", "llm.requests_running", "llm.requests_waiting",
|
||||
"llm.spec_accept_rate",
|
||||
"llm.prefix_cache_hit_ratio", "llm.tpot_seconds",
|
||||
"llm.ttft_seconds", "llm.ttft_p95_seconds",
|
||||
"llm.queue_depth", "llm.queue_cap",
|
||||
"llm.decode_step_seconds_recent", "llm.vram_used_pct"):
|
||||
assert expected in names, f"{expected} missing from the radiance samples"
|
||||
# `tokens_total` is a payload key, not a history series (it is not an existing
|
||||
# HISTORY_METRICS key, and the task's additions are the four above): the fleet
|
||||
# block reads it from the payload, the chart never drew it
|
||||
assert "llm.tokens_total" not in names
|
||||
# the stored values are the capture's own numbers (volatile — pinned as passthrough,
|
||||
# never as frozen values)
|
||||
prom = Prom(text)
|
||||
stored = {s.metric: s.value for s in result.samples}
|
||||
assert stored["llm.decode_tokens_per_second"] == \
|
||||
prom.scalar(["radiance:decode_tokens_per_second"], agg="avg")
|
||||
assert stored["llm.prompt_tokens_per_second"] == \
|
||||
prom.scalar(["radiance:prefill_tokens_per_second"], agg="avg")
|
||||
assert stored["llm.queue_cap"] == stats["queue_cap"]
|
||||
assert stored["llm.queue_depth"] == prom.scalar(["radiance:http_queue_depth"], agg="max")
|
||||
assert stored["llm.decode_step_seconds_recent"] == \
|
||||
prom.scalar(["radiance:decode_step_seconds_recent"], agg="max")
|
||||
assert stored["llm.kv_cache_usage"] == \
|
||||
prom.scalar(["vllm:kv_cache_usage_perc"], agg="max")
|
||||
# the per-position ratios are snapshot-only (decision 2): no history series
|
||||
assert not any("position" in name for name in names)
|
||||
# no request_queue_time on this build: queue_seconds is n/a by absence
|
||||
assert "llm.queue_seconds" not in names
|
||||
|
||||
|
||||
def test_radiance_history_does_not_touch_gufo_or_vllm_samples(metrics_text) -> None:
|
||||
"""The existing keys and their candidate order are untouched: a gufo or vLLM
|
||||
payload never grows a radiance series."""
|
||||
for engine in ("gufo", "vllm"):
|
||||
result = engine_result(engine, "http://x:1", metrics_text(engine),
|
||||
RateEstimator(), now=1.0)
|
||||
names = {s.metric for s in result.samples}
|
||||
for foreign in ("llm.queue_depth", "llm.queue_cap",
|
||||
"llm.decode_step_seconds_recent", "llm.vram_used_pct"):
|
||||
assert foreign not in names, f"{engine} grew a radiance series: {foreign}"
|
||||
|
||||
|
||||
def test_series_names_radiance_is_its_own_map() -> None:
|
||||
"""`series_names` is the parse catalog — the map's keys, per engine."""
|
||||
names = series_names("radiance")
|
||||
assert "spec_accept_rate" in names
|
||||
assert "gpu_cache_usage" in names
|
||||
assert "queue_depth" in names
|
||||
assert "tokens_generated_total" not in names, "that is the gufo key"
|
||||
assert "spec_accept_rate" not in series_names("vllm")
|
||||
|
||||
|
||||
# ---- the fleet totals ---------------------------------------------------- #
|
||||
|
||||
def test_fleet_totals_with_a_gufo_and_radiance_payload(metrics_text) -> None:
|
||||
"""The code needed no change: the aliases and the KV max already cover a radiance
|
||||
payload; the seams sentence says the known asymmetries in its own words.
|
||||
The numbers are re-derived from the two captures (volatile between re-captures).
|
||||
"""
|
||||
gufo_text = metrics_text("gufo")
|
||||
rad_text = metrics_text("radiance")
|
||||
gufo = engine_result("gufo", "http://g:1", gufo_text, RateEstimator(), now=1.0)
|
||||
rad = engine_result("radiance", "http://r:8000", rad_text,
|
||||
RateEstimator(), stats=_radiance_stats(), now=1.0)
|
||||
totals = fleet_totals([gufo.payload, rad.payload])
|
||||
assert totals["engines_total"] == 2
|
||||
assert totals["engines_live"] == 2
|
||||
# running/waiting: the sum of each capture's own counters (gufo's aliases + the gauges)
|
||||
gp = Prom(gufo_text)
|
||||
rp = Prom(rad_text)
|
||||
assert totals["requests_active"] == pytest.approx(
|
||||
(gp.scalar(["llamacpp:requests_processing"]) or 0.0)
|
||||
+ (rp.scalar(["vllm:num_requests_running"]) or 0.0))
|
||||
assert totals["requests_waiting"] == pytest.approx(
|
||||
(gp.scalar(["llamacpp:requests_deferred"]) or 0.0)
|
||||
+ (rp.scalar(["vllm:num_requests_waiting"]) or 0.0))
|
||||
gufo_kv = gp.scalar(["llamacpp:kv_cache_usage_ratio"], agg="max") or 0.0
|
||||
rad_kv = rp.scalar(["vllm:kv_cache_usage_perc"], agg="max") or 0.0
|
||||
assert totals["kv_cache_usage_max"] == pytest.approx(max(gufo_kv, rad_kv)), \
|
||||
"the true max: the lower engine's value must not hide the other's pressure"
|
||||
assert totals["tokens_total"] == pytest.approx(
|
||||
gufo.payload["metrics"]["tokens_total"]["value"]
|
||||
+ rad.payload["metrics"]["tokens_total"]["value"])
|
||||
# a fresh estimator: gufo's decode is n/a on its first poll, radiance's gauge wins
|
||||
assert totals["decode_tokens_per_second"] == pytest.approx(
|
||||
rad.payload["metrics"]["decode_tokens_per_second"]["value"])
|
||||
assert totals["prompt_tokens_per_second"] == pytest.approx(
|
||||
rad.payload["metrics"]["prompt_tokens_per_second"]["value"])
|
||||
seams = totals["seams"]
|
||||
assert "cache-hit" in seams, "gufo's prompt-counter asymmetry stays named"
|
||||
assert "provenance" in seams, "the seams name the provenance labelling"
|
||||
assert "vllm:prompt_tokens_total" in seams, \
|
||||
"and the thing the dashboard deliberately does not claim"
|
||||
|
||||
Reference in New Issue
Block a user