phase: 34_radiance_llm_engine

Everything is verified and green. Final report:

**Phase 34 final verification pass — complete**

- Verified all 6 completed tasks' work against the live fleet and hermetic suite: `llm:radiance` job registered, `llm_engines()` registry drives 404 gate / history defaults / `/api/config` (additive, byte-identical gufo/vllm keys), radiance field map + `/stats` sidecar + 7-position spec curve, `/llm` strip/curve/rows.
- **Defect found & fixed (pre-existing, task 05):** the vLLM-only sentence "vLLM exposes no per-slot JSON…" rendered on the **radiance** panel (`slotSection`'s non-gufo branch). Fix: vLLM line gated to `engine === 'vllm'`; radiance's panel slot section renders nothing (its session home is the strip) — `page-llm.js` + the served-byte pin in `tests/test_api.py` (now asserts the radiance branch is empty and the sentence can't leak). Browser-verified; re-shot `data/logs/34-llm-radiance-rest.png` (the 4.5-min wait found no in-flight request, so the busy shot remains the earlier pass's — its strip rows still stand; only the panel text predates the fix).
- Tests: `uv run pytest` → **2031 passed, 19 skipped** (100%); `uv run pytest --cov` → **98.73%** (floor 94%; config 100%, llm 99%, collectors 98%, api 99%); `uv run ruff check src tests agent && uv run pyright` → clean; `node --check` OK; `uv pip list --outdated` → empty.
- `uv run pytest -m live --live` → **13 passed, 2 skipped** (vLLM unset — correct; live-write opt-in — correct); the read-only radiance test passed.
- Completion criteria: (1) engines `gufo`+`radiance`, `fleet.engines_live` 2, `config.llm.radiance.enabled` true — **pass (live curl)**; (2) unset → 404 no-dial + no job — **pass (spy-pinned, green)**; (3) fixtures present, both families (79 `radiance:` / 198 `vllm:` lines), drift pin — **pass**; (4) panel/curve/rows — **pass (re-shot rest screenshot)**; (5) live radiance test — **pass**; (6) suite/coverage/lint/`bash .agents/validate.sh` (exit 0) — **pass**; (7) docs — below; (8) gufo/vLLM behaviour unchanged — **pass (existing pins untouched)**.
- Documentation: **README.md** full review — env table, LLM-engines section, architecture diagram, capture-script and `/llm` section all name radiance correctly; no inconsistencies found, no changes needed. **AGENTS.md** — the three task-06 fixes (fleet line, route row, module bullets) are in place and all other statements match reality; no changes needed.
- Dev server healthy (`./scripts/dev-server.sh status`: all sources ok, `llm:radiance` answering); all changes left in the working tree for the harness.
- Next pending phase: none — 34 was the last phase in `todo/`.
This commit is contained in:
2026-11-01 18:53:46 -05:00
parent 86c8bf4992
commit 219d7a7c3b
77 changed files with 3517 additions and 149 deletions
+646 -13
View File
@@ -5,17 +5,25 @@ the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentati
"""
from __future__ import annotations
import json
import re
from collections.abc import Callable, Mapping, Sequence
from pathlib import Path
from typing import Any
import pytest
from dashboards.prom import Prom
from dashboards.rates import MIN_WINDOW_SECONDS, RateEstimator
from dashboards.severity import classify
from dashboards.sources.llm import (
RADIANCE_HISTOGRAMS,
RADIANCE_OPTIONAL,
RADIANCE_SERIES,
engine_result,
fleet_totals,
parse_gufo_metrics,
parse_radiance_metrics,
parse_vllm_metrics,
rate_key,
series_names,
@@ -62,19 +70,31 @@ def poll_engine(metrics_text: Any, engine: str, span: float, stamps: Sequence[fl
# ---- gufo / llama.cpp ----------------------------------------------------- #
def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
metrics = parse_gufo_metrics(metrics_text("gufo"))
assert metrics["tokens_generated_total"]["value"] == 877_336.0
assert metrics["tokens_prompt_total"]["value"] == 1_350_666.0
assert metrics["tokens_prompt_cached_total"]["value"] == 124_529_105.0
assert metrics["requests_processing"]["value"] == 0.0
assert metrics["device_lost_total"]["value"] == 0.0
assert metrics["kv_cache_usage_ratio"]["value"] == 0.0
"""The cell's value is the number the exposition declares, verbatim: no
conversion, no scaling, and the series' own name is the source. Compared against
the raw `Prom` parse of the same capture — never a value frozen at capture time
(the counters reset on every engine restart)."""
text = metrics_text("gufo")
prom = Prom(text)
metrics = parse_gufo_metrics(text)
for key, name in (
("tokens_generated_total", "llamacpp:tokens_predicted_total"),
("tokens_prompt_total", "llamacpp:prompt_tokens_total"),
("tokens_prompt_cached_total", "llamacpp:prompt_tokens_cached_total"),
("requests_processing", "llamacpp:requests_processing"),
("device_lost_total", "gufo_device_lost_total"),
("kv_cache_usage_ratio", "llamacpp:kv_cache_usage_ratio"),
):
assert metrics[key]["value"] == prom.scalar([name]), key
assert metrics[key]["source"] == name, "the counter's own name is the source"
def test_gufo_speculative_accept_rate_is_accepted_over_draft(metrics_text) -> None:
metrics = parse_gufo_metrics(metrics_text("gufo"))
accept = metrics["spec_accept_rate"]["value"]
assert accept == pytest.approx(578_514 / 789_028)
accepted = metrics["spec_accepted_tokens_total"]["value"]
draft_tokens = metrics["spec_draft_tokens_total"]["value"]
assert accept == pytest.approx(accepted / draft_tokens)
assert str(metrics["spec_accept_rate"]["source"]).startswith("computed")
@@ -82,7 +102,10 @@ def test_gufo_lifetime_tokens_include_cache_hits(metrics_text) -> None:
"""gufo's prompt counter excludes cache hits; the lifetime total must not."""
metrics = parse_gufo_metrics(metrics_text("gufo"))
total = metrics["tokens_total"]["value"]
assert total == 1_350_666 + 124_529_105 + 877_336
prompt = metrics["tokens_prompt_total"]["value"]
cached = metrics["tokens_prompt_cached_total"]["value"] or 0.0
generated = metrics["tokens_generated_total"]["value"]
assert total == pytest.approx(prompt + cached + generated)
def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_text) -> None:
@@ -99,8 +122,11 @@ def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_te
assert decode["value"] == pytest.approx(40.0)
assert decode["window"] == pytest.approx(10.0)
assert "Δ" in decode["source"] and "tokens_predicted_total" in decode["source"]
# the native gauge is still there, labelled, because it answers a different question
assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)
# the native gauge is still there, labelled, because it answers a different question —
# verbatim the exposition's own number (a volatile gauge: no frozen value)
assert metrics["decode_tokens_per_second_native"]["value"] == \
Prom(text).scalar(["llamacpp:predicted_tokens_seconds"])
assert metrics["decode_tokens_per_second_native"]["source"] == "llamacpp:predicted_tokens_seconds"
def test_gufo_the_window_the_llm_jobs_are_spanned_by_is_one_that_fills(metrics_text) -> None:
@@ -323,10 +349,12 @@ def test_absent_gufo_metrics_are_none_with_no_source() -> None:
def test_gufo_kv_cache_cell_keeps_provenance_and_gains_severity(metrics_text) -> None:
metrics = parse_gufo_metrics(metrics_text("gufo"))
cell = metrics["kv_cache_usage_ratio"]
assert cell["value"] == 0.0
assert cell["value"] == Prom(metrics_text("gufo")).scalar(["llamacpp:kv_cache_usage_ratio"]), \
"verbatim the exposition's own ratio — the cell stays what it was"
assert cell["source"] == "llamacpp:kv_cache_usage_ratio"
assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
assert cell["severity"] == "ok"
assert cell["severity"] == classify("llm.kv_cache_usage_pct", cell["value"] * 100.0), \
"the verdict is the table's for this value (percent scale)"
@pytest.mark.parametrize("ratio,expected", [
@@ -667,3 +695,608 @@ def test_vllm_a_measured_zero_beats_the_running_estimate(metrics_text) -> None:
decode = metrics["decode_tokens_per_second"]
assert decode["value"] == 0.0
assert "running/tpot" not in str(decode["source"])
# ---- radiance: the ground-truth fixtures (phase 34, task 01) ---------------- #
#
# Radiance is a vLLM derivative: its /metrics carries the `radiance:` native series plus
# a `vllm:` compatibility family, and the capture records /stats and /v1/models in the
# same pass. These tests pin the *shape and the name set* of those captures, never the
# numbers (the nut_*.vars precedent): a metric rename or a new build's series fails here
# before a dashboard does.
def _load_radiance_json(name: str) -> Any:
"""Load a radiance JSON fixture, skipping with the capture command like the others."""
path = Path(__file__).resolve().parent / "fixtures" / name
if not path.exists():
pytest.skip(f"missing fixture {path.name} — run scripts/capture-fixtures.sh --with-llm")
# the capture script writes its verbatim header as `#` comment lines; the document
# is what follows them
body = "\n".join(line for line in path.read_text().splitlines() if not line.startswith("#"))
return json.loads(body)
def _radiance_map_names() -> set[str]:
return ({name for names, _ in RADIANCE_SERIES.values() for name in names}
| {name for names in RADIANCE_HISTOGRAMS.values() for name in names})
def test_radiance_fixture_name_set_is_pinned_by_the_map_and_optionals(metrics_text) -> None:
"""The drift pin: every name in the capture's radiance:/vllm: families is either a
payload cell (the map) or a declared non-cell (RADIANCE_OPTIONAL), and vice versa —
so a rename, a dropped series, or a typo in a map name fails the suite before a
dashboard does."""
names = set()
for line in metrics_text("radiance").splitlines():
if line and not line.startswith("#"):
names.add(line.split("{")[0].split()[0])
contract = {name for name in names if name.startswith(("radiance:", "vllm:"))}
assert contract, "the capture lost its series — re-run the capture with the engine up"
map_names = _radiance_map_names()
unexplained = contract - map_names - RADIANCE_OPTIONAL
assert not unexplained, (
f"names in the capture that are neither a payload cell nor declared optional: "
f"{sorted(unexplained)}")
absent_everywhere = map_names - contract - RADIANCE_OPTIONAL
assert not absent_everywhere, (
f"map names in neither the capture nor RADIANCE_OPTIONAL (a rename or a typo): "
f"{sorted(absent_everywhere)}")
stale = RADIANCE_OPTIONAL - contract - map_names
assert not stale, f"RADIANCE_OPTIONAL entries that are neither real names: {sorted(stale)}"
def test_radiance_fixture_carries_both_families(metrics_text) -> None:
"""The vLLM-derived shape is part of the contract: the capture must hold both the
native series and the compatibility family, or the name set has drifted."""
text = metrics_text("radiance")
native = sum(1 for line in text.splitlines() if line.startswith("radiance:"))
compat = sum(1 for line in text.splitlines() if line.startswith("vllm:"))
assert native > 0, "the capture lost its radiance: native series"
assert compat > 0, "the capture lost its vllm: compatibility family"
def test_radiance_per_position_series_has_the_seven_positions_the_phase_pins(
metrics_text,
) -> None:
"""The per-position spec-acceptance series is the 7-bar curve: its label set is
position="0".."6", each position a series of its own."""
positions = set()
for line in metrics_text("radiance").splitlines():
if line.startswith("radiance:draft_position_acceptance_ratio{"):
m = re.search(r'position="(\d+)"', line)
assert m, f"a per-position line without a position label: {line}"
positions.add(m.group(1))
assert positions == {str(i) for i in range(7)}, (
f"the phase pins seven draft positions; the capture has {sorted(positions)}")
STATS_TOP_LEVEL_KEYS = (
"cards", "checkpoints_written", "ckpt_evictions", "ckpt_refused", "decode_tps",
"draft_accept", "draft_positions", "hit_cut_tokens", "hit_cuts", "in_flight",
"kv_bytes_carved", "kv_bytes_total", "kv_bytes_used", "kv_tokens_cached",
"kv_tokens_max", "kv_tokens_total", "kv_tokens_used", "kv_util",
"linear_cached_tokens", "linear_hit_rate", "link", "model", "now_ms", "passes",
"preempted", "preemptions", "prefill_chunks_capped", "prefill_tps",
"prefix_evictions", "prefix_hit_rate", "queue_cap", "released", "requests",
"rows_unspec", "running", "started_at", "startup_s", "step_ns", "step_ns_decode",
"step_ns_decode_best", "step_ns_decode_recent", "steps", "steps_decode", "steps_held",
"steps_mixed", "steps_prefill", "total_requests", "uptime_s", "version", "waiting",
)
def test_radiance_stats_fixture_shape() -> None:
"""The /stats document: the top-level key set, the per-card VRAM rows, the requests
list and the per-position spec-acceptance list — shape, not numbers. A capture at
rest records `requests: []` (the header comment says so); a row shape is recorded
by re-running the capture with a request in flight."""
stats = _load_radiance_json("stats_radiance.json")
assert isinstance(stats, dict)
assert tuple(sorted(stats.keys())) == STATS_TOP_LEVEL_KEYS, (
"the /stats top-level key set changed — the shape the page reads drifted; "
"re-capture and re-decide the payload set")
cards = stats["cards"]
assert isinstance(cards, list) and cards, "cards[] is the per-card VRAM panel's source"
for card in cards:
assert isinstance(card, dict)
assert isinstance(card["index"], int)
assert isinstance(card["name"], str) and card["name"]
assert isinstance(card["vram_used"], int)
assert isinstance(card["vram_total"], int)
assert 0 < card["vram_used"] <= card["vram_total"]
assert isinstance(stats["requests"], list)
assert isinstance(stats["queue_cap"], int)
assert isinstance(stats["in_flight"], int)
positions = stats["draft_positions"]
assert isinstance(positions, list)
assert len(positions) == 7, (
"the phase pins seven draft positions; the capture has "
f"{len(positions)} — the 7-bar curve needs re-deciding")
for pos in positions:
assert isinstance(pos, dict)
assert isinstance(pos["reached"], int)
assert isinstance(pos["accepted"], int)
assert isinstance(pos["rate"], float)
def test_radiance_models_fixture_shape() -> None:
"""The /v1/models document: one model, and the capabilities / max_model_len it
publishes — the only place the engine states what it can do."""
models = _load_radiance_json("models_radiance.json")
assert isinstance(models, dict)
assert models["object"] == "list"
data = models["data"]
assert isinstance(data, list) and data
model = data[0]
assert model["owned_by"] == "radiance", "the engine's own word for its family"
assert isinstance(model["id"], str) and model["id"]
assert isinstance(model["max_model_len"], int)
assert isinstance(model["capabilities"], dict)
# ---- radiance: the field map (phase 34, task 03) ----------------------- #
#
# The map is built from the task-01 fixture (rule 7); the tests below pin the three
# throughput arms (gauge wins / measured fills / n/a on a refused window), the severity
# pass-through on the existing KV row, the /stats session arms, the whitelisted stats
# block and the additive history series. As always: shape and name set, never the
# numbers of a live engine.
def _radiance_stats() -> dict:
return _load_radiance_json("stats_radiance.json")
# ---- the fixture parse -------------------------------------------------- #
def test_radiance_fixture_parse_every_mapped_key_resolves(metrics_text) -> None:
"""Every mapped key resolves on the verbatim capture, with its pinned source string."""
metrics = parse_radiance_metrics(metrics_text("radiance"))
expected_source = {
"decode_tokens_per_second_native": "radiance:decode_tokens_per_second",
"prompt_tokens_per_second_native": "radiance:prefill_tokens_per_second",
"num_requests_running": "vllm:num_requests_running",
"num_requests_waiting": "vllm:num_requests_waiting",
"gpu_cache_usage": "vllm:kv_cache_usage_perc",
"kv_cache_max_tokens": "radiance:kv_cache_max_tokens",
"kv_cache_max_bytes": "radiance:kv_cache_max_bytes",
"queue_depth": "radiance:http_queue_depth",
"admission_rejected_total": "radiance:http_admission_rejected_total",
"num_preemptions_total": "vllm:num_preemptions_total",
"prompt_tokens_total": "vllm:prompt_tokens_total",
"generation_tokens_total": "vllm:generation_tokens_total",
"gpu_prefix_cache_hit_rate": "vllm:gpu_prefix_cache_hit_rate",
"spec_accept_rate": "radiance:draft_acceptance_ratio",
"decode_step_seconds_recent": "radiance:decode_step_seconds_recent",
"decode_step_seconds_best": "radiance:decode_step_seconds_best",
}
for key, source in expected_source.items():
cell = metrics[key]
assert set(cell) >= {"value", "source", "derived"}, f"{key} lost its cell shape"
assert cell["source"] == source, f"{key} resolved to {cell['source']!r}, not {source!r}"
assert cell["value"] is not None, f"{key} should resolve on the capture"
# a map name this build does not carry: n/a, never 0
assert metrics["num_requests_swapped"]["value"] is None
assert metrics["num_requests_swapped"]["source"] is None
def test_radiance_a_series_removed_from_the_capture_is_n_a_never_zero(metrics_text) -> None:
"""The completion-criteria arm: a line removed from a fixture copy answers
`{\"value\": null}` — 0 would claim \"the engine published zero\", which it did not."""
by_key = {
"radiance:http_queue_depth": "queue_depth",
"radiance:kv_cache_max_bytes": "kv_cache_max_bytes",
"radiance:decode_step_seconds_best": "decode_step_seconds_best",
"vllm:num_preemptions_total": "num_preemptions_total",
"vllm:gpu_prefix_cache_hit_rate": "gpu_prefix_cache_hit_rate",
"radiance:draft_acceptance_ratio": "spec_accept_rate",
}
for name, key in by_key.items():
text = "\n".join(line for line in metrics_text("radiance").splitlines()
if not line.startswith(name))
cell = parse_radiance_metrics(text)[key]
assert cell["value"] is None, f"{key} answered a value for a removed series: {cell}"
assert cell["source"] is None
def test_radiance_spec_positions_are_the_ordered_seven(metrics_text) -> None:
metrics = parse_radiance_metrics(metrics_text("radiance"))
positions = metrics["spec_positions"]
assert [p["position"] for p in positions] == list(range(7)), \
"the phase pins seven draft positions, in the order the engine publishes them"
for p in positions:
assert 0.0 <= p["ratio"] <= 1.0, "a per-position ratio is a 0-1 gauge"
# the ratios are a running-window aggregate: volatile between captures, so the
# pin is the shape (seven, ordered, in-range) — the name set is pinned above
prom = Prom(metrics_text("radiance"))
by_position = prom.by_label(["radiance:draft_position_acceptance_ratio"], "position")
for p in positions:
assert p["ratio"] == by_position[str(p["position"])], \
f"position {p['position']} is not the exposition's own number"
def test_radiance_an_absent_position_stays_absent_not_zeroed(metrics_text) -> None:
text = "\n".join(
line for line in metrics_text("radiance").splitlines()
if not ('radiance:draft_position_acceptance_ratio{' in line
and 'position="6"' in line)
)
positions = parse_radiance_metrics(text)["spec_positions"]
assert [p["position"] for p in positions] == [0, 1, 2, 3, 4, 5], \
"position 6 is absent from this capture: the list holds what the engine published"
assert all(p["ratio"] is not None and p["ratio"] > 0.0 for p in positions), \
"an absent position must not be filled with a 0"
def test_radiance_tokens_total_is_the_two_vllm_counters(metrics_text) -> None:
metrics = parse_radiance_metrics(metrics_text("radiance"))
prompt = metrics["prompt_tokens_total"]["value"]
generated = metrics["generation_tokens_total"]["value"]
assert metrics["tokens_total"]["value"] == pytest.approx(prompt + generated)
assert metrics["tokens_total"]["source"] == "computed:prompt+generated"
# both counters removed: n/a, never 0
text = metrics_text("radiance")
for name in ("vllm:prompt_tokens_total", "vllm:generation_tokens_total"):
text = "\n".join(line for line in text.splitlines()
if not line.startswith(name))
assert parse_radiance_metrics(text)["tokens_total"]["value"] is None
def test_radiance_prefix_hit_ratio_is_the_gauge_source_named_not_derived(metrics_text) -> None:
"""This build publishes no prefix counters, so a hits/queries ratio would be an
invented number — the gauge is the answer, under the shared UI key."""
text = metrics_text("radiance")
metrics = parse_radiance_metrics(text)
gauge = Prom(text).scalar(["vllm:gpu_prefix_cache_hit_rate"])
cell = metrics["prefix_cache_hit_ratio"]
assert cell["value"] == pytest.approx(gauge)
assert cell["source"] == "vllm:gpu_prefix_cache_hit_rate"
assert cell["derived"] is False, "the gauge is not a computed ratio"
# the gauge cell itself is the payload cell too (the map entry)
assert metrics["gpu_prefix_cache_hit_rate"]["value"] == pytest.approx(gauge)
def test_radiance_latency_histograms_are_the_vllm_shaped_family(metrics_text) -> None:
metrics = parse_radiance_metrics(metrics_text("radiance"))
ttft = metrics["time_to_first_token_seconds"]
assert ttft and ttft["count"] > 0 and ttft["mean"] > 0
assert metrics["time_to_first_token_seconds__p95"]["value"] >= ttft["mean"]
assert metrics["time_per_output_token_seconds"]["mean"] > 0
assert metrics["e2e_request_latency_seconds"]["count"] > 0
# no request_queue_time on this build: the cell is absent, never back-filled
assert "request_queue_time_seconds" not in metrics
assert "request_queue_time_seconds__mean" not in metrics
# ---- rule 2, the three throughput arms ----------------------------------- #
def test_radiance_throughput_arm_gauge_wins(metrics_text) -> None:
"""The native gauge is the scheduler's recent-window aggregate, so it wins."""
text = metrics_text("radiance")
metrics = parse_radiance_metrics(text, RateEstimator(span=10), 1_000.0)
decode = metrics["decode_tokens_per_second"]
# the gauge's value is the exposition's own number (volatile — the pin is the
# passthrough, not a frozen value), and it wins the cell
assert decode["value"] == metrics["decode_tokens_per_second_native"]["value"]
assert decode["value"] == Prom(text).scalar(["radiance:decode_tokens_per_second"], agg="avg")
assert decode["source"] == "radiance:decode_tokens_per_second"
assert decode["derived"] is False, "a scraped gauge is not a computed Δ"
assert "window" not in decode, "a scraped gauge names no Δ window"
prompt = metrics["prompt_tokens_per_second"]
assert prompt["value"] == metrics["prompt_tokens_per_second_native"]["value"]
assert prompt["source"] == "radiance:prefill_tokens_per_second"
def test_radiance_throughput_arm_measured_delta_when_the_gauge_is_absent(metrics_text) -> None:
"""A gauge this build does not publish falls back to the measured Δ, per label set."""
text = metrics_text("radiance")
text = "\n".join(line for line in text.splitlines()
if not line.startswith("radiance:decode_tokens_per_second"))
rates = RateEstimator(span=10)
parse_radiance_metrics(text, rates, 1_000.0)
text = bump_metric(text, "vllm:generation_tokens_total", 250.0)
metrics = parse_radiance_metrics(text, rates, 1_010.0)
decode = metrics["decode_tokens_per_second"]
assert decode["value"] == pytest.approx(25.0)
assert decode["derived"] is True
assert decode["source"] == "computed:Δvllm:generation_tokens_total/Δt"
assert decode["window"] == pytest.approx(10.0), \
"the label must name the window the value used"
# the arms are independent: the prompt gauge still wins its own cell
prompt = metrics["prompt_tokens_per_second"]
assert prompt["value"] == metrics["prompt_tokens_per_second_native"]["value"]
assert prompt["derived"] is False
def test_radiance_throughput_arm_a_refused_window_is_n_a_with_its_label(metrics_text) -> None:
"""A refused window is still an answer: n/a beside the honest Δ<window> it refused."""
text = metrics_text("radiance")
text = "\n".join(line for line in text.splitlines()
if not line.startswith("radiance:decode_tokens_per_second"))
rates = RateEstimator(span=10)
parse_radiance_metrics(text, rates, 1_000.0)
text = bump_metric(text, "vllm:generation_tokens_total", 50.0)
parse_radiance_metrics(text, rates, 1_010.0)
text = bump_metric(text, "vllm:generation_tokens_total", 60.0)
decode = parse_radiance_metrics(text, rates, 1_010.3)["decode_tokens_per_second"]
assert decode["value"] is None, "0.3 s is the loop's catch-up gap, not a rate"
assert decode["window"] == pytest.approx(0.3)
assert decode["window"] < MIN_WINDOW_SECONDS
assert decode["source"] == "computed:Δvllm:generation_tokens_total/Δt"
def test_radiance_throughput_arm_both_absent_is_n_a_never_zero(metrics_text) -> None:
text = metrics_text("radiance")
for name in ("radiance:decode_tokens_per_second", "radiance:prefill_tokens_per_second",
"vllm:generation_tokens_total", "vllm:prompt_tokens_total"):
text = "\n".join(line for line in text.splitlines() if not line.startswith(name))
metrics = parse_radiance_metrics(text) # no estimator: no measured arm either
for key in ("decode_tokens_per_second", "prompt_tokens_per_second"):
cell = metrics[key]
assert cell["value"] is None, f"{key} printed a value with no source: {cell}"
assert cell["source"] is None
# ---- severity: the KV cell is rated, the spec cell is not --------------- #
def test_radiance_kv_cell_carries_the_severity_verdict(metrics_text) -> None:
"""Radiance publishes the same 0-1 ratio the vLLM map feeds the table: the
existing `llm.kv_cache_usage_pct` row rates it, no new threshold anywhere."""
text = metrics_text("radiance")
base = Prom(text).scalar(["vllm:kv_cache_usage_perc"], agg="max")
assert base is not None
cell = parse_radiance_metrics(text)["gpu_cache_usage"]
assert cell["value"] == pytest.approx(base)
assert cell["source"] == "vllm:kv_cache_usage_perc"
assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
assert cell["severity"] == classify("llm.kv_cache_usage_pct", base * 100.0), \
"the table's verdict for this value (percent scale)"
# push it above the table's fail bound (90 %): the verdict follows the value
target = 0.99
pressed = bump_metric(text, "vllm:kv_cache_usage_perc", target - base)
cell = parse_radiance_metrics(pressed)["gpu_cache_usage"]
assert cell["value"] == pytest.approx(target)
assert cell["severity"] == "fail"
def test_radiance_spec_accept_carries_no_severity(metrics_text) -> None:
"""Decision 4 pinned both ways: the KV cell is rated, the spec cell is not — an
efficiency number is not a verdict."""
text = metrics_text("radiance")
metrics = parse_radiance_metrics(text)
assert metrics["spec_accept_rate"]["value"] == \
Prom(text).scalar(["radiance:draft_acceptance_ratio"], agg="avg")
assert "severity" not in metrics["spec_accept_rate"]
assert metrics["gpu_cache_usage"]["severity"] is not None, \
"the other direction: the KV cell is rated"
# ---- /stats as the session source ---------------------------------------- #
def test_radiance_sessions_stats_arm_at_rest_counts_and_cap(metrics_text) -> None:
"""The at-rest capture: requests[] empty → the counts-otherwise sentence, the cap
from /stats, `active` from the /metrics gauge (one definition in every arm).
The numbers come from the capture itself — volatile between re-captures."""
stats = _radiance_stats()
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
RateEstimator(), stats=stats, now=1.0)
sessions = result.payload["sessions"]
active_expected = Prom(metrics_text("radiance")).scalar(["vllm:num_requests_running"])
assert sessions["active"] == active_expected, "active stays the /metrics gauge in every arm"
assert sessions["running"] == stats["running"]
assert sessions["waiting"] == stats["waiting"]
assert sessions["queue_cap"] == stats["queue_cap"]
assert sessions["list"] == stats["requests"]
assert sessions["source"] == "radiance /stats"
if not stats["requests"]:
assert sessions["detail"] == (
f"radiance reports counts, not sessions: {stats['running']:g} running"
f" of cap {stats['queue_cap']:g}. Per-request detail appears here while "
f"requests are in flight.")
else:
assert sessions["detail"] is None, "rows in flight: no at-rest sentence"
def test_radiance_sessions_stats_arm_non_empty_rows_verbatim(metrics_text) -> None:
"""A request in flight: the rows are the engine's own shape, verbatim."""
stats = _radiance_stats()
rows = [{"id": 42, "model": "juggernaut", "state": "running",
"prompt_tokens": 1_200, "completed_tokens": 250}]
stats = {**stats, "requests": rows, "running": 1, "waiting": 2}
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
RateEstimator(), stats=stats, now=1.0)
sessions = result.payload["sessions"]
assert sessions["list"] == rows, "the rows pass through untouched"
assert sessions["source"] == "radiance /stats"
assert sessions["detail"] is None
assert sessions["running"] == 1
assert sessions["waiting"] == 2
assert sessions["queue_cap"] == stats["queue_cap"] # the capture's cap, untouched
def test_radiance_sessions_missing_stats_fall_back_to_the_gauges(metrics_text) -> None:
"""The sidecar failed: the poll is still healthy, the counts come from the
/metrics gauges (a different provenance, said so), and the cap is null — never 0."""
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
RateEstimator(), now=1.0) # stats=None: /stats did not answer
sessions = result.payload["sessions"]
assert sessions["running"] == 0.0, "the /metrics gauge's value"
assert sessions["waiting"] == 0.0
assert sessions["queue_cap"] is None, "null, not 0 — rule 2"
assert sessions["list"] == []
assert "did not answer" in sessions["detail"]
assert "gauges" in sessions["detail"]
assert "stats" not in result.payload, "no stats block: the sidecar never answered"
# the metrics passthroughs degrade with it
assert result.payload["metrics"]["queue_cap"]["value"] is None
assert result.payload["metrics"]["vram_used_pct"]["value"] is None
# ---- the whitelisted stats block ------------------------------------------ #
def test_radiance_payload_stats_block_is_whitelisted(metrics_text) -> None:
stats = _radiance_stats()
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
RateEstimator(), stats=stats, now=1.0)
block = result.payload["stats"]
assert set(block) == {"model", "version", "uptime_s", "startup_s",
"total_requests", "cards"}, \
"the engine internals (link / passes / steps* / …) stay out of the payload"
assert block["model"] == "juggernaut"
assert block["version"] == "1.15.1"
assert isinstance(block["uptime_s"], int) and block["uptime_s"] >= 0, \
"uptime is the engine's own seconds — volatile between captures"
card = block["cards"][0]
src = stats["cards"][0]
assert set(card) == {"index", "name", "vram_used", "vram_total", "vram_used_pct"}
assert card["vram_used"] == src["vram_used"], \
"the raw bytes travel as the engine published them"
assert card["vram_total"] == src["vram_total"]
pct = card["vram_used_pct"]
assert pct["derived"] is True, "the one computed number in the block"
assert pct["source"] == "computed:vram_used/vram_total*100"
assert pct["value"] == pytest.approx(src["vram_used"] / src["vram_total"] * 100.0)
def test_radiance_a_renamed_stats_key_degrades_to_absent(metrics_text) -> None:
"""A radiance upgrade that renames a /stats key degrades it to absent, not a
crash — the poll still returns its /metrics payload."""
stats = _radiance_stats()
stats["engine_version"] = stats.pop("version")
result = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
RateEstimator(), stats=stats, now=1.0)
block = result.payload["stats"]
assert "version" not in block, "renamed away: absent, never a guessed value"
assert "engine_version" not in block, "and not carried under the new name either"
assert block["model"] == "juggernaut", "the rest of the block is intact"
assert result.payload["metrics"]["spec_accept_rate"]["value"] is not None, \
"the poll still returns"
# and a renamed card key degrades the computed pct, not the whole block
stats2 = _radiance_stats()
card = {**stats2["cards"][0], "vram_in_use": stats2["cards"][0]["vram_used"]}
del card["vram_used"]
stats2["cards"] = [card]
result2 = engine_result("radiance", "http://r:8000", metrics_text("radiance"),
RateEstimator(), stats=stats2, now=1.0)
row = result2.payload["stats"]["cards"][0]
assert "vram_used" not in row and "vram_in_use" not in row
assert row["vram_used_pct"]["value"] is None, "the computed number degrades to n/a"
# ---- the /stats passthroughs in the metrics dict ------------------------- #
def test_radiance_queue_cap_is_the_stats_passthrough(metrics_text) -> None:
stats = _radiance_stats()
metrics = parse_radiance_metrics(metrics_text("radiance"), stats=stats)
cell = metrics["queue_cap"]
assert cell["value"] == stats["queue_cap"], "the /stats number, verbatim"
assert cell["source"] == "radiance /stats"
assert cell["derived"] is False
absent = parse_radiance_metrics(metrics_text("radiance"))
assert absent["queue_cap"]["value"] is None, "null, not 0 — rule 2"
assert absent["queue_cap"]["source"] is None
# ---- the history samples ------------------------------------------------- #
def test_radiance_history_samples_include_the_new_series(metrics_text) -> None:
text = metrics_text("radiance")
stats = _radiance_stats()
result = engine_result("radiance", "http://r:8000", text,
RateEstimator(), stats=stats, now=1.0)
names = {s.metric for s in result.samples}
for expected in ("llm.decode_tokens_per_second", "llm.prompt_tokens_per_second",
"llm.kv_cache_usage", "llm.requests_running", "llm.requests_waiting",
"llm.spec_accept_rate",
"llm.prefix_cache_hit_ratio", "llm.tpot_seconds",
"llm.ttft_seconds", "llm.ttft_p95_seconds",
"llm.queue_depth", "llm.queue_cap",
"llm.decode_step_seconds_recent", "llm.vram_used_pct"):
assert expected in names, f"{expected} missing from the radiance samples"
# `tokens_total` is a payload key, not a history series (it is not an existing
# HISTORY_METRICS key, and the task's additions are the four above): the fleet
# block reads it from the payload, the chart never drew it
assert "llm.tokens_total" not in names
# the stored values are the capture's own numbers (volatile — pinned as passthrough,
# never as frozen values)
prom = Prom(text)
stored = {s.metric: s.value for s in result.samples}
assert stored["llm.decode_tokens_per_second"] == \
prom.scalar(["radiance:decode_tokens_per_second"], agg="avg")
assert stored["llm.prompt_tokens_per_second"] == \
prom.scalar(["radiance:prefill_tokens_per_second"], agg="avg")
assert stored["llm.queue_cap"] == stats["queue_cap"]
assert stored["llm.queue_depth"] == prom.scalar(["radiance:http_queue_depth"], agg="max")
assert stored["llm.decode_step_seconds_recent"] == \
prom.scalar(["radiance:decode_step_seconds_recent"], agg="max")
assert stored["llm.kv_cache_usage"] == \
prom.scalar(["vllm:kv_cache_usage_perc"], agg="max")
# the per-position ratios are snapshot-only (decision 2): no history series
assert not any("position" in name for name in names)
# no request_queue_time on this build: queue_seconds is n/a by absence
assert "llm.queue_seconds" not in names
def test_radiance_history_does_not_touch_gufo_or_vllm_samples(metrics_text) -> None:
"""The existing keys and their candidate order are untouched: a gufo or vLLM
payload never grows a radiance series."""
for engine in ("gufo", "vllm"):
result = engine_result(engine, "http://x:1", metrics_text(engine),
RateEstimator(), now=1.0)
names = {s.metric for s in result.samples}
for foreign in ("llm.queue_depth", "llm.queue_cap",
"llm.decode_step_seconds_recent", "llm.vram_used_pct"):
assert foreign not in names, f"{engine} grew a radiance series: {foreign}"
def test_series_names_radiance_is_its_own_map() -> None:
"""`series_names` is the parse catalog — the map's keys, per engine."""
names = series_names("radiance")
assert "spec_accept_rate" in names
assert "gpu_cache_usage" in names
assert "queue_depth" in names
assert "tokens_generated_total" not in names, "that is the gufo key"
assert "spec_accept_rate" not in series_names("vllm")
# ---- the fleet totals ---------------------------------------------------- #
def test_fleet_totals_with_a_gufo_and_radiance_payload(metrics_text) -> None:
"""The code needed no change: the aliases and the KV max already cover a radiance
payload; the seams sentence says the known asymmetries in its own words.
The numbers are re-derived from the two captures (volatile between re-captures).
"""
gufo_text = metrics_text("gufo")
rad_text = metrics_text("radiance")
gufo = engine_result("gufo", "http://g:1", gufo_text, RateEstimator(), now=1.0)
rad = engine_result("radiance", "http://r:8000", rad_text,
RateEstimator(), stats=_radiance_stats(), now=1.0)
totals = fleet_totals([gufo.payload, rad.payload])
assert totals["engines_total"] == 2
assert totals["engines_live"] == 2
# running/waiting: the sum of each capture's own counters (gufo's aliases + the gauges)
gp = Prom(gufo_text)
rp = Prom(rad_text)
assert totals["requests_active"] == pytest.approx(
(gp.scalar(["llamacpp:requests_processing"]) or 0.0)
+ (rp.scalar(["vllm:num_requests_running"]) or 0.0))
assert totals["requests_waiting"] == pytest.approx(
(gp.scalar(["llamacpp:requests_deferred"]) or 0.0)
+ (rp.scalar(["vllm:num_requests_waiting"]) or 0.0))
gufo_kv = gp.scalar(["llamacpp:kv_cache_usage_ratio"], agg="max") or 0.0
rad_kv = rp.scalar(["vllm:kv_cache_usage_perc"], agg="max") or 0.0
assert totals["kv_cache_usage_max"] == pytest.approx(max(gufo_kv, rad_kv)), \
"the true max: the lower engine's value must not hide the other's pressure"
assert totals["tokens_total"] == pytest.approx(
gufo.payload["metrics"]["tokens_total"]["value"]
+ rad.payload["metrics"]["tokens_total"]["value"])
# a fresh estimator: gufo's decode is n/a on its first poll, radiance's gauge wins
assert totals["decode_tokens_per_second"] == pytest.approx(
rad.payload["metrics"]["decode_tokens_per_second"]["value"])
assert totals["prompt_tokens_per_second"] == pytest.approx(
rad.payload["metrics"]["prompt_tokens_per_second"]["value"])
seams = totals["seams"]
assert "cache-hit" in seams, "gufo's prompt-counter asymmetry stays named"
assert "provenance" in seams, "the seams name the provenance labelling"
assert "vllm:prompt_tokens_total" in seams, \
"and the thing the dashboard deliberately does not claim"