phase: 23_llm_rate_responsiveness
Phase 23 final verification complete — all green.
```
VERIFIED (criteria)
1 /api/llm/snapshot windows ≈2 s live (1.98–2.00), /api/system window_seconds ≈10 s (9.996–10.005);
/api/config llm.rate_window_seconds=2=LLM_INTERVAL_SECONDS; browser re-check ALL CHECKS PASSED
(data/logs/23-window-{llm,system}.png — /llm prints Δ2.0s, /system footnote Δ10.0s)
2 Phase-13 pins unchanged: floor, 2×span ceiling, 8× jump gate + independent anchor — all pass
(only import/span-line diffs in tests/; git diff confirms no assertion rewritten)
3 Live proof re-derived from artifacts: 780 rows, 0 unlabelled, windows 0.61–3.38 s, 5 starts +
5 ends settled by poll 2 (0 unsettled), 1 m rollups max 42.3 vs raw 201.6, 0 over 300 tok/s;
counterfactual 10 s window: median settle poll 3, |ln| 0.924 vs 0.037
4 uv run pytest: 1494 passed / 15 skipped · coverage 98.94% (floor 94) · ruff clean · pyright 0
errors · node --check clean · bash .agents/validate.sh exit 0 · live 11 passed · store_perf 4 passed
Dev server up, all nine sources ok. No git add/commit run; everything left in the working tree.
DOCS PASS — 4 inconsistencies fixed in AGENTS.md (last two runs above include them)
- UI conventions hero example still said the `.sub` footnote reads `Δ10.0s`; the only hero that
prints a Δ is /llm's → now `Δ2.0s` on /llm, "at the current load" on /ups
- gap 9: "found 10 busy-edges" contradicted its own artifact (17 edges, 10 judged, 7 reported-not-
judged) → 17; and the poll-2 ratios were a 3-of-5 cherry-pick → full five values both windows
- gap 12: the responsive-table fix is now scheduled — names 27_mobile_tables_and_audit
README, config table, .env.example, rates/prom docstrings, app/api bullets and the testing-policy
pins all re-checked against code and found consistent (store pragmas, budgets, throttle, fleet).
NOTES
- Advisory only, not phase 23: fastapi 0.153.0 → 0.156.8 available; dependencies untouched here,
so rule 8's "update on purpose" is left to the operator.
- NOTES.md is empty in the working tree (its items became phases 24–27); left as-is, untracked-file
rules forbid me removing it.
- Next phase: 24_dev_mtls_rollout (25–27 follow).
```
This commit is contained in:
+195
-1
@@ -6,10 +6,12 @@ the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentati
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.rates import MIN_WINDOW_SECONDS, RateEstimator
|
||||
from dashboards.sources.llm import (
|
||||
engine_result,
|
||||
fleet_totals,
|
||||
@@ -18,6 +20,7 @@ from dashboards.sources.llm import (
|
||||
rate_key,
|
||||
series_names,
|
||||
)
|
||||
from rate_window_contract import LLM_WINDOW, assert_window, delta_label
|
||||
from slots_contract import load_slots_capture
|
||||
|
||||
|
||||
@@ -33,6 +36,29 @@ def bump_metric(text: str, name: str, delta: float) -> str:
|
||||
return bumped
|
||||
|
||||
|
||||
def poll_engine(metrics_text: Any, engine: str, span: float, stamps: Sequence[float],
|
||||
growth: Mapping[str, Callable[[float], float]]) -> list[dict[str, Any]]:
|
||||
"""Re-scrape one capture at `stamps` with counters growing as f(t), through one estimator.
|
||||
|
||||
The production shape in miniature: **one** `RateEstimator(span=span)` for every poll (the
|
||||
object `app.build_hub` hands that source), each poll stamped with its own `now`, the
|
||||
exposition carrying the tokens produced since the last scrape. Returns the parsed metrics per
|
||||
poll so a pin reads the very cells `/api/llm/snapshot` serves — including their `window`.
|
||||
"""
|
||||
parse = parse_gufo_metrics if engine == "gufo" else parse_vllm_metrics
|
||||
rates = RateEstimator(span=span)
|
||||
body = metrics_text(engine)
|
||||
carried = dict.fromkeys(growth, 0.0)
|
||||
scrapes: list[dict[str, Any]] = []
|
||||
for now in stamps:
|
||||
for name, produced in growth.items():
|
||||
value = produced(now)
|
||||
body = bump_metric(body, name, value - carried[name])
|
||||
carried[name] = value
|
||||
scrapes.append(parse(body, rates, now))
|
||||
return scrapes
|
||||
|
||||
|
||||
# ---- gufo / llama.cpp ----------------------------------------------------- #
|
||||
|
||||
def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None:
|
||||
@@ -77,6 +103,144 @@ def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_te
|
||||
assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597)
|
||||
|
||||
|
||||
def test_gufo_the_window_the_llm_jobs_are_spanned_by_is_one_that_fills(metrics_text) -> None:
|
||||
"""Phase 23: a 2 s span is a real window at the LLM cadence, and it kills the ramp.
|
||||
|
||||
`app.build_hub` spans the LLM estimator by `LLM_INTERVAL_SECONDS` (the wiring is pinned in
|
||||
`tests/test_api.py`); this is what that span buys. A slot that starts mid-window is spread
|
||||
over the whole span by the wider window, so the hero climbs one fifth of the true rate per
|
||||
poll — the "ramp up/ramp down" of `TODO.md:8` — while a window of one poll interval reads
|
||||
the engine's speed on the first poll after it changes and 0 again on the first after it ends.
|
||||
|
||||
The engine here is the capture plus a steady 50 tok/s that begins at t=1006, polled every
|
||||
2 s, which is the production cadence rather than a fast scrape: the estimator's
|
||||
`MIN_WINDOW_SECONDS` floor is 4x below this window, so nothing is being refused here.
|
||||
"""
|
||||
text = metrics_text("gufo")
|
||||
started = 1_006.0 # the slot starts here, at a steady 50 tok/s
|
||||
polls = [1_000.0, 1_002.0, 1_004.0, 1_006.0, 1_008.0, 1_010.0, 1_012.0]
|
||||
|
||||
def series(span: float) -> list[tuple[float | None, float | None]]:
|
||||
rates = RateEstimator(span=span)
|
||||
body = text
|
||||
cells: list[tuple[float | None, float | None]] = []
|
||||
previous = 0.0
|
||||
for now in polls:
|
||||
generated = 50.0 * max(0.0, now - started)
|
||||
body = bump_metric(body, "llamacpp:tokens_predicted_total", generated - previous)
|
||||
previous = generated
|
||||
cell = parse_gufo_metrics(body, rates, now)["decode_tokens_per_second"]
|
||||
cells.append((cell["value"], cell.get("window")))
|
||||
return cells
|
||||
|
||||
narrow = series(2.0)
|
||||
wide = series(10.0)
|
||||
# the new window: n/a on the first poll (no window yet), then the engine's real speed, and
|
||||
# every one of them labelled with the 2 s window it actually measured
|
||||
assert narrow[0] == (None, None)
|
||||
for _value, window in narrow[1:]:
|
||||
assert window == pytest.approx(2.0), "the label must name the window the value used"
|
||||
assert [v for v, _ in narrow[4:]] == [pytest.approx(50.0)] * 3, (
|
||||
f"a 2 s window reads the slot's speed at once, got {narrow}")
|
||||
# the old one: the same work, averaged across the polls the operator is watching
|
||||
assert [v for v, _ in wide[4:]] != [pytest.approx(50.0)] * 3, \
|
||||
f"the 10 s window no longer smooths, so this test's premise moved: {wide}"
|
||||
assert wide[4][0] is not None and wide[4][0] < 30.0, \
|
||||
f"expected the ramp to print well under the true rate, got {wide}"
|
||||
|
||||
|
||||
def test_gufo_rate_cells_name_the_window_the_llm_jobs_are_measured_over(metrics_text) -> None:
|
||||
"""Phase 23's payload pin: an LLM rate cell's `window` is the LLM cadence, label and all.
|
||||
|
||||
`build_hub` spans the `llm` estimator by `LLM_INTERVAL_SECONDS` (the object-level pin is
|
||||
`tests/test_api.py`'s); this is the same span where the browser reads it — the `window` beside
|
||||
the value, the `computed:Δ<series>/Δt` provenance printed with it, and the `Δ2.0s` label
|
||||
`provenance()` builds from that number. **Both** throughput cells are checked, on the
|
||||
production 2 s cadence. `tests/test_host_source.py` is the other direction of the same pin:
|
||||
the host cells keep 10 s in the same payload shape.
|
||||
"""
|
||||
started = 1_006.0
|
||||
stamps = [1_000.0, 1_002.0, 1_004.0, 1_006.0, 1_008.0, 1_010.0]
|
||||
scrapes = poll_engine(
|
||||
metrics_text, "gufo", LLM_WINDOW, stamps,
|
||||
{"llamacpp:tokens_predicted_total": lambda t: 50.0 * max(0.0, t - started),
|
||||
"llamacpp:prompt_tokens_total": lambda t: 200.0 * max(0.0, t - started)},
|
||||
)
|
||||
assert "window" not in scrapes[0]["decode_tokens_per_second"], \
|
||||
"one sample is not a window, so it must not grow a Δ label"
|
||||
for key, series, speed in (
|
||||
("decode_tokens_per_second", "llamacpp:tokens_predicted_total", 50.0),
|
||||
("prompt_tokens_per_second", "llamacpp:prompt_tokens_total", 200.0),
|
||||
):
|
||||
cells = [scrape[key] for scrape in scrapes[1:]]
|
||||
for cell in cells:
|
||||
assert cell["value"] is not None, f"{key} went n/a on a healthy cadence: {cell}"
|
||||
assert_window(cell.get("window"), LLM_WINDOW, where=f"gufo {key}")
|
||||
assert cell["derived"] is True
|
||||
assert cell["source"] == f"computed:Δ{series}/Δt", \
|
||||
"the provenance string travels with the value whatever window it used"
|
||||
assert cells[-1]["value"] == pytest.approx(speed), (
|
||||
f"a one-interval window reads the engine's speed on the next poll, got {cells}")
|
||||
|
||||
|
||||
def test_gufo_the_floor_still_refuses_a_catch_up_gap_at_the_llm_window(metrics_text) -> None:
|
||||
"""The phase-13 floor belongs to the estimator, not to the old 10 s span.
|
||||
|
||||
Shrinking the window to one poll interval must not shrink the guard: `poller.Job._loop` still
|
||||
sleeps `max(0.2, interval - elapsed)`, so a slow fetch is still followed by a catch-up poll
|
||||
~0.2 s later, and 0.2 s of a 2 s window is the same lie a 0.2 s slice of a 10 s one was (the
|
||||
3284.59 tok/s row for a 53 tok/s engine). `test_rates.py` pins the gate at both spans; this
|
||||
is the refusal through the production parse path, where the `n/a` keeps the window it refused
|
||||
**and** its `source` so the cell still says which Δ it is talking about.
|
||||
"""
|
||||
scrapes = poll_engine(
|
||||
metrics_text, "gufo", LLM_WINDOW,
|
||||
[1_000.0, 1_002.0, 1_004.0, 1_014.25, 1_014.47, 1_016.47],
|
||||
{"llamacpp:tokens_predicted_total": lambda t: 50.0 * (t - 1_000.0)},
|
||||
)
|
||||
stalled = scrapes[3]["decode_tokens_per_second"]
|
||||
assert stalled["value"] is None, \
|
||||
"a 10 s gap is past a 2 s span's ceiling (2 x span): n/a, not a diluted rate"
|
||||
assert "window" not in stalled, "the ceiling answers no window because it measured none"
|
||||
refused = scrapes[4]["decode_tokens_per_second"]
|
||||
assert refused["value"] is None, f"0.22 s is the loop's catch-up gap, not a rate: {refused}"
|
||||
assert refused["window"] == pytest.approx(0.22)
|
||||
assert refused["window"] < MIN_WINDOW_SECONDS
|
||||
assert refused["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt"
|
||||
assert delta_label(refused["window"]) == "Δ0.2s", "the label names the window it refused"
|
||||
recovered = scrapes[5]["decode_tokens_per_second"]
|
||||
assert recovered["value"] == pytest.approx(50.0), (
|
||||
f"the refusal re-bases, so the next honest poll reads the engine again: {recovered}")
|
||||
assert_window(recovered.get("window"), LLM_WINDOW, where="gufo decode")
|
||||
|
||||
|
||||
def test_gufo_a_poll_gap_a_hair_wider_than_the_span_is_still_the_2s_claim(metrics_text) -> None:
|
||||
"""Live reality: a poll lands slightly after its interval, and that is still `Δ2.0s`.
|
||||
|
||||
With the span equal to the cadence (phase 23) only the current sample is strictly *inside* the
|
||||
window, so the base comes from the estimator's window-empty fallback — the previous sample,
|
||||
one poll back — and the published window is the real gap. Read off the running dev server the
|
||||
LLM cells report 2.009-2.026 s and the host payload's `window_seconds` 9.978-10.002 s, so a
|
||||
pin demanding `window == span` exactly would fail against working software. This drives the
|
||||
jittered shape and asserts what the operator sees: the same `Δ2.0s` label and the engine's real
|
||||
speed. A gap of a whole extra interval is the other story — the `2 x span` ceiling refuses it,
|
||||
which is what the test above pins.
|
||||
"""
|
||||
started = 1_006.0
|
||||
stamps = [1_000.0, 1_002.03, 1_004.07, 1_006.1, 1_008.13, 1_010.16]
|
||||
scrapes = poll_engine(
|
||||
metrics_text, "gufo", LLM_WINDOW, stamps,
|
||||
{"llamacpp:tokens_predicted_total": lambda t: 50.0 * max(0.0, t - started)},
|
||||
)
|
||||
cells = [scrape["decode_tokens_per_second"] for scrape in scrapes[1:]]
|
||||
for cell in cells:
|
||||
assert cell["value"] is not None
|
||||
assert_window(cell.get("window"), LLM_WINDOW, where="gufo decode on a jittered cadence")
|
||||
assert delta_label(cell["window"]) == "Δ2.0s", \
|
||||
"a 2.03 s gap still prints the window the panel claims"
|
||||
assert cells[-1]["value"] == pytest.approx(50.0, rel=0.02)
|
||||
|
||||
|
||||
def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None:
|
||||
text = metrics_text("gufo")
|
||||
rates = RateEstimator(span=10)
|
||||
@@ -244,6 +408,36 @@ def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text)
|
||||
assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0)
|
||||
|
||||
|
||||
def test_vllm_rate_cells_name_the_same_window_the_llm_jobs_are_measured_over(
|
||||
metrics_text) -> None:
|
||||
"""The pin is the **source's** span, so both engines answer it the same way.
|
||||
|
||||
The capture exports no throughput gauge, which is what makes these two cells measured (see
|
||||
`test_vllm_measured_rate_when_the_build_has_no_throughput_gauge`), and vLLM's counters are
|
||||
labelled — the estimator keys each label set separately while the cell's `source` names the
|
||||
series. Same two rules as the gufo pin: the `window` inside the LLM span, and
|
||||
`computed:Δ<series>/Δt` beside the value.
|
||||
"""
|
||||
started = 2_006.0
|
||||
scrapes = poll_engine(
|
||||
metrics_text, "vllm", LLM_WINDOW,
|
||||
[2_000.0, 2_002.0, 2_004.0, 2_006.0, 2_008.0, 2_010.0],
|
||||
{"vllm:generation_tokens_total": lambda t: 40.0 * max(0.0, t - started),
|
||||
"vllm:prompt_tokens_total": lambda t: 300.0 * max(0.0, t - started)},
|
||||
)
|
||||
for key, series, speed in (
|
||||
("decode_tokens_per_second", "vllm:generation_tokens_total", 40.0),
|
||||
("prompt_tokens_per_second", "vllm:prompt_tokens_total", 300.0),
|
||||
):
|
||||
cells = [scrape[key] for scrape in scrapes[1:]]
|
||||
for cell in cells:
|
||||
assert cell["value"] is not None, f"{key} went n/a on a healthy cadence: {cell}"
|
||||
assert_window(cell.get("window"), LLM_WINDOW, where=f"vllm {key}")
|
||||
assert cell["source"] == f"computed:Δ{series}/Δt"
|
||||
assert cells[-1]["value"] == pytest.approx(speed), (
|
||||
f"the labelled series still reads its own speed over one interval, got {cells}")
|
||||
|
||||
|
||||
def test_vllm_a_joining_label_set_does_not_lend_its_carried_counter(metrics_text) -> None:
|
||||
"""A second `engine=`/`model_name=` series appearing mid-window must not be a rate.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user