phase: 23_llm_rate_responsiveness
Phase 23 final verification complete — all green.
```
VERIFIED (criteria)
1 /api/llm/snapshot windows ≈2 s live (1.98–2.00), /api/system window_seconds ≈10 s (9.996–10.005);
/api/config llm.rate_window_seconds=2=LLM_INTERVAL_SECONDS; browser re-check ALL CHECKS PASSED
(data/logs/23-window-{llm,system}.png — /llm prints Δ2.0s, /system footnote Δ10.0s)
2 Phase-13 pins unchanged: floor, 2×span ceiling, 8× jump gate + independent anchor — all pass
(only import/span-line diffs in tests/; git diff confirms no assertion rewritten)
3 Live proof re-derived from artifacts: 780 rows, 0 unlabelled, windows 0.61–3.38 s, 5 starts +
5 ends settled by poll 2 (0 unsettled), 1 m rollups max 42.3 vs raw 201.6, 0 over 300 tok/s;
counterfactual 10 s window: median settle poll 3, |ln| 0.924 vs 0.037
4 uv run pytest: 1494 passed / 15 skipped · coverage 98.94% (floor 94) · ruff clean · pyright 0
errors · node --check clean · bash .agents/validate.sh exit 0 · live 11 passed · store_perf 4 passed
Dev server up, all nine sources ok. No git add/commit run; everything left in the working tree.
DOCS PASS — 4 inconsistencies fixed in AGENTS.md (last two runs above include them)
- UI conventions hero example still said the `.sub` footnote reads `Δ10.0s`; the only hero that
prints a Δ is /llm's → now `Δ2.0s` on /llm, "at the current load" on /ups
- gap 9: "found 10 busy-edges" contradicted its own artifact (17 edges, 10 judged, 7 reported-not-
judged) → 17; and the poll-2 ratios were a 3-of-5 cherry-pick → full five values both windows
- gap 12: the responsive-table fix is now scheduled — names 27_mobile_tables_and_audit
README, config table, .env.example, rates/prom docstrings, app/api bullets and the testing-policy
pins all re-checked against code and found consistent (store pragmas, budgets, throttle, fleet).
NOTES
- Advisory only, not phase 23: fastapi 0.153.0 → 0.156.8 available; dependencies untouched here,
so rule 8's "update on purpose" is left to the operator.
- NOTES.md is empty in the working tree (its items became phases 24–27); left as-is, untracked-file
rules forbid me removing it.
- Next phase: 24_dev_mtls_rollout (25–27 follow).
```
This commit is contained in:
+93
-1
@@ -28,11 +28,12 @@ from auth_client import sign_in
|
||||
from auth_contract import ADMIN_PASSWORD, FAST_PBKDF2
|
||||
from card_contract import HOST_SHAPES, assert_card_contract
|
||||
from dashboards import alerts, auth, certs
|
||||
from dashboards import app as app_module
|
||||
from dashboards.api import MIN_PASSWORD_LENGTH, RANGE_SECONDS, _disk_visible, parse_range
|
||||
from dashboards.app import create_app
|
||||
from dashboards.collectors import UpstreamError
|
||||
from dashboards.config import Settings, Target
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.rates import MIN_WINDOW_SECONDS, RateEstimator
|
||||
from dashboards.severity import classify
|
||||
from dashboards.sources import nut
|
||||
from dashboards.sources.disks import SMART_SERIES, DisksCollector, parse_disks_snapshot
|
||||
@@ -43,6 +44,7 @@ from dashboards.sources.llm import (
|
||||
parse_vllm_metrics,
|
||||
)
|
||||
from dashboards.store import RES_RAW, Event, Sample, SampleStore
|
||||
from rate_window_contract import LLM_WINDOW, delta_label
|
||||
from slots_contract import (
|
||||
CANONICAL_TASK_ID_FIELD,
|
||||
OPTIONAL_NEXT_TOKEN_FIELDS,
|
||||
@@ -815,6 +817,77 @@ def test_engine_urls_are_redacted_the_same_way_hosts_are(tmp_path: Path) -> None
|
||||
store.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("llm_interval", "rate_window"), [(2.0, 10.0), (4.0, 20.0)])
|
||||
def test_build_hub_gives_each_source_the_estimator_its_own_cadence_fills(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
llm_interval: float, rate_window: float) -> None:
|
||||
"""Two rate estimators, one per measuring cadence (phase 23, `TODO.md:8`).
|
||||
|
||||
The spy is the point: the numbers themselves are `config` defaults and a test on the default
|
||||
would pass even if `build_hub` handed one estimator to both sources. So this asserts the two
|
||||
**objects** the jobs receive, and does it at a non-default cadence pair to prove the LLM span
|
||||
tracks `LLM_INTERVAL_SECONDS` rather than a second knob nobody configured — one 10 s window
|
||||
smoothed a slot start across the five polls the operator was watching (the ramp), and a 2 s
|
||||
window over a 10 s `host` poll would answer `n/a` every poll.
|
||||
|
||||
The payload-level twin of this pin is the `Δ<window>` label each rate cell carries: this is
|
||||
the wiring, that is what the browser prints (phase 23 task 02 pins the labels).
|
||||
"""
|
||||
seen: dict[str, RateEstimator] = {}
|
||||
real_host_jobs = app_module.host_jobs
|
||||
real_llm_jobs = app_module.llm_jobs
|
||||
|
||||
def spy_host_jobs(targets: Any, http: Any, host_collector: Any, interval: float,
|
||||
**kwargs: Any) -> list[Any]:
|
||||
seen["host"] = host_collector.rates
|
||||
return real_host_jobs(targets, http, host_collector, interval, **kwargs)
|
||||
|
||||
def spy_llm_jobs(cfg: Any, http: Any, rates: RateEstimator) -> list[Any]:
|
||||
seen["llm"] = rates
|
||||
return real_llm_jobs(cfg, http, rates)
|
||||
|
||||
monkeypatch.setattr(app_module, "host_jobs", spy_host_jobs)
|
||||
monkeypatch.setattr(app_module, "llm_jobs", spy_llm_jobs)
|
||||
|
||||
db = str(tmp_path / "rate-windows.sqlite3")
|
||||
store = SampleStore(db)
|
||||
settings = Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
||||
pbkdf2_iterations=FAST_PBKDF2,
|
||||
host_targets=(Target("deskwork", DEAD_URL),),
|
||||
gufo_url="http://deskwork:8731", vllm_url="http://driveripper:8000",
|
||||
llm_interval=llm_interval, rate_window=rate_window)
|
||||
# `build_hub` only wires jobs; nothing here dials, so the shared client is never started
|
||||
# and there is no event loop to close one in.
|
||||
hub, _nut_monitor = app_module.build_hub(settings, store, httpx.AsyncClient())
|
||||
store.close()
|
||||
|
||||
assert sorted(seen) == ["host", "llm"], "a source stopped being handed an estimator"
|
||||
assert sorted(hub.jobs) == ["disks:deskwork", "host:deskwork", "llm:gufo", "llm:vllm"]
|
||||
assert seen["llm"] is not seen["host"], \
|
||||
"one estimator for both cadences is the bug this phase exists to fix"
|
||||
assert seen["llm"].span == pytest.approx(llm_interval), \
|
||||
"the LLM window must track LLM_INTERVAL_SECONDS, not a second knob"
|
||||
assert seen["host"].span == pytest.approx(rate_window), \
|
||||
"the host rates keep RATE_WINDOW_SECONDS: a 2 s span over a 10 s poll is `n/a` forever"
|
||||
# …and the smaller window still carries the production floor, so the phase-13 refusal of the
|
||||
# poll loop's ~0.2 s catch-up gap is not a consequence of the wider span.
|
||||
assert seen["llm"].min_window == MIN_WINDOW_SECONDS
|
||||
assert seen["host"].min_window == MIN_WINDOW_SECONDS
|
||||
assert seen["llm"].span > MIN_WINDOW_SECONDS, "the LLM window would refuse every poll"
|
||||
|
||||
|
||||
def test_config_reports_the_window_the_llm_rates_are_actually_measured_over(
|
||||
client: TestClient) -> None:
|
||||
"""`/api/config`'s llm window is a measurement claim, so it is the LLM's own span.
|
||||
|
||||
It used to echo `RATE_WINDOW_SECONDS` for both sources, which was harmless while one estimator
|
||||
served them all. Since phase 23 that same field would have advertised a 10 s window beside
|
||||
rates measured over 2 s — a number the page can print and no poll ever used (rule 2).
|
||||
"""
|
||||
llm = client.get("/api/config").json()["llm"]
|
||||
assert llm["rate_window_seconds"] == llm["interval_seconds"] == 2.0
|
||||
|
||||
|
||||
def wait_for_state(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
@@ -2586,6 +2659,25 @@ def test_llm_engine_panel_leads_with_two_throughput_heroes(client) -> None:
|
||||
assert "node.title = value.source" in js
|
||||
|
||||
|
||||
def test_llm_provenance_tag_prints_the_window_the_payload_sent(client: TestClient) -> None:
|
||||
"""`Δ2.0s` on `/llm` is the cell's own `window`, never a config echo (phase 23).
|
||||
|
||||
The window moved from 10 s to one poll interval without touching the page: `provenance()`
|
||||
reads `cell.window`, which is the number `tests/test_llm_source.py` pins to the LLM span, and
|
||||
both spell the label with the same one-decimal rounding (`delta_label` in
|
||||
`tests/rate_window_contract.py` is that rounding, used by both payload pins). A page printing
|
||||
the configured interval instead would advertise a window no poll measured (rule 2), and a
|
||||
literal `Δ10.0s` in the script is the same lie with the number typed in.
|
||||
"""
|
||||
js = client.get("/static/page-llm.js").text
|
||||
prov = js[js.index("function provenance("):js.index("function humanNumber(")]
|
||||
assert "`Δ${cell.window.toFixed(1)}s`" in prov, \
|
||||
f"the Δ tag is no longer built from the window the cell carries: {prov}"
|
||||
assert delta_label(LLM_WINDOW) == "Δ2.0s", "the rounding the page and the pins share moved"
|
||||
assert "rate_window_seconds" not in js, "the page echoes a config window beside a measurement"
|
||||
assert "Δ10.0s" not in js, "a hard-coded Δ is an invented window (rule 2)"
|
||||
|
||||
|
||||
def test_every_llm_cell_carries_a_window_sparkline_with_percentile_feet(client) -> None:
|
||||
"""Each panel row draws its own sparkline over the *selected* window and prints mean
|
||||
**and** p95 without a hover.
|
||||
|
||||
Reference in New Issue
Block a user