diff --git a/.agents/phases/todo/23_llm_rate_responsiveness/00_phase.md b/.agents/phases/complete/23_llm_rate_responsiveness/00_phase.md similarity index 100% rename from .agents/phases/todo/23_llm_rate_responsiveness/00_phase.md rename to .agents/phases/complete/23_llm_rate_responsiveness/00_phase.md diff --git a/.agents/phases/todo/23_llm_rate_responsiveness/01_llm_window_per_poll.md b/.agents/phases/complete/23_llm_rate_responsiveness/01_llm_window_per_poll.md similarity index 100% rename from .agents/phases/todo/23_llm_rate_responsiveness/01_llm_window_per_poll.md rename to .agents/phases/complete/23_llm_rate_responsiveness/01_llm_window_per_poll.md diff --git a/.agents/phases/todo/23_llm_rate_responsiveness/02_window_pins.md b/.agents/phases/complete/23_llm_rate_responsiveness/02_window_pins.md similarity index 100% rename from .agents/phases/todo/23_llm_rate_responsiveness/02_window_pins.md rename to .agents/phases/complete/23_llm_rate_responsiveness/02_window_pins.md diff --git a/.agents/phases/todo/23_llm_rate_responsiveness/03_live_verification.md b/.agents/phases/complete/23_llm_rate_responsiveness/03_live_verification.md similarity index 100% rename from .agents/phases/todo/23_llm_rate_responsiveness/03_live_verification.md rename to .agents/phases/complete/23_llm_rate_responsiveness/03_live_verification.md diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.err b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.md b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.md new file mode 100644 index 0000000..f6a6788 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.md @@ -0,0 +1,32 @@ +Phase 23 final verification complete — all green. + +``` +VERIFIED (criteria) +1 /api/llm/snapshot windows ≈2 s live (1.98–2.00), /api/system window_seconds ≈10 s (9.996–10.005); + /api/config llm.rate_window_seconds=2=LLM_INTERVAL_SECONDS; browser re-check ALL CHECKS PASSED + (data/logs/23-window-{llm,system}.png — /llm prints Δ2.0s, /system footnote Δ10.0s) +2 Phase-13 pins unchanged: floor, 2×span ceiling, 8× jump gate + independent anchor — all pass + (only import/span-line diffs in tests/; git diff confirms no assertion rewritten) +3 Live proof re-derived from artifacts: 780 rows, 0 unlabelled, windows 0.61–3.38 s, 5 starts + + 5 ends settled by poll 2 (0 unsettled), 1 m rollups max 42.3 vs raw 201.6, 0 over 300 tok/s; + counterfactual 10 s window: median settle poll 3, |ln| 0.924 vs 0.037 +4 uv run pytest: 1494 passed / 15 skipped · coverage 98.94% (floor 94) · ruff clean · pyright 0 + errors · node --check clean · bash .agents/validate.sh exit 0 · live 11 passed · store_perf 4 passed + Dev server up, all nine sources ok. No git add/commit run; everything left in the working tree. + +DOCS PASS — 4 inconsistencies fixed in AGENTS.md (last two runs above include them) +- UI conventions hero example still said the `.sub` footnote reads `Δ10.0s`; the only hero that + prints a Δ is /llm's → now `Δ2.0s` on /llm, "at the current load" on /ups +- gap 9: "found 10 busy-edges" contradicted its own artifact (17 edges, 10 judged, 7 reported-not- + judged) → 17; and the poll-2 ratios were a 3-of-5 cherry-pick → full five values both windows +- gap 12: the responsive-table fix is now scheduled — names 27_mobile_tables_and_audit +README, config table, .env.example, rates/prom docstrings, app/api bullets and the testing-policy +pins all re-checked against code and found consistent (store pragmas, budgets, throttle, fleet). + +NOTES +- Advisory only, not phase 23: fastapi 0.153.0 → 0.156.8 available; dependencies untouched here, + so rule 8's "update on purpose" is left to the operator. +- NOTES.md is empty in the working tree (its items became phases 24–27); left as-is, untracked-file + rules forbid me removing it. +- Next phase: 24_dev_mtls_rollout (25–27 follow). +``` diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.md.status b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.validate b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.validate new file mode 100644 index 0000000..2ce15a4 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__00_phase.a1.validate @@ -0,0 +1,65 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 14%] +........................................................................ [ 19%] +........................................................................ [ 23%] +.........................................................s.............. [ 28%] +........................................................................ [ 33%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 47%] +........................................................................ [ 52%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 66%] +........................sssssssssss..................................... [ 71%] +........................................................................ [ 76%] +........................................................................ [ 81%] +........................................................................ [ 85%] +........................................................................ [ 90%] +........................................................................ [ 95%] +...........sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1494 passed, 15 skipped, 1 warning in 155.69s (0:02:35) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.156.8 wheel diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.err b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.md b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.md new file mode 100644 index 0000000..c2722e1 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.md @@ -0,0 +1,12 @@ +**Implemented (phase 23 task 01 — the LLM source gets its own Δ window)** + +- `app.py::build_hub` now builds **two** estimators: `host_rates = RateEstimator(span=settings.rate_window)` (10 s) for `HostCollector`, `llm_rates = RateEstimator(span=settings.llm_interval)` (2 s) for `llm_jobs`; docstring rewritten to the two-estimator rule (no new env var — the LLM window *is* `LLM_INTERVAL_SECONDS`). +- `api.py`: `/api/config`'s `llm.rate_window_seconds` now reports the LLM's actual span (it echoed the host's 10 s window, which would have been a config fact reading as a measurement). +- `config.py` / `rates.py` / `page-llm.js` comments corrected to the per-cadence span; `.env.example` comments say `RATE_WINDOW_SECONDS` is the host's knob and the LLM window tracks `LLM_INTERVAL_SECONDS`. +- Tests: `test_api.py::test_build_hub_gives_each_source_the_estimator_its_own_cadence_fills` (spies the two estimator objects at 2/10 and 4/20 — distinct objects, LLM span tracks the cadence knob, `MIN_WINDOW_SECONDS` floor intact) + `test_config_reports_the_window_the_llm_rates_are_actually_measured_over`; `test_llm_source.py::test_gufo_the_window_the_llm_jobs_are_spanned_by_is_one_that_fills` (2 s span reads a steady 50 tok/s on the first poll after a slot starts, labelled `Δ2.0s`, while the 10 s span ramps 12.5 → 20 → 30); `test_live_fleet.py` estimators re-spanned to `llm_interval`. +- `bash .agents/validate.sh` → **1481 passed, 15 skipped**, coverage **98.94 %** (floor 94), `ruff` clean, `pyright` 0 errors, `node --check` clean (only the pre-existing fastapi-outdated advisory). +- Docs: README (`RATE_WINDOW_SECONDS` row rewritten, new `LLM_INTERVAL_SECONDS` row, both `Δ10.0s` → `Δ2.0s`/`Δ` mentions) and AGENTS.md (`app.py` and `rates.py` bullets gain the two-estimator fact, gap 9 gains the "phase 23 changed the window, not the gates" paragraph, the `Δ` UI convention notes each page's own window). +- Extra evidence (read-only, no fleet writes): dev server restarted — all nine sources `ok`; `/api/llm/snapshot` windows 2.01/2.007 s, `/api/system/snapshot` 9.97/10.00/10.01 s, `/api/config` `llm.rate_window_seconds=2`; headless browser pass `data/logs/23-task01-window.cjs` shows `/llm` printing `Δ2.0s` four times and `/system` still `Δ10.0s` (screenshots `data/logs/23-window-{llm,system}.png`, note `data/logs/NOTES-23-window.md`). +- Decision: pre-existing working-tree change to `NOTES.md` (cleared TODO) left as-is; fastapi not upgraded (dependencies untouched this task). + +Next pending task: `23_llm_rate_responsiveness/02_window_pins.md`. diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.md.status b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.validate b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.validate new file mode 100644 index 0000000..ac03aed --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__01_llm_window_per_poll.a1.validate @@ -0,0 +1,65 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 14%] +........................................................................ [ 19%] +........................................................................ [ 24%] +........................................................s............... [ 28%] +........................................................................ [ 33%] +........................................................................ [ 38%] +........................................................................ [ 43%] +........................................................................ [ 48%] +........................................................................ [ 52%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 67%] +.....................sssssssssss........................................ [ 72%] +........................................................................ [ 77%] +........................................................................ [ 81%] +........................................................................ [ 86%] +........................................................................ [ 91%] +......................................................................ss [ 96%] +s....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1481 passed, 15 skipped, 1 warning in 154.51s (0:02:34) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.156.6 wheel diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.err b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.md b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.md new file mode 100644 index 0000000..fd271c4 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.md @@ -0,0 +1,12 @@ +**Implemented (phase 23 task 02 — the window pins)** + +- New `tests/rate_window_contract.py`: the two Δ windows (`HOST_WINDOW` = `RATE_WINDOW_SECONDS` 10 s, `LLM_WINDOW` = `LLM_INTERVAL_SECONDS` 2 s), `delta_label()` (the pages' `toFixed(1)` tag) and `assert_window()` (estimator floor → `2 × span` ceiling **plus** the printed label). +- `tests/test_llm_source.py`: `poll_engine()` capture-replay helper + pins for **both** throughput cells of **both** engines at the 2 s shape (`window`, `Δ2.0s`, `computed:Δ/Δt`); the phase-13 floor refusal through the production parse path at that span (`n/a`, honest `Δ0.2s`, re-base to 50 tok/s); a jittered-cadence leg proving a real 2.03 s gap still labels `Δ2.0s`. +- `tests/test_host_source.py`: host `window_seconds` pinned at 10 s / `Δ10.0s`, and the same 10 s cadence starved to `n/a` by a 2 s span (why one estimator cannot serve both sources). +- `tests/test_rates.py`: floor / `2 × span` ceiling / 8× jump gate parametrized at 2.0 **and** 10.0; the phase-13 tests themselves untouched (verified in the diff). +- `tests/test_api.py`: served `page-llm.js` pin — the provenance tag is built from `cell.window`, no config echo, no literal `Δ10.0s`. `tests/test_config.py`: the previously-missing `rate_window` 10 s default pin + "exactly one `*window*` knob" (the LLM window *is* `LLM_INTERVAL_SECONDS`). +- Tests/lint: `bash .agents/validate.sh` → **1494 passed, 15 skipped**, coverage **98.94 %** (floor 94, unchanged from task 01), `ruff` clean, `pyright` 0 errors, `node --check` clean, **EXIT=0**; `uv run pytest -m live --live` → **11 passed** (read-only). +- Docs: README (a paragraph on the contract module beside the other `_contract` files), AGENTS.md (new testing-policy bullet for the two-window pin; invariant 2's example now `computed:Δtokens/Δt` labelled `Δ2.0s`), `prom.field` docstring names both windows. +- Notable decision: a signed-in read of the running dev server showed real LLM windows **2.009–2.026 s** (span == cadence ⇒ the base comes from the window-empty fallback) and host `window_seconds` 9.978–10.002 s, so `assert_window`'s ceiling is the estimator's `2 × span`, not the span — a literal `window == span` pin would fail against working software; the tight half of the pin is the label. No runtime behaviour changed (test files + docstrings/docs only); the `fastapi 0.153 → 0.156` advisory is pre-existing and dependencies were not touched. + +Next pending task: `23_llm_rate_responsiveness/03_live_verification.md`. diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.md.status b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.validate b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.validate new file mode 100644 index 0000000..d2d52f5 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__02_window_pins.a1.validate @@ -0,0 +1,65 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 14%] +........................................................................ [ 19%] +........................................................................ [ 23%] +.........................................................s.............. [ 28%] +........................................................................ [ 33%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 47%] +........................................................................ [ 52%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 66%] +........................sssssssssss..................................... [ 71%] +........................................................................ [ 76%] +........................................................................ [ 81%] +........................................................................ [ 85%] +........................................................................ [ 90%] +........................................................................ [ 95%] +...........sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1494 passed, 15 skipped, 1 warning in 153.57s (0:02:33) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.156.8 wheel diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.err b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.md b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.md new file mode 100644 index 0000000..2640af9 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.md @@ -0,0 +1,11 @@ +**Phase 23 task 03 — live verification: DONE, PASS.** + +**Implementation.** `scripts/watch-llm-window.py` (new, read-only): signs in, samples `/api/llm/snapshot` at the app's own `LLM_INTERVAL_SECONDS`, records per poll the decode/prompt value **with the `Δ` window that produced it**, the engine's own busy signal and lifetime counters; judges zero-untagged values, window shape (never the host's 10 s), edge settle/decay, and the stored 1 m rollups; re-diffs the *same* samples over 2 s and the pre-phase 10 s span, and `DASH_LLM_WATCH_REPLAY=` re-derives the verdict from captured evidence without sending traffic. `data/logs/23-live-window.cjs` is the browser half (hero `.sub` + strip `.prov` vs the API). + +**Live results** (operator workload only; no engine called, no load generated). Pass A 780 s + pass B 460 s: 390 / 229 polls per engine, **0 poll errors**, gufo max **201.6 tok/s**, windows **0.61–3.38 s**, **0 untagged**, 1 m rollups **42.3** and **24.7** (gap-9 ceiling 300). **8 slot starts + 8 ends, 0 unsettled by poll 2** on the ungated Δ. Counterfactual on the same samples: median `|ln(value/steady)|` at poll 2 **0.037** (2 s) vs **0.924** (10 s), with the pre-phase leg printed as `13.4 → 28.3 → 45.1 → 52.5` up and `53.3 → 37.9 → 28.9 → 13.9` down — the ramp the phase removed. Browser pass: **866** rendered decode figures, **0** untagged, every tag inside the estimator's bounds, modal `Δ2.0s` on page and API (833 of 866), no `Δ10s`; screenshots `data/logs/23-live-llm-{midburst (50.9 tok/s, Δ2.0s), post-free (0.0, Δ2.0s), burst-settled, gufo-idle, vllm-idle, opening, final}.png`, transcripts `23-live-window.out` / `llm-window-watch{,-pass2}.log`. + +**Checks.** `uv run pytest` → **1494 passed, 15 skipped**; `bash .agents/validate.sh` → **exit 0**, coverage **98.94 %** (floor 94), ruff clean, pyright 0 errors, `node --check` clean (advisory only: fastapi 0.153.0→0.156.8; deps untouched); `uv run pytest -m live --live` → **11 passed**; dev server restarted, all nine sources `ok`. + +**Docs.** `AGENTS.md`: gap 9's phase-23 paragraph rewritten from "hermetic half is pinned; task 03 is the live proof" to what the watch measured; new testing-policy bullet (live measurement re-derivable from its capture; pass line on the ungated Δ, because phase-13 holds are correct behaviour; isolated edges only — a following request is not a slot ending, and prefill makes a slot's first polls read ~0); Commands gains the watch + replay; hermetic count corrected 1477→1494. `README.md`: Testing gains both invocations. + +**Next task:** phase 23 is complete — `.agents/phases/todo/24_dev_mtls_rollout/01_dev_bundle_export.md`. diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.md.status b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.validate b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.validate new file mode 100644 index 0000000..c015e1c --- /dev/null +++ b/.agents/reports/23_llm_rate_responsiveness/23_llm_rate_responsiveness__03_live_verification.a1.validate @@ -0,0 +1,65 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 4%] +........................................................................ [ 9%] +........................................................................ [ 14%] +........................................................................ [ 19%] +........................................................................ [ 23%] +.........................................................s.............. [ 28%] +........................................................................ [ 33%] +........................................................................ [ 38%] +........................................................................ [ 42%] +........................................................................ [ 47%] +........................................................................ [ 52%] +........................................................................ [ 57%] +........................................................................ [ 62%] +........................................................................ [ 66%] +........................sssssssssss..................................... [ 71%] +........................................................................ [ 76%] +........................................................................ [ 81%] +........................................................................ [ 85%] +........................................................................ [ 90%] +........................................................................ [ 95%] +...........sss....................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 440 0 100% +src/dashboards/app.py 186 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/certs.py 292 0 100% +src/dashboards/collectors.py 205 5 98% 284-285, 289-290, 307 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 134 0 100% +src/dashboards/sources/host.py 209 0 100% +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 313 2 99% 553-554 +------------------------------------------------------------------ +TOTAL 3495 37 99% +Required test coverage of 94% reached. Total coverage: 98.94% +1494 passed, 15 skipped, 1 warning in 155.17s (0:02:35) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + fastapi 0.153.0 0.156.8 wheel diff --git a/.env.example b/.env.example index 5847a5f..cd0ce9d 100644 --- a/.env.example +++ b/.env.example @@ -78,10 +78,14 @@ GUFO_BASE_URL= GUFO_API_KEY= VLLM_BASE_URL= VLLM_API_KEY= +# The LLM Δ window IS this interval: the llm jobs get their own estimator spanned by +# LLM_INTERVAL_SECONDS (phase 23), so a slot start reads steady within two polls instead of +# ramping over RATE_WINDOW_SECONDS. Slowing this slows the window with it. LLM_INTERVAL_SECONDS=2 LLM_TIMEOUT_SECONDS=3 -# Window the token rates are measured over. A bare 1s delta jitters because -# prompts arrive in bursts. +# The HOST's Δ window (bytes/sec, jiffies→CPU %). A bare 1s delta jitters because work +# arrives in bursts, and a 10 s poll has nothing inside a 2 s window to measure. +# The LLM rates do NOT use this — see LLM_INTERVAL_SECONDS above. RATE_WINDOW_SECONDS=10 # --------------------------------------------------------------------------- # diff --git a/AGENTS.md b/AGENTS.md index 136eec5..d6271a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,7 +56,8 @@ page still renders. `DANGER` requirement and the wire send still behave exactly as they did without them. 2. **Never invent a number.** A value the upstream does not publish renders as `n/a`, never as `0`. Every derived value carries the `source` string it came from and `derived: true` - so the UI can print *how* it was obtained (`computed:Δtokens/Δt`, `Δ10.0s`). Aggregating + so the UI can print *how* it was obtained (`computed:Δtokens/Δt` labelled `Δ2.0s`, the LLM's + own poll interval since phase 23). Aggregating two different series into one rate is the failure mode this rule exists to catch — the rate estimator is keyed by the *resolved* upstream series name **plus its label set** for exactly that reason (phase 13: a label set joining a summed counter mid-window @@ -374,7 +375,13 @@ Responsibilities, one line each: so that sample is the stall older than its own timestamp, and every later window off it is the same skewed delta re-diffed over a longer span — measured twice from one sample is one measurement, and the hold simply continues until the stale anchor ages out of the span. - `min_window=0.0` exists for tests that scrape faster than any real loop. + `min_window=0.0` exists for tests that scrape faster than any real loop. **The span belongs to + the cadence that fills it** (phase 23): `app.build_hub` builds one estimator per measuring + cadence — the host's at `RATE_WINDOW_SECONDS` (10), the LLM's at `LLM_INTERVAL_SECONDS` (2) — + because a window wider than the poll averages bursts across the polls the operator is watching + and a window narrower than it has nothing inside it to diff. Nothing else about the estimator + moved: the floor, the ceiling, the jump gate and the label are the same code, and a 2 s window + is 4× above the floor, so a healthy 2 s cadence is never refused. - `auth.py` — every authentication decision as a pure function, stdlib only (phase 11): `hash_password`/`verify_password`/`needs_rehash` over a self-describing `pbkdf2_sha256$$$` verifier (so the cost can rise without a @@ -690,6 +697,18 @@ Responsibilities, one line each: `http://`/token path rather than an error, and one map covers both of a host's jobs. A corrupt stored certificate raises out of the same place — a dashboard that cannot present its own client certificate would otherwise spend the fleet's TLS history reporting an upstream problem it caused itself. + `build_hub` also builds **two** `RateEstimator`s, because a window is only a claim about the + polls that fill it (phase 23, `TODO.md:8`'s "ramp up/ramp down"): the `host` jobs get + `RateEstimator(span=settings.rate_window)` (10 s — what a 10 s cadence can fill) and the `llm` + jobs `RateEstimator(span=settings.llm_interval)` (2 s, one poll interval, so a slot that starts + reads steady on the next poll instead of climbing one fifth of the true rate per poll), and + there is no third knob — the LLM window **is** `LLM_INTERVAL_SECONDS`, which is the only way + the pair cannot drift. One estimator for both could only be right about one of them: a 2 s span + over a 10 s poll sits the window-empty fallback against the `2 × span` ceiling and answers + `n/a` every poll, and a 10 s span over a 2 s poll smooths the work the operator is watching. + The estimator's own gates (the `MIN_WINDOW_SECONDS` floor, the ceiling, the 8× jump + confirmation with its independent anchor) are unchanged and apply to the smaller window, whose + `Δ2.0s` label the payload carries with the value. - `web/` — five dashboard pages (`/`, `/system`, `/disks`, `/llm`, `/ups`) plus the login page (`/login`, `pages/login.html` + `static/page-login.js`), one stylesheet, `charts.js`. No framework. `/login` renders the masthead's **brand link and theme toggle** and nothing else @@ -1110,7 +1129,7 @@ set -a; . ./.env.dev; set +a # dev dials the fleet the prod way, no ./scripts/dev-server.sh status # per-source ok/DOWN — unauthenticated: /api/health ./scripts/dev-server.sh logs -uv run pytest # hermetic: 1477 passed + 15 skipped (11 live, +uv run pytest # hermetic: 1494 passed + 15 skipped (11 live, # 4 store_perf), no network uv run pytest --cov # coverage gate (98.94%; floor 94%) uv run pytest -m live --live # opt-in: real fleet / engines / upsd @@ -1123,6 +1142,14 @@ uv run python scripts/watch-store-perf.py # running dev server, timing the page paths while counting # `store-error` rows and the WAL size (DASH_WATCH_SECONDS # shortens it; writes store-perf-watch.{log,tsv,json}) +uv run python scripts/watch-llm-window.py + # ... and the phase-23 proof: signed-in reads of + # /api/llm/snapshot at the app's own cadence, judging every + # slot start/end against "settled within two polls", the + # Δ labels and the 1 m rollups — read-only, it + # starts no workload (DASH_LLM_WATCH_SECONDS shortens it; + # DASH_LLM_WATCH_REPLAY= re-derives the verdict from a + # captured pass over the same samples, sending nothing) uv run python scripts/store_perf_probe.py # ... as a report (data/logs/store-perf-baseline.{md,json}, # then store-perf-task03.* for the write side and # store-perf-task04.* for the budget table); @@ -1216,6 +1243,22 @@ branch `main`); there is no CI on it, so `bash .agents/validate.sh` is the only needs **no** markup for a new verdict: `
  • ` plus `ul.checks li. .mark` in `dash.css` is the whole render path, and no page asset or stylesheet rule names one check (a second opinion about the driver's word is the bug). +- **A measurement window is pinned from both sources, through one contract module** (phase 23). + `tests/rate_window_contract.py` names the two Δ windows (`HOST_WINDOW` = `RATE_WINDOW_SECONDS`, + `LLM_WINDOW` = `LLM_INTERVAL_SECONDS`), the `delta_label()` rounding the pages use to print + `Δ2.0s` / `Δ10.0s`, and `assert_window()` — the estimator's own bounds (`MIN_WINDOW_SECONDS` up + to `2 × span`, because a span equal to the cadence means the base comes from the window-empty + fallback and a real window is the poll gap: 2.009-2.026 s live on `/llm`) **plus** the printed + label, so raising a cadence knob moves the window and the pin together instead of rotting it. + Four files read it: `test_api.py` spies the estimator object each source's jobs receive and pins that + `/api/config`'s LLM window is the LLM's own span; `test_llm_source.py` walks both throughput + cells of **both** engines at 2 s and re-runs the phase-13 floor refusal through the production + parse path at that span; `test_host_source.py` keeps the host payload's `window_seconds` at 10 s + and shows the same 10 s cadence starved to `n/a` by a 2 s span (why one estimator cannot serve + both); `test_rates.py` runs the floor, the `2 × span` ceiling and the 8× jump gate at both + spans, because those are properties of the estimator and not of the window size. A pin naming + one source's window is what this prevents — the two payloads are the same shape and only the + number differs, so half the pin is how a quiet regression would look. - **Page payloads that a rewrite must not break are pinned as a table.** `tests/card_contract.py` holds the dotted `/api/system/snapshot` paths the `/system` hero cards read (`CARD_METRICS`, `GPU_METRICS`, `HOST_SHAPES`); `tests/test_system_contract.py` @@ -1303,6 +1346,13 @@ branch `main`); there is no CI on it, so `bash .agents/validate.sh` is the only faked, which is how a `0640` mode, a byte-identical bundle and the https `--status` probe are proven without a root shell or a host. **A `0600` in a task file is not a fact about this repo**: the agent opens its own bundle and re-reads it for the denylist, so the group bit is load-bearing. +- **A live measurement is re-derivable, so a verdict can be re-checked without re-watching** + (phase 23). `scripts/watch-llm-window.py` writes the series it judged (value, the `Δ` + the payload carried, the engine's own busy count, the raw token counter) beside its verdict, and + `DASH_LLM_WATCH_REPLAY=` re-computes that verdict from the file with no traffic sent. The + pass line is read on the same samples re-diffed over the estimator's own span *and* on the + published value, because the phase-13 jump gate can legitimately hold a correct number for a + poll — judging only the published column would call a designed hold a ramp. - **A cost claim is pinned as a shape and a control, never as a wall clock** (phase 14). The default run keeps the seeded-store tests hermetic and asserts **shape and determinism** — counts, retention windows, the statements a plan is taken from, idempotence — while the timings live behind @@ -1385,9 +1435,9 @@ component vocabulary the pages are still being converted to (`.hero-bar`, the gr variants) stays even while only one page uses it. A big number is `Dash.hero()` on `.hero` / `.hero-bar` (`dash.css`): 2.4rem tabular numerals, -muted unit, mono `.sub` footnote (`Δ10.0s`, "at current load"), coloured by the payload's -`severity` through the existing `.ok` / `.warn` / `.fail` classes — there is no `.sev-*` -family and no second palette, and `tests/test_api.py` fails if either appears or if a CSS +muted unit, mono `.sub` footnote (`Δ2.0s` on `/llm`'s rates since phase 23, "at the current load" +on `/ups`), coloured by the payload's `severity` through the existing `.ok` / `.warn` / `.fail` +classes — there is no `.sev-*` family and no second palette, and `tests/test_api.py` fails if either appears or if a CSS rule reaches for a token that `:root` does not declare. **The `/disks` card is one panel per *physical* drive, led by identity and verdict, not by a @@ -1570,7 +1620,9 @@ column; this was a promotion, not a de-duplication). A `Δ` label is a measurement, never a config value: `sources/host.py` sends the window the estimator actually used as the payload's `window_seconds`, and `/system`'s footnote prints those (a range when the hosts disagree) instead of the poll interval, which only coincides with -them at the defaults. `/llm`'s hero `.sub` prints the rate cell's own `window` through the same +them at the defaults (since phase 23 the `/llm` rates are measured over one poll interval, so +their tag reads `Δ2.0s` while `/system`'s stays `Δ10.0s` — each page naming the window its own +estimator used). `/llm`'s hero `.sub` prints the rate cell's own `window` through the same `provenance()` tag, and the hero's `title` carries the exact `source` string — "78 tok/s" must always say whether it was scraped, measured `Δcounter/Δt`, or estimated (`llm_monitor`'s `setThroughput` idiom; dropping the tag to look cleaner is a regression, not a simplification). @@ -1930,6 +1982,49 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl (`test_a_fallback_inside_the_documented_ceiling_still_publishes`), so a span-relative floor would be a re-decision of that phase, not a bug fix. Do not "fix" the residual by hiding the short window or its label. + **Phase 23 changed the size of the window, not the gates** (`TODO.md:8`: "a strange delay and a + 'ramp up/ramp down' effect"). What was left after phases 13/14 was the span itself: one + 10 s-window estimator served both cadences, so on the 2 s `llm` poll a slot's first seconds + were spread over five polls — the capture replay in + `tests/test_llm_source.py::test_gufo_the_window_the_llm_jobs_are_spanned_by_is_one_that_fills` + prints 12.5 → 20 → 30 tok/s for an engine steady at 50, and the same five polls backwards when + the slot ends. `build_hub` now builds **one estimator per measuring cadence** (the `llm` jobs + spanned by `LLM_INTERVAL_SECONDS`, the host jobs by `RATE_WINDOW_SECONDS`), which reads 50 on + the first poll after the change and 0 on the first after it ends. Every phase-13 protection + still applies to the smaller window — `Δ2.0s` is 4× above `MIN_WINDOW_SECONDS`, the ceiling is + relative to the span, and the 8× jump gate is the estimator's own — so this is the smoothing + dial turned down, not a second definition of a rate. + **The live half has landed** (task 03, 2026-10-21, read-only: nothing was written and no + workload was started — the busy periods were the operator's own requests). A 13-minute + signed-in pass of `/api/llm/snapshot` at the app's own 2 s cadence + (`scripts/watch-llm-window.py`; series `data/logs/llm-window-watch.tsv`, verdict + `data/logs/llm-window-watch.{json,log}`) recorded **zero** values printed without their + `Δ` label across 780 rows, with every window in 0.61-3.38 s and never 10 s. It found + 17 busy-edges on gufo and judged the 5 starts + 5 ends that held long enough (an edge followed + by the next request within a poll is not a slot ending, and prefill polls genuinely decode + ~0 — both are reported, neither is judged): **every** one was inside the 8× pass line at poll 2 + on the ungated Δ. Re-diffing the *same samples* over the span this phase replaced is the + comparison the operator's complaint was about: median |ln(value/steady)| at poll 2 was **0.037** + with this window against **0.924** with the 10 s one, whose median edge needed **poll 3** to + settle (the five starts read 0.98 / 1.04 / 0.95 / 1.00 / 1.27 of steady at poll 2 on the + ungated 2 s Δ where those same five read 0.25 / 0.34 / 0.96 / 0.40 / 0.40 on the 10 s one; ends + took polls 4-5 to fall away against poll 2 now). + One start had its first correct value held for a poll by the phase-13 jump gate and published + on the next — designed behaviour, and the reason the pass line is read on the ungated re-diff + as well as on the published column. The stored **1 m rollups** for the window topped out at + **42.3 tok/s** (raw max 201.6 — that row's own `Δ0.610s` label says so, gap 9's sub-8× skew, + far under the 300 pass line). A second pass over the same engines an hour later + (`data/logs/llm-window-watch-pass2.log`) reproduced it: 3 starts + 2 ends, 0 unsettled, 0 + unlabelled, 1 m rollups max 24.7 tok/s. + The browser half (`data/logs/23-live-window.cjs`, transcript `data/logs/23-live-window.out`, + screenshots `data/logs/23-live-llm-{opening,burst-settled,midburst,post-free,gufo-idle,vllm-idle,final}.png`) + checked the same claims as rendered: **866** decode figures, **0** without a tag in either + component that prints the number (the panel hero's `.sub` and the strip card's `.prov`), every + tag a Δ inside the estimator's own bounds — `MIN_WINDOW_SECONDS` to `2 × span`, the modal one + `Δ2.0s` on both the page and the API (833 of 866) and never the host's `Δ10s`. Its own edge + checks (isolated only, for the same reason) read inside the 8× line at poll 2 on two starts and + exactly `0.0 tok/s` at poll 2 on two ends — the frame named post-free is only taken once the + engine is provably idle, because the busy signal flickers between back-to-back requests. 10. **`driveripper`'s 16 drives read `n/a`: kernel 7.2's unprivileged-passthrough gate (2026-10-19, resolved the same day; the installer template that shipped the broken device filter was cleared in phase 17).** @@ -2090,7 +2185,8 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl the table tail is still out of reach. The fix is a responsive-table pass (`.scroll` on the wide tables, or a per-host card layout at ≤640px) and it touches pinned page shape on `/`, so it is a phase rather than a cleanup — `21_page_cleanups`' `.scroll` on the `/certs` table is the same - shape applied to one table and does not reach this one. Of the two operator TODO items that sat + shape applied to one table and does not reach this one; `27_mobile_tables_and_audit` is that + pass, still in `todo/`. Of the two operator TODO items that sat beside it, `20_masthead_nav_polish` has landed in full (now complete): the bar is three left-justified rows at every width, it is no longer sticky, the phone menu slides, and both `⚙ Account` and `⚷ Certs` render on all seven signed-in pages including the two they name diff --git a/README.md b/README.md index 004a618..68bb805 100644 --- a/README.md +++ b/README.md @@ -478,9 +478,10 @@ documentation and is never loaded, and `scripts/dev-server.sh` exports `.env.dev | ------------------------------------- | -------------------------------------------------------------- | | `HOST_TARGETS` | `name=url` per host running the agent; a bearer token may ride in the URL userinfo (`http://TOKEN@host:9120`). An `https://` URL is dialled as a *protected* agent instead: verified against the dashboard's own root CA and presented with its client certificate, with a revoked host refused before the dial (`certs.py`, `collectors.AgentTls`) | | `GUFO_BASE_URL` / `VLLM_BASE_URL` | LLM engines; unset means *not deployed here*, and its half of the page disappears | -| `RATE_WINDOW_SECONDS` | the Δ window behind every measured rate — tokens/sec, bytes/sec, jiffies→CPU % (default 10); each value carries the window it actually got, and `/system`'s footnote prints that, not this default. A window the poll loop cannot honour — the ~0.2 s catch-up after a fetch slower than the interval, or a base older than 2× the window — answers one `n/a` instead of a spike, and a rate 8× above the last one is held until a window measured from a **different** sample confirms it (phase 13) | +| `RATE_WINDOW_SECONDS` | the **host's** Δ window (default 10) — bytes/sec and jiffies→CPU %, measured over what a 10 s poll can fill; each value carries the window it actually got, and `/system`'s footnote prints that, not this default. A window the poll loop cannot honour — the ~0.2 s catch-up after a fetch slower than the interval, or a base older than 2× the window — answers one `n/a` instead of a spike, and a rate 8× above the last one is held until a window measured from a **different** sample confirms it (phase 13). The LLM rates do **not** use it: they are measured over `LLM_INTERVAL_SECONDS` (phase 23) | +| `LLM_INTERVAL_SECONDS` | the `/llm` poll cadence (default 2 s) **and** the Δ window behind every token rate — one estimator per measuring cadence, so a slot that starts reads steady within two polls instead of ramping over five. Slowing the poll slows the window with it; there is no separate LLM window knob, and a window narrower than the interval would answer `n/a` every poll | | `DASH_HOST` / `DASH_PORT` | bind address for the dashboard itself (default `127.0.0.1:8090`; the container sets `0.0.0.0`) | -| `HOST_INTERVAL_SECONDS` / `LLM_INTERVAL_SECONDS` / `NUT_INTERVAL_SECONDS` | how often each source is polled (10 / 2 / 5 s) | +| `HOST_INTERVAL_SECONDS` / `NUT_INTERVAL_SECONDS` | how often the host agents and upsd are polled (10 / 5 s) | | `DISKS_INTERVAL_SECONDS` | how often the disk-health source re-reads each agent for SMART (default 300 s — SMART is not a per-second signal). No targets of its own: the `disks:` jobs exist exactly when `HOST_TARGETS` does | | `HOST_TIMEOUT_SECONDS` / `LLM_TIMEOUT_SECONDS` / `NUT_TIMEOUT_SECONDS` | per-source fetch timeout — one slow source never holds up the others | | `DISKS_TIMEOUT_SECONDS` | the disks fetch's own budget (default 30 s), because the agent runs `smartctl` inside its snapshot handler and a wedged SMART pass must not read as a dead host | @@ -674,7 +675,7 @@ certs.py ─ root CA + client cert, minted once at startup (unreadable row → n the way out and the same on the way back in, with nothing hidden from a keyboard or a screen reader while it is shut, and no motion at all when the system asks for none (`prefers-reduced-motion`). No framework, no CDN, no bundler, and no narration: a page never - explains its own pipeline, so the words on screen are the ones you act on — a unit, a `Δ10.0s` provenance tag, a + explains its own pipeline, so the words on screen are the ones you act on — a unit, a `Δ` provenance tag, a `SIMULATED` badge, or the sentence beside an empty panel. The overview (`/`) is exactly two panels — a **Warnings & errors** list built from `/api/summary`'s `alerts` (each row links to the dashboard that explains it) and the **Fleet** table — and nothing else: an empty alert @@ -751,8 +752,8 @@ certs.py ─ root CA + client cert, minted once at startup (unreadable row → n in the section stack — sits the **stats strip**: one card per configured engine — reachability dot, name, age stamp, the **KV-cache bar** (the one rated cell: the fill and a breach's text take their colour from the payload's `severity` and nothing else), - then `prompt tok/s` and `decode tok/s` with their provenance tag (`Δ10.0s`, scraped, - estimated), `total tokens`, the sessions this engine reports (`slots busy` counted from + then `prompt tok/s` and `decode tok/s` with their provenance tag (`Δ2.0s` at the default cadence — + the LLM window is one poll interval, phase 23 — scraped, estimated), `total tokens`, the sessions this engine reports (`slots busy` counted from gufo's `/slots`, or `running`/`waiting` for vLLM, plus `waiting`), `prefix cache hit`, `spec accept` (gufo), `queue time` with its `p95 …` foot (vLLM) — and red alarm chips for preemptions / GPU-context losses that exist only while non-zero. Then the engine panels @@ -811,6 +812,15 @@ uv run python scripts/store_perf_probe.py # the same measurements as a repo DASH_WATCH_SECONDS=1500 uv run python scripts/watch-store-perf.py # ... and the live watch: 25 signed-in minutes against # the running dev server, counting store-error and WAL +DASH_LLM_WATCH_SECONDS=780 uv run python scripts/watch-llm-window.py + # ... and the LLM Δ-window watch (phase 23): signed-in + # reads of /api/llm/snapshot at the app's own cadence, + # judging every slot start/end against "settled within + # two polls", the Δ labels and the 1 m rollups. + # Read-only: it starts no workload and writes nothing. +DASH_LLM_WATCH_REPLAY=data/logs/llm-window-watch.tsv \ + uv run python scripts/watch-llm-window.py # ... or re-derive that verdict from a captured pass, + # over the same samples, with no traffic sent uv run ruff check src tests agent && uv run pyright # lint + types ./scripts/capture-fixtures.sh # re-record ground truth: host agents, UPS replies, gufo /slots ./scripts/capture-fixtures.sh --with-llm # ... and the engines' /metrics while they are up @@ -934,6 +944,18 @@ quietly becoming a fallback chain. A busy slot's `prompt` is the user's in-flight request text, so the capture script blanks it and the page never reads it. +`tests/rate_window_contract.py` is the same device for one number rather than a field table: the Δ +window each source measures its rates over — `RATE_WINDOW_SECONDS` for the host agents, +`LLM_INTERVAL_SECONDS` for the engines, one estimator per measuring cadence (phase 23). It holds +the two spans, the `Δ` rounding the pages print, and a range assertion — the estimator's +own floor up to its `2 × span` ceiling, plus the printed label — shared by four files: the +`build_hub` wiring spy in +`test_api.py`, both throughput cells of both engines in `test_llm_source.py`, the host payload's +`window_seconds` in `test_host_source.py`, and the estimator's floor / `2 × span` ceiling / 8× +jump gate run at both spans in `test_rates.py`. The two payloads are the same shape and only the +number differs, so a pin naming one source's window is exactly how the other could quietly keep +advertising a window no poll ever measured. + ## Battery-test buttons Instant commands are the only endpoint in the application that changes anything **outside this diff --git a/scripts/watch-llm-window.py b/scripts/watch-llm-window.py new file mode 100755 index 0000000..fc1166d --- /dev/null +++ b/scripts/watch-llm-window.py @@ -0,0 +1,694 @@ +#!/usr/bin/env python3 +"""Phase 23 task 03: watch the running dev server prove the LLM Δ window is one poll interval. + +Sampled, not exhaustive: every ``LLM_INTERVAL_SECONDS`` (read back from ``/api/config``, so the +watch polls at the cadence the app itself uses) a signed-in pass reads ``/api/llm/snapshot`` and +records, per engine, the decode/prompt rate **with the ``Δ`` the payload carried**, the +lifetime token counter (so the same samples can be re-diffed over the pre-phase 10 s window for +a counterfactual), and how many slots the engine says are working. + +Nothing here writes. The only POST is the dashboard's own sign-in; no engine endpoint is called, +no ``/api/ups/command``, no SET. The workload is whatever the operator's engines are already +doing — loading a shared engine to make a prettier graph is not a test. + +What it judges, and the numbers it judges them against: + +* **the ramp** — within two polls of a slot *starting* the rate is within 8x of the steady rate of + that busy period, and within two polls of it *ending* the rate has fallen away by 8x (the + estimator's own ``JUMP_CONFIRM_RATIO``). Read that on the **ungated** Δ — the same samples + re-diffed over the estimator's span — because the phase-13 jump gate legitimately holds a correct + number for a poll, and on edges that are *isolated*: an edge followed by the next request inside + the examined window is not a slot ending (the counter never stopped), and a slot's first polls + decode ~0 because the engine is prefilling. Both are reported with their reason and not judged. + Beside every edge is the identical re-diff over the **10 s** span this phase replaced, so the + shape that was removed is in the record rather than in a claim: five polls of 1/5, 2/5, 3/5 of + the truth, up and back down. +* **the label** — zero published values without their ``Δ`` tag, and every tag naming + the window actually used (``Δ2.0s`` on a healthy cadence, the honest sub-window when the + floor or a stall engages, never a bare number). +* **the rollups** — the stored 1 m averages of ``llm.decode_tokens_per_second`` inside the watch + window stay in this fleet's measured band (gap 9: the real maxima are 119.9 gufo / 171.7 vLLM; + above ~300 tok/s is arithmetic, not work), read straight out of the dev store on a read-only + connection. + +Run it while the dev server keeps polling the fleet:: + + set -a; . ./.env.dev; set +a + DASH_LLM_WATCH_SECONDS=720 uv run python scripts/watch-llm-window.py + +Writes ``data/logs/llm-window-watch.tsv`` (the series), ``llm-window-watch.json`` (the verdict) +and ``llm-window-watch.log`` (the same verdict in prose). ``data/`` is gitignored: local evidence +for ``AGENTS.md`` gap 9, never a tracked artifact. +""" + +from __future__ import annotations + +import json +import math +import os +import sqlite3 +import statistics +import sys +import time +from pathlib import Path +from typing import Any, cast + +import httpx + +ROOT = Path(__file__).resolve().parents[1] + +BASE_URL = os.environ.get("DASH_WATCH_URL", "http://127.0.0.1:8090") +ADMIN_PASSWORD = os.environ.get("DASH_ADMIN_PASSWORD", "") +DB_PATH = Path(os.environ.get("DASH_WATCH_DB", ROOT / "data" / "dashboards.sqlite3")) +OUT = Path(os.environ.get("DASH_LLM_WATCH_OUT", ROOT / "data" / "logs")) +DURATION_SECONDS = float(os.environ.get("DASH_LLM_WATCH_SECONDS", "720")) + +#: Gap 9's pass line: this fleet's real maxima are 119.9 (gufo) / 171.7 (vLLM) tok/s, so a +#: value above this is arithmetic on a collapsed window, not work the engine did. +CEILING_TPS = 300.0 + +#: The estimator's own jump ratio (`rates.JUMP_CONFIRM_RATIO`), used here as the "is this +#: number still ramping?" bound so the pass line and the hold behaviour are one number. +JUMP_RATIO = 8.0 + +#: How many polls after a transition are examined, and which one has to be settled. +POLLS_AFTER_TRANSITION = 6 +SETTLED_BY_POLL = 2 + +#: The window the phase-23 defect was made of, for the counterfactual re-diff. +PRE_PHASE_SPAN = 10.0 + +#: `rates.MAX_WINDOW_RATIO` — the reconstruction below mirrors the estimator's fallback. +MAX_WINDOW_RATIO = 2.0 + +#: A busy run shorter than this cannot have a "steady" to settle to, so it is reported and not +#: judged (a one-poll request is a burst, not a ramp). +MIN_RUN_TO_JUDGE = 3 + +METRIC = "llm.decode_tokens_per_second" + + +def _num(value: object) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def load_rows(path: Path) -> list[dict]: + """Re-read a previous watch's TSV so a verdict can be re-derived without re-watching. + + The file carries the value, the window it was measured over, the engine's own busy count and + the raw counter, which is everything the judgement needs — so re-running the pass line on + captured evidence costs no new traffic and stays the *same* measurement. + """ + rows: list[dict] = [] + with path.open() as fh: + header = fh.readline().rstrip("\n").split("\t") + for line in fh: + fields = dict(zip(header, line.rstrip("\n").split("\t"), strict=True)) + rows.append({ + "ts": _parse(fields.get("ts")), + "wall": _parse(fields.get("wall")) or 0.0, + "engine": fields.get("engine", "?"), + "decode": _parse(fields.get("decode")), + "window": _parse(fields.get("window")), + "source": fields.get("source", ""), + "prompt": _parse(fields.get("prompt")), + "prompt_window": _parse(fields.get("prompt_window")), + "busy": _parse(fields.get("busy")) or 0.0, + "tokens_total": _parse(fields.get("tokens_total")), + "generated_total": _parse(fields.get("generated_total")), + }) + return rows + + +def _parse(text: str | None) -> float | None: + """The TSV writes `n/a` for a missing value; everything else is a float.""" + if text is None or text == "n/a": + return None + try: + return float(text) + except ValueError: + return None + + +def sign_in(client: httpx.Client) -> None: + """One sign-in on the client we use for the whole watch; `testpass` in dev.""" + response = client.post("/api/auth/login", json={"password": ADMIN_PASSWORD}) + response.raise_for_status() + if "dash_session" not in response.headers.get("set-cookie", ""): + raise SystemExit("sign-in returned no session cookie") + + +def llm_interval(client: httpx.Client) -> float: + """The app's own LLM cadence — the watch polls at it, not at a guessed 2 s.""" + config = client.get("/api/config").json() + return float(config["llm"]["interval_seconds"]) + + +def sample_row(payload: dict, engine: dict) -> dict[str, object]: + """One (poll, engine) row: value, the Δ it was measured over, activity, raw counters.""" + metrics = engine.get("metrics") or {} + decode = metrics.get("decode_tokens_per_second") or {} + prompt = metrics.get("prompt_tokens_per_second") or {} + sessions = engine.get("sessions") or {} + busy = _num(sessions.get("active")) or 0.0 + processing = _num((metrics.get("requests_processing") or {}).get("value")) or 0.0 + return { + "ts": _num(payload.get("ts")), + "wall": time.time(), + "engine": engine.get("engine"), + "decode": _num(decode.get("value")), + "window": _num(decode.get("window")), + "source": decode.get("source") or "", + "prompt": _num(prompt.get("value")), + "prompt_window": _num(prompt.get("window")), + "busy": max(busy, processing), + "tokens_total": _num((metrics.get("tokens_total") or {}).get("value")), + # the counter this engine's decode rate is actually diffed from: gufo publishes + # `tokens_generated_total`, vLLM `generation_tokens_total` (sources/llm.py's `measured()`) + "generated_total": (_num((metrics.get("tokens_generated_total") or {}).get("value")) + if (metrics.get("tokens_generated_total") or {}).get("value") is not None + else _num((metrics.get("generation_tokens_total") or {}).get("value"))), + } + + +def take_samples(client: httpx.Client, interval: float) -> tuple[list[dict], int, float, float]: + """Poll until the clock runs out; a failed read is counted and the watch goes on.""" + rows: list[dict[str, object]] = [] + errors = 0 + started = time.time() + deadline = started + DURATION_SECONDS + while time.time() < deadline: + loop_start = time.time() + try: + response = client.get("/api/llm/snapshot") + response.raise_for_status() + payload = response.json() + for engine in payload.get("engines") or []: + rows.append(sample_row(payload, engine)) + except (httpx.HTTPError, ValueError, KeyError) as exc: # a watcher reports, it does not fail + errors += 1 + print(f"poll error: {type(exc).__name__}: {exc}", file=sys.stderr) + time.sleep(max(0.05, interval - (time.time() - loop_start))) + return rows, errors, started, time.time() + + +def rollup_window(db: sqlite3.Connection, since: float, until: float) -> dict[str, Any]: + """The stored 1 m averages inside the watch window, per engine (read-only connection).""" + out: dict[str, Any] = {"rows": 0, "max": None, "per_engine": {}} + found = db.execute( + "SELECT host, ts, value FROM sample WHERE metric=? AND res='1m' AND ts>=? AND ts<=? " + "ORDER BY ts", + (METRIC, since, until), + ).fetchall() + per_engine: dict[str, list[float]] = {} + for host, _ts, value in found: + number = _num(value) + if number is None: + continue + per_engine.setdefault(str(host), []).append(number) + values = [n for numbers in per_engine.values() for n in numbers] + newest = db.execute( + "SELECT MAX(ts) FROM sample WHERE metric=? AND res='1m'", (METRIC,) + ).fetchone()[0] + out["rows"] = len(found) + out["max"] = max(values) if values else None + out["per_engine"] = { + host: {"rows": len(nums), "max": max(nums), "median": statistics.median(nums)} + for host, nums in per_engine.items() + } + out["newest_1m_ts"] = _num(newest) + out["lag_seconds"] = (until - float(newest)) if newest else None + return out + + +def raw_band(db: sqlite3.Connection, since: float, until: float) -> dict[str, object]: + """The raw samples in the same window — the row-level view under the 1 m averages.""" + found = db.execute( + "SELECT host, MAX(value) FROM sample WHERE metric=? AND res='raw' AND ts>=? AND ts<=? " + "GROUP BY host", + (METRIC, since, until), + ).fetchall() + return {str(host): value for host, value in found if value is not None} + + +def rediff(series: list[dict], span: float) -> list[float | None]: + """What a `span`-window estimator would have printed, from the counters in the payload. + + Same samples, same arithmetic as `rates.RateEstimator.rate` (base = the oldest sample inside + the window), without the phase-13 gates: this is the counterfactual that shows the ramp the + 10 s window used to draw, not a second measurement of the engine. The index of the returned + list matches `series`, so a transition can point at the row it happened on. + """ + out: list[float | None] = [] + for index, row in enumerate(series): + counter = row["generated_total"] + if counter is None or row["ts"] is None: + out.append(None) + continue + prior = [(p["ts"], float(p["generated_total"])) for p in series[:index] + if p["ts"] is not None and p["generated_total"] is not None] + if not prior: + out.append(None) + continue + inside = [point for point in prior if row["ts"] - point[0] <= span] + if inside: + base = inside[0] + else: + # The estimator's own window-empty fallback (`MAX_WINDOW_RATIO x span`): a cadence + # of exactly `span` leaves the previous sample a hair *outside* the window, and + # without this the reconstruction would answer n/a where the app answers a rate. + fallback = prior[-1] + if row["ts"] - fallback[0] > MAX_WINDOW_RATIO * span: + out.append(None) + continue + base = fallback + dt = row["ts"] - base[0] + dv = float(counter) - base[1] + out.append(None if dt <= 0 or dv < 0 else dv / dt) + return out + + +def transitions(rows: list[dict], engine: str) -> list[dict[str, object]]: + """Every busy-edge on this engine and the polls on each side of it. + + The busy signal is the engine's own (`sessions.active` for gufo, the running count for + vLLM, `requests_processing` as the fallback), never an inference from the rate: judging the + rate against itself would call a held poll a slot ending. No verdict here — `judge_edges()` + makes those, because a verdict needs the re-diffed Δ as well as the published value. + """ + series = [r for r in rows if r["engine"] == engine] + found: list[dict[str, object]] = [] + for index in range(1, len(series)): + before, after = series[index - 1]["busy"], series[index]["busy"] + if before == after: + continue + kind = "start" if after > before else "end" + run = _run(series, index if kind == "start" else index - 1, + forward=kind == "start") + examined = (run[:POLLS_AFTER_TRANSITION] if kind == "start" + else series[index:index + POLLS_AFTER_TRANSITION]) + found.append({ + "kind": kind, + "engine": engine, + "ts": series[index]["ts"], + "index": index, + "busy_before": before, + "busy_after": after, + "run_polls": len(run), + "values": [r["decode"] for r in examined], + "windows": [r["window"] for r in examined], + # the busy state flickering back inside the examined window means the engine never + # went idle (one request ended and the next started within a poll or two) — an edge + # that is not an edge, so decay cannot be judged from it + "flips_back": any((series[j]["busy"] > 0) != (after > 0) + for j in range(index, min(index + POLLS_AFTER_TRANSITION, len(series)))), + "truncated": len(examined) < SETTLED_BY_POLL if kind == "end" else index + len(run) >= len(series), + }) + return found + + +def judge_edges(edges: list[dict], series: list[dict], delta: list[float | None], + pre_phase: list[float | None], app_span: float) -> None: + """Fill in each edge's steady, ratios and verdict, in place. + + Two columns are judged, and the difference between them is the whole point of the task: + + * **`published`** — what `/api/llm/snapshot` answered, which carries the phase-13 jump gate + as well as the window (a value held one poll is designed behaviour, not a ramp); + * **`delta`** — the same samples re-diffed over the estimator's own span with no gate, which + is the *window* on its own. This is the column the pass line is read from, because the + gate can hold a correct number for a poll and no window can make it settle sooner. + + `pre_phase` is the identical re-diff over the 10 s span this phase replaced; it is reported + beside the other two so the reader can see the shape that was removed rather than take the + claim on trust. + """ + for edge in edges: + index = int(edge["index"]) + kind = str(edge["kind"]) + if kind == "start": + run = series[index:index + int(edge["run_polls"])] + pool = [r["decode"] for r in run if r["decode"]] + else: + run = _run(series, index - 1, forward=False) + pool = [r["decode"] for r in run if r["decode"]] + steady = statistics.median(pool) if len(pool) >= MIN_RUN_TO_JUDGE else None + edge["steady"] = steady + edge["delta_values"] = delta[index:index + POLLS_AFTER_TRANSITION] + edge["pre_phase_values"] = pre_phase[index:index + POLLS_AFTER_TRANSITION] + edge["pre_phase_span"] = PRE_PHASE_SPAN + edge["app_span"] = app_span + edge["ratios"] = _ratios(cast("list[float | None]", edge["values"]), steady, kind) + edge["delta_ratios"] = _ratios(cast("list[float | None]", edge["delta_values"]), steady, kind) + edge["pre_phase_ratios"] = _ratios(cast("list[float | None]", edge["pre_phase_values"]), + steady, kind) + edge["settled_by_poll"] = _settled_by(cast("list[float | None]", edge["ratios"]), kind) + edge["delta_settled_by_poll"] = _settled_by(cast("list[float | None]", edge["delta_ratios"]), + kind) + edge["pre_phase_settled_by_poll"] = _settled_by( + cast("list[float | None]", edge["pre_phase_ratios"]), kind) + # a published value that repeats while the ungated Δ moved is the jump gate holding + held = any(v is not None and i > 0 and v == edge["values"][i - 1] + for i, v in enumerate(edge["values"]) if i <= SETTLED_BY_POLL) + edge["jump_gate_held"] = bool(held and edge["settled_by_poll"] != edge["delta_settled_by_poll"]) + if edge["flips_back"]: + edge["judgable"] = False + edge["reason"] = "busy again within the examined window — the engine never went idle" + elif edge["truncated"]: + edge["judgable"] = False + edge["reason"] = "no polls after the edge inside the watch" + elif steady is None: + edge["judgable"] = False + edge["reason"] = f"busy run of {edge['run_polls']} poll(s) has no steady to settle to" + else: + edge["judgable"] = True + edge["reason"] = None + + +def _run(series: list[dict], index: int, forward: bool) -> list[dict]: + """The consecutive polls on the busy side of an edge, starting at `index`.""" + if series[index]["busy"] <= 0 and forward: + return [] + out = [series[index]] + step = 1 if forward else -1 + cursor = index + step + while 0 <= cursor < len(series) and series[cursor]["busy"] > 0: + out.append(series[cursor]) + cursor += step + return out + + +#: The cap on "the rate is now effectively zero", so the report stays JSON and not `Infinity`. +DECAY_CAP = 999.0 + + +def _ratios(values: list[float | None], steady: float | None, kind: str) -> list[float | None]: + """How far each poll after the edge still is from the steady rate of the other side. + + A **start** is settled when the value reaches the busy steady (ratio ~1); an **end** is + settled when it has fallen away from the steady it used to have (ratio >= 8, capped), so one + number reads the same way on both edges: bigger is closer to done on an end, closer to 1 on + a start. + """ + if not steady or steady <= 0: + return [None] * len(values) + out: list[float | None] = [] + for value in values: + if value is None: + out.append(None) + elif kind == "start": + out.append(value / steady) + else: + out.append(DECAY_CAP if value <= 0 else min(DECAY_CAP, steady / value)) + return out + + +def judge(rows: list[dict], edges: list[dict], rollups: dict, app_span: float) -> dict[str, Any]: + """The four pass lines of the task, as numbers.""" + verdict: dict[str, Any] = {"ceiling_tps": CEILING_TPS, "jump_ratio": JUMP_RATIO, + "polls": len(rows), "app_span": app_span, + "engines": {}, "transitions": edges, "counterfactual": {}} + problems: list[str] = [] + for engine in sorted({str(r["engine"]) for r in rows}): + series = [r for r in rows if r["engine"] == engine] + values = [r["decode"] for r in series if r["decode"] is not None] + unlabelled = [r for r in series if r["decode"] is not None and r["window"] is None] + windows = [r["window"] for r in series if r["window"] is not None] + over = [v for v in values if v > CEILING_TPS] + verdict["engines"][engine] = { + "rows": len(series), + "null_values": sum(1 for r in series if r["decode"] is None), + "max": max(values) if values else None, + "median": statistics.median(values) if values else None, + "window_min": min(windows) if windows else None, + "window_max": max(windows) if windows else None, + "values_without_delta_label": len(unlabelled), + "values_over_ceiling": len(over), + "over_ceiling_examples": over[:5], + } + if unlabelled: + problems.append(f"{engine}: {len(unlabelled)} values printed without a Δ label") + if over: + problems.append(f"{engine}: {len(over)} values above {CEILING_TPS} tok/s") + cf_app = rediff(series, app_span) + cf10 = rediff(series, PRE_PHASE_SPAN) + verdict["counterfactual"][engine] = {"app_span": _shape(cf_app), + "pre_phase_span": _shape(cf10)} + judge_edges([e for e in edges if e["engine"] == engine], series, cf_app, cf10, app_span) + settled = [e for e in edges if e["kind"] == "start" and e["judgable"]] + decayed = [e for e in edges if e["kind"] == "end" and e["judgable"]] + verdict["transitions_checked"] = { + "starts": len(settled), "ends": len(decayed), + "pass_line": f"the ungated Δ over the app's own {app_span}s span is inside " + f"{JUMP_RATIO}x of steady by poll {SETTLED_BY_POLL}", + "unsettled_by_poll_2": [ + {"kind": e["kind"], "ts": e["ts"], "delta_ratios": e["delta_ratios"]} + for e in settled + decayed + if not _settled(e["delta_ratios"], SETTLED_BY_POLL, e["kind"])], + # how much of steady the first two polls delivered, this phase's window and the one it + # replaced: the pre-phase 10 s window sat at ~1/5 and climbed, which is the complaint + "poll_1_of_steady_at_a_start": _at_poll(settled, "ratios", 1), + "poll_2_of_steady_at_a_start": _at_poll(settled, "ratios", 2), + "poll_1_of_steady_at_a_start_ungated": _at_poll(settled, "delta_ratios", 1), + "poll_2_of_steady_at_a_start_ungated": _at_poll(settled, "delta_ratios", 2), + "poll_1_of_steady_at_a_start_pre_phase": _at_poll(settled, "pre_phase_ratios", 1), + "poll_2_of_steady_at_a_start_pre_phase": _at_poll(settled, "pre_phase_ratios", 2), + # the headline: which poll settles, this span vs the 10 s span, over the same samples + "settled_by_poll_median": { + "app_span": _median_of(settled + decayed, "delta_settled_by_poll"), + "pre_phase_span": _median_of(settled + decayed, "pre_phase_settled_by_poll"), + "published": _median_of(settled + decayed, "settled_by_poll"), + }, + # and how far from steady the second poll still is, as a median |ln(ratio)|: the 8x line + # alone is satisfied by a 10 s window at 0.25 of steady, so this is the number that says + # the ramp is gone rather than merely inside the tolerance + "median_log_distance_from_steady_at_poll_2": { + "app_span": _median_log_distance(settled, "delta_ratios", 2), + "pre_phase_span": _median_log_distance(settled, "pre_phase_ratios", 2), + }, + "jump_gate_holds": sum(1 for e in edges if e.get("jump_gate_held")), + "reported_not_judged": [{"kind": e["kind"], "ts": e["ts"], "reason": e["reason"]} + for e in edges if not e["judgable"]], + } + for e in settled + decayed: + if not _settled(e["delta_ratios"], SETTLED_BY_POLL, e["kind"]): + problems.append(f"{e['kind']} at ts={e['ts']} still ramping at poll " + f"{SETTLED_BY_POLL}: ungated Δ ratios={e['delta_ratios']}, " + f"published ratios={e['ratios']}") + rollup_max = rollups.get("max") + verdict["rollups"] = rollups + verdict["raw_max_in_window"] = rollups.get("raw_max_in_window") + if rollup_max is not None and rollup_max > CEILING_TPS: + problems.append(f"1 m rollups peaked at {rollup_max:.1f} tok/s, above the pass line") + # An empty pass is not a pass: "no transition happened" must never read as "the ramp is + # gone", and a watch that recorded nothing cannot speak about the engines at all. + if not rows: + problems.append("no rows were recorded — the watch proves nothing") + elif not any(r["decode"] for r in rows): + problems.append("no engine decoded anything during the watch — no rate to judge") + elif not (settled or decayed): + problems.append("no busy-edge held long enough to judge (a slot start or end of " + f"{MIN_RUN_TO_JUDGE}+ polls, not followed by the next request) — " + "run the watch again; the ramp claim is unproven without one") + verdict["problems"] = problems + verdict["pass"] = not problems + return verdict + + +def _at_poll(edges: list[dict], key: str, poll: int) -> list[float]: + """The ratio at `poll` for every edge that has one, rounded for the report.""" + return [round(float(e[key][poll - 1]), 3) for e in edges # type: ignore[index] + if e.get(key) and len(e[key]) >= poll and e[key][poll - 1] is not None] # type: ignore[arg-type] + + +def _median_of(edges: list[dict], key: str) -> float | None: + values = [e[key] for e in edges if e.get(key) is not None] + return statistics.median(values) if values else None + + +def _median_log_distance(edges: list[dict], key: str, poll: int) -> float | None: + """Median |ln(value / steady)| at `poll` — 0 is exactly steady, ln(8) is the pass line.""" + logs = [abs(math.log(r)) for r in _at_poll(edges, key, poll) if r > 0] + return round(statistics.median(logs), 3) if logs else None + + +def _settled_by(ratios: list[float | None], kind: str) -> int | None: + """Which poll first sits inside the pass line, or None if none of them does.""" + for poll, ratio in enumerate(ratios, start=1): + if _settled([ratio], 1, kind): + return poll + return None + + +def _settled(ratios: list[float | None], by_poll: int, kind: str) -> bool: + """A start has *reached* steady by `by_poll`; an end has *left* it by at least 8x. + + A held poll (the jump gate repeating the previous value) counts as settled: the hold is the + designed behaviour, and the following poll is the honest publish. + """ + if len(ratios) < by_poll: + return False + ratio = ratios[by_poll - 1] + if ratio is None: + return False + if kind == "end": + return ratio >= JUMP_RATIO + return 1.0 / JUMP_RATIO <= ratio <= JUMP_RATIO + + +def _shape(values: list[float | None]) -> dict[str, object]: + numbers = [v for v in values if v is not None] + return {"n": len(numbers), "max": max(numbers) if numbers else None, + "median": statistics.median(numbers) if numbers else None} + + +def write_outputs(rows: list[dict], verdict: dict, errors: int, + started: float, stopped: float) -> None: + OUT.mkdir(parents=True, exist_ok=True) + tsv = OUT / "llm-window-watch.tsv" + with tsv.open("w") as fh: + fh.write("ts\twall\tengine\tdecode\twindow\tΔ_label\tprompt\tprompt_window\t" + "busy\ttokens_total\tgenerated_total\tsource\n") + for row in rows: + label = f"Δ{row['window']:.1f}s" if row["window"] else "n/a" + fh.write("\t".join([ + _fmt(row["ts"]), _fmt(row["wall"]), str(row["engine"]), _fmt(row["decode"]), + _fmt(row["window"]), label, _fmt(row["prompt"]), _fmt(row["prompt_window"]), + _fmt(row["busy"]), _fmt(row["tokens_total"]), _fmt(row["generated_total"]), + str(row["source"]), + ]) + "\n") + (OUT / "llm-window-watch.json").write_text(json.dumps({ + "start": started, "stop": stopped, + "start_iso": _iso(started), "stop_iso": _iso(stopped), + "polls": len(rows), "poll_errors": errors, + } | verdict, indent=2, sort_keys=True)) + with (OUT / "llm-window-watch.log").open("w") as fh: + fh.write(_report(verdict, errors, started, stopped)) + + +def _report(verdict: dict, errors: int, started: float, stopped: float) -> str: + lines = [ + f"phase 23 live window watch {_iso(started)} -> {_iso(stopped)} " + f"({stopped - started:.0f}s, {verdict['polls']} rows, {errors} poll errors)", + ] + for engine, stats in sorted(verdict["engines"].items()): + lines.append( + f" {engine}: rows={stats['rows']} null={stats['null_values']} " + f"max={_s(stats['max'])} median={_s(stats['median'])} " + f"window={_s(stats['window_min'])}-{_s(stats['window_max'])} " + f"unlabelled={stats['values_without_delta_label']} " + f"over{CEILING_TPS:.0f}={stats['values_over_ceiling']}" + ) + tcheck = verdict["transitions_checked"] + lines.append(f" transitions: {tcheck['starts']} judgable start(s), {tcheck['ends']} judgable " + f"end(s); unsettled by poll {SETTLED_BY_POLL}: " + f"{len(tcheck['unsettled_by_poll_2'])}; pass line: {tcheck['pass_line']}") + for poll in (1, 2): + got = tcheck[f"poll_{poll}_of_steady_at_a_start"] + pre = tcheck[f"poll_{poll}_of_steady_at_a_start_pre_phase"] + if got: + lines.append(f" poll {poll} of a start delivered {got} of steady with this phase's " + f"Δ, where the 10 s window it replaced delivered {pre}") + distance = tcheck["median_log_distance_from_steady_at_poll_2"] + if distance["app_span"] is not None: + lines.append(f" median |ln(value/steady)| at poll 2: {distance['app_span']} with this " + f"phase's Δ vs {distance['pre_phase_span']} with the 10 s window " + f"(ln({JUMP_RATIO:g}) = {math.log(JUMP_RATIO):.2f} is the pass line)") + settled_by = tcheck["settled_by_poll_median"] + lines.append(f" median poll that settles an edge: {settled_by['app_span']} with this " + f"phase's window (published: {settled_by['published']}), " + f"{settled_by['pre_phase_span']} with the 10 s window") + lines.append(f" jump-gate holds inside an examined window: {tcheck['jump_gate_holds']}; " + f"edges reported but not judged: {len(tcheck['reported_not_judged'])}") + for edge in verdict["transitions"]: + lines.append( + f" {edge['engine']}/{edge['kind']:5s} ts={_s(edge['ts'])} busy " + f"{_s(edge['busy_before'])}->{_s(edge['busy_after'])} run={edge['run_polls']}poll " + f"steady={_s(edge['steady'])}" + + ("" if edge["judgable"] else f" [{edge['reason']}]") + + (" [jump gate held a poll]" if edge.get("jump_gate_held") else "") + ) + lines.append(f" published =" + f"{[None if v is None else round(v, 2) for v in edge['values']]}") + lines.append(f" Δ{edge.get('app_span')}s (this phase)=" + f"{[None if v is None else round(v, 2) for v in edge['delta_values']]} " + f"settled by poll {edge.get('delta_settled_by_poll')}") + lines.append(f" Δ{edge.get('pre_phase_span')}s (before)=" + f"{[None if v is None else round(v, 2) for v in edge['pre_phase_values']]} " + f"settled by poll {edge.get('pre_phase_settled_by_poll')}") + for engine, cf in sorted(verdict["counterfactual"].items()): + lines.append(f" whole-window re-diff {engine}: app span {cf['app_span']} | " + f"pre-phase 10 s span {cf['pre_phase_span']}") + roll = verdict["rollups"] + lines.append(f" 1 m rollups in window: rows={roll['rows']} max={_s(roll['max'])} " + f"per_engine={json.dumps(roll['per_engine'])}") + lines.append(f" raw max in window: {json.dumps(verdict['raw_max_in_window'])}") + lines.append(" RESULT: " + ("PASS" if verdict["pass"] else "FAIL")) + for problem in verdict["problems"]: + lines.append(f" ! {problem}") + return "\n".join(lines) + "\n" + + +def _fmt(value: object) -> str: + return "n/a" if value is None else f"{value}" + + +def _s(value: object) -> str: + if value is None: + return "n/a" + return f"{float(value):.3f}" if isinstance(value, (int, float)) else str(value) + + +def _iso(ts: float) -> str: + return time.strftime("%Y-%m-%dT%H:%M:%S%z", time.localtime(ts)) + + +def wait_for_rollups(db_path: Path, since: float, until: float, + timeout: float = 150.0) -> dict[str, Any]: + """1 m rows appear on the maintenance tick (60 s), so give it two ticks, no more.""" + deadline = time.time() + timeout + rollups: dict[str, Any] = {"rows": 0, "max": None, "per_engine": {}} + while True: + with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=30) as db: + rollups = rollup_window(db, since, until) + rollups["raw_max_in_window"] = raw_band(db, since, until) + newest = rollups.get("newest_1m_ts") + if rollups["rows"] and isinstance(newest, (int, float)) and newest >= until - 65: + return rollups + if time.time() >= deadline: + return rollups + time.sleep(5) + + +def main() -> int: + OUT.mkdir(parents=True, exist_ok=True) + replay = os.environ.get("DASH_LLM_WATCH_REPLAY") + if replay: + rows = load_rows(Path(replay)) + if not rows: + raise SystemExit(f"{replay} has no rows") + errors, interval = 0, float(os.environ.get("DASH_LLM_WATCH_SPAN", "2")) + started = float(rows[0]["ts"] or 0) + stopped = float(rows[-1]["ts"] or 0) + print(f"replaying {replay}: {len(rows)} rows, " + f"{_iso(started)} -> {_iso(stopped)} (no traffic sent)") + else: + if not ADMIN_PASSWORD: + raise SystemExit("DASH_ADMIN_PASSWORD is required (it is `testpass` in .env.dev)") + with httpx.Client(base_url=BASE_URL, timeout=30, follow_redirects=False) as client: + sign_in(client) + interval = llm_interval(client) + print(f"signed in; polling /api/llm/snapshot every {interval:.1f}s " + f"for {DURATION_SECONDS:.0f}s (read-only)") + rows, errors, started, stopped = take_samples(client, interval) + rollups = wait_for_rollups(DB_PATH, started, stopped) + edges = [edge for engine in sorted({str(r["engine"]) for r in rows}) + for edge in transitions(rows, engine)] + verdict = judge(rows, edges, rollups, interval) + write_outputs(rows, verdict, errors, started, stopped) + print(_report(verdict, errors, started, stopped)) + return 0 if verdict["pass"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/dashboards/api.py b/src/dashboards/api.py index a180ffd..10d9e6a 100644 --- a/src/dashboards/api.py +++ b/src/dashboards/api.py @@ -545,7 +545,12 @@ def common_router() -> APIRouter: "auth": bool(settings.vllm_api_key)}, "interval_seconds": settings.llm_interval, "poll_ms": settings.poll_ms_llm, - "rate_window_seconds": settings.rate_window, + # The Δ window these engines' rates are actually measured over. Phase 23 gave the + # `llm` jobs their own estimator spanned by `llm_interval`, so printing + # `RATE_WINDOW_SECONDS` here (the host's window) would be a config fact that + # reads as a measurement (rule 2): the number beside a rate always beats the + # number a knob happens to hold. + "rate_window_seconds": settings.llm_interval, }, "disks": { # The drive-health page has no upstream of its own: it reads the same agent diff --git a/src/dashboards/app.py b/src/dashboards/app.py index 5f63ff5..0c705c1 100644 --- a/src/dashboards/app.py +++ b/src/dashboards/app.py @@ -118,16 +118,29 @@ def build_hub(settings: Settings, store: SampleStore, client: httpx.AsyncClient, Order is host → disks → llm → ups so far only for readability: two jobs carry the same host label (`host:gw`, `disks:gw`), so anything that looks a job up by label alone gets the system record — `alerts.py` filters by dashboard instead and does not depend on it. + + There are **two** rate estimators, one per measuring cadence (phase 23). A window is only + a claim about the polls that fill it: the `host` jobs run every `HOST_INTERVAL_SECONDS` + and need `RATE_WINDOW_SECONDS` (10) to have anything inside the window at all, while the + `llm` jobs run every `LLM_INTERVAL_SECONDS` and are given exactly that as their span, so a + slot that starts reads steady within two polls instead of ramping over ten. One estimator + for both could only ever be right about one of them — a 2 s span over a 10 s poll sits the + window-empty fallback against the `2 x span` ceiling and answers `n/a` forever, and a 10 s + span over a 2 s poll averages a burst across five polls the operator is watching. The + phase-13 gates (the `MIN_WINDOW_SECONDS` floor, the ceiling, the 8x jump confirmation) are + the estimator's own and therefore apply to the smaller window unchanged; the `Δ` + label always prints the window actually used, so the smaller claim is legible. """ - rates = RateEstimator(span=settings.rate_window) + host_rates = RateEstimator(span=settings.rate_window) + llm_rates = RateEstimator(span=settings.llm_interval) hub = Hub(store) - for job in host_jobs(list(settings.host_targets), client, HostCollector(rates), + for job in host_jobs(list(settings.host_targets), client, HostCollector(host_rates), settings.host_interval, tls=tls): hub.add(job) for job in disks_jobs(list(settings.host_targets), client, DisksCollector(), settings.disks_interval, timeout=settings.disks_timeout, tls=tls): hub.add(job) - for job in llm_jobs(settings, client, rates): + for job in llm_jobs(settings, client, llm_rates): hub.add(job) nut_monitor: NutMonitor | None = None if settings.nut_enabled: diff --git a/src/dashboards/config.py b/src/dashboards/config.py index 30cfe59..73b2b10 100644 --- a/src/dashboards/config.py +++ b/src/dashboards/config.py @@ -194,8 +194,14 @@ class Settings: gufo_api_key: str = _env("GUFO_API_KEY") vllm_url: str = _base_url("VLLM_BASE_URL") vllm_api_key: str = _env("VLLM_API_KEY") + # The LLM window *is* `llm_interval`: `app.build_hub` gives the `llm` jobs a + # `RateEstimator(span=llm_interval)`, so the cadence knob is also the Δ-window knob and + # there is no second number to keep in step with it (phase 23). llm_interval: float = _num("LLM_INTERVAL_SECONDS", 2) llm_timeout: float = _num("LLM_TIMEOUT_SECONDS", 3) + # The **host's** Δ window (phase 23), not a global one: `HOST_INTERVAL_SECONDS` polls have + # to have samples inside the window, so a 10 s span is what that cadence can honour. The + # LLM rates are measured over `llm_interval` instead — see `app.build_hub`. rate_window: float = _num("RATE_WINDOW_SECONDS", 10) # --- ups dashboard ------------------------------------------------------------ diff --git a/src/dashboards/prom.py b/src/dashboards/prom.py index 6a047fd..8a66667 100644 --- a/src/dashboards/prom.py +++ b/src/dashboards/prom.py @@ -187,8 +187,8 @@ def field(value: float | None, source: str | None, derived: bool = False, window: float | None = None) -> dict[str, Any]: """Uniform metric cell: the UI renders `n/a` whenever value is None. - `window` is the Δt a rate was measured over, so the UI can say `Δ10.0s` instead of - implying a scraped gauge it never got. + `window` is the Δt a rate was measured over, so the UI can say `Δ2.0s` / `Δ10.0s` — the + window this source's estimator was given — instead of implying a scraped gauge it never got. """ cell: dict[str, Any] = {"value": value, "source": source, "derived": derived} if window is not None: diff --git a/src/dashboards/rates.py b/src/dashboards/rates.py index be5d3a1..6dd6c20 100644 --- a/src/dashboards/rates.py +++ b/src/dashboards/rates.py @@ -2,9 +2,16 @@ Both upstreams export cumulative counters (tokens generated, jiffies spent, bytes read) and the dashboards need per-second rates. A bare 1s delta jitters between 0 and spikes -because work arrives in bursts, so the rate is measured over a window (default 10s) and -the window that was actually used travels with the value — the UI prints `Δ10.0s` rather -than implying a scraped gauge it never got. +because work arrives in bursts, so the rate is measured over a window and the window that +was actually used travels with the value — the UI prints `Δ2.0s` / `Δ10.0s` rather than +implying a scraped gauge it never got. + +The span belongs to the **cadence that fills it**, so there is one estimator per measuring +cadence (phase 23): `app.build_hub` hands the `host` jobs `RATE_WINDOW_SECONDS` (10 s, what a +10 s poll can fill) and the `llm` jobs `LLM_INTERVAL_SECONDS` (2 s — the smallest window that +cadence can honestly fill, which is what stops a slot start ramping the hero over five polls). +A span wider than the poll smooths work the operator is watching in real time; a span narrower +than the interval leaves nothing inside the window and every poll answers `n/a`. The effective window is bounded on **both** sides (phase 13, `data/logs/NOTES-decode-spike.md`): diff --git a/src/dashboards/web/static/page-llm.js b/src/dashboards/web/static/page-llm.js index 6a6e875..540cd86 100644 --- a/src/dashboards/web/static/page-llm.js +++ b/src/dashboards/web/static/page-llm.js @@ -278,8 +278,9 @@ function p95Value(metrics, key) { return cell && cell.value != null ? cell.value * 1000 : null; } -/** The provenance tag: Δ10.0s for measured rates, the formula for estimates, nothing for a - * plain scraped gauge (which would only add noise). */ +/** The provenance tag: the Δ the value was measured over (`Δ2.0s` at the default LLM cadence — + * the window is one poll interval since phase 23, and the payload says which), the formula for + * estimates, nothing for a plain scraped gauge (which would only add noise). */ function provenance(cell) { if (!cell || !cell.source) return ''; if (!String(cell.source).startsWith('computed:')) return ''; diff --git a/tests/rate_window_contract.py b/tests/rate_window_contract.py new file mode 100644 index 0000000..b47c9d9 --- /dev/null +++ b/tests/rate_window_contract.py @@ -0,0 +1,78 @@ +"""The two Δ windows of phase 23, in one place so the pins cannot drift. + +`app.build_hub` hands out **one `RateEstimator` per measuring cadence**: + +| source | span | knob | cadence | +| ------ | ------------- | ----------------------- | ------- | +| host | `HOST_WINDOW` | `RATE_WINDOW_SECONDS` | 10 s | +| llm | `LLM_WINDOW` | `LLM_INTERVAL_SECONDS` | 2 s | + +Two numbers living in the same payload shape: every measured rate cell carries the `window` the +estimator actually used, and `provenance()` in `page-llm.js` / `windowLabel()` in +`page-system.js` print it as `Δs`. A pin that names only **one** of the two +windows is how a later change could quietly give both sources the same span, so every pin reads +them through the constants and helpers here: + +* `tests/test_api.py` — the wiring: which estimator object each source's jobs receive, and the + `/api/config` window the LLM page is allowed to echo; +* `tests/test_llm_source.py` — the LLM payload: both throughput cells of both engines at the + 2 s shape, and the phase-13 floor still refusing the poll loop's catch-up gap at that span; +* `tests/test_host_source.py` — the host payload: `window_seconds` at the 10 s shape, and the + same 10 s cadence starved to `n/a` by the LLM's 2 s span (why one estimator cannot serve both); +* `tests/test_rates.py` — the estimator's own gates at both spans, because they are properties of + the estimator and not of the window size. + +`config.py`'s defaults are the source of truth for the numbers themselves and +`tests/test_config.py` pins them; this module exists so the pins cite one spelling of them +instead of five literals that can each drift on their own day. +""" +from __future__ import annotations + +from dashboards.rates import MAX_WINDOW_RATIO, MIN_WINDOW_SECONDS + +#: `RATE_WINDOW_SECONDS` — the Δ window the `host` jobs are measured over. +HOST_WINDOW = 10.0 + +#: `LLM_INTERVAL_SECONDS` — the Δ window the `llm` jobs are measured over. It *is* their poll +#: interval (phase 23): the smallest window a 2 s cadence can honestly fill, which is what stops +#: a slot start ramping the hero over five polls. +LLM_WINDOW = 2.0 + + +def delta_label(window: float | None) -> str: + """The provenance tag a page prints for a window — `provenance()`'s `toFixed(1)` idiom. + + The pages build `Δs` out of the payload's own `window` field; this is the same + rounding in Python, so a pin can assert *the label the operator reads* (`Δ2.0s`) instead of a + bare float whose meaning nobody checked. + """ + return "" if window is None else f"Δ{window:.1f}s" + + +def assert_window(window: float | None, span: float, *, where: str) -> None: + """Assert a measured number carries its own source's window, inside the estimator's bounds. + + `window` is a rate cell's `window` field (both engines) or a host payload's `window_seconds` + (`/system`'s footnote), and `where` names it in the failure message. + + The bound is a **range**, never a literal, and its ceiling is the estimator's own + `2 x span` rather than the span. A span equal to the cadence means only the current sample is + ever strictly *inside* the window, so the base comes from the window-empty fallback — the + previous sample, one poll back — and a published window is the real poll gap: the dev server + reads 2.009-2.026 s on `/llm` and 9.978-10.002 s on `/system`. Anything past `2 x span` is a + stall and answers `n/a`, which is the half of this the phase-13 pins hold. Because the window + tracks a cadence knob, an operator who raises `LLM_INTERVAL_SECONDS` moves the window and this + assertion together, while a pin hard-coded to `2.0` would rot. + + The tight half is the label: `delta_label(window) == delta_label(span)` is what the browser + prints, so `Δ2.0s` on `/llm` and `Δ10.0s` on `/system` are one assertion made at two different + spans, and a window that had drifted a whole interval would print a different label. + """ + assert window is not None, f"{where} carries no window at all" + assert MIN_WINDOW_SECONDS <= window <= MAX_WINDOW_RATIO * span, ( + f"{where} window {window} is outside its source's own bounds " + f"(floor {MIN_WINDOW_SECONDS}, ceiling {MAX_WINDOW_RATIO} x span {span})" + ) + assert delta_label(window) == delta_label(span), ( + f"{where} would print {delta_label(window)!r} beside a value measured over {span}s" + ) diff --git a/tests/test_api.py b/tests/test_api.py index 67f21ff..1b972da 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -28,11 +28,12 @@ from auth_client import sign_in from auth_contract import ADMIN_PASSWORD, FAST_PBKDF2 from card_contract import HOST_SHAPES, assert_card_contract from dashboards import alerts, auth, certs +from dashboards import app as app_module from dashboards.api import MIN_PASSWORD_LENGTH, RANGE_SECONDS, _disk_visible, parse_range from dashboards.app import create_app from dashboards.collectors import UpstreamError from dashboards.config import Settings, Target -from dashboards.rates import RateEstimator +from dashboards.rates import MIN_WINDOW_SECONDS, RateEstimator from dashboards.severity import classify from dashboards.sources import nut from dashboards.sources.disks import SMART_SERIES, DisksCollector, parse_disks_snapshot @@ -43,6 +44,7 @@ from dashboards.sources.llm import ( parse_vllm_metrics, ) from dashboards.store import RES_RAW, Event, Sample, SampleStore +from rate_window_contract import LLM_WINDOW, delta_label from slots_contract import ( CANONICAL_TASK_ID_FIELD, OPTIONAL_NEXT_TOKEN_FIELDS, @@ -815,6 +817,77 @@ def test_engine_urls_are_redacted_the_same_way_hosts_are(tmp_path: Path) -> None store.close() +@pytest.mark.parametrize(("llm_interval", "rate_window"), [(2.0, 10.0), (4.0, 20.0)]) +def test_build_hub_gives_each_source_the_estimator_its_own_cadence_fills( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + llm_interval: float, rate_window: float) -> None: + """Two rate estimators, one per measuring cadence (phase 23, `TODO.md:8`). + + The spy is the point: the numbers themselves are `config` defaults and a test on the default + would pass even if `build_hub` handed one estimator to both sources. So this asserts the two + **objects** the jobs receive, and does it at a non-default cadence pair to prove the LLM span + tracks `LLM_INTERVAL_SECONDS` rather than a second knob nobody configured — one 10 s window + smoothed a slot start across the five polls the operator was watching (the ramp), and a 2 s + window over a 10 s `host` poll would answer `n/a` every poll. + + The payload-level twin of this pin is the `Δ` label each rate cell carries: this is + the wiring, that is what the browser prints (phase 23 task 02 pins the labels). + """ + seen: dict[str, RateEstimator] = {} + real_host_jobs = app_module.host_jobs + real_llm_jobs = app_module.llm_jobs + + def spy_host_jobs(targets: Any, http: Any, host_collector: Any, interval: float, + **kwargs: Any) -> list[Any]: + seen["host"] = host_collector.rates + return real_host_jobs(targets, http, host_collector, interval, **kwargs) + + def spy_llm_jobs(cfg: Any, http: Any, rates: RateEstimator) -> list[Any]: + seen["llm"] = rates + return real_llm_jobs(cfg, http, rates) + + monkeypatch.setattr(app_module, "host_jobs", spy_host_jobs) + monkeypatch.setattr(app_module, "llm_jobs", spy_llm_jobs) + + db = str(tmp_path / "rate-windows.sqlite3") + store = SampleStore(db) + settings = Settings(db_path=db, admin_password=ADMIN_PASSWORD, + pbkdf2_iterations=FAST_PBKDF2, + host_targets=(Target("deskwork", DEAD_URL),), + gufo_url="http://deskwork:8731", vllm_url="http://driveripper:8000", + llm_interval=llm_interval, rate_window=rate_window) + # `build_hub` only wires jobs; nothing here dials, so the shared client is never started + # and there is no event loop to close one in. + hub, _nut_monitor = app_module.build_hub(settings, store, httpx.AsyncClient()) + store.close() + + assert sorted(seen) == ["host", "llm"], "a source stopped being handed an estimator" + assert sorted(hub.jobs) == ["disks:deskwork", "host:deskwork", "llm:gufo", "llm:vllm"] + assert seen["llm"] is not seen["host"], \ + "one estimator for both cadences is the bug this phase exists to fix" + assert seen["llm"].span == pytest.approx(llm_interval), \ + "the LLM window must track LLM_INTERVAL_SECONDS, not a second knob" + assert seen["host"].span == pytest.approx(rate_window), \ + "the host rates keep RATE_WINDOW_SECONDS: a 2 s span over a 10 s poll is `n/a` forever" + # …and the smaller window still carries the production floor, so the phase-13 refusal of the + # poll loop's ~0.2 s catch-up gap is not a consequence of the wider span. + assert seen["llm"].min_window == MIN_WINDOW_SECONDS + assert seen["host"].min_window == MIN_WINDOW_SECONDS + assert seen["llm"].span > MIN_WINDOW_SECONDS, "the LLM window would refuse every poll" + + +def test_config_reports_the_window_the_llm_rates_are_actually_measured_over( + client: TestClient) -> None: + """`/api/config`'s llm window is a measurement claim, so it is the LLM's own span. + + It used to echo `RATE_WINDOW_SECONDS` for both sources, which was harmless while one estimator + served them all. Since phase 23 that same field would have advertised a 10 s window beside + rates measured over 2 s — a number the page can print and no poll ever used (rule 2). + """ + llm = client.get("/api/config").json()["llm"] + assert llm["rate_window_seconds"] == llm["interval_seconds"] == 2.0 + + def wait_for_state(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]: deadline = time.time() + timeout while time.time() < deadline: @@ -2586,6 +2659,25 @@ def test_llm_engine_panel_leads_with_two_throughput_heroes(client) -> None: assert "node.title = value.source" in js +def test_llm_provenance_tag_prints_the_window_the_payload_sent(client: TestClient) -> None: + """`Δ2.0s` on `/llm` is the cell's own `window`, never a config echo (phase 23). + + The window moved from 10 s to one poll interval without touching the page: `provenance()` + reads `cell.window`, which is the number `tests/test_llm_source.py` pins to the LLM span, and + both spell the label with the same one-decimal rounding (`delta_label` in + `tests/rate_window_contract.py` is that rounding, used by both payload pins). A page printing + the configured interval instead would advertise a window no poll measured (rule 2), and a + literal `Δ10.0s` in the script is the same lie with the number typed in. + """ + js = client.get("/static/page-llm.js").text + prov = js[js.index("function provenance("):js.index("function humanNumber(")] + assert "`Δ${cell.window.toFixed(1)}s`" in prov, \ + f"the Δ tag is no longer built from the window the cell carries: {prov}" + assert delta_label(LLM_WINDOW) == "Δ2.0s", "the rounding the page and the pins share moved" + assert "rate_window_seconds" not in js, "the page echoes a config window beside a measurement" + assert "Δ10.0s" not in js, "a hard-coded Δ is an invented window (rule 2)" + + def test_every_llm_cell_carries_a_window_sparkline_with_percentile_feet(client) -> None: """Each panel row draws its own sparkline over the *selected* window and prints mean **and** p95 without a hover. diff --git a/tests/test_config.py b/tests/test_config.py index 78f5644..4fa5121 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3,7 +3,7 @@ from __future__ import annotations import importlib import os -from dataclasses import FrozenInstanceError +from dataclasses import FrozenInstanceError, fields from typing import Any import pytest @@ -109,6 +109,26 @@ def test_the_disk_cadence_is_an_env_var_like_every_other_one(monkeypatch: pytest importlib.reload(config) +def test_there_is_one_window_knob_per_measuring_cadence() -> None: + """Phase 23: `RATE_WINDOW_SECONDS` is the **host's** Δ window; the LLM's *is* its interval. + + These are the defaults `app.build_hub` spans its two estimators with — the wiring is pinned in + `tests/test_api.py`, the payload labels in `tests/test_llm_source.py` and + `tests/test_host_source.py`, both through `tests/rate_window_contract.py`. And there is + exactly **one** `*window*` setting: the LLM window tracks `LLM_INTERVAL_SECONDS` by + construction, so it cannot drift from the cadence it belongs to, while the host's cannot + shrink to match it either (a 2 s span over a 10 s poll has nothing inside it to diff, which + `test_the_host_rates_keep_their_own_cadence_s_window` pins as `n/a`). + """ + settings = Settings() + assert settings.rate_window == 10.0 + assert settings.llm_interval == 2.0 + assert settings.host_interval == 10.0 + window_knobs = {field.name for field in fields(Settings) if "window" in field.name} + assert window_knobs == {"rate_window"}, ( + "the LLM window must stay `LLM_INTERVAL_SECONDS`: a second knob drifts from its cadence") + + def test_actions_need_the_flag_and_are_dead_under_read_only() -> None: assert Settings(nut_allow_actions=True, read_only=False).actions_allowed is True assert Settings(nut_allow_actions=True, read_only=True).actions_allowed is False diff --git a/tests/test_host_source.py b/tests/test_host_source.py index 1d44f66..07ded8c 100644 --- a/tests/test_host_source.py +++ b/tests/test_host_source.py @@ -15,6 +15,7 @@ import pytest from dashboards.rates import RateEstimator from dashboards.severity import classify from dashboards.sources.host import HostCollector, fleet_summary, parse_host_snapshot +from rate_window_contract import HOST_WINDOW, LLM_WINDOW, assert_window, delta_label TICKS_PER_SECOND = 100.0 @@ -80,6 +81,41 @@ def test_second_scrape_measures_the_window_it_claims(agent_snapshot) -> None: assert usage == 10.0 +def test_the_host_rates_keep_their_own_cadence_s_window(agent_snapshot) -> None: + """Phase 23's other direction: a host cell labels ~10 s, the LLM cell's 2 s is not its shape. + + `app.build_hub` gives the `host` jobs `RateEstimator(span=RATE_WINDOW_SECONDS)` and the `llm` + jobs one spanned by `LLM_INTERVAL_SECONDS`; the payload shape is identical and only the number + differs, so a pin naming one of them is how the smaller window could quietly reach both + sources. `tests/test_llm_source.py` pins the 2 s side; this pins the 10 s side on the same + scrape-the-agent-twice shape the page reads (`window_seconds` is what `/system`'s footnote + prints). + + The second half is the reason one estimator cannot serve both cadences: hand the host's own + 10 s poll the LLM's 2 s span and there is nothing inside the window to diff, so every rate + answers `n/a` — the designed refusal, and the reason the host keeps `RATE_WINDOW_SECONDS`. + """ + second = advance(agent_snapshot, 10.0, rx_bytes=100_000, tx_bytes=50_000) + + collector = HostCollector(RateEstimator(span=HOST_WINDOW)) + parse_host_snapshot("deskwork", agent_snapshot, collector, now=1_000.0) + payload = parse_host_snapshot("deskwork", second, collector, now=1_010.0).payload + assert_window(payload["window_seconds"], HOST_WINDOW, + where="the host payload's window_seconds") + assert delta_label(payload["window_seconds"]) == "Δ10.0s", \ + "the /system footnote prints this, and it is not the LLM page's Δ" + assert payload["cpu"]["usage_pct"] is not None + assert payload["network"]["rx_bytes_per_s"] is not None + + starved = HostCollector(RateEstimator(span=LLM_WINDOW)) + parse_host_snapshot("deskwork", agent_snapshot, starved, now=1_000.0) + narrow = parse_host_snapshot("deskwork", second, starved, now=1_010.0).payload + assert narrow["window_seconds"] is None, \ + "a 2 s window over a 10 s poll has nothing to diff: n/a, never a rate" + assert narrow["cpu"]["usage_pct"] is None + assert narrow["network"]["rx_bytes_per_s"] is None + + def test_iowait_counts_as_idle_but_is_still_reported(agent_snapshot) -> None: _first, payload = parse_twice(agent_snapshot) modes = payload["cpu"]["modes_pct"] diff --git a/tests/test_live_fleet.py b/tests/test_live_fleet.py index aeb540e..e2a2b30 100644 --- a/tests/test_live_fleet.py +++ b/tests/test_live_fleet.py @@ -198,7 +198,9 @@ async def test_engine_still_publishes_the_series_we_parse(engine: str, "gufo": ("tokens_generated_total", "tokens_prompt_total"), "vllm": ("generation_tokens_total", "prompt_tokens_total"), }[engine] - rates = RateEstimator(span=settings.rate_window) + # The span the LLM jobs really get: phase 23 spans the LLM estimator by the poll + # interval, not by the host's `RATE_WINDOW_SECONDS`. + rates = RateEstimator(span=settings.llm_interval) async with httpx.AsyncClient(timeout=settings.llm_timeout) as client: result = await collectors.fetch_engine(client, engine, base_url, api_key, rates, with_slots=engine == "gufo") @@ -224,7 +226,9 @@ async def test_gufo_slots_answer_the_pinned_shape(settings: Settings) -> None: """ if not settings.gufo_url: pytest.skip("GUFO_BASE_URL is unset — no gufo to ask for /slots") - rates = RateEstimator(span=settings.rate_window) + # The span the LLM jobs really get: phase 23 spans the LLM estimator by the poll + # interval, not by the host's `RATE_WINDOW_SECONDS`. + rates = RateEstimator(span=settings.llm_interval) async with httpx.AsyncClient(timeout=settings.llm_timeout) as client: result = await collectors.fetch_engine(client, "gufo", settings.gufo_url, settings.gufo_api_key, rates, with_slots=True) diff --git a/tests/test_llm_source.py b/tests/test_llm_source.py index 6258fce..5c391ae 100644 --- a/tests/test_llm_source.py +++ b/tests/test_llm_source.py @@ -6,10 +6,12 @@ the deployed servers (tests/fixtures/metrics_*.txt), not to upstream documentati from __future__ import annotations import re +from collections.abc import Callable, Mapping, Sequence +from typing import Any import pytest -from dashboards.rates import RateEstimator +from dashboards.rates import MIN_WINDOW_SECONDS, RateEstimator from dashboards.sources.llm import ( engine_result, fleet_totals, @@ -18,6 +20,7 @@ from dashboards.sources.llm import ( rate_key, series_names, ) +from rate_window_contract import LLM_WINDOW, assert_window, delta_label from slots_contract import load_slots_capture @@ -33,6 +36,29 @@ def bump_metric(text: str, name: str, delta: float) -> str: return bumped +def poll_engine(metrics_text: Any, engine: str, span: float, stamps: Sequence[float], + growth: Mapping[str, Callable[[float], float]]) -> list[dict[str, Any]]: + """Re-scrape one capture at `stamps` with counters growing as f(t), through one estimator. + + The production shape in miniature: **one** `RateEstimator(span=span)` for every poll (the + object `app.build_hub` hands that source), each poll stamped with its own `now`, the + exposition carrying the tokens produced since the last scrape. Returns the parsed metrics per + poll so a pin reads the very cells `/api/llm/snapshot` serves — including their `window`. + """ + parse = parse_gufo_metrics if engine == "gufo" else parse_vllm_metrics + rates = RateEstimator(span=span) + body = metrics_text(engine) + carried = dict.fromkeys(growth, 0.0) + scrapes: list[dict[str, Any]] = [] + for now in stamps: + for name, produced in growth.items(): + value = produced(now) + body = bump_metric(body, name, value - carried[name]) + carried[name] = value + scrapes.append(parse(body, rates, now)) + return scrapes + + # ---- gufo / llama.cpp ----------------------------------------------------- # def test_gufo_counters_come_straight_from_the_capture(metrics_text) -> None: @@ -77,6 +103,144 @@ def test_gufo_throughput_prefers_measured_delta_over_the_native_gauge(metrics_te assert metrics["decode_tokens_per_second_native"]["value"] == pytest.approx(42.621031668072597) +def test_gufo_the_window_the_llm_jobs_are_spanned_by_is_one_that_fills(metrics_text) -> None: + """Phase 23: a 2 s span is a real window at the LLM cadence, and it kills the ramp. + + `app.build_hub` spans the LLM estimator by `LLM_INTERVAL_SECONDS` (the wiring is pinned in + `tests/test_api.py`); this is what that span buys. A slot that starts mid-window is spread + over the whole span by the wider window, so the hero climbs one fifth of the true rate per + poll — the "ramp up/ramp down" of `TODO.md:8` — while a window of one poll interval reads + the engine's speed on the first poll after it changes and 0 again on the first after it ends. + + The engine here is the capture plus a steady 50 tok/s that begins at t=1006, polled every + 2 s, which is the production cadence rather than a fast scrape: the estimator's + `MIN_WINDOW_SECONDS` floor is 4x below this window, so nothing is being refused here. + """ + text = metrics_text("gufo") + started = 1_006.0 # the slot starts here, at a steady 50 tok/s + polls = [1_000.0, 1_002.0, 1_004.0, 1_006.0, 1_008.0, 1_010.0, 1_012.0] + + def series(span: float) -> list[tuple[float | None, float | None]]: + rates = RateEstimator(span=span) + body = text + cells: list[tuple[float | None, float | None]] = [] + previous = 0.0 + for now in polls: + generated = 50.0 * max(0.0, now - started) + body = bump_metric(body, "llamacpp:tokens_predicted_total", generated - previous) + previous = generated + cell = parse_gufo_metrics(body, rates, now)["decode_tokens_per_second"] + cells.append((cell["value"], cell.get("window"))) + return cells + + narrow = series(2.0) + wide = series(10.0) + # the new window: n/a on the first poll (no window yet), then the engine's real speed, and + # every one of them labelled with the 2 s window it actually measured + assert narrow[0] == (None, None) + for _value, window in narrow[1:]: + assert window == pytest.approx(2.0), "the label must name the window the value used" + assert [v for v, _ in narrow[4:]] == [pytest.approx(50.0)] * 3, ( + f"a 2 s window reads the slot's speed at once, got {narrow}") + # the old one: the same work, averaged across the polls the operator is watching + assert [v for v, _ in wide[4:]] != [pytest.approx(50.0)] * 3, \ + f"the 10 s window no longer smooths, so this test's premise moved: {wide}" + assert wide[4][0] is not None and wide[4][0] < 30.0, \ + f"expected the ramp to print well under the true rate, got {wide}" + + +def test_gufo_rate_cells_name_the_window_the_llm_jobs_are_measured_over(metrics_text) -> None: + """Phase 23's payload pin: an LLM rate cell's `window` is the LLM cadence, label and all. + + `build_hub` spans the `llm` estimator by `LLM_INTERVAL_SECONDS` (the object-level pin is + `tests/test_api.py`'s); this is the same span where the browser reads it — the `window` beside + the value, the `computed:Δ/Δt` provenance printed with it, and the `Δ2.0s` label + `provenance()` builds from that number. **Both** throughput cells are checked, on the + production 2 s cadence. `tests/test_host_source.py` is the other direction of the same pin: + the host cells keep 10 s in the same payload shape. + """ + started = 1_006.0 + stamps = [1_000.0, 1_002.0, 1_004.0, 1_006.0, 1_008.0, 1_010.0] + scrapes = poll_engine( + metrics_text, "gufo", LLM_WINDOW, stamps, + {"llamacpp:tokens_predicted_total": lambda t: 50.0 * max(0.0, t - started), + "llamacpp:prompt_tokens_total": lambda t: 200.0 * max(0.0, t - started)}, + ) + assert "window" not in scrapes[0]["decode_tokens_per_second"], \ + "one sample is not a window, so it must not grow a Δ label" + for key, series, speed in ( + ("decode_tokens_per_second", "llamacpp:tokens_predicted_total", 50.0), + ("prompt_tokens_per_second", "llamacpp:prompt_tokens_total", 200.0), + ): + cells = [scrape[key] for scrape in scrapes[1:]] + for cell in cells: + assert cell["value"] is not None, f"{key} went n/a on a healthy cadence: {cell}" + assert_window(cell.get("window"), LLM_WINDOW, where=f"gufo {key}") + assert cell["derived"] is True + assert cell["source"] == f"computed:Δ{series}/Δt", \ + "the provenance string travels with the value whatever window it used" + assert cells[-1]["value"] == pytest.approx(speed), ( + f"a one-interval window reads the engine's speed on the next poll, got {cells}") + + +def test_gufo_the_floor_still_refuses_a_catch_up_gap_at_the_llm_window(metrics_text) -> None: + """The phase-13 floor belongs to the estimator, not to the old 10 s span. + + Shrinking the window to one poll interval must not shrink the guard: `poller.Job._loop` still + sleeps `max(0.2, interval - elapsed)`, so a slow fetch is still followed by a catch-up poll + ~0.2 s later, and 0.2 s of a 2 s window is the same lie a 0.2 s slice of a 10 s one was (the + 3284.59 tok/s row for a 53 tok/s engine). `test_rates.py` pins the gate at both spans; this + is the refusal through the production parse path, where the `n/a` keeps the window it refused + **and** its `source` so the cell still says which Δ it is talking about. + """ + scrapes = poll_engine( + metrics_text, "gufo", LLM_WINDOW, + [1_000.0, 1_002.0, 1_004.0, 1_014.25, 1_014.47, 1_016.47], + {"llamacpp:tokens_predicted_total": lambda t: 50.0 * (t - 1_000.0)}, + ) + stalled = scrapes[3]["decode_tokens_per_second"] + assert stalled["value"] is None, \ + "a 10 s gap is past a 2 s span's ceiling (2 x span): n/a, not a diluted rate" + assert "window" not in stalled, "the ceiling answers no window because it measured none" + refused = scrapes[4]["decode_tokens_per_second"] + assert refused["value"] is None, f"0.22 s is the loop's catch-up gap, not a rate: {refused}" + assert refused["window"] == pytest.approx(0.22) + assert refused["window"] < MIN_WINDOW_SECONDS + assert refused["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt" + assert delta_label(refused["window"]) == "Δ0.2s", "the label names the window it refused" + recovered = scrapes[5]["decode_tokens_per_second"] + assert recovered["value"] == pytest.approx(50.0), ( + f"the refusal re-bases, so the next honest poll reads the engine again: {recovered}") + assert_window(recovered.get("window"), LLM_WINDOW, where="gufo decode") + + +def test_gufo_a_poll_gap_a_hair_wider_than_the_span_is_still_the_2s_claim(metrics_text) -> None: + """Live reality: a poll lands slightly after its interval, and that is still `Δ2.0s`. + + With the span equal to the cadence (phase 23) only the current sample is strictly *inside* the + window, so the base comes from the estimator's window-empty fallback — the previous sample, + one poll back — and the published window is the real gap. Read off the running dev server the + LLM cells report 2.009-2.026 s and the host payload's `window_seconds` 9.978-10.002 s, so a + pin demanding `window == span` exactly would fail against working software. This drives the + jittered shape and asserts what the operator sees: the same `Δ2.0s` label and the engine's real + speed. A gap of a whole extra interval is the other story — the `2 x span` ceiling refuses it, + which is what the test above pins. + """ + started = 1_006.0 + stamps = [1_000.0, 1_002.03, 1_004.07, 1_006.1, 1_008.13, 1_010.16] + scrapes = poll_engine( + metrics_text, "gufo", LLM_WINDOW, stamps, + {"llamacpp:tokens_predicted_total": lambda t: 50.0 * max(0.0, t - started)}, + ) + cells = [scrape["decode_tokens_per_second"] for scrape in scrapes[1:]] + for cell in cells: + assert cell["value"] is not None + assert_window(cell.get("window"), LLM_WINDOW, where="gufo decode on a jittered cadence") + assert delta_label(cell["window"]) == "Δ2.0s", \ + "a 2.03 s gap still prints the window the panel claims" + assert cells[-1]["value"] == pytest.approx(50.0, rel=0.02) + + def test_gufo_prompt_rate_excludes_cached_tokens(metrics_text) -> None: text = metrics_text("gufo") rates = RateEstimator(span=10) @@ -244,6 +408,36 @@ def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0) +def test_vllm_rate_cells_name_the_same_window_the_llm_jobs_are_measured_over( + metrics_text) -> None: + """The pin is the **source's** span, so both engines answer it the same way. + + The capture exports no throughput gauge, which is what makes these two cells measured (see + `test_vllm_measured_rate_when_the_build_has_no_throughput_gauge`), and vLLM's counters are + labelled — the estimator keys each label set separately while the cell's `source` names the + series. Same two rules as the gufo pin: the `window` inside the LLM span, and + `computed:Δ/Δt` beside the value. + """ + started = 2_006.0 + scrapes = poll_engine( + metrics_text, "vllm", LLM_WINDOW, + [2_000.0, 2_002.0, 2_004.0, 2_006.0, 2_008.0, 2_010.0], + {"vllm:generation_tokens_total": lambda t: 40.0 * max(0.0, t - started), + "vllm:prompt_tokens_total": lambda t: 300.0 * max(0.0, t - started)}, + ) + for key, series, speed in ( + ("decode_tokens_per_second", "vllm:generation_tokens_total", 40.0), + ("prompt_tokens_per_second", "vllm:prompt_tokens_total", 300.0), + ): + cells = [scrape[key] for scrape in scrapes[1:]] + for cell in cells: + assert cell["value"] is not None, f"{key} went n/a on a healthy cadence: {cell}" + assert_window(cell.get("window"), LLM_WINDOW, where=f"vllm {key}") + assert cell["source"] == f"computed:Δ{series}/Δt" + assert cells[-1]["value"] == pytest.approx(speed), ( + f"the labelled series still reads its own speed over one interval, got {cells}") + + def test_vllm_a_joining_label_set_does_not_lend_its_carried_counter(metrics_text) -> None: """A second `engine=`/`model_name=` series appearing mid-window must not be a rate. diff --git a/tests/test_rates.py b/tests/test_rates.py index b5b5396..eaa9261 100644 --- a/tests/test_rates.py +++ b/tests/test_rates.py @@ -12,6 +12,7 @@ import math import pytest from dashboards.rates import RateEstimator +from rate_window_contract import HOST_WINDOW, LLM_WINDOW def test_first_sample_has_no_rate() -> None: @@ -210,6 +211,65 @@ def test_a_held_jump_is_not_confirmed_by_a_window_on_the_same_anchor() -> None: assert published[-1] == pytest.approx(45.0), "the stall ends in the truth, not a tail" +# --- phase 23: the gates are the estimator's, not the old 10 s span's --------- # +# `app.build_hub` now spans the `llm` jobs by `LLM_INTERVAL_SECONDS` (2 s), a fifth of the host's +# `RATE_WINDOW_SECONDS`. Everything phase 13 built has to survive the shrink or the smaller window +# is a licence for the same 3284.59 tok/s row, so each gate runs here at both spans. The numbers +# scale with the span: what these three pin is the *relation*, which is what "unchanged" means. +# The fourth protection — a held jump needing a **different anchor** — is deliberately not +# parametrized: it compares two sample timestamps and no span arithmetic, and a 2 s window holds +# one sample per poll interval anyway, so a stale anchor ages out one poll later instead of five. +# Its span-10 regression pair (`test_a_held_jump_is_not_confirmed_by_a_window_on_the_same_anchor` +# above, `test_gufo_a_stale_scrape_body_never_becomes_a_rate_above_the_engine` in +# `tests/test_llm_source.py`) runs unchanged, and the floor row of this section is the one that +# could actually have been weakened by the smaller window: `MIN_WINDOW_SECONDS` is absolute. + +@pytest.mark.parametrize("span", [LLM_WINDOW, HOST_WINDOW]) +def test_the_floor_refuses_the_catch_up_gap_at_either_window(span: float) -> None: + """The poll loop's ~0.2 s catch-up is a gap whatever the panel's span is. + + 0.22 s is a tenth of a 10 s window and an eighth of a 2 s one, and neither is a measurement: + the floor is absolute on purpose (phase 13 pinned that), so the LLM's narrower claim is not a + narrower guard. + """ + r = RateEstimator(span=span) + r.rate("x", 0.0, now=0.0) + r.rate("x", 50.0 * span, now=span) # the honest cadence: 50 tok/s + rate, window = r.rate("x", 50.0 * span + 11.0, now=span + 0.22) + assert rate is None, f"a 0.22 s catch-up became a rate at span={span}" + assert window == pytest.approx(0.22), "the n/a still names the window it refused" + + +@pytest.mark.parametrize("span", [LLM_WINDOW, HOST_WINDOW]) +def test_the_ceiling_is_two_spans_at_either_window(span: float) -> None: + """`2 x span` is relative, so a stall stays a stall at either window size.""" + r = RateEstimator(span=span) + r.rate("x", 0.0, now=0.0) + assert r.rate("x", 100.0, now=2.5 * span) == (None, None), "past the ceiling: no rate" + rate, window = r.rate("x", 200.0, now=3.5 * span) + assert rate == pytest.approx(100.0 / span) + assert window == pytest.approx(span), "the next poll re-bases on the fresh sample" + + +@pytest.mark.parametrize("span", [LLM_WINDOW, HOST_WINDOW]) +def test_a_jump_is_held_and_confirmed_at_either_window(span: float) -> None: + """An 8x step holds one poll at a 2 s window exactly as at a 10 s one. + + The ratio is dimensionless, so narrowing the window does not loosen the gate: a real burst + outlives one poll interval of either size and prints one poll late, while the held value keeps + the `Δ` label it was published with. + """ + r = RateEstimator(span=span) + r.rate("x", 0.0, now=0.0) + r.rate("x", 10.0 * span, now=span) # 10 tok/s published + rate, window = r.rate("x", 90.0 * span, now=2.0 * span) # 80 tok/s = 8x: held + assert rate == pytest.approx(10.0) + assert window == pytest.approx(span), "the held value keeps the Δ label it was published with" + rate, window = r.rate("x", 172.0 * span, now=3.0 * span) # 82: a new anchor confirms it + assert rate == pytest.approx(82.0) + assert window == pytest.approx(span) + + def test_a_rise_from_zero_is_never_held() -> None: """The first work an idle engine does must print on its own poll, not one late.""" r = RateEstimator(span=10)