From 1bbd868401777450df5191cf47932b68c394f0e2 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Mon, 19 Oct 2026 14:53:58 -0400 Subject: [PATCH] phase: 13_llm_page_and_decode_rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All green. Final verification summary: **Verified / implemented (phase 13 final pass)** - All 4 tasks already in `complete/`; this was the verification pass. Confirmed the CSS rhythm rules (`#strip`, `#banners:empty ~ #fleet`) are served and pinned, `data/logs/NOTES-decode-spike.md` names the mechanism, and the phase-07/04 surfaces still pass their pins. - **Found and fixed a residual defect in this phase's own code**: a stalled fetch stamps its body with the poll's *finish* time, so that sample anchors every window for the rest of the span; the jump gate "confirmed" a jump measured from that same anchor → live gufo rows 232.28 tok/s (one request at ~50) + a decaying 146/114/95 tail, repeated at 200.6. `rates.py::_publish` now requires an **independent anchor** (`_pending` stores the base sample's ts; the hold continues until it ages out). Pinned by `test_a_held_jump_is_not_confirmed_by_a_window_on_the_same_anchor` + `test_gufo_a_stale_scrape_body_never_becomes_a_rate_above_the_engine` (both fail on task 03's gate). - Re-verified live after a dev-server restart: 10 min / 540 signed-in `/api/llm/snapshot` polls + read-only SQLite pass; **0 values > 300, 0 without a `Δ` label**; 0 post-fix stored rows over the ceiling except the two pre-fix-shape rows above. **Tests / lint (exact, all 100 % green)** - `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov --cov-fail-under=94` → **1195 passed, 11 skipped, coverage 98.70 %**; `ruff check src tests agent` → All checks passed; `pyright` → 0 errors; `node --check` clean; repo hygiene clean. - `uv run pytest -m live --live` → **11 passed / 0 failed**. `/api/health` → **all nine sources ok**. **Completion criteria** - Fleet/strip + head/fleet gap with screenshots (`llm-fleet-spacing{,-top,-banner-case}.png`) — met (CSS unchanged this pass, pinned on served bytes). - Task-02 findings file names the confirmed mechanism — met. - `test_rates.py`/`test_llm_source.py` pin label-set, both window bounds and the always-honest `Δ` — met (+2 new pins). - Sustained watch under the recorded ceiling (300 tok/s) — met (worst 203.05, all labelled). - Full suite + validate green; no phase-07/04 behaviour change — met. **Documentation** - README: `RATE_WINDOW_SECONDS` row now states the hold/confirm rule (only change needed; rest verified consistent). - AGENTS.md: `rates.py` bullet + gap 9 rewritten (residual closed *and* the remaining sub-8× skew measured, root-caused to gap 4, explicitly not fixed by inventing a new threshold). - **Fleet facts changed under this pass and were corrected**: operator added driveripper's bearer token → `host:driveripper`/`disks:driveripper` `ok`, live suite 9/2→11/0; but driveripper reads **identity only, zero SMART numbers** (root `smartctl` gives 55 °C/14 %; `/dev/nvme0`, `/dev/ng0n1`, `/dev/bsg/*` are root-only; its unit is `DevicePolicy=auto`, not the installer's `strict`). Evidence appended to `data/logs/NOTES-agent-install.md` + `NOTES-decode-stable.md`. **Notable decisions / deviations** - Did **not** add a span-relative window floor: that would re-decide task 03's pinned `MIN_WINDOW_SECONDS` and its "in-bounds diluted row publishes" test. Recorded as the next phase's call, with the root cause left at gap 4 / `14_store_performance` (the ~10 s stall). - No live fleet change made (read-only probes + one dev-server restart). Fixtures deliberately not re-captured (no parse change, task 02 proved no shape drift). Dependency-currency advisory left untouched (no dependency moved in this phase): certifi/fastapi/starlette/websockets have newer releases. **Next pending phase:** `.agents/phases/todo/14_store_performance` (then `15_agent_mtls`). --- .../13_llm_page_and_decode_rate/00_phase.md | 0 .../01_fleet_strip_spacing.md | 0 .../02_pin_decode_spike_mechanism.md | 0 ..._bounded_estimator_and_per_series_rates.md | 0 .../04_live_stable_verification.md | 0 ..._llm_page_and_decode_rate__00_phase.a1.err | 0 ...3_llm_page_and_decode_rate__00_phase.a1.md | 28 +++ ...age_and_decode_rate__00_phase.a1.md.status | 1 + ...page_and_decode_rate__00_phase.a1.validate | 63 ++++++ ...decode_rate__01_fleet_strip_spacing.a1.err | 0 ..._decode_rate__01_fleet_strip_spacing.a1.md | 20 ++ ..._rate__01_fleet_strip_spacing.a1.md.status | 1 + ...e_rate__01_fleet_strip_spacing.a1.validate | 62 ++++++ ...rate__02_pin_decode_spike_mechanism.a1.err | 0 ..._rate__02_pin_decode_spike_mechanism.a1.md | 15 ++ ...02_pin_decode_spike_mechanism.a1.md.status | 1 + ..._02_pin_decode_spike_mechanism.a1.validate | 62 ++++++ ...nded_estimator_and_per_series_rates.a1.err | 0 ...unded_estimator_and_per_series_rates.a1.md | 20 ++ ...stimator_and_per_series_rates.a1.md.status | 1 + ...estimator_and_per_series_rates.a1.validate | 62 ++++++ ...e_rate__04_live_stable_verification.a1.err | 0 ...de_rate__04_live_stable_verification.a1.md | 18 ++ ...__04_live_stable_verification.a1.md.status | 1 + ...e__04_live_stable_verification.a1.validate | 63 ++++++ AGENTS.md | 178 +++++++++++++--- README.md | 5 +- src/dashboards/prom.py | 19 ++ src/dashboards/rates.py | 122 ++++++++++- src/dashboards/sources/llm.py | 85 ++++++-- src/dashboards/web/static/dash.css | 16 ++ tests/test_api.py | 90 ++++++++ tests/test_collectors.py | 9 +- tests/test_llm_source.py | 126 +++++++++++ tests/test_prom.py | 17 ++ tests/test_rates.py | 197 ++++++++++++++++++ 36 files changed, 1231 insertions(+), 51 deletions(-) rename .agents/phases/{todo => complete}/13_llm_page_and_decode_rate/00_phase.md (100%) rename .agents/phases/{todo => complete}/13_llm_page_and_decode_rate/01_fleet_strip_spacing.md (100%) rename .agents/phases/{todo => complete}/13_llm_page_and_decode_rate/02_pin_decode_spike_mechanism.md (100%) rename .agents/phases/{todo => complete}/13_llm_page_and_decode_rate/03_bounded_estimator_and_per_series_rates.md (100%) rename .agents/phases/{todo => complete}/13_llm_page_and_decode_rate/04_live_stable_verification.md (100%) create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.err create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.md create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.md.status create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.validate create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.err create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.md create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.md.status create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.validate create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.err create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.md create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.md.status create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.validate create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.err create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.md create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.md.status create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.validate create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.err create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.md create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.md.status create mode 100644 .agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.validate diff --git a/.agents/phases/todo/13_llm_page_and_decode_rate/00_phase.md b/.agents/phases/complete/13_llm_page_and_decode_rate/00_phase.md similarity index 100% rename from .agents/phases/todo/13_llm_page_and_decode_rate/00_phase.md rename to .agents/phases/complete/13_llm_page_and_decode_rate/00_phase.md diff --git a/.agents/phases/todo/13_llm_page_and_decode_rate/01_fleet_strip_spacing.md b/.agents/phases/complete/13_llm_page_and_decode_rate/01_fleet_strip_spacing.md similarity index 100% rename from .agents/phases/todo/13_llm_page_and_decode_rate/01_fleet_strip_spacing.md rename to .agents/phases/complete/13_llm_page_and_decode_rate/01_fleet_strip_spacing.md diff --git a/.agents/phases/todo/13_llm_page_and_decode_rate/02_pin_decode_spike_mechanism.md b/.agents/phases/complete/13_llm_page_and_decode_rate/02_pin_decode_spike_mechanism.md similarity index 100% rename from .agents/phases/todo/13_llm_page_and_decode_rate/02_pin_decode_spike_mechanism.md rename to .agents/phases/complete/13_llm_page_and_decode_rate/02_pin_decode_spike_mechanism.md diff --git a/.agents/phases/todo/13_llm_page_and_decode_rate/03_bounded_estimator_and_per_series_rates.md b/.agents/phases/complete/13_llm_page_and_decode_rate/03_bounded_estimator_and_per_series_rates.md similarity index 100% rename from .agents/phases/todo/13_llm_page_and_decode_rate/03_bounded_estimator_and_per_series_rates.md rename to .agents/phases/complete/13_llm_page_and_decode_rate/03_bounded_estimator_and_per_series_rates.md diff --git a/.agents/phases/todo/13_llm_page_and_decode_rate/04_live_stable_verification.md b/.agents/phases/complete/13_llm_page_and_decode_rate/04_live_stable_verification.md similarity index 100% rename from .agents/phases/todo/13_llm_page_and_decode_rate/04_live_stable_verification.md rename to .agents/phases/complete/13_llm_page_and_decode_rate/04_live_stable_verification.md diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.err b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.md b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.md new file mode 100644 index 0000000..35a1747 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.md @@ -0,0 +1,28 @@ +All green. Final verification summary: + +**Verified / implemented (phase 13 final pass)** +- All 4 tasks already in `complete/`; this was the verification pass. Confirmed the CSS rhythm rules (`#strip`, `#banners:empty ~ #fleet`) are served and pinned, `data/logs/NOTES-decode-spike.md` names the mechanism, and the phase-07/04 surfaces still pass their pins. +- **Found and fixed a residual defect in this phase's own code**: a stalled fetch stamps its body with the poll's *finish* time, so that sample anchors every window for the rest of the span; the jump gate "confirmed" a jump measured from that same anchor → live gufo rows 232.28 tok/s (one request at ~50) + a decaying 146/114/95 tail, repeated at 200.6. `rates.py::_publish` now requires an **independent anchor** (`_pending` stores the base sample's ts; the hold continues until it ages out). Pinned by `test_a_held_jump_is_not_confirmed_by_a_window_on_the_same_anchor` + `test_gufo_a_stale_scrape_body_never_becomes_a_rate_above_the_engine` (both fail on task 03's gate). +- Re-verified live after a dev-server restart: 10 min / 540 signed-in `/api/llm/snapshot` polls + read-only SQLite pass; **0 values > 300, 0 without a `Δ` label**; 0 post-fix stored rows over the ceiling except the two pre-fix-shape rows above. + +**Tests / lint (exact, all 100 % green)** +- `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov --cov-fail-under=94` → **1195 passed, 11 skipped, coverage 98.70 %**; `ruff check src tests agent` → All checks passed; `pyright` → 0 errors; `node --check` clean; repo hygiene clean. +- `uv run pytest -m live --live` → **11 passed / 0 failed**. `/api/health` → **all nine sources ok**. + +**Completion criteria** +- Fleet/strip + head/fleet gap with screenshots (`llm-fleet-spacing{,-top,-banner-case}.png`) — met (CSS unchanged this pass, pinned on served bytes). +- Task-02 findings file names the confirmed mechanism — met. +- `test_rates.py`/`test_llm_source.py` pin label-set, both window bounds and the always-honest `Δ` — met (+2 new pins). +- Sustained watch under the recorded ceiling (300 tok/s) — met (worst 203.05, all labelled). +- Full suite + validate green; no phase-07/04 behaviour change — met. + +**Documentation** +- README: `RATE_WINDOW_SECONDS` row now states the hold/confirm rule (only change needed; rest verified consistent). +- AGENTS.md: `rates.py` bullet + gap 9 rewritten (residual closed *and* the remaining sub-8× skew measured, root-caused to gap 4, explicitly not fixed by inventing a new threshold). +- **Fleet facts changed under this pass and were corrected**: operator added driveripper's bearer token → `host:driveripper`/`disks:driveripper` `ok`, live suite 9/2→11/0; but driveripper reads **identity only, zero SMART numbers** (root `smartctl` gives 55 °C/14 %; `/dev/nvme0`, `/dev/ng0n1`, `/dev/bsg/*` are root-only; its unit is `DevicePolicy=auto`, not the installer's `strict`). Evidence appended to `data/logs/NOTES-agent-install.md` + `NOTES-decode-stable.md`. + +**Notable decisions / deviations** +- Did **not** add a span-relative window floor: that would re-decide task 03's pinned `MIN_WINDOW_SECONDS` and its "in-bounds diluted row publishes" test. Recorded as the next phase's call, with the root cause left at gap 4 / `14_store_performance` (the ~10 s stall). +- No live fleet change made (read-only probes + one dev-server restart). Fixtures deliberately not re-captured (no parse change, task 02 proved no shape drift). Dependency-currency advisory left untouched (no dependency moved in this phase): certifi/fastapi/starlette/websockets have newer releases. + +**Next pending phase:** `.agents/phases/todo/14_store_performance` (then `15_agent_mtls`). diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.md.status b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.validate b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.validate new file mode 100644 index 0000000..cfd500b --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__00_phase.a1.validate @@ -0,0 +1,63 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +....................sssssssssss......................................... [ 71%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 89%] +........................................................................ [ 95%] +...................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 348 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 73 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 188 2 99% 255-256 +------------------------------------------------------------------ +TOTAL 2914 38 99% +Required test coverage of 94% reached. Total coverage: 98.70% +1195 passed, 11 skipped, 1 warning in 92.75s (0:01:32) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.149.0 wheel +starlette 1.8.0 1.9.0 wheel +websockets 17.2 17.3 wheel diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.err b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.md b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.md new file mode 100644 index 0000000..9fc2a54 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.md @@ -0,0 +1,20 @@ +Everything is green. Final state: + +**Implemented** +- `src/dashboards/web/static/dash.css` — added the two rhythm rules in the fleet-totals section: `#strip { margin-top: .85rem; }` (unconditional, so the strip keeps the gap when `#fleet` is `[hidden]`) and `#banners:empty ~ #fleet { margin-top: .85rem; }` (gap above the block only when the banner slot renders nothing), with a comment naming why each half is shaped that way. No markup/JS change; the sections were already siblings of `
`. +- `tests/test_api.py::test_llm_section_stack_keeps_the_shared_vertical_rhythm` — pins both rules on the served stylesheet, pins that **exactly one** rule styles each of `#strip`/`#fleet` (an unguarded second one = the double gap), pins the `~`-sibling structure and the inline `.85rem` margins the rules are written around. Verified to fail both ways (rules deleted; extra `#fleet` rule added). +- Live capture with **two engines answering**: `data/logs/llm-fleet-spacing.png` (full page), `-top.png`, `-banner-case.png`, and `data/logs/NOTES-llm-fleet-spacing.md` with the before/after table. + +**Tests / lint (exact)** +- `bash .agents/validate.sh` → **EXIT=0**: `uv run pytest --cov --cov-fail-under=94` → **1173 passed, 11 skipped, coverage 98.63 %** (baseline 1172; +1 new test, coverage unchanged); `uv run ruff check src tests agent` → All checks passed; `uv run pyright` → 0 errors; `node --check web/static/*.js` → clean. +- `uv run pytest -m live --live` → 8 passed / 3 failed — the three `driveripper` agent tests, i.e. the transport state AGENTS.md already records; unchanged by a CSS rule. +- Chromium measurements (`.85rem` = 13.6px): `#fleet → #strip` **0 → 13.59px**; `#strip → panels` 13.59px; banner slot → fleet 13.59px with a banner up (`#fleet` computes `margin-top: 0px`, i.e. no double gap) and with `#fleet` hidden. + +**Docs**: AGENTS.md (`/llm` UI-conventions paragraph: the shared `.85rem` rule, both rule shapes, the test pins) and README.md (`/llm` walkthrough clause). + +**Decisions/deviations** +- Measured honesty: only fleet→strip actually moves. The `:empty`-guarded rule changes no rendered distance today because the *empty* `#banners` slot carries its own inline `margin-bottom:.85rem` (the same 1rem + .85rem head spacing `/` and `/ups` already have); I kept the prescribed rule — it moves the gap from markup-that-renders-nothing onto the block and prevents the double gap — and recorded that measurement in the test docstring and the notes instead of claiming a second fix. +- Left the `/`–`/ups` page-head rhythm and `/disks`'s analogous fleet→"Per host" adjacency alone (out of this task's scope); noted `/disks` here as the same pattern for a future pass. +- `validate.sh` advisory (not a gate): certifi/fastapi/starlette have newer releases; this task moved no dependency. + +**Next pending task**: `.agents/phases/todo/13_llm_page_and_decode_rate/02_pin_decode_spike_mechanism.md`. diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.md.status b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.validate b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.validate new file mode 100644 index 0000000..c1746bb --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__01_fleet_strip_spacing.a1.validate @@ -0,0 +1,62 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 6%] +........................................................................ [ 12%] +........................................................................ [ 18%] +........................................................................ [ 24%] +........................................................................ [ 30%] +........................................................................ [ 36%] +........................................................................ [ 42%] +........................................................................ [ 48%] +........................................................................ [ 54%] +........................................................................ [ 60%] +........................................................................ [ 66%] +....................sssssssssss......................................... [ 72%] +........................................................................ [ 79%] +........................................................................ [ 85%] +........................................................................ [ 91%] +........................................................................ [ 97%] +................................ [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 348 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 188 2 99% 255-256 +------------------------------------------------------------------ +TOTAL 2843 39 99% +Required test coverage of 94% reached. Total coverage: 98.63% +1173 passed, 11 skipped, 1 warning in 93.51s (0:01:33) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.148.0 wheel +starlette 1.8.0 1.9.0 wheel diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.err b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.md b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.md new file mode 100644 index 0000000..4617cd7 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.md @@ -0,0 +1,15 @@ +**Implemented** +- `data/logs/NOTES-decode-spike.md` — mechanism **confirmed, not guessed**: `RateEstimator` has no lower bound on its window (`base = inside[0] if len(inside)>1 else samples[-2]`) **and** `poller.Job._loop`'s `max(0.2, interval-elapsed)` catch-up makes a 0.2 s window after any slow fetch, so a stalled fetch's carried tokens are diffed over 0.2 s. Candidates (a) label-set sum, (c) lineage jump and (d) engine gauge are all excluded by measurement; (a) stays a latent defect with its own red test. +- Evidence: 277/277 stored gufo values >1000 tok/s are catch-up rows (every one with a >3 s fetch gap in the prior 15 s); no spike's numerator exceeds 1.12× what its elapsed span can produce; the 15.47 s incident totals 827 tokens = 53.5 tok/s, the engine's real rate; all sources gap at the same instant (process-wide stall, gap-4/phase-14 territory). +- Live read-only probes (`data/logs/probe-decode-spike{,2}.py`, `/metrics`+`/slots` only): gufo counters unlabelled + monotonic (max +76 tokens/read); vLLM has one label set and **no** throughput gauge; probe 1 reproduced a 0.297 s window, probe 2 reproduced the spike live (2043.66 tok/s, `Δ0.4s`) with the same tail as the stored incident. +- Fixture↔live shape: **no drift** (gufo 15/15, vLLM 128/128) → no re-capture needed for task 04. +- 4 red tests: 2 in `tests/test_rates.py` (spike + the inflated tail), 2 in `tests/test_llm_source.py` (gufo incident through `parse_gufo_metrics`; vLLM joining label set). +- Physical ceiling for task 04: spike-free max **119.9** (gufo) / **171.7** (vLLM) tok/s, engine lifetime means 30.0 / 2.8, native gauge 27.3–49.5, bandwidth bounds 256/640 GB/s → **>300 tok/s is arithmetic**, ≥1000 impossible. + +**Tests/lint/coverage** — `uv run pytest --cov --cov-fail-under=94` → **1173 passed, 11 skipped, 4 xfailed, coverage 98.66%**; `uv run ruff check src tests agent` → All checks passed; `uv run pyright` → 0 errors; `bash .agents/validate.sh` → exit 0 (incl. `node --check`, hygiene). No deps moved. + +**Docs** — AGENTS.md: new known gap 9 (confirmed mechanism, evidence file, the four xfails that XPASS when fixed) + the `rates.py` architecture line now says the window has no floor yet. README: no doc impact (nothing user-visible changed yet). + +**Decisions/deviations** — (1) "Red tests" ship as `xfail(strict=True)` so the suite is 100% green now and fails loudly (XPASS) when task 03 fixes the behaviour — markers must be removed, assertions kept. (2) No live change of any kind was needed or made; the ~10–15 s stall's root cause is left to gap 4 / `14_store_performance`, with a pointer in the notes. + +**Next pending task** — `.agents/phases/todo/13_llm_page_and_decode_rate/03_bounded_estimator_and_per_series_rates.md`. diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.md.status b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.validate b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.validate new file mode 100644 index 0000000..274fb15 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__02_pin_decode_spike_mechanism.a1.validate @@ -0,0 +1,62 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 6%] +........................................................................ [ 12%] +........................................................................ [ 18%] +........................................................................ [ 24%] +........................................................................ [ 30%] +........................................................................ [ 36%] +........................................................................ [ 42%] +........................................................................ [ 48%] +........................................................................ [ 54%] +........................................................................ [ 60%] +........................................................................ [ 66%] +....................sssssssssss....................................x.... [ 72%] +...........x............................................................ [ 78%] +........................................................................ [ 84%] +.................xx..................................................... [ 90%] +........................................................................ [ 96%] +.................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 348 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 188 2 99% 255-256 +------------------------------------------------------------------ +TOTAL 2843 38 99% +Required test coverage of 94% reached. Total coverage: 98.66% +1173 passed, 11 skipped, 4 xfailed, 1 warning in 95.47s (0:01:35) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.148.0 wheel +starlette 1.8.0 1.9.0 wheel diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.err b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.md b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.md new file mode 100644 index 0000000..362fe85 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.md @@ -0,0 +1,20 @@ +All green. Final state verified against the recorded fleet baseline (the 3 live failures are the documented `driveripper`-no-agent ConnectErrors, unchanged by this task). + +## Report — task 03 `bounded_estimator_and_per_series_rates` + +**Implemented** +- `src/dashboards/rates.py`: effective window now bounded both sides — `MIN_WINDOW_SECONDS=0.5` floor (the poll loop's ~0.2 s catch-up answers `(None, dt)` — one `n/a` with its honest label — **and drops the stale base** so the inflated tail re-bases) and the documented `2 × span` ceiling enforced via `MAX_WINDOW_RATIO` on the window-empty fallback; `JUMP_CONFIRM_RATIO=8.0` one-poll hold (previous value + label repeat; jump publishes only when the next interval confirms, else dropped); reset and `forget()` clear the judgement state; `min_window=0.0` constructor escape for sub-cadence tests. +- `src/dashboards/prom.py`: `values_by_series()` per-label-set accessor; `scalar` untouched. +- `src/dashboards/sources/llm.py`: gufo + vLLM `measured()` now diff **each label set under its own key** (`rate_key()` = resolved name + sorted, quoted labels); cell = Σ per-series rates over the **max** contributing window; a baseline-less series contributes 0 (never its carried value); a refused window keeps `computed:Δ/Δt` + `Δ` beside `n/a`. +- Removed all four task-02 `xfail(strict=True)` markers, assertions kept; added pins for floor, ceiling, hold/confirm/drop, per-key state, per-series joins, provenance shape, `values_by_series`. + +**Tests / lint / coverage** — `uv run pytest`: **1193 passed, 11 skipped**; `bash .agents/validate.sh` exit 0 (coverage **98.69%** ≥ 94; rates.py 100%, llm.py 99%, prom.py 96% — misses pre-existing in `_quantile`); `ruff` + `pyright` clean; dev server restarted, `/api/health` as recorded (driveripper DOWN = missing agent), live `/api/llm/snapshot` reads e.g. `42.5 tok/s @ Δ9.9s computed:Δllamacpp:tokens_predicted_total/Δt`; `pytest -m live --live` = 8 passed / 3 failed, all three the documented `driveripper:9120` ConnectError. + +**Docs** — `AGENTS.md` (invariant 2 keyed-by-name+label-set, `rates.py`/`prom.py`/`sources/llm.py` bullets, gap 9 rewritten as fixed) and `README.md` (`RATE_WINDOW_SECONDS` row: a stall answers one `n/a`). + +**Decisions/deviations** +- The task file's "accept fallback base ≤ 1.5 × span" contradicts its own pinned red test (asserts the diluted 7.47 row at a 15.25 s window, span 10). The test wins: ceiling enforced at the estimator's documented `2 × span`; the confirmed mechanism (NOTES candidate b, *lower*-bound form) is fixed by the floor + stale-base drop, as the findings file prescribes. +- A refused window is treated as an answer (skips vLLM's weaker `running/tpot` estimate for that poll) so the stall is visible as one self-labelled `n/a`. +- Two `test_collectors.py` real-time tests (fetches µs apart) now pass `min_window=0.0` — they pin window semantics, not the floor. + +**Next pending task:** `13_llm_page_and_decode_rate/04_live_stable_verification.md` (sustained live watch ≤ 300 tok/s, screenshots). diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.md.status b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.validate b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.validate new file mode 100644 index 0000000..d41d2f1 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__03_bounded_estimator_and_per_series_rates.a1.validate @@ -0,0 +1,62 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +....................sssssssssss......................................... [ 71%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 89%] +........................................................................ [ 95%] +.................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 348 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 69 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 188 2 99% 255-256 +------------------------------------------------------------------ +TOTAL 2910 38 99% +Required test coverage of 94% reached. Total coverage: 98.69% +1193 passed, 11 skipped, 1 warning in 97.58s (0:01:37) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.148.0 wheel +starlette 1.8.0 1.9.0 wheel diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.err b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.md b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.md new file mode 100644 index 0000000..a03af1b --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.md @@ -0,0 +1,18 @@ +Everything is green. Final report: + +**Implemented (task 04 — live proof; no production code needed changing)** +- `data/logs/watch-decode-stable.py` — signed-in watcher: 11.0 min / 285 `/api/llm/snapshot` polls, logging `decode_tokens_per_second` value + `Δ` + `source` per engine (no load generated). +- `data/logs/analyze-watch.py` — judge: TSV + `mode=ro` SQLite over the identical window; ceilings, label check, jump/hold check, 1 m rollups → `VERDICT: PASS`. +- `data/logs/NOTES-decode-stable.md` — the completion evidence (row-by-row holds/confirmations/refusals, fixtures decision, live tallies, screenshots, gap measurements). +- Screenshots: `llm-decode-stable.png` (full page), `llm-decode-stable-top.png` (fleet block + **13.59 px** `.85rem` gaps + strip + `DECODE` heroes with `Δ8.2s`/`Δ8.0s`), `llm-decode-burst.png` (real two-slot burst: 62.1 tok/s `Δ8.5s`, fleet sum 102.4). + +**Results** — gufo max **117.57**, vLLM max **98.00** tok/s (ceiling 300): 0 payload and 0 stored raw rows over it, 0 values printed without a `Δ` label; 3 floor refusals (`Δ0.387/0.399/0.363 s`) and 2 ≥8× jumps held-then-confirmed ran live; 1 m rollups top out at 51.9/34.1 (task 02's bad minute was 222.7). `bash .agents/validate.sh` → **rc 0** (1193 passed, 11 skipped; coverage 98.69 % ≥ 94; ruff "All checks passed!"; pyright 0 errors; `node --check` clean). `pytest -m live --live` → **9 passed / 2 failed**, both `driveripper` agent tests. + +**Docs** — `AGENTS.md`: gap 9 rewritten as *fixed and verified live* (numbers, screenshots, fixtures-not-recaptured), stale `1172 passed`/`98.6 %` counts corrected, and the fleet facts updated for a change that happened mid-task (see below). README: no doc impact (task 03 already carried the operator-facing line). + +**Decisions / deviations** +- **Fixtures not re-captured**: task 02 proved no shape drift (gufo 15/15, vLLM 128/128); task 13 changed no parsing, so `capture-fixtures.sh` was deliberately not run. +- **Fleet fact moved under the task**: the operator installed `driveripper`'s `dashboards-agent` at 13:27 (`active`, `/health` 200, `/snapshot` 401) — the two `driveripper` jobs are still DOWN but now on a **missing bearer token** (`HOST_TARGETS` has none for driveripper), which is why the live tally moved 8/3 → 9/2. I made **no live change**: reading/rotating that root-owned token is an operator action; AGENTS.md now records it and the one step that turns those rows green. +- The ~10 s process-wide stall persists (38 catch-up polls in 11 min) — gap 4 / phase 14's problem; the estimator absorbs it, so no re-open of task 03. + +**Next pending task:** none in phase 13 — `13_llm_page_and_decode_rate/04` was the last; next is `14_store_performance` (and `15_agent_mtls` now in `todo/`). diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.md.status b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.validate b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.validate new file mode 100644 index 0000000..14588f2 --- /dev/null +++ b/.agents/reports/13_llm_page_and_decode_rate/13_llm_page_and_decode_rate__04_live_stable_verification.a1.validate @@ -0,0 +1,63 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 5%] +........................................................................ [ 11%] +........................................................................ [ 17%] +........................................................................ [ 23%] +........................................................................ [ 29%] +........................................................................ [ 35%] +........................................................................ [ 41%] +........................................................................ [ 47%] +........................................................................ [ 53%] +........................................................................ [ 59%] +........................................................................ [ 65%] +....................sssssssssss......................................... [ 71%] +........................................................................ [ 77%] +........................................................................ [ 83%] +........................................................................ [ 89%] +........................................................................ [ 95%] +.................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 32 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 348 0 100% +src/dashboards/app.py 178 0 100% +src/dashboards/auth.py 265 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 120 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 112 5 96% 126, 163, 177, 180, 183 +src/dashboards/rates.py 69 0 100% +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 178 1 99% 285 +src/dashboards/sources/nut.py 413 20 95% 153, 172, 186, 198, 204, 217, 225, 241, 319-320, 322, 579, 591, 626, 657, 661, 718-719, 724, 736 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 188 2 99% 255-256 +------------------------------------------------------------------ +TOTAL 2910 38 99% +Required test coverage of 94% reached. Total coverage: 98.69% +1193 passed, 11 skipped, 1 warning in 95.31s (0:01:35) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == +advisory: dependencies are out of date (AGENTS.md rule 8 — update on purpose): + certifi 2026.7.22 2026.10.18 wheel +fastapi 0.143.5 0.149.0 wheel +starlette 1.8.0 1.9.0 wheel +websockets 17.2 17.3 wheel diff --git a/AGENTS.md b/AGENTS.md index b4bcd2b..283e4e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,7 +56,10 @@ page still renders. as `0`. Every derived value carries the `source` string it came from and `derived: true` so the UI can print *how* it was obtained (`computed:Δtokens/Δt`, `Δ10.0s`). Aggregating two different series into one rate is the failure mode this rule exists to catch — the - rate estimator is keyed by the *resolved* upstream series name for exactly that reason. + rate estimator is keyed by the *resolved* upstream series name **plus its label set** + for exactly that reason (phase 13: a label set joining a summed counter mid-window + contributes nothing until it has its own baseline, never its carried cumulative value), + and the estimator refuses a window too short or too stale to carry the claim. 3. **Unconfigured is not broken.** A source with no env config answers `404 {"configured": false}` **without dialling**, and the page says "not configured". A red panel for a server the operator never installed destroys the signal from the @@ -253,8 +256,13 @@ Responsibilities, one line each: counting it in `hosts_total` — the `/llm` fleet-block rule, so the page never does arithmetic and a dead agent never shrinks a fleet total. - `sources/llm.py` — gufo + vLLM field maps (`GUFO_SERIES`, `VLLM_SERIES`, - `VLLM_HISTOGRAMS`), measured Δ rates, prefill speed as - `mean(prompt tokens)/mean(prefill time)`, session list from gufo `/slots`. + `VLLM_HISTOGRAMS`), measured Δ rates **diffed per label set** (`rate_key()` keys the + estimator on the resolved name plus its labels; the cell is the Σ of the per-series + rates over the widest window one of them actually used — phase 13, so a joining + `{engine="1"}` series can never lend its carried counter to one interval's delta), + prefill speed as `mean(prompt tokens)/mean(prefill time)`, session list from gufo + `/slots`. A refused window (the estimator's floor) is still an answer: the cell keeps + its `computed:Δ/Δt` source and its honest `Δ` beside a `null` value. - `sources/nut.py` — upsd client, `UpsReading`, `VAR_MAP`, `assess()` (explicit checks, not a mystery score), `SELF_TEST_OUTCOMES` + `self_test_outcome()` (the `Last self-test` verdict follows the *outcome* of the driver's `*.test.result` word, not its phrasing: pass words → @@ -287,7 +295,23 @@ Responsibilities, one line each: `from_env()` is the single place the `NUT_SIMULATE` truthy rule lives — `app.py` builds the monitor through it, so the assembly and the model cannot disagree about what is enabled. - `prom.py` — Prometheus text-exposition parser (in-house; only for engines we don't own). -- `rates.py` — Δ/window estimator, keyed by resolved series name. + `scalar(agg="sum")` stays right for **gauges**; `values_by_series()` is the per-label-set + view **counter** deltas need, because a summed counter mixes whatever label sets exist now. +- `rates.py` — Δ/window estimator, keyed by resolved series name + label set. Its effective + window is `now - base[0]` **bounded on both sides** (phase 13): under + `MIN_WINDOW_SECONDS` (the poll loop's 0.2 s catch-up gap after any slow fetch) it answers + `(None, dt)` — one `n/a` with the window it refused — and drops that base so the polls + *after* the stall cannot re-diff it; the window-empty fallback accepts a base no older + than `MAX_WINDOW_RATIO × span` (the `2 × span` ceiling the docstring always claimed, now + enforced). A rate ≥ 8× the previously published one is held for one + poll (the previous value repeats, label unchanged) and publishes only if the next interval + confirms it (`JUMP_CONFIRM_RATIO`) — and **confirmation means a different anchor**: a held + jump remembers the sample its window was diffed from, and stays held while that same sample + still anchors the window. A stalled fetch stamps its body with the time the poll *finished*, + so that sample is the stall older than its own timestamp, and every later window off it is + the same skewed delta re-diffed over a longer span — measured twice from one sample is one + measurement, and the hold simply continues until the stale anchor ages out of the span. + `min_window=0.0` exists for tests that scrape faster than any real loop. - `auth.py` — every authentication decision as a pure function, stdlib only (phase 11): `hash_password`/`verify_password`/`needs_rehash` over a self-describing `pbkdf2_sha256$$$` verifier (so the cost can rise without a @@ -530,15 +554,26 @@ That scheme is the history API's key — treat it as a public interface. **Fleet transport, re-verified from the laptop on 2026-10-19.** The port-selective inter-VLAN drop phase 10 recorded on the lab gateway `10.2.0.1` is **gone**: `gateway` (`10.1.1.27:9120`) and `deskwork` (`10.1.0.51:9120`) both take a plain TCP connect and answer `/snapshot`, and - `driveripper`'s upsd answers `LIST UPS` on `10.1.0.126:3051` from this same laptop. Two hosts - run the **installed** service — `systemctl is-active dashboards-agent` is `active` on both, - with `/usr/local/lib/dashboards-agent/host_agent.py` and a `--token-file` unit bound to the - address the dashboard dials. `driveripper` is the one host with no agent at all (unit - `inactive`, no installed file, nothing listening on `:9120` → `ConnectionRefused`), and it - renders as an `unreachable` stub row: the designed degradation, recorded rather than hidden. - `/api/health` today reads `host:deskwork ok`, `host:gateway ok`, `disks:deskwork ok`, - `disks:gateway ok`, `nut ok`, `llm:gufo ok`, `llm:vllm ok` with the two `driveripper` jobs - `DOWN`, and the `live` suite is 8 passed / 3 failed with every failure that one transport. + `driveripper`'s upsd answers `LIST UPS` on `10.1.0.126:3051` from this same laptop. All three + hosts now run the **installed** service — `systemctl is-active dashboards-agent` is `active` on + each, with `/usr/local/lib/dashboards-agent/host_agent.py` and a `--token-file` unit bound to + the address the dashboard dials. **`driveripper` joined during phase 13 on 2026-10-19, in two + operator steps.** It went from *no agent at all* (unit `inactive`, nothing listening → + `ConnectionRefused`, an `unreachable` stub row) to **`active`** with + `/usr/local/lib/dashboards-agent/host_agent.py` installed at 13:27, and then its bearer token + went into `HOST_TARGETS`' URL userinfo — so `host:driveripper` and `disks:driveripper` are + **`ok`** and the `live` suite is **11 passed / 0 failed** (it was 9/2 on the missing token that + morning, and 8/3 on the ConnectError before that). The unit reads + `--host 10.1.0.126 --port 9120 --token-file /etc/dashboards-agent.token` (`640 + root:dashboards-agent`, the mode `deploy/install-agent.sh` writes) with `User=dashboards-agent`, + and `getent group disk` → `disk:x:6:dashboards-agent`. + **Two things about that install are not what this repo writes, and both cost SMART data on + `/disks` (see the SMART block below): `systemctl show -p DevicePolicy dashboards-agent` answers + `auto` with no `DeviceAllow` lines** (the installer's `strict` + node allow-list is not what is + running there), **and its `smartctl` cannot read drives as that user.** + `/api/health` today reads **all nine sources `ok`**: `host:driveripper`, `host:deskwork`, + `host:gateway`, `disks:driveripper`, `disks:deskwork`, `disks:gateway`, `nut`, `llm:gufo`, + `llm:vllm`. Phase 10's proof that the old drop was not this repo's firewall stands as history in `data/logs/NOTES-verify.md` — do not re-add a tunnel on the strength of it, and re-run the probe rather than trusting either note. @@ -597,8 +632,13 @@ That scheme is the history API's key — treat it as a public interface. `systemctl restart|reload nut-server` pair documented in `deploy/nut-sudoers`); `deskwork` grants an unrelated set (`btrfs scrub`, `snapper`, `setfacl /srv/*`, `osbuild-auto.sh`). Neither host grants `ops` the installer or `visudo` (re-checked 2026-10-19), so every install - that has happened on those two went through root by the operator — including the one still - outstanding, `driveripper`'s. + that has happened on those two went through root by the operator — `driveripper`'s included, + which is why its unit is not the one this repo writes (see the SMART block: `DevicePolicy=auto`, + no `DeviceAllow`). Worth knowing before the next install: the allow-list `install-agent.sh` + does write (`/dev/nvme*`, `/dev/sd*`, `/dev/vd*`, `/dev/sg*`) has **no `/dev/ng*n*` and no + `/dev/bsg/*`**, the nodes smartctl 7.5 reaches for NVMe admin and SATA pass-through, so a + `strict` unit there would deny exactly what a SMART read needs — whatever phase takes + driveripper's SMART on has to settle the node list and the grant together. - **SMART: real captures, and one live host finally reads them.** `smartctl` 7.5 is installed on deskwork and driveripper. The `ops` account (which is who runs a hand-started dev agent) is in no `disk` group, driveripper grants it `(root) NOPASSWD: /usr/sbin/smartctl*` and deskwork @@ -607,12 +647,25 @@ That scheme is the history API's key — treat it as a public interface. What changed by 2026-10-19 is that `deskwork`'s **installed** service runs as `dashboards-agent`, which is in the `disk` group (`getent group disk` → `disk:dashboards-agent`) on an agent installed after the SMART pass: `/disks` shows real numbers there — - `smartctl_available: true`, `nvme0n1` healthy at 52 °C, 19 160 power-on hours (an odometer, + `smartctl_available: true`, `nvme0n1` healthy at 55 °C (amber, past `disk.temp_c`'s 50 °C + threshold), 19 166 power-on hours (an odometer, so that figure is a capture date, not a fact about the drive), 11 % of rated wear — so the wear bar and the alarm chips are drawn from live hardware, not only fixtures. `gateway` answers `smartctl_available: false` (no smartmontools in its VM: the *install it* - half of the three-way flag), and `driveripper` answers nothing at all until its agent exists, - which is where `smartctl_available: null` — "nobody told us either way" — still comes from. + half of the three-way flag), and **`driveripper` is the fourth case the flag was built for: + `smartctl_available: true` with almost no numbers.** Its agent answers (16 whole-disk rows, + `nvme0n1`…`nvme7n1` + `sda`…`sdh`) and `smart` is non-null for the 8 NVMes, but every one of + them carries only identity — `health`, `temp_c`, `power_on_hours`, `percentage_used` and the + counters are all `null` — while the 8 spinning drives come back `smart: null` each with the + agent's own line `smart: sda: Try an additional '-d ata' or '-d sat' argument.` Measured + read-only on 2026-10-19: the same `smartctl -i -A -H /dev/nvme0n1` **as root** prints + `PASSED`, 55 °C and 14 % used, so the drives answer and the *agent user* is what is locked out + — `/dev/nvme0`, `/dev/ng0n1` and `/dev/bsg/*` (the nodes NVMe admin and SATA pass-through + actually go through) are `crw------- root root`, and only `/dev/nvme0n1` / `/dev/sg0` carry the + `disk` group the agent is in. So `/disks` shows driveripper's drive inventory with `n/a` values + and a why-line, which is the designed degradation, and **lighting it up is a new phase, not a + token**: a udev rule granting those nodes, and/or `host_agent.smart()` learning the + `-d sat` / `/dev/ngXnY` forms smartctl needs when auto-detection fails. **Real data is in the repo now:** `tests/fixtures/smartctl_driveripper_.txt` is all 16 drives (8 NVMe, 8 spinning) read through that sudo rule, and `tests/test_agent.py` parses every one — which is what found the parser's one real bug: this binary annotates the @@ -622,13 +675,15 @@ That scheme is the history API's key — treat it as a public interface. honest look at this fleet's drives: `nvme5n1` reports 13 media-and-data-integrity errors and `nvme1n1` is at 58 % of its rated wear, so the alarm chip and the amber wear bar on `/disks` are real features with real data behind them, not demo furniture. - **Lighting up the rest of it is the same two operator actions that travel together:** install - the agent on `driveripper`, and give its agent user device read access — the `disk` group + **Lighting up the rest of it was the same two operator actions that travel together** — install + the agent on `driveripper`, and give its agent user device read access: the `disk` group (`sudo usermod -aG disk `, which is what `deploy/install-agent.sh` does on install), *not* a sudo rule, because the agent must read the device itself — on an `agent/host_agent.py` from phase 09 or later (never redeploy without the grant: 16 permission-denied lines would age nothing but are pure noise, while with the grant they are 16 drives' worth of temperature, - hours and sector counters). `gateway` needs smartmontools first and will still read no SMART + hours and sector counters). **The agent and the `disk` group are now both in place on + `driveripper`, and it still reads no SMART numbers** — the group only covers nodes that carry + it (see above), so what is left is the node grant / `-d sat` work, not an install. `gateway` needs smartmontools first and will still read no SMART after that, being a VM with no physical disk behind `vda`. A host that cannot read its drives renders the designed degradation: one dimmed card per physical drive, every number `n/a`, and the one-line why. @@ -656,8 +711,8 @@ set -a; . ./.env.dev; set +a # dev dials the fleet the prod way, no ./scripts/dev-server.sh status # per-source ok/DOWN — unauthenticated: /api/health ./scripts/dev-server.sh logs -uv run pytest # hermetic: 1172 passed + 11 live-skipped, no network -uv run pytest --cov # coverage gate (98.6%; floor 94%) +uv run pytest # hermetic: 1195 passed + 11 live-skipped, no network +uv run pytest --cov # coverage gate (98.7%; floor 94%) uv run pytest -m live --live # opt-in: real fleet / engines / upsd uv run ruff check src tests agent && uv run pyright ./scripts/capture-fixtures.sh # re-record ground truth (agents + UPS + gufo /slots) @@ -1036,7 +1091,16 @@ carries **no severity colour anywhere**: `fleet_totals` has no `severity` to ren summed-KV bar is gone for that reason) and `severity.py` rates no throughput or queue depth, so the numbers stay ink and only the dot speaks reachability. `fleet.seams` — the API's own note on how the two engines count prompt tokens differently — is the prompt sum's tooltip -rather than another line of ink. The +rather than another line of ink. **The block, the strip and the panels keep the page's shared +`.85rem` gap** (phase 13, `TODO.md:4`: fleet and strip rendered flush into one wall): +`#strip { margin-top: .85rem }` is unconditional because below two engines `#fleet` is `[hidden]` +and the strip *is* the first visible section, and `#banners:empty ~ #fleet { margin-top: .85rem }` +gives the block its top gap only when the banner slot renders nothing, so a banner's own +`margin-bottom:.85rem` never stacks a second gap under it. `tests/test_api.py` pins both rules on +the served stylesheet, pins that **exactly one** rule styles each of the two ids (a second, +unguarded one is the double gap), and pins that `#banners`/`#fleet`/`#strip` are still direct +siblings of `
` — the `~` combinator resolves between siblings, so re-flowing that markup has +to rewrite the selectors instead of losing the gap. The four trend charts stay but at height 110 *below* the panels, each with an `ariaLabel`. Panels sit at their own height (`#engines` carries `.items-start`): a two-hero gufo card next to a latency-row vLLM one otherwise grows a blank bottom, and a blank panel reads as "no data". An @@ -1216,6 +1280,72 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl the operator's own machine is the threat model here, not a shared kiosk — and the reason a lost laptop is a password change and not a sign-out. A real revocation list would be a row per session with a prune rule, which is the thing this design traded away. +9. **The `/llm` `decode tok/s` spike is fixed, and verified live** (phase 13, 2026-10-19; + the diagnosis is in the git-ignored `data/logs/NOTES-decode-spike.md`, the live proof in + `data/logs/NOTES-decode-stable.md`). **Task 04's sustained watch passed the ceiling test:** + 11 minutes / 285 signed-in `/api/llm/snapshot` polls against the live engines, worst value + **117.57 tok/s** (gufo) and **98.00** (vLLM) — both in the payload and in the stored raw + series, 2.6× under the 300 tok/s pass line, with **zero** values printed without their + `Δ` label. The stall is still there (38 catch-up gaps < 1.5 s in 11 minutes, a + 10.6 s gap between two samples) and the estimator now absorbs it: three refusals each + labelled with the sub-0.5 s window it refused (`Δ0.387s`, `Δ0.399s`, `Δ0.363s`), two ≥ 8× + steps held for one poll and published only when the next interval confirmed them, and — the + point of the whole fix — **1 m rollups topping out at 51.9 / 34.1 tok/s** where task 02 found + a 222.7 tok/s minute, so the trend chart's y-scale stops being set by arithmetic. Fixtures + were **not** re-captured: task 02 compared the live exposure with `tests/fixtures/` series + by series (gufo 15/15, vLLM 128/128, identical sets), task 13 changed no parsing, and the + page still reads `15 series · 0 unparsed` / `128 series · 0 unparsed`. Screenshots: + `data/logs/llm-decode-stable.png` (+ `-top`, the fleet-block gap and both `Δ` tags) and + `data/logs/llm-decode-burst.png` (a real two-slot burst: 62.1 tok/s `Δ8.5s`, fleet sum 102.4). + The confirmed mechanism was neither a counter bug nor a label-set sum: `poller.Job._loop` + sleeps `max(0.2, interval - elapsed)`, so any fetch slower than `LLM_INTERVAL_SECONDS` + is followed by a poll **0.2 s later**, and the estimator happily diffed a whole stall's + tokens over that gap — the 2026-10-18 09:16 gufo row read **3284.59 tok/s for an engine + doing 53**, with an inflated tail because the stale sample stayed `inside[0]`. Task 03 + made that impossible by construction (see the `rates.py` bullet above): the window floor + turns the catch-up poll into one `n/a` **and drops the stale base** so the tail re-bases; + the fallback's `2 × span` ceiling is enforced (the honest 15.25 s diluted row still + publishes); jumps ≥ 8× hold one poll and publish only on confirmation; and + `values_by_series()` + `rate_key()` diff every counter + per label set, so the latent joining-series defect (candidate (a)) contributes 0 tokens + until it has its own baseline. The four task-02 `xfail(strict=True)` tests are green with + their markers removed and every assertion kept. Spike-free across two days of raw history + the fleet's real maximum is **119.9 tok/s** (gufo / deskwork) and **171.7** + (vLLM / driveripper) — anything above ~300 is arithmetic, not work, which was task 04's + pass line and what the watch above stayed under. The `Δ` label stayed honest throughout — + both before and after the fix it names the window actually used; do not "fix" anything + here by hiding or rewriting it. The ~10–15 s process-wide **stall** that triggers a + refused poll is still gap 4's and `14_store_performance`'s problem; the estimator no longer + turns a slow fetch into 0.2 s arithmetic, and one `n/a` per stall is the designed degradation + (the residual the verification pass measured is the paragraph below). + **The verification pass found one residual path and closed it** (same day, live rows in the + store the fix was running): a stalled fetch stamps its body with the time the poll + *finished*, so the sample it stores is the stall older than its own timestamp. The floor + cannot see that (the window is honest, the value is old), the sample then anchors every + window for the rest of the span, and the jump gate published the second one because it was + measured from the same anchor — gufo 12:42:10 read **232.28 tok/s** with exactly one request + running at ~50, followed by the tell-tale decaying tail 146.4 / 114.1 / 94.8 and the true + 47.5 (a 13:40:58 row repeats it at 200.6). `rates.py` now requires an **independent anchor** + for a confirmation (`_pending` stores the base sample's timestamp and the hold continues + while the window is still diffed from it), so a skewed sample is held until it ages out and + the honest rate publishes — the reconstruction of both rows is pinned by + `test_a_held_jump_is_not_confirmed_by_a_window_on_the_same_anchor` (estimator) and + `test_gufo_a_stale_scrape_body_never_becomes_a_rate_above_the_engine` (through + `parse_gufo_metrics`), each of which fails on the pre-fix gate and prints the live shape. + **That closes the gate's path, not the skew itself, and the verification pass measured the + difference.** A slow reply's counter is true somewhere between the request and the reply and + nothing in the payload says where, so when the resulting jump is *under* 8× the last published + rate the gate never engages: the post-fix watch (10 min / 540 reads, `data/logs/ + llm-decode-verify-pass.tsv`) caught gufo at **203.05 tok/s labelled `Δ2.595s`** where two busy + slots were really doing ~110, then 127.4 / 100.2 / 89.6 as the window grew back to the span and + 35.6 once the freed slot settled — every value labelled, **0 of 540 above the 300 pass line**, + and no stored raw row over it since the restart either. A collapsed window is what makes a + burst *look* like a rate, so the fix for this family is the ~10 s stall itself (gap 4 / + `14_store_performance`), not a fresh threshold invented in the estimator: task 03 pinned + `MIN_WINDOW_SECONDS` as an absolute floor and pinned that an in-bounds diluted row publishes + (`test_a_fallback_inside_the_documented_ceiling_still_publishes`), so a span-relative floor + would be a re-decision of that phase, not a bug fix. Do not "fix" the residual by hiding the + short window or its label. ## Locked decisions (and why) diff --git a/README.md b/README.md index 10e3ee2..695619b 100644 --- a/README.md +++ b/README.md @@ -329,7 +329,7 @@ documentation and is never loaded, and `scripts/dev-server.sh` exports `.env.dev | ------------------------------------- | -------------------------------------------------------------- | | `HOST_TARGETS` | `name=url` per host running the agent; a bearer token may ride in the URL userinfo (`http://TOKEN@host:9120`) | | `GUFO_BASE_URL` / `VLLM_BASE_URL` | LLM engines; unset means *not deployed here*, and its half of the page disappears | -| `RATE_WINDOW_SECONDS` | the Δ window behind every measured rate — tokens/sec, bytes/sec, jiffies→CPU % (default 10); each value carries the window it actually got, and `/system`'s footnote prints that, not this default | +| `RATE_WINDOW_SECONDS` | the Δ window behind every measured rate — tokens/sec, bytes/sec, jiffies→CPU % (default 10); each value carries the window it actually got, and `/system`'s footnote prints that, not this default. A window the poll loop cannot honour — the ~0.2 s catch-up after a fetch slower than the interval, or a base older than 2× the window — answers one `n/a` instead of a spike, and a rate 8× above the last one is held until a window measured from a **different** sample confirms it (phase 13) | | `DASH_HOST` / `DASH_PORT` | bind address for the dashboard itself (default `127.0.0.1:8090`; the container sets `0.0.0.0`) | | `HOST_INTERVAL_SECONDS` / `LLM_INTERVAL_SECONDS` / `NUT_INTERVAL_SECONDS` | how often each source is polled (10 / 2 / 5 s) | | `DISKS_INTERVAL_SECONDS` | how often the disk-health source re-reads each agent for SMART (default 300 s — SMART is not a per-second signal). No targets of its own: the `disks:` jobs exist exactly when `HOST_TARGETS` does | @@ -502,7 +502,8 @@ auth.py ── verifier + session key, loaded once at startup (no row and no see healthy — the dot is reachability, and `fleet.seams` (how the two engines count prompt tokens differently) is a tooltip on the prompt sum rather than another line of ink. Below two configured engines the block is hidden: a cross-engine sum of one engine duplicates that - engine's panel. Below the fleet block sits the **stats strip**: one card per configured + engine's panel. Below the fleet block — with the page's shared `.85rem` gap, as everywhere else + in the section stack — sits the **stats strip**: one card per configured engine — reachability dot, name, age stamp, the **KV-cache bar** (the one rated cell: the fill and a breach's text take their colour from the payload's `severity` and nothing else), then `prompt tok/s` and `decode tok/s` with their provenance tag (`Δ10.0s`, scraped, diff --git a/src/dashboards/prom.py b/src/dashboards/prom.py index 1b610e4..6a047fd 100644 --- a/src/dashboards/prom.py +++ b/src/dashboards/prom.py @@ -97,6 +97,25 @@ class Prom: return sum(finite) / len(finite) return sum(finite) + def values_by_series(self, names: Sequence[str], + label_match: dict[str, str] | None = None) -> list[tuple[dict[str, str], float]]: + """Every label set of the resolved name as `[(labels, value), ...]`. + + `scalar(agg="sum")` is right for a *gauge* — a sum over label sets is a real + aggregate — but a **counter delta** may not mix series: a label set that joins + mid-window lends its carried cumulative value to one interval's delta, and a + counter never goes down, so the restart check never fires. Diffing each series + individually (keyed by name + label set in `sources/llm.py`) is what keeps a + joining series at zero until it has its own baseline. + """ + name = self.resolve(names) + if name is None: + return [] + if not label_match: + return list(self.series[name]) + return [(labels, value) for labels, value in self.series[name] + if all(labels.get(k) == x for k, x in label_match.items())] + def by_label(self, names: Sequence[str], label: str) -> dict[str, float]: name = self.resolve(names) if name is None: diff --git a/src/dashboards/rates.py b/src/dashboards/rates.py index b9569b7..be5d3a1 100644 --- a/src/dashboards/rates.py +++ b/src/dashboards/rates.py @@ -6,24 +6,83 @@ because work arrives in bursts, so the rate is measured over a window (default 1 the window that was actually used travels with the value — the UI prints `Δ10.0s` rather than implying a scraped gauge it never got. -Keyed by the *resolved* upstream series name (e.g. `llamacpp:tokens_predicted_total`, or -`driveripper:cpu.total`), so a renamed counter starts a fresh window instead of diffing -two different series into a fabricated rate, and two hosts can never read each other's -samples. +The effective window is bounded on **both** sides (phase 13, `data/logs/NOTES-decode-spike.md`): + +* **floor** — `poller.Job._loop` sleeps `max(0.2, interval - elapsed)`, so any fetch + slower than the interval is followed by a catch-up poll ~0.2 s later. Whatever the + stalled fetch carried is then diffed over that gap: the tokens are real, the window is + a lie (that was the 3284.59 tok/s row for a 53 tok/s engine). A window under + `MIN_WINDOW_SECONDS` answers `(None, dt)` — one `n/a` that still labels the window it + measured — and the base it was measured from is dropped, so the *next* poll re-bases + instead of re-diffing the same stalled delta (the tail rows after a spike were as + wrong as the spike, because the stale sample stayed `inside[0]`). +* **ceiling** — the window-empty fallback (`base = samples[-2]`) accepts a base no older + than `MAX_WINDOW_RATIO x span`; older than that, the window is a different claim than + the panel makes ("a 20-second window on a 10-second panel"), so this poll answers + `(None, None)` and the next one recovers from the fresh sample. + +And a jump needs two polls to publish: a rate ≥ `JUMP_CONFIRM_RATIO` x the previously +published rate (previous > 0) is held for one poll — the previous value repeats, its +`Δ` label unchanged — and publishes only if the next interval confirms it. A +genuine step-up outlives a poll interval and appears one poll late; a single-interval +artifact (a carried counter, a mis-timed stall) never appears at all. + +*Confirmation means a second, different measurement*, so the held jump also remembers the +**base sample** it was measured from and is confirmed only by a window anchored on another +one. A stalled fetch stamps its body with the time the poll *finished*, so that body is +older than its own timestamp by the length of the stall; the sample that carries it then +anchors every window for the rest of the span, and each poll re-diffs the same skewed delta +over a longer window (a decaying tail). Measured twice against the same anchor is one +measurement, not a confirmation: the hold simply continues until the anchor ages out. + +Keyed by the *resolved* upstream series name plus its label set (e.g. +`llamacpp:tokens_predicted_total`, or `vllm:generation_tokens_total{engine="0",...}` — +`sources/llm.py::rate_key`), so a renamed counter — or a label set joining a summed +counter mid-window — starts a fresh window instead of diffing two different series into +a fabricated rate, and two hosts can never read each other's samples. """ from __future__ import annotations +import contextlib import math import time from collections import deque +#: A window shorter than this is the poll loop's catch-up gap, not a measurement over +#: the panel's span: answer `n/a`, never a rate. The spike rows were 0.2-0.4 s windows +#: on a 10 s span; every real cadence in this app polls at ≥ 1 s. +MIN_WINDOW_SECONDS = 0.5 + +#: The window-empty fallback may reach back for a base no older than this many spans. +#: This is the ceiling the estimator always documented (`2 x span`) made true: the trim +#: keeps the last two samples however old, so without this check any length of outage +#: between two samples became a rate. +MAX_WINDOW_RATIO = 2.0 + +#: A rate this many times above the previously published one is held for one poll and +#: publishes only when the next interval measures it again. A real decode step persists +#: across a poll interval; a carried-counter artifact does not. +JUMP_CONFIRM_RATIO = 8.0 + class RateEstimator: - def __init__(self, span: float = 10.0, max_samples: int = 512) -> None: + def __init__(self, span: float = 10.0, max_samples: int = 512, + min_window: float | None = None) -> None: + """`min_window` exists for tests that scrape faster than any real poll loop + (two fetches milliseconds apart have no rate to measure); every production + estimator answers with the `MIN_WINDOW_SECONDS` floor.""" self.span = span self.max_samples = max_samples + self.min_window = MIN_WINDOW_SECONDS if min_window is None else min_window self._samples: dict[str, deque[tuple[float, float]]] = {} + # key -> the last (rate, window) actually published, and a jump held pending + # confirmation (keyed by the timestamp of the sample it was measured from, which is + # the anchor the confirming window must move off). Both are judgement, not + # measurement, and a counter restart or a `forget()` drops them together with the + # window they were made of. + self._published: dict[str, tuple[float, float]] = {} + self._pending: dict[str, float] = {} def rate(self, key: str, value: float | None, now: float | None = None) -> tuple[float | None, float | None]: """(per-second, window actually used). (None, window|None) when not computable. @@ -37,6 +96,10 @@ class RateEstimator: samples = self._samples.setdefault(key, deque()) if samples and value < samples[-1][1]: samples.clear() + # A restart is a fresh counter with no history to be judged against: the + # previous published rate must not veto or hold the new lineage's numbers. + self._published.pop(key, None) + self._pending.pop(key, None) samples.append((now, value)) while len(samples) > self.max_samples: samples.popleft() @@ -44,20 +107,59 @@ class RateEstimator: return None, None inside = [point for point in samples if now - point[0] <= self.span] - base = inside[0] if len(inside) > 1 else samples[-2] + if len(inside) > 1: + base = inside[0] + else: + fallback = samples[-2] + if now - fallback[0] > MAX_WINDOW_RATIO * self.span: + return None, None # a stall, not a window: one n/a, next poll recovers + base = fallback dt, dv = now - base[0], value - base[1] - cutoff = now - 2 * self.span + cutoff = now - MAX_WINDOW_RATIO * self.span while samples and samples[0][0] < cutoff and len(samples) > 2: samples.popleft() - if dt <= 0 or dv < 0: - return None, dt if dt > 0 else None - return dv / dt, dt + if dt <= 0: + return None, None + if dt < self.min_window: + # The catch-up poll's gap: refuse to turn it into a rate, and drop the base + # so the stall's tokens cannot be re-diffed by the polls after it either. + with contextlib.suppress(ValueError): # base is always a point of this deque + samples.remove(base) + return None, dt + # dv can never be negative here: a decrease clears the window before the append, + # so every window this function diffs is non-decreasing by construction. + return self._publish(key, dv / dt, dt, base_ts=base[0]) + + def _publish(self, key: str, rate: float, window: float, base_ts: float) -> tuple[float | None, float | None]: + """The confirmation gate: a jump ≥ ratio x previous publishes one poll late. + + `base_ts` is the timestamp of the sample this window was diffed from, and a held + jump stores it: a poll anchored on the same sample re-measures the same delta over + a longer window, which confirms nothing. + """ + last = self._published.get(key) + held_from = self._pending.get(key) + if held_from is not None and last is not None: + if held_from == base_ts: + return last # same anchor, same measurement: the hold simply continues + del self._pending[key] + if rate >= JUMP_CONFIRM_RATIO * last[0]: + self._published[key] = (rate, window) # the step is real: publish it + return rate, window + return last # unconfirmed artifact: the cell keeps printing the held value + if last is not None and last[0] > 0 and rate >= JUMP_CONFIRM_RATIO * last[0]: + self._pending[key] = base_ts # hold: the previous value prints once more + return last + self._published[key] = (rate, window) + return rate, window def forget(self, prefix: str) -> int: """Drop every window whose key starts with `prefix` (a target that went away).""" keys = [k for k in self._samples if k.startswith(prefix)] for key in keys: del self._samples[key] + self._published.pop(key, None) + self._pending.pop(key, None) return len(keys) def keys(self) -> list[str]: diff --git a/src/dashboards/sources/llm.py b/src/dashboards/sources/llm.py index 1f2b1d2..2c86764 100644 --- a/src/dashboards/sources/llm.py +++ b/src/dashboards/sources/llm.py @@ -138,6 +138,54 @@ def _histogram_mean(out: dict[str, Any], key: str) -> float | None: return mean if isinstance(mean, float) and math.isfinite(mean) else None +def _quote_label(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def rate_key(name: str, labels: dict[str, str]) -> str: + """The estimator key of ONE labelled series: the resolved name plus its label set. + + Keying a summed counter by the name alone lets a label set that joins between two + polls (`{engine="1"}` on a DP rank, a reloaded model still exporting its old series) + lend its carried cumulative value to one interval's delta — an *increase* never + trips the restart check. Under its own key a joining series simply has no baseline + and contributes 0 tokens, which is the "a renamed series starts a fresh window" + doctrine extended from names to label sets (rule 2). + """ + if not labels: + return name + inner = ",".join(f'{k}={_quote_label(labels[k])}' for k in sorted(labels)) + return f"{name}{{{inner}}}" + + +def _series_delta(parsed: Prom, rates: RateEstimator, name: str, + now: float | None) -> tuple[float | None, float | None]: + """Σ of the per-label-set Δrates of one cumulative counter. + + Returns (sum-of-rates, widest window a contributing series actually used). A series + with no previous sample contributes nothing this interval — never its carried value. + When no series contributed but one refused the window (phase 13's floor), the refused + window travels back so the cell can print `n/a` beside its honest `Δ` label. + """ + total = 0.0 + contributions = 0 + window: float | None = None + refusal: float | None = None + for labels, value in parsed.values_by_series([name]): + rate, used = rates.rate(rate_key(name, labels), value, now) + if rate is not None: + contributions += 1 + total += rate + if used is not None: + window = used if window is None else max(window, used) + elif used is not None: + refusal = used if refusal is None else max(refusal, used) + if contributions: + return total, window + return None, refusal + + def parse_gufo_metrics(text: str, rates: RateEstimator | None = None, now: float | None = None) -> dict[str, Any]: """`rates` enables the Δ-based aggregate rates; omitted, the parse stays pure.""" @@ -147,27 +195,30 @@ def parse_gufo_metrics(text: str, rates: RateEstimator | None = None, out[key] = field(parsed.scalar(names, agg), parsed.resolve(names)) def measured(total_key: str) -> dict[str, Any] | None: - """Δcounter/Δt for one cumulative counter, keyed by the RESOLVED series name so an - alias rename starts a fresh window instead of diffing two different series.""" + """Δcounter/Δt for one cumulative counter, per label set, keyed by the RESOLVED + series name so an alias rename starts a fresh window instead of diffing two + different series.""" if rates is None: return None source = out[total_key]["source"] if source is None: return None - rate, window = rates.rate(source, out[total_key]["value"], now) - if rate is None: + rate, window = _series_delta(parsed, rates, source, now) + if rate is None and window is None: return None return field(rate, f"computed:Δ{source}/Δt", derived=True, window=window) # The counters add every slot; gufo's single gauge cannot. Measured therefore wins for # both speeds here (the inverse of vLLM, where the native gauge is the aggregate) — and # when no window exists yet the cell stays n/a instead of borrowing one request's speed. + # A *refused* window (value None, window present) is also an answer: the cell prints + # n/a with the short Δ it refused, rather than silently waiting a poll to say anything. for ui_key, total_key in ( ("decode_tokens_per_second", "tokens_generated_total"), ("prompt_tokens_per_second", "tokens_prompt_total"), ): - rate = measured(total_key) - out[ui_key] = rate if rate is not None else field(None, None) + cell = measured(total_key) + out[ui_key] = cell if cell is not None else field(None, None) prompt = out["tokens_prompt_total"]["value"] cached = out["tokens_prompt_cached_total"]["value"] or 0.0 @@ -214,13 +265,19 @@ def parse_vllm_metrics(text: str, rates: RateEstimator | None = None, per_seq = (1.0 / tpot) if tpot else None out["per_request_decode_tokens_per_second"] = field(per_seq, "computed:1/tpot", derived=True) - def measured(key: str, total_key: str) -> dict[str, Any] | None: + def measured(total_key: str) -> dict[str, Any] | None: + """Δcounter/Δt per label set, keyed by the resolved series name plus its labels — + vLLM's counters are labelled (`{engine="0",model_name=...}`), so a summed diff + over a changing label set is exactly the carried-counter bug phase 13 pinned.""" if rates is None: return None - rate, window = rates.rate(key, out[total_key]["value"], now) - if rate is None: + source = out[total_key]["source"] + if source is None: return None - return field(rate, f"computed:Δ{VLLM_SERIES[total_key][0][0]}/Δt", derived=True, window=window) + rate, window = _series_delta(parsed, rates, source, now) + if rate is None and window is None: + return None + return field(rate, f"computed:Δ{source}/Δt", derived=True, window=window) decode = out["decode_tokens_per_second_native"]["value"] running = out["num_requests_running"]["value"] @@ -229,8 +286,10 @@ def parse_vllm_metrics(text: str, rates: RateEstimator | None = None, out["decode_tokens_per_second_native"]["source"]) else: # measured beats the running/tpot estimate: it counts real output tokens, the - # estimate assumes every running request is decoding this instant - measured_decode = measured("vllm:generation_tokens_total", "generation_tokens_total") + # estimate assumes every running request is decoding this instant. A refused + # window counts as an answer too: papering over a stall with the lifetime + # estimate would hide exactly the poll the operator needs to see as n/a. + measured_decode = measured("generation_tokens_total") if measured_decode is not None: # `or` here would treat a genuine 0 tok/s as missing and print the lifetime # estimate for an engine that is provably idle @@ -249,7 +308,7 @@ def parse_vllm_metrics(text: str, rates: RateEstimator | None = None, "computed:mean(request_prompt_tokens)/mean(request_prefill_time)", derived=True) if out["prompt_tokens_per_second"]["value"] is None: - measured_prefill = measured("vllm:prompt_tokens_total", "prompt_tokens_total") + measured_prefill = measured("prompt_tokens_total") if measured_prefill is not None: out["prompt_tokens_per_second"] = measured_prefill elif prefill_avg is not None: diff --git a/src/dashboards/web/static/dash.css b/src/dashboards/web/static/dash.css index 91ed13c..19858b4 100644 --- a/src/dashboards/web/static/dash.css +++ b/src/dashboards/web/static/dash.css @@ -416,6 +416,22 @@ main { padding: 1.15rem 1.25rem 3rem; max-width: 1680px; margin: 0 auto; } } .fleet .metrics { margin-top: .55rem; } +/* The `/llm` section stack's `.85rem` rhythm (`TODO.md:4`: the fleet block and the engine + * summaries rendered flush into one wall; the section under `#strip` already carried + * `margin-top:.85rem` inline). + * + * `#strip`'s gap is unconditional: below two engines `#fleet` is `[hidden]` and the strip *is* + * the first visible section, and a summary strip whose gap depended on a block above it + * rendering is one layout change away from flush again. + * The `:empty` guard is the no-double-gap half of the same idea — with a banner up, that + * banner's own inline `margin-bottom:.85rem` **is** the gap above the block and this rule must + * not stack a second one; with the slot empty the block owns its gap instead of borrowing the + * empty slot's margin. `tests/test_api.py` pins both rules, and pins that nothing else styles + * either id. + */ +#strip { margin-top: .85rem; } +#banners:empty ~ #fleet { margin-top: .85rem; } + /* ---- drive panels ----------------------------------------------------------- * `/disks`: one `.disk` mini-panel per **physical** drive inside its host's `.card`, in the * `.slot` shape (bordered mini-panel, `.chip` for the drive's own word) because a drive is an diff --git a/tests/test_api.py b/tests/test_api.py index 12c3975..5e65a4a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2421,6 +2421,96 @@ def test_llm_fleet_block_colours_no_number(client) -> None: "the fleet block's head row lost its styling" +#: Elements the browser treats as empty, so they never open a nesting level. +_VOID_TAGS = {"meta", "link", "br", "img", "input", "hr", "source", "track", "wbr"} + + +def _depth_at(html: str, needle: str) -> int: + """How deeply the `needle` sits inside `
` (1 = a direct child of it). + + The `~` in `#banners:empty ~ #fleet` only resolves between **siblings**, so the gap rule + depends on the markup as much as on the stylesheet. Counting tags is enough here and keeps + the check on the served bytes: no HTML parser, no dependency, and comments are already + gone by the time `_code` hands the page over. + """ + body = html[html.index("
"):html.index(needle, html.index("
"))] + depth = 0 + for closing, tag, _attrs, self_closed in re.findall( + r"<(/?)([a-zA-Z][\w-]*)([^>]*?)(/?)>", body): + if self_closed or tag.lower() in _VOID_TAGS: + continue + depth += -1 if closing else 1 + return depth + + +def test_llm_section_stack_keeps_the_shared_vertical_rhythm(client) -> None: + """Fleet block → engine summaries → engine panels sit the shared `.85rem` apart. + + `TODO.md:4`: the fleet section had no spacing between it and the engine summaries. Every + boundary below it was already handled inline — the section under `#strip` carries + `margin-top:.85rem` — so `#fleet` and `#strip` skipped the rhythm and the top of the page + read as one unbroken wall. Measured in Chromium at the 16px root (`data/logs/` + `llm-fleet-spacing*.png`), before → after: + + | gap | before | after | + |-----|--------|-------| + | `#fleet` → `#strip` | **0 (flush)** | 13.59px | + | `#strip` → engine panels | 13.59px | 13.59px | + | banner slot → `#fleet`, no banner | 13.59px | 13.59px | + | banner slot → `#fleet`, one banner up | 13.59px | 13.59px (`margin-top: 0px`) | + + One rendered distance moves, and it is the one the operator named. What the *shape* of the + two rules buys is that no other state can put it back: + + * `#strip`'s gap is **unconditional**, because below two engines `#fleet` is `[hidden]` + (`display:none !important`, so it contributes nothing) and the strip is then the first + visible section: the 13.59px below the banner slot in that state is now the strip's own + margin rather than one borrowed from a slot above it, which is what `#fleet ~ #strip` + would have been; + * `#fleet`'s gap is guarded by `#banners:empty`, so the block owns its top gap exactly when + nothing occupies the slot above it, and a banner's own `margin-bottom:.85rem` stays the + single gap under it. An unguarded `#fleet { margin-top }` is a second rule for one gap: + margin collapsing hides it at `.85rem`/`.85rem` today, and it doubles the moment either + margin is changed. The empty slot keeps its inline bottom margin, so head → fleet stays + the 1rem + .85rem the other two banner pages (`/`, `/ups`) already render. + + Pinned on the served bytes because a stylesheet "cleanup" that drops either rule is + invisible in a diff and only shows up as a page that reads worse. + """ + raw = _css(client) + section = _css_section(raw, "fleet totals") + assert "#strip { margin-top: .85rem; }" in section, \ + "the fleet block and the engine summaries render flush again (TODO.md:4)" + assert "#banners:empty ~ #fleet { margin-top: .85rem; }" in section, ( + "the gap above the fleet block stopped belonging to the block: a banner-less page " + "would then have it only as long as the empty banner slot keeps its inline margin") + + # one rule per element: a second one (or an unguarded `#fleet`) is the double gap + css = _code(raw) + for element, expected in (("#strip", {"#strip"}), + ("#fleet", {"#banners:empty ~ #fleet"})): + selectors = {s.strip() for s in re.findall(rf"([^{{}}]*{element}[^{{}}]*)\{{", css)} + assert selectors == expected, ( + f"dash.css styles {element} with {sorted(selectors)} — exactly one rule, the " + "guarded one, or the page grows a double gap or loses the single one") + + html = _code(client.get("/llm").text) + for element in ("banners", "fleet", "strip"): + assert _depth_at(html, f'id="{element}"') == 1, ( + f"`#{element}` is no longer a direct child of
, so the `~` gap rule cannot " + "match: rewrite the selectors for the new structure instead of dropping the gap") + assert html.index('id="banners"') < html.index('id="fleet"') < html.index('id="strip"') \ + < html.index('id="engines"'), "the section stack was re-ordered" + + # the inline half of the rhythm, which the two rules are written around + assert 'style="margin-bottom:.85rem"' in html[:html.index('id="fleet"')], ( + "#banners lost the inline bottom margin that is the banner → fleet gap, and today the " + "only thing spacing the page head from the block") + after_strip = html[html.index('id="strip"'):] + assert after_strip.count('style="margin-top:.85rem"') == 3, \ + "a section below the strip lost the inline gap the strip's rule hands off to" + + # ---- the `/llm` gufo slot grid (phase 07 task 04) ------------------------- # # `llm_monitor`'s slot cards (`index.html` ~L1179), rebuilt against the capture instead of a diff --git a/tests/test_collectors.py b/tests/test_collectors.py index 491bc2e..c4bbd31 100644 --- a/tests/test_collectors.py +++ b/tests/test_collectors.py @@ -62,7 +62,10 @@ async def test_a_real_agent_snapshot_parses() -> None: return httpx.Response(200, json=body) target = Target("driveripper", "http://agent.test") - collector = HostCollector(RateEstimator()) + # min_window=0: these fetches land microseconds apart, far under any real poll + # cadence; this test pins the Δjiffies window semantics, not the phase-13 floor + # (tests/test_rates.py pins the floor at a real cadence). + collector = HostCollector(RateEstimator(min_window=0.0)) async with client_for(handler) as client: first = (await fetch_host_snapshot(client, target, collector)).payload # CPU usage is Δjiffies/Δt: one scrape has no window, so it is n/a and not 0 % @@ -392,7 +395,9 @@ async def test_second_gufo_poll_measures_throughput_over_a_window() -> None: return httpx.Response(200, json=[{"index": 0, "n_ctx": 8192, "predicted": 12}]) return httpx.Response(404) - rates = RateEstimator(span=10.0) + # min_window=0: the two fetches are microseconds apart (no real loop polls this + # fast); this test pins that the measured cell divides by the window it labels. + rates = RateEstimator(span=10.0, min_window=0.0) async with client_for(handler) as client: await fetch_engine(client, "gufo", "http://engine.test", "", rates) state["text"] = bump(GUFO, "llamacpp:tokens_predicted_total", 40.0) diff --git a/tests/test_llm_source.py b/tests/test_llm_source.py index 5426999..6258fce 100644 --- a/tests/test_llm_source.py +++ b/tests/test_llm_source.py @@ -15,6 +15,7 @@ from dashboards.sources.llm import ( fleet_totals, parse_gufo_metrics, parse_vllm_metrics, + rate_key, series_names, ) from slots_contract import load_slots_capture @@ -94,6 +95,58 @@ def test_gufo_counter_reset_produces_n_a_not_a_spike(metrics_text) -> None: assert metrics.get("decode_tokens_per_second", {}).get("value") is None +def test_gufo_a_stalled_poll_then_the_catch_up_is_not_a_spike(metrics_text) -> None: + """The exact stored incident, through the production parse path. + + 2026-10-18 09:16:31 on deskwork: a 15 s gap in every source, a first result carrying 114 + tokens (its body was generated ~2 s into the stall), then the poll loop's 0.2 s catch-up + printing **3284.59 tok/s** for an engine doing 53. `test_rates.py` holds the estimator-level + pair; this is the same defect where the operator sees it. + """ + text = metrics_text("gufo") + rates = RateEstimator(span=10.0) + parse_gufo_metrics(text, rates, 1_000.0) + text = bump_metric(text, "llamacpp:tokens_predicted_total", 114.0) + parse_gufo_metrics(text, rates, 1_015.25) + text = bump_metric(text, "llamacpp:tokens_predicted_total", 713.0) + metrics = parse_gufo_metrics(text, rates, 1_015.47) + cell = metrics["decode_tokens_per_second"] + assert cell["window"] == pytest.approx(0.22) + assert cell["value"] is None or cell["value"] < 100.0, ( + f"53 tok/s of real work printed as {cell['value']:.0f} tok/s") + + +def test_gufo_a_stale_scrape_body_never_becomes_a_rate_above_the_engine(metrics_text) -> None: + """The other half of the stall: the reply is stamped when the poll *finished*, not when the + engine rendered it, so its counter is the stall's length older than its own timestamp. + + That sample then anchors every window for the rest of the span and each poll re-diffs the + same skewed delta over a longer one — the live 2026-10-19 12:42 incident printed 232 tok/s + for one request running at ~50, then a decaying 146 / 114 / 95 tail. The estimator's jump + gate holds the first of those and now refuses to call a window anchored on the same sample a + confirmation, so nothing above the engine's real rate reaches the payload. + """ + text = metrics_text("gufo") + rates = RateEstimator(span=10.0) + parse_gufo_metrics(text, rates, 1_000.0) + # (stamp, tokens the engine had really produced by then): the 1011 reply is the stalled one — + # its body was rendered ~9 s before the poll stamped it — and from there the counter advances + # at a steady 45 tok/s, so 45 is the only rate that is true to print. + timeline = [(1_011.0, 90.0), (1_012.0, 540.0), (1_014.0, 630.0), (1_016.0, 720.0), + (1_018.0, 810.0), (1_022.0, 990.0), (1_026.0, 1_170.0)] + produced = 0.0 + cells = [] + for stamp, total in timeline: + text = bump_metric(text, "llamacpp:tokens_predicted_total", total - produced) + produced = total + cells.append(parse_gufo_metrics(text, rates, stamp)["decode_tokens_per_second"]) + for cell in cells: + assert cell["value"] is None or cell["value"] <= 100.0, \ + f"a 45 tok/s engine printed {cell['value']:.0f} tok/s ({cells})" + assert cells[-1]["value"] == pytest.approx(45.0), "the stall ends in the truth, not a tail" + assert cells[-1]["window"] == pytest.approx(10.0) + + def test_absent_gufo_metrics_are_none_with_no_source() -> None: metrics = parse_gufo_metrics("# nothing here\n") assert metrics["requests_processing"]["value"] is None @@ -191,6 +244,79 @@ def test_vllm_measured_rate_when_the_build_has_no_throughput_gauge(metrics_text) assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(25.0) +def test_vllm_a_joining_label_set_does_not_lend_its_carried_counter(metrics_text) -> None: + """A second `engine=`/`model_name=` series appearing mid-window must not be a rate. + + vLLM's counters are labelled, so `scalar(agg="sum")` added whatever label sets exist right + now. A rank joining (or a reloaded model whose old series is still exported) carries its own + cumulative counter, and `value < samples[-1]` never fires for an *increase*, so the carried + value was diffed. Confirmed as a defect, not as this fleet's trigger: `generation_tokens_total` + has exactly one label set live (`{engine="0",model_name="juggernaut"}`). Fixed in task 03 by + diffing each label set under its own estimator key — the joining series has no baseline and + contributes 0 tokens until it has one. + """ + text = metrics_text("vllm") + rates = RateEstimator(span=10.0) + parse_vllm_metrics(text, rates, 2_000.0) + gained = text + 'vllm:generation_tokens_total{engine="1",model_name="juggernaut"} 500000.0\n' + metrics = parse_vllm_metrics(gained, rates, 2_010.0) + cell = metrics["decode_tokens_per_second"] + assert cell["value"] is None or cell["value"] < 100.0, ( + f"a joining label set printed {cell['value']:.0f} tok/s") + + +def test_vllm_a_label_set_that_survives_starts_contributing_its_own_delta(metrics_text) -> None: + """Fresh series contribute nothing on their first poll and their real Δ on the second.""" + text = metrics_text("vllm") + rates = RateEstimator(span=10.0) + parse_vllm_metrics(text, rates, 2_000.0) + gained = text + 'vllm:generation_tokens_total{engine="1",model_name="juggernaut"} 500000.0\n' + metrics = parse_vllm_metrics(gained, rates, 2_010.0) + assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(0.0), \ + "the joining series is baseline-less: 500 000 carried, 0 tokens diffed" + grown = bump_metric(gained, "vllm:generation_tokens_total", 100.0) # bumps every label set + metrics = parse_vllm_metrics(grown, rates, 2_020.0) + assert metrics["decode_tokens_per_second"]["value"] == pytest.approx(20.0), \ + "two series x 100 tokens / 10 s: the rate is the sum of the per-series rates" + assert metrics["decode_tokens_per_second"]["window"] == pytest.approx(10.0) + + +def test_gufo_measured_cell_keeps_the_provenance_shape(metrics_text) -> None: + """Rule 2 after the per-series rewrite: value, `computed:Δ/Δt`, `Δ`.""" + text = metrics_text("gufo") + rates = RateEstimator(span=10) + parse_gufo_metrics(text, rates, 1_000.0) + text = bump_metric(text, "llamacpp:tokens_predicted_total", 400.0) + decode = parse_gufo_metrics(text, rates, 1_010.0)["decode_tokens_per_second"] + assert decode["value"] == pytest.approx(40.0) + assert decode["derived"] is True + assert decode["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt" + assert decode["window"] == pytest.approx(10.0), "the widest window a contributing used" + + +def test_gufo_a_refused_window_is_n_a_and_names_the_window_it_refused(metrics_text) -> None: + """The stall refusal is an answer: value None, but the series and Δ still travel.""" + text = metrics_text("gufo") + rates = RateEstimator(span=10) + parse_gufo_metrics(text, rates, 1_000.0) + text = bump_metric(text, "llamacpp:tokens_predicted_total", 50.0) + parse_gufo_metrics(text, rates, 1_010.0) + text = bump_metric(text, "llamacpp:tokens_predicted_total", 60.0) + decode = parse_gufo_metrics(text, rates, 1_010.3)["decode_tokens_per_second"] + assert decode["value"] is None, "0.3 s is the loop's catch-up gap, not a rate" + assert decode["window"] == pytest.approx(0.3) + assert decode["source"] == "computed:Δllamacpp:tokens_predicted_total/Δt" + + +def test_rate_key_puts_the_label_set_in_the_estimator_key() -> None: + """The key is the resolved name plus its labels, label order included-stable.""" + assert rate_key("llamacpp:tokens_predicted_total", {}) == "llamacpp:tokens_predicted_total" + assert rate_key("vllm:generation_tokens_total", {"model_name": "m", "engine": "0"}) == \ + 'vllm:generation_tokens_total{engine="0",model_name="m"}' + assert rate_key("m", {"weird": 'a"b\\c'}) == 'm{weird="a\\"b\\\\c"}', \ + "a quote in a label value must not splice two keys into one" + + def test_vllm_waiting_requests_are_broken_down_by_reason(metrics_text) -> None: metrics = parse_vllm_metrics(metrics_text("vllm")) reasons = metrics["waiting_by_reason"]["value"] diff --git a/tests/test_prom.py b/tests/test_prom.py index 0bb16a8..2698cb8 100644 --- a/tests/test_prom.py +++ b/tests/test_prom.py @@ -33,6 +33,23 @@ def test_labels_are_parsed_and_filterable(metrics_text) -> None: assert by_model and all(model for model in by_model) +def test_values_by_series_keeps_the_label_sets_apart() -> None: + """Counter deltas diff per label set; the summed view would let a joining series + lend its carried value to one interval (phase 13). This is the accessor that view.""" + prom = Prom('x{a="1"} 5.0\nx{a="2"} 7.0\ny 1.0\n') + assert prom.values_by_series(["x"]) == [({"a": "1"}, 5.0), ({"a": "2"}, 7.0)] + assert prom.values_by_series(["y"]) == [({}, 1.0)] + assert prom.values_by_series(["x"], {"a": "2"}) == [({"a": "2"}, 7.0)] + assert prom.values_by_series(["nope"]) == [] + + +def test_values_by_series_on_a_real_labelled_capture(metrics_text) -> None: + prom = Prom(metrics_text("vllm")) + series = prom.values_by_series(["vllm:generation_tokens_total"]) + assert series, "vLLM exposes generation_tokens_total" + assert all(labels.get("model_name") for labels, _v in series), "the capture lost its labels" + + def test_aggregations(metrics_text) -> None: prom = Prom(metrics_text("vllm")) total = prom.scalar(["vllm:prompt_tokens_total"], agg="sum") diff --git a/tests/test_rates.py b/tests/test_rates.py index ac32e72..b5b5396 100644 --- a/tests/test_rates.py +++ b/tests/test_rates.py @@ -52,6 +52,203 @@ def test_nan_and_inf_are_treated_as_missing() -> None: assert r.rate("x", bad, now=1.0) == (None, None) +# --- phase 13: the live decode-tok/s spike ---------------------------------- # +# Confirmed against the stored history and reproduced on live hardware; the findings are in +# data/logs/NOTES-decode-spike.md. These two tests were written RED (as strict xfails) by +# task 02 and turned GREEN by task 03's bounded window; the markers were removed and every +# assertion kept verbatim, as the phase mandated. They are permanent pins now. +# +# The incident, from `sample` (source=llm, host=gufo, metric=llm.decode_tokens_per_second, +# res=raw) on 2026-10-18, with the counter value reconstructed from the rows' own windows: +# +# 09:16:15.839 53.64 tok/s window ~10 s (steady work on deskwork: 53 tok/s) +# ... 15.25 s with no sample at all: every source in the process gaps at the same instant ... +# 09:16:31.091 7.47 tok/s window 15.25 s (114 tokens — a body generated ~2 s into it) +# 09:16:31.308 3284.59 tok/s window 0.217 s (713 tokens — the rest of the stall) +# 09:16:33.311 371.23 tok/s window 2.22 s (the stale sample is still base[0]) +# 09:16:35.315 222.32, 09:16:37.318 166.23, 09:16:39.316 138.37, then back to ~50 +# +# 827 tokens across the whole 15.47 s is 53.5 tok/s — the engine's true rate. The tokens are +# real; the window is the lie. `probe-decode-spike2.py` produced the same row pair (and nearly +# the same tail: 379 / 232 / 174 / 140) against live gufo with the production parser. + +GUFO_TOKENS = "llamacpp:tokens_predicted_total" + + +def stalled_estimator(span: float = 10.0) -> RateEstimator: + """A window fed the steady 2 s cadence at exactly 50 tok/s.""" + r = RateEstimator(span=span) + for i in range(6): + r.rate(GUFO_TOKENS, 100.0 * i, now=1_000.0 + 2.0 * i) + return r + + +def test_a_catch_up_poll_after_a_stall_answers_na_not_a_spike() -> None: + """A fetch that takes 15 s and the loop's 0.2 s catch-up floor must not make a rate. + + `poller.Job._loop` sleeps `max(0.2, interval - elapsed)`, so any poll slower than the + interval is followed by a second one ~0.2 s later. Whatever the stalled fetch carried is + then diffed over that 0.2 s. + """ + r = stalled_estimator() # steady 50 tok/s, last sample (1010.0, 500.0) + # the stalled fetch: 15.25 s after the last sample, but its body was generated ~2 s in, + # so it carries 114 tokens and gets stamped 13 s later than the engine measured them + diluted, _window = r.rate(GUFO_TOKENS, 614.0, now=1_025.25) + assert diluted == pytest.approx(7.47, abs=0.01) # the diluted row the DB actually holds + # the catch-up: 0.22 s later, the current counter (713 tokens of stall still unaccounted) + rate, window = r.rate(GUFO_TOKENS, 1_327.0, now=1_025.47) + assert window == pytest.approx(0.22) + assert rate is None, "a 0.22 s window is not a rate over the span; it must be n/a" + + +def test_a_stalled_sample_does_not_inflate_the_polls_after_it() -> None: + """The tail is as wrong as the spike: base[0] is the stale sample for the next ~span.""" + r = stalled_estimator() + r.rate(GUFO_TOKENS, 614.0, now=1_025.25) + r.rate(GUFO_TOKENS, 1_327.0, now=1_025.47) + # back to a normal 2 s cadence and the same honest 50 tok/s + tail = [r.rate(GUFO_TOKENS, 1_327.0 + 100.0 * n, now=1_029.47 + 2.0 * (n - 1))[0] + for n in range(1, 5)] + assert all(v is None or v <= 60.0 for v in tail), f"inflated tail, real rate is 50: {tail}" + + +# --- phase 13 task 03: both window bounds and the confirmed jump ------------- # + +def test_window_below_the_floor_answers_na_with_its_honest_label() -> None: + """A 0.4 s catch-up gap is not a measurement over a 10 s span; it labels itself.""" + r = RateEstimator(span=10) + r.rate("x", 100.0, now=0.0) + rate, window = r.rate("x", 103.0, now=0.4) + assert rate is None + assert window == pytest.approx(0.4), "the n/a still names the window it refused" + # the refused base was dropped: the next normal poll measures from the catch-up sample + rate, window = r.rate("x", 203.0, now=4.4) + assert rate == pytest.approx(25.0) + assert window == pytest.approx(4.0) + + +def test_fallback_base_older_than_two_spans_answers_na_and_recovers() -> None: + """A 20-second window on a 10-second panel is a different claim — one n/a, then normal.""" + r = RateEstimator(span=10) + r.rate("x", 0.0, now=0.0) + assert r.rate("x", 50.0, now=30.0) == (None, None) + rate, window = r.rate("x", 100.0, now=32.0) + assert rate == pytest.approx(25.0) + assert window == pytest.approx(2.0), "the next poll re-bases on the fresh sample" + + +def test_a_fallback_inside_the_documented_ceiling_still_publishes() -> None: + """The stall row the DB actually holds (15.25 s ≤ 2 x 10 s) keeps its diluted rate.""" + r = RateEstimator(span=10) + r.rate("x", 0.0, now=0.0) + rate, window = r.rate("x", 100.0, now=15.25) + assert rate == pytest.approx(100.0 / 15.25) + assert window == pytest.approx(15.25) + + +def test_two_samples_at_the_same_instant_are_not_a_rate() -> None: + r = RateEstimator(span=10) + r.rate("x", 100.0, now=5.0) + assert r.rate("x", 200.0, now=5.0) == (None, None) + + +def test_a_jump_is_held_for_one_poll_and_publishes_when_confirmed() -> None: + """A real step-up survives one more interval and then prints — one poll late. + + The confirming window is anchored on the sample 10 s back, which has moved on from the one + the held jump was measured from — that is what makes it a second measurement. The other + half is `test_a_held_jump_is_not_confirmed_by_a_window_on_the_same_anchor`. + """ + r = RateEstimator(span=10) + r.rate("x", 0.0, now=0.0) + r.rate("x", 100.0, now=10.0) # 10 tok/s published + rate, window = r.rate("x", 900.0, now=20.0) # 80 tok/s = 8x: held + assert rate == pytest.approx(10.0) + assert window == pytest.approx(10.0), "the held value keeps the Δ label it was published with" + rate, window = r.rate("x", 1_720.0, now=30.0) # 82 tok/s: the step is real — publish + assert rate == pytest.approx(82.0) + assert window == pytest.approx(10.0) + + +def test_an_unconfirmed_jump_is_dropped_and_the_held_value_keeps_printing() -> None: + """A one-interval artifact never publishes; the cell kept printing through it anyway.""" + r = RateEstimator(span=10) + r.rate("x", 0.0, now=0.0) + r.rate("x", 100.0, now=10.0) # 10 tok/s + assert r.rate("x", 900.0, now=20.0)[0] == pytest.approx(10.0) # 80: held + assert r.rate("x", 1_005.0, now=30.0)[0] == pytest.approx(10.0) # 10.5: artifact dropped + rate, _ = r.rate("x", 1_105.0, now=40.0) # ordinary publishing resumes + assert rate == pytest.approx(10.0) + + +def test_a_held_jump_is_not_confirmed_by_a_window_on_the_same_anchor() -> None: + """A stalled fetch stamps its body late, and that body then anchors the whole span. + + Measured on live gufo at 2026-10-19 12:42 (`data/logs/NOTES-decode-stable.md`'s window, + one request running at ~50 tok/s): the process stalled 11 s, the first reply back carried + the counter gufo had rendered ~2 s into it, and the estimator diffed the rest of the stall + over a 3 s window — 232 tok/s — then re-diffed **the same delta** over 5, 7 and 9 s + (146, 114, 95) because the stale sample stayed `inside[0]`. Measured twice from one sample + is one measurement, so the confirmation gate requires a window anchored on a different + sample; the hold simply continues until the stale anchor ages out of the span. + """ + r = RateEstimator(span=10.0) + tokens = lambda t: 45.0 * t # noqa: E731 the engine: a steady 45 tok/s + polls = [(0.0, tokens(0.0)), + (11.0, tokens(2.2)), # the stalled reply: honest body, 9 s-old stamp + (12.0, tokens(12.0)), # the catch-up: ≥8x the diluted row, so it is held + (14.0, tokens(14.0)), # …and every poll still anchored on that stale sample + (16.0, tokens(16.0)), + (18.0, tokens(18.0)), + (20.0, tokens(20.0)), + (22.0, tokens(22.0)), # the anchor has aged out; one poll still denies the jump + (24.0, tokens(24.0)), + (26.0, tokens(26.0))] + published = [r.rate("g", value, now=1_000.0 + t)[0] for t, value in polls] + assert all(v is None or v <= 60.0 for v in published), \ + f"a 45 tok/s engine printed {max(v for v in published if v is not None):.0f}: {published}" + assert published[-1] == pytest.approx(45.0), "the stall ends in the truth, not a tail" + + +def test_a_rise_from_zero_is_never_held() -> None: + """The first work an idle engine does must print on its own poll, not one late.""" + r = RateEstimator(span=10) + r.rate("x", 0.0, now=0.0) + r.rate("x", 0.0, now=10.0) # 0 tok/s published + assert r.rate("x", 500.0, now=20.0)[0] == pytest.approx(50.0) + + +def test_confirmation_state_is_per_key() -> None: + """One key's pending jump must not hold another key's numbers.""" + r = RateEstimator(span=10) + r.rate("busy", 0.0, now=0.0) + r.rate("busy", 100.0, now=10.0) # 10 tok/s — a jump base exists + r.rate("idle", 5.0, now=0.0) + r.rate("idle", 5.0, now=10.0) # 0 tok/s — previous is 0, nothing to confirm against + assert r.rate("busy", 900.0, now=20.0)[0] == pytest.approx(10.0) + assert r.rate("idle", 505.0, now=20.0)[0] == pytest.approx(50.0) + + +def test_a_counter_restart_clears_the_confirmation_state() -> None: + """A restarted counter has no previous lineage to be judged against.""" + r = RateEstimator(span=10) + r.rate("x", 0.0, now=0.0) + r.rate("x", 100.0, now=10.0) # 10 tok/s + assert r.rate("x", 10.0, now=20.0) == (None, None) # restart drops the window + rate, _ = r.rate("x", 910.0, now=30.0) # 90 tok/s on the new counter + assert rate == pytest.approx(90.0) + + +def test_forget_drops_the_judgement_state_with_the_window() -> None: + r = RateEstimator(span=10) + r.rate("host:x", 0.0, now=0.0) + r.rate("host:x", 100.0, now=10.0) + r.rate("host:x", 900.0, now=20.0) # held: one pending jump, one published rate + assert r._published and r._pending + assert r.forget("host:") == 1 + assert r._published == {} and r._pending == {} + + def test_counter_reset_yields_none_once_then_recovers() -> None: """A restart must not report a huge negative — or, worse, a huge positive — rate.""" r = RateEstimator(span=60)