diff --git a/.agents/phases/todo/06_system_page_gpu_verdicts/00_phase.md b/.agents/phases/complete/06_system_page_gpu_verdicts/00_phase.md similarity index 100% rename from .agents/phases/todo/06_system_page_gpu_verdicts/00_phase.md rename to .agents/phases/complete/06_system_page_gpu_verdicts/00_phase.md diff --git a/.agents/phases/todo/06_system_page_gpu_verdicts/01_drop_gpu_severity_verdicts.md b/.agents/phases/complete/06_system_page_gpu_verdicts/01_drop_gpu_severity_verdicts.md similarity index 100% rename from .agents/phases/todo/06_system_page_gpu_verdicts/01_drop_gpu_severity_verdicts.md rename to .agents/phases/complete/06_system_page_gpu_verdicts/01_drop_gpu_severity_verdicts.md diff --git a/.agents/phases/todo/06_system_page_gpu_verdicts/02_gpu_temp_on_hero_row.md b/.agents/phases/complete/06_system_page_gpu_verdicts/02_gpu_temp_on_hero_row.md similarity index 100% rename from .agents/phases/todo/06_system_page_gpu_verdicts/02_gpu_temp_on_hero_row.md rename to .agents/phases/complete/06_system_page_gpu_verdicts/02_gpu_temp_on_hero_row.md diff --git a/.agents/phases/todo/06_system_page_gpu_verdicts/03_collapse_host_history_charts.md b/.agents/phases/complete/06_system_page_gpu_verdicts/03_collapse_host_history_charts.md similarity index 100% rename from .agents/phases/todo/06_system_page_gpu_verdicts/03_collapse_host_history_charts.md rename to .agents/phases/complete/06_system_page_gpu_verdicts/03_collapse_host_history_charts.md diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.err b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.md b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.md new file mode 100644 index 0000000..4c50a81 --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.md @@ -0,0 +1,20 @@ +All criteria verified and the project is fully green. + +**Verified (final pass — all 3 phase-06 tasks were already in `complete/`)** +- `classify("gpu.busy_pct", 99)` / `gpu.mem_used_pct` → `None`; `THRESHOLDS` has no `gpu.*` row; `alerts.py`/`api.py` carry no GPU verdict; live browser: deskwork GPU at **91 %** and **98 %** renders `class="hero"`, `color=rgb(22,25,31)` (plain text), no bar, no `.sev-*`. +- GPU temp on the hero `.sub` from live agents: `Radeon 8060S Graphics · VRAM 165.1 MiB / 512.0 MiB · pool 78% · 48 °C`, R9700 `· 40 °C`, gateway prints none (no `n/a`, no `0 °C`). +- Charts collapsed by default: fresh load = 3 panels / 3 toggles / **0 open / 0 ``**; expand → 3 SVGs (510 px wide), repaints across polls (legend 3.3 %→5.6 %), collapsed siblings keep their own height (803 vs 358/339). +- `gpu.busy_pct@0` still answers from `/api/system/history` (345 points) — verdict gone, data intact. + +**Tests / lint / coverage** +- `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov --cov-fail-under=94` → **550 passed, 8 skipped, 97.91 %**; `uv run ruff check src tests agent` → clean; `uv run pyright` → 0 errors; `node --check` all scripts clean; `uv pip list --outdated` empty. +- `uv run pytest -m live --live` → **8 passed** (real agents/engines/upsd); dev server restarted, `/api/health` all sources `ok`. +- New screenshots: `data/logs/phase06-final-collapsed-default.png`, `data/logs/phase06-final-expanded-gpu-busy-uncoloured.png`. + +**Completion criteria:** classify→None ✅ · pytest+94 % gate ✅ · ruff+pyright ✅ · validate.sh ✅ · screenshots of the three behaviours on live data ✅ · AGENTS.md severity section claims no GPU rows ✅ + +**Docs:** fixed 3 items — (1) `AGENTS.md` pytest count stale (546 → 550); (2) `app.js` `severityFor` docstring still claimed "`page-ups.js` still calls it (phase 05 converts it)", contradicting the code/tests/AGENTS.md (comment-only); rest of README/AGENTS reviewed in full (env table vs `config.py`+`.env.example`, agent routes incl. `/schema`, alerts rules, page descriptions, contracts) — consistent. + +**Notable:** fixed a real pre-existing defect in `scripts/dev-server.sh` — `stop` slept 1 s and deleted the pidfile without waiting for exit, so `restart` with a browser open overlapped two servers on one SQLite file, producing a `database is locked` store refusal that `/system` rendered as a red event. Now waits (bounded 20 s) then SIGKILLs; verified a restart with live polling logs no such error/event. No live infra config was changed. + +**Next pending phase:** `.agents/phases/todo/07_llm_monitor_style_pages` diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.md.status b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.validate b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.validate new file mode 100644 index 0000000..0045e4e --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__00_phase.a1.validate @@ -0,0 +1,47 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 12%] +........................................................................ [ 25%] +........................................................................ [ 38%] +.......................................................ssssssss......... [ 51%] +........................................................................ [ 64%] +........................................................................ [ 77%] +........................................................................ [ 90%] +...................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 166 0 100% +src/dashboards/api.py 193 0 100% +src/dashboards/app.py 107 6 94% 50-51, 56, 58, 60, 127 +src/dashboards/collectors.py 142 5 96% 97-98, 102-103, 120 +src/dashboards/config.py 113 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 243-244 +------------------------------------------------------------------ +TOTAL 2158 45 98% +Required test coverage of 94% reached. Total coverage: 97.91% +550 passed, 8 skipped, 1 warning in 50.39s +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.err b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.md b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.md new file mode 100644 index 0000000..eb639fe --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.md @@ -0,0 +1,17 @@ +**Task 01 complete — GPU usage/VRAM no longer carry a verdict.** + +**Implemented** +- `severity.py`: deleted `gpu.busy_pct` / `gpu.mem_used_pct` rows from `THRESHOLDS` (no verdict at any level; `classify` → `None`), with a comment recording why. +- `sources/host.py`: dropped the per-GPU `severity` dict (samples/series untouched); `alerts.py`: removed `_GPU_CELLS`, the GPU alert loop, and the two `_DISPLAY` GPU entries (permanently unreachable once the table row is gone — a busy GPU now yields zero alerts); `api.py`: `_fleet_severity` no longer emits the always-null `gpu_busy_pct_avg` (matches the task's "remove the empty dict" rule for host rows; the numeric `gpu_busy_pct_avg` field stays). +- `page-system.js`: GPU hero built with no `severity`, no `bar`, no `hero-bar`, `data-sev` hook removed. +- Contract pins: `card_contract.py` pins the *absence* of `gpus[].severity` on every GPU row (like `network`), the two verdict paths removed from `GPU_METRICS`; live fleet contract follows automatically. + +**Tests / lint (all green)** +- `uv run pytest` → **544 passed, 8 skipped**; `--cov` → 97.91% (gate 94%; severity/alerts/api 100%, host.py 99%). +- `uv run ruff check src tests agent` clean · `uv run pyright` 0 errors · `bash .agents/validate.sh` exit 0. +- `uv run pytest -m live --live` → **8 passed** (real agents + `gpu.busy_pct@0` still answers from history, 97 %/100 %-busy data intact). +- Browser: `data/logs/system-01-gpu-no-verdict-uncoloured.png` — driveripper R9700 at **97 %** renders plain dark text, no bar; gateway's no-GPU state unchanged. + +**Docs**: AGENTS.md (severity bullet: no `gpu.*` row, verdict-≠-data rule; `/system` GPU-slot prose; testing-policy line; 544-count) and README.md (severity + hero descriptions) updated. + +**Next pending task**: `06_system_page_gpu_verdicts/02_gpu_temp_on_hero_row.md`. diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.md.status b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.validate b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.validate new file mode 100644 index 0000000..1f9d856 --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__01_drop_gpu_severity_verdicts.a1.validate @@ -0,0 +1,47 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 13%] +........................................................................ [ 26%] +........................................................................ [ 39%] +..................................................ssssssss.............. [ 52%] +........................................................................ [ 65%] +........................................................................ [ 78%] +........................................................................ [ 91%] +................................................ [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 166 0 100% +src/dashboards/api.py 193 0 100% +src/dashboards/app.py 107 6 94% 50-51, 56, 58, 60, 127 +src/dashboards/collectors.py 142 5 96% 97-98, 102-103, 120 +src/dashboards/config.py 113 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 243-244 +------------------------------------------------------------------ +TOTAL 2158 45 98% +Required test coverage of 94% reached. Total coverage: 97.91% +544 passed, 8 skipped, 1 warning in 48.66s +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.err b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.md b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.md new file mode 100644 index 0000000..c0c437c --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.md @@ -0,0 +1,19 @@ +**Task 02 complete — GPU temperature is on the hero row.** + +**Implemented** +- `page-system.js` `gpuSub()`: appends `· °C` from `gpus[].temp_c` (the agent's max over that card's hwmon labels) — no new field, no new metric name; the bit is omitted entirely when the sensor is unreadable (never `n/a`, never `0 °C`). "More detail" GPU table untouched (still carries the per-label Temp column). +- No backend change: `sources/host.py` already publishes `temp_c` and the `gpu.temp_c@N` sample. +- Contract: `tests/card_contract.py` `GPU_METRICS` now pins `temp_c` (present, may be `None`), so a rename fails the hermetic + live contracts. +- Docs: AGENTS.md (`/system` GPU-slot prose incl. the no-sensor rule and "promotion, not de-duplication"; pytest count 544→546) and README.md (hero footnote + "More detail" contents). + +**Tests / lint / coverage (all green)** +- `uv run pytest` → **546 passed, 8 skipped** (new: `test_gpu_hero_footnote_prints_the_temperature_the_agent_read` on served bytes; `test_a_gpu_with_no_temperature_input_reports_none_not_zero`; max-over-labels pin). +- `uv run pytest --cov --cov-fail-under=94` → **97.91%**, gate reached. +- `uv run ruff check src tests agent` clean · `uv run pyright` 0 errors · `node --check` all page scripts clean · `bash .agents/validate.sh` **exit 0**. +- `uv run pytest -m live --live` → **8 passed** (hero-card contract now includes `gpus[].temp_c` on the real agents). + +**Browser (live data)** +- `data/logs/system-gpu-temp-hero.png` — deskwork GPU at **99 %** renders uncoloured with `… pool 78% · 68 °C`; driveripper R9700 `… pool 32% · 41 °C`; gateway prints no temperature (`No GPU reported by this agent`). +- `data/logs/system-gpu-detail-table-kept.png` — GPU detail table still shows its Temp column (41 °C). + +**Next pending task**: `06_system_page_gpu_verdicts/03_collapse_host_history_charts.md`. diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.md.status b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.validate b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.validate new file mode 100644 index 0000000..b7917bb --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__02_gpu_temp_on_hero_row.a1.validate @@ -0,0 +1,47 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 12%] +........................................................................ [ 25%] +........................................................................ [ 38%] +...................................................ssssssss............. [ 51%] +........................................................................ [ 64%] +........................................................................ [ 77%] +........................................................................ [ 90%] +.................................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 166 0 100% +src/dashboards/api.py 193 0 100% +src/dashboards/app.py 107 6 94% 50-51, 56, 58, 60, 127 +src/dashboards/collectors.py 142 5 96% 97-98, 102-103, 120 +src/dashboards/config.py 113 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 243-244 +------------------------------------------------------------------ +TOTAL 2158 45 98% +Required test coverage of 94% reached. Total coverage: 97.91% +546 passed, 8 skipped, 1 warning in 48.93s +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.err b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.md b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.md new file mode 100644 index 0000000..c2f16f3 --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.md @@ -0,0 +1,18 @@ +**Implemented (task 03 — collapse per-host history charts)** +- `.host-charts` now sits inside a collapsed `
`; summary reads `Charts` + a window chip (`state.rangeKey`, kept in sync by the page's range control). +- Chart construction moved into `drawHostCharts()`, the single gate that returns early when the stack is shut — no `LineChart`, no SVG, no pixels while closed; instances built on first expand only. +- `renderHostCharts()` still issues **one** `/api/system/history` request per host per refresh naming every series; the response is cached per host (`{range, response}`) so the first open paints instantly (browser: 0 extra requests on expand), and a response for a window the operator left is filed, not drawn under the new label. +- `chartsToggled()` paints from cache on open, re-fetches on window mismatch; `pruneHosts()` now also drops `state.payloads`/`state.history` (toggle is a card child, so a dead host leaves no orphan control). +- Quiet `details.charts-box` summary in `dash.css` (caret shape for open/closed, `--text-faint`/`--text-muted` only, no `.sev-*`, no new tokens). +- Extra: `#hosts` gained the existing `.grid.items-start` — an expanded panel otherwise stretched collapsed siblings into blank cards (same defect phase 05 fixed on `/llm`); pinned by its own test. + +**Tests / lint / coverage** +- `uv run pytest`: **550 passed, 8 skipped** (4 new served-bytes tests in `tests/test_api.py`: collapse + no `open`, lazy build/paint gating, prune removes toggle, quiet-disclosure CSS, grid stretch). +- `uv run pytest --cov --cov-fail-under=94`: **97.91%** (Python untouched by this task; JS-only change). +- `uv run ruff check src tests agent` clean · `uv run pyright` 0 errors · `node --check` all static scripts OK. +- `uv run pytest -m live --live`: 8 passed · `uv pip list --outdated`: empty · `bash .agents/validate.sh`: **EXIT=0**. +- Browser (live, dev server restarted, `/api/health` all ok): `data/logs/system-charts-collapsed-default.png` (3 hosts, hero rows only, 0 ``), `system-charts-expanded.png` (3 charts drawn, siblings not stretched), `system-charts-open-repaint.png` (repaint + `24h`/`6h` window change while open; `pruneHosts()` verified live: toggles 3→2). + +**Docs**: `README.md` `/system` paragraph and `AGENTS.md` "Charts belong to the host they chart" updated (collapse, lazy paint, unchanged request discipline, `.grid.items-start`, prune takes the toggle). + +**Next pending task**: phase 06 is complete with this file; next is `.agents/phases/todo/07_llm_monitor_style_pages/01_slots_schema_and_table_removal.md`. diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.md.status b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.validate b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.validate new file mode 100644 index 0000000..5531ece --- /dev/null +++ b/.agents/reports/06_system_page_gpu_verdicts/06_system_page_gpu_verdicts__03_collapse_host_history_charts.a1.validate @@ -0,0 +1,47 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 12%] +........................................................................ [ 25%] +........................................................................ [ 38%] +.......................................................ssssssss......... [ 51%] +........................................................................ [ 64%] +........................................................................ [ 77%] +........................................................................ [ 90%] +...................................................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 166 0 100% +src/dashboards/api.py 193 0 100% +src/dashboards/app.py 107 6 94% 50-51, 56, 58, 60, 127 +src/dashboards/collectors.py 142 5 96% 97-98, 102-103, 120 +src/dashboards/config.py 113 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 243-244 +------------------------------------------------------------------ +TOTAL 2158 45 98% +Required test coverage of 94% reached. Total coverage: 97.91% +550 passed, 8 skipped, 1 warning in 48.86s +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/AGENTS.md b/AGENTS.md index 7cd4674..5082904 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,10 +160,14 @@ Responsibilities, one line each: `classify()` → `ok`/`warn`/`fail`/`None`, plus `attach()`/`with_severity()` to put the verdict beside a value. No measurement, no verdict: `None` in → `None` out, never a green pass for an unknown. The verdicts travel **additively** in payloads — `cpu.severity`, - `memory.severity`, `disks.severity`, `gpus[].severity`, each UPS card's `severity` - (charge/load/runtime), `severity` inside rated LLM cells (KV pressure, classified on a + `memory.severity`, `disks.severity`, each UPS card's `severity` (charge/load/runtime), + `severity` inside rated LLM cells (KV pressure, classified on a percent scale because the engines publish a 0-1 ratio), and `severity` per host/fleet/UPS row in `/api/summary`. Pages render the class; they never compare a number to a threshold. + There is **no `gpu.*` row and no `gpus[].severity`** (phase 06): these are LLM servers, + a busy GPU is the job at every level, so the GPU card and the fleet GPU column render + uncoloured — like `net.*`, whose colourlessness predates it. Removing the verdict is not + removing the data: `gpu.busy_pct@N` / `gpu.mem_used_pct@N` are still stored and charted. - `alerts.py` — `build(settings, job_states, host_payloads, engine_payloads, ups_payload, recent_events)` → the "what is wrong right now" list `/api/summary` serves as `alerts` (`{id, severity, source, message, href, at}`, fail first then warn, then by source). A @@ -250,7 +254,7 @@ cp .env.dev.example .env.dev # dev-only: ssh tunnels + live targets ./scripts/dev-server.sh status # per-source ok/DOWN ./scripts/dev-server.sh logs -uv run pytest # hermetic: 543 passed + 8 live-skipped, no network +uv run pytest # hermetic: 550 passed + 8 live-skipped, no network uv run pytest --cov # coverage gate (97.9%; floor 94%) uv run pytest -m live --live # opt-in: real fleet / engines / upsd uv run ruff check src tests agent && uv run pyright @@ -287,8 +291,9 @@ branch `main`); there is no CI on it, so `bash .agents/validate.sh` is the only asserts it against the three captures (discrete GPU, APU, no GPU) and over HTTP, and `test_every_host_serves_the_hero_card_contract` asserts the same table against the live agents — so a rename fails a test, and a card that shows `n/a` cannot be "fixed" by - inventing a field. `network` carries no `severity` on purpose (throughput has no good/bad), - and the contract pins its absence. `tests/ups_contract.py` is the same device for the `/ups` + inventing a field. `network` carries no `severity` on purpose (throughput has no good/bad) + and neither does a GPU row (phase 06: a busy GPU is the job on an LLM server) — the + contract pins both absences. `tests/ups_contract.py` is the same device for the `/ups` hero row and battery glyph (`HERO_FIELDS`, `UPS_SHAPES`, `assert_ups_hero_contract`): `tests/test_ups_contract.py` drives it from `tests/fixtures/nut_.vars` — verbatim `LIST UPS`/`LIST VAR`/`LIST CMD` replies from the two real UPSes — and @@ -414,10 +419,18 @@ Colour comes from the payload's `severity` dicts and nowhere else; a missing num verdict — gateway's CPU temp, any host's absent GPU — prints `n/a` **uncoloured**. NET is one slot with two lines (`.hero-lines`, `↓` in / `↑` out) and is deliberately colourless: throughput has no good/bad, so `network` carries no severity. The GPU slot holds one hero -per card (model, VRAM used/total and the honest APU `pool` view in the `.sub`), and -everything not glance-worthy — load averages, clock, disk read/write/busy, uptime, GPU -power/temp, filesystems, processes — folds into a collapsed "More detail" `
`: a -simpler surface, not a less complete page. +per card whose `.sub` carries model, VRAM used/total, the honest APU `pool` view and the +card's temperature (`gpus[].temp_c` — the agent's max over that card's hwmon labels, the +same value stored as `gpu.temp_c@N`, so the hero and the history cannot disagree), and it is +colourless for the same reason (phase 06): a 95 %-busy card on an LLM server is the job, so +the hero carries no `severity` and no utilisation bar — a bar under a verdictless number +would be a verdict by shape. A card whose driver exposes no temperature input prints **no** +temperature in that footnote rather than `n/a` or `0 °C`: "where you can" is the operator's +wording, and `n/a` is reserved for numbers the card is supposed to have. Everything not +glance-worthy — load averages, clock, disk read/write/busy, uptime, GPU power and the +per-label temp breakdown, filesystems, processes — folds into a collapsed "More detail" +`
`: a simpler surface, not a less complete page (the table keeps its Temp column; +this was a promotion, not a de-duplication). A `Δ` label is a measurement, never a config value: `sources/host.py` sends the window the estimator actually used as the payload's `window_seconds`, and `/system`'s footnote prints @@ -453,14 +466,28 @@ panel and the host/metric detail picker are deleted, along with the `#chart-*` / the `select` / `label.field` styling only those pickers used. Each panel stacks its own `.host-charts` — utilisation (CPU, memory, worst disk), network (in/out), then GPU busy — where the GPU slot is created and destroyed with `host.gpus`, so gateway shows neither a phantom card -nor a hole. **One `/api/system/history` request per host per refresh** +nor a hole. **That stack is collapsed behind a per-host `Charts` `
`** (phase 06: "only +visible if expanded… there will be lots of hosts"): the default `/system` view is the hero row, +the summary carries the window label the stack will draw (the chips themselves stay in the page +head, one control per page), and `drawHostCharts()` is the single place that decides whether to +paint — so nothing is built or drawn while the stack is shut (`LineChart` inside a +`display:none` container measures 0 px, and rebuilding three charts per host every poll for +plots nobody sees is the waste this removes). The **history request is deliberately unchanged**: +one per host per refresh, naming every series the panel could draw, and its response is cached +per host so the first expand paints from data in hand rather than waiting a poll; a cached +response from another window is re-fetched instead of being drawn under the wrong label. +Expanding one panel must not stretch its collapsed siblings into blank cards, so `#hosts` +carries the existing `.grid.items-start`. **One `/api/system/history` request per host per refresh** (`hosts=&metrics=&range=…`, pinned by -`test_every_system_chart_is_drawn_inside_its_own_host_panel`): unknown metrics come back as +`test_every_system_chart_is_drawn_inside_its_own_host_panel` and the collapse by +`test_system_history_charts_are_collapsed_behind_a_toggle`): unknown metrics come back as empty series on purpose, so a panel asks for every series it could draw and asks once. A legend here names metrics (or one GPU's model), never hosts — the panel heading already says who this is, and a legend that grew a second host is the bug this replaced. A host that stops reporting -loses its panel *and* its charts (`pruneHosts`), never a chart of nothing. Per-core visuals are -gone from `/system` and `renderCores` is **deleted** from `charts.js` with its last caller. +loses its panel, its `Charts` toggle *and* its charts (`pruneHosts` deletes the card plus the +payload/chart/history maps), never a chart of nothing or an orphan control in the grid. Per-core +visuals are gone from `/system` and `renderCores` is **deleted** from `charts.js` with its last +caller. `drawHostChart()` shades a chart only when one series has points — two or three filled areas stack into one block and bury the quieter line, which is how a idle CPU would hide behind a busy memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryable from diff --git a/README.md b/README.md index 1e342ca..0aa3ccd 100644 --- a/README.md +++ b/README.md @@ -139,12 +139,14 @@ store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── pruned, so a year of history is a few hundred MB rather than a disk emergency. * `severity.py` — the single green/amber/red table (`THRESHOLDS`, plus `INVERTED` for the metrics where lower is worse). Rated values in the API payloads travel with a sibling - `severity` (`cpu.severity.usage_pct`, `memory.severity`, `disks.severity`, - `gpus[].severity`, a UPS card's `severity.charge_pct`, `severity` inside the rated LLM + `severity` (`cpu.severity.usage_pct`, `memory.severity`, `disks.severity`, a UPS card's + `severity.charge_pct`, `severity` inside the rated LLM cells such as KV pressure, `/api/summary` rows), computed server-side, so every page colours the same way and a value nobody reported renders `n/a` instead of green. A metric published as a 0-1 ratio is rescaled *for the verdict only* — the number printed stays - what the upstream published. + what the upstream published. There is no `gpu.*` row: these are LLM servers, a busy GPU + is the job, so the GPU card (and the fleet GPU column) renders uncoloured at every + utilisation — the `gpu.*` history series are still stored and charted. * `alerts.py` — "what is wrong right now", as data. `alerts.build(settings, job_states, host_payloads, engine_payloads, ups_payload, recent_events)` is a pure projection of the state the poller already keeps plus the verdicts `severity.py` and the UPS health checks @@ -171,18 +173,27 @@ store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── charging, dashed outline on battery, a dash through the body when there is no reading) and a `` / `aria-label` that names it ("apc: Charge 100 percent, on line") unless the number beside it already does. Each `/system` host is one panel led - by exactly six of them — CPU %, MEM %, TEMP, DISK % (mount underneath), GPU busy (VRAM and - the APU pool in the footnote), and NET on two lines (↓ in / ↑ out, never coloured: a byte - rate has no good/bad). Under the numbers, **each panel charts only its own host**: utilisation + by exactly six of them — CPU %, MEM %, TEMP, DISK % (mount underneath), GPU busy (VRAM, the + APU pool and the card's temperature in the footnote — no temperature is printed when the + driver exposes no sensor, never `0 °C`; and likewise never coloured and with no bar: a busy + GPU is the job on an LLM server), and NET on two lines (↓ in / ↑ out, never coloured: a byte + rate has no good/bad). Under the numbers, **each panel charts only its own host** — behind a + collapsed **Charts** toggle, because the default view is current stats and a fleet of ten + hosts each drawing three plots is noise: the summary reads `▸ Charts` with the window in it + (`1h`, `24h` …), and opening it is what builds and paints the charts — instantly, from the + response the last refresh already fetched, and repainted on every refresh after that while it + stays open. Expanding one panel never stretches its collapsed siblings + (`.grid.items-start`). The stack is utilisation (CPU, memory, worst disk), network (↓ in / ↑ out), and GPU busy for a host that reports cards — three 120px SVG charts, each with its own legend and its own `aria-label` naming the host and the plotted metrics (`LineChart`'s `ariaLabel` option), never a second host in sight. The fleet-wide charts, the nine-figure fleet-totals panel and the host/metric detail picker are gone, and so are the per-core charts, while `cpu.usage_pct@coreN` stays in SQLite. Everything else — - load, clock, disk I/O, uptime, GPU power/temp, filesystems, processes — folds into a collapsed + load, clock, disk I/O, uptime, GPU power and the per-label temp breakdown, filesystems, + processes — folds into a collapsed "More detail" disclosure. Each panel pulls its series in **one** `/api/system/history` request - (`hosts=&metrics=…&range=…`), and a host that stops reporting takes its panel and - its charts off the page. The LLM page (`/llm`) is per engine too: two big + (`hosts=&metrics=…&range=…`) every refresh, stack open or not, and a host that stops + reporting takes its panel, its Charts toggle and its charts off the page. The LLM page (`/llm`) is per engine too: two big throughput numbers (decode, prompt — uncoloured, because 0 tok/s on an idle server is healthy) with their provenance tag (`Δ10.0s`, scraped, estimated), then a compact list of metric rows — sessions, queue, KV pressure (the only rated cell), prefix/spec percentages, diff --git a/scripts/dev-server.sh b/scripts/dev-server.sh index aee0cc4..2232198 100755 --- a/scripts/dev-server.sh +++ b/scripts/dev-server.sh @@ -39,8 +39,26 @@ start() { stop() { if running; then - kill "$(cat "$PIDFILE")" 2>/dev/null - sleep 1 + local pid + pid="$(cat "$PIDFILE")" + kill "$pid" 2>/dev/null + # Wait for the process to actually go away before handing the pidfile back. A bare + # `sleep 1` was not enough: uvicorn's graceful shutdown waits for in-flight requests, + # so `restart` with a browser open started the new server while the old one still + # held the SQLite file — the dying process then logged `database is locked` for a poll + # it had already fetched, which the app correctly turned into a red store-error event. + # Two servers on one DB is the one way this script can fabricate an incident, so the + # wait is bounded rather than optimistic, and SIGKILL is the last resort. + local i=0 + while kill -0 "$pid" 2>/dev/null; do + i=$((i + 1)) + if [ "$i" -gt 60 ]; then + echo "pid $pid ignored SIGTERM after 20s; sending SIGKILL" + kill -9 "$pid" 2>/dev/null + break + fi + sleep 0.25 + done echo "stopped" else echo "not running" diff --git a/src/dashboards/alerts.py b/src/dashboards/alerts.py index 0baa78b..c49ac06 100644 --- a/src/dashboards/alerts.py +++ b/src/dashboards/alerts.py @@ -71,23 +71,19 @@ _DISPLAY: dict[str, tuple[str, str, int]] = { "cpu.temp_c": ("CPU temperature", "°C", 0), "mem.used_pct": ("memory", "%", 0), "disk.used_pct": ("disk", "%", 0), - "gpu.busy_pct": ("GPU utilisation", "%", 0), - "gpu.mem_used_pct": ("GPU memory", "%", 0), "llm.kv_cache_usage_pct": ("KV cache", "%", 0), } #: host payload section -> (field, severity metric). Mirrors the `severity` sub-dicts that -#: `sources/host.py` attaches, so a new rated field is one row here. +#: `sources/host.py` attaches, so a new rated field is one row here. There is no GPU row: +#: these are LLM servers, a busy card is the job, so `severity.THRESHOLDS` has no `gpu.*` +#: entry and `sources/host.py` attaches no GPU verdict — nothing here may invent one. _HOST_CELLS: tuple[tuple[str, str, str], ...] = ( ("cpu", "usage_pct", "cpu.usage_pct"), ("cpu", "temp_c", "cpu.temp_c"), ("memory", "used_pct", "mem.used_pct"), ("disks", "worst_used_pct", "disk.used_pct"), ) -_GPU_CELLS: tuple[tuple[str, str], ...] = ( - ("busy_pct", "gpu.busy_pct"), - ("mem_used_pct", "gpu.mem_used_pct"), -) # --------------------------------------------------------------------------- # @@ -310,16 +306,6 @@ def _host_alerts(sink: _Sink, host: Mapping[str, Any], sink.add(verdict, name, metric, f"{name}: {_value(metric, number)}{where}{_trip(metric, verdict)}", HREF["system"], at) - for gpu in host.get("gpus") or []: - card = str(gpu.get("model") or f"GPU {gpu.get('index', 0)}") - for field, metric in _GPU_CELLS: - number = gpu.get(field) - verdict = (gpu.get("severity") or {}).get(field) - if number is None or verdict not in _RANK: - continue - sink.add(verdict, name, f"{metric}@{gpu.get('index', 0)}", - f"{name}: {_value(metric, number)} on {card}{_trip(metric, verdict)}", - HREF["system"], at) degraded = host.get("degraded") if degraded: # a /proc read failed on the box: the numbers that did come back are real, but the diff --git a/src/dashboards/api.py b/src/dashboards/api.py index e762fe6..9c1508d 100644 --- a/src/dashboards/api.py +++ b/src/dashboards/api.py @@ -96,11 +96,12 @@ def _host_severity(host: dict[str, Any]) -> dict[str, str | None]: def _fleet_severity(fleet: dict[str, Any]) -> dict[str, str | None]: """Fleet aggregates judged by the same rows as the per-host values — no fleet-only thresholds, so "the fleet is amber" means the same thing as one amber host.""" + # No GPU verdict, matching `sources/host.py`: the fleet average of a metric no host + # is graded on must not grow one here (phase 06 — busy GPUs are the job on LLM servers). return { "cpu_usage_pct_avg": classify("cpu.usage_pct", fleet.get("cpu_usage_pct_avg")), "memory_used_pct": classify("mem.used_pct", fleet.get("memory_used_pct")), "disk_worst_used_pct": classify("disk.used_pct", fleet.get("disk_worst_used_pct")), - "gpu_busy_pct_avg": classify("gpu.busy_pct", fleet.get("gpu_busy_pct_avg")), } diff --git a/src/dashboards/severity.py b/src/dashboards/severity.py index 8d6ccf1..1562240 100644 --- a/src/dashboards/severity.py +++ b/src/dashboards/severity.py @@ -42,14 +42,16 @@ FAIL = "fail" #: pages only where they had already drifted. The two UPS metrics are new: a battery at #: half charge or with ten minutes of runtime left is worth amber before it is worth red. #: Keys are the dotted metric names of the history namespace; ``@stream`` suffixes -#: (``disk.used_pct@/srv``, ``gpu.busy_pct@0``) inherit their base metric's row. +#: (``disk.used_pct@/srv``, ``cpu.usage_pct@core7``) inherit their base metric's row. +#: There is deliberately **no ``gpu.*`` row**: these are LLM servers, where a 95 %-busy +#: card is the job, not a fault. A busy-at-every-level metric has no honest amber, so the +#: table simply does not judge it — ``classify`` returns ``None`` and the page renders the +#: number uncoloured (the same rule ``net.*`` follows for throughput). THRESHOLDS: dict[str, tuple[float, float]] = { "cpu.usage_pct": (75.0, 90.0), "mem.used_pct": (80.0, 92.0), "disk.used_pct": (80.0, 90.0), "cpu.temp_c": (75.0, 90.0), - "gpu.busy_pct": (85.0, 95.0), - "gpu.mem_used_pct": (85.0, 95.0), "llm.kv_cache_usage_pct": (75.0, 90.0), "ups.charge_pct": (50.0, 20.0), "ups.battery_runtime": (600.0, 300.0), # seconds of runtime left diff --git a/src/dashboards/sources/host.py b/src/dashboards/sources/host.py index 43227e4..365c319 100644 --- a/src/dashboards/sources/host.py +++ b/src/dashboards/sources/host.py @@ -257,9 +257,9 @@ def parse_host_snapshot(name: str, snapshot: dict[str, Any], collector: HostColl "temps": temps, "temp_c": temp_c, "clocks": gpu.get("clocks") or {}, - # the verdict travels with the card; keys mirror the rated fields beside them - "severity": {"busy_pct": classify("gpu.busy_pct", gpu.get("busy_pct")), - "mem_used_pct": classify("gpu.mem_used_pct", pool_used_pct)}, + # deliberately no `severity` section here, unlike cpu/memory/disks (and like + # `network` below): these are LLM servers, a busy GPU is the job, and a + # verdict-less number is the honest render at every utilisation level. } gpus.append(row) for metric, value in (("busy_pct", row["busy_pct"]), ("vram_used_bytes", vram_used), @@ -368,7 +368,7 @@ def parse_host_snapshot(name: str, snapshot: dict[str, Any], collector: HostColl "rx_bytes_per_s": rx_total if any_net else None, "tx_bytes_per_s": tx_total if any_net else None, "interfaces": net_rows, - # Deliberately no `severity` section here, unlike cpu/memory/disks/gpus: a byte + # Deliberately no `severity` section here, unlike cpu/memory/disks: a byte # rate has no good/bad (900 MB/s is idle on 10 GbE, 2 KB/s can be an incident), # so `severity.THRESHOLDS` has no `net.*` row and a colour here would be a # verdict nobody measured. tests/test_system_contract.py pins its absence. diff --git a/src/dashboards/web/pages/system.html b/src/dashboards/web/pages/system.html index 003a3fb..e7b1ca0 100644 --- a/src/dashboards/web/pages/system.html +++ b/src/dashboards/web/pages/system.html @@ -44,7 +44,9 @@

Per host

agent-reported values; rates are measured over the Δ window in the footer, and each panel's charts cover only that host
-
+ +
-
-
-
-
+
+ Charts +
+
+
+
+
+
+
+
+
-
-
-
-
-
+
More detail
@@ -90,9 +105,26 @@ function hostCard(name) {
`; + // The chart stack ships collapsed (`TODO.md:4` — "only visible if expanded"), and the + // native `toggle` event is what turns the paint pass on. It fires on close too; a closed + // stack simply has nothing to draw, so collapsing costs the page nothing to undo. + card.querySelector('[data-k="charts"]').addEventListener('toggle', + () => chartsToggled(card, name)); + setChartsWindow(card); return card; } +/** The window the expanded stack draws, echoed inside the toggle. + * + * The window chips themselves stay in the page head — one control for every panel, one + * history request per host — so a panel opened a week into a session has to say for itself + * which window it is about to plot. The chip's text is the chips' own label (`state.rangeKey`), + * never a re-derived duration, so the toggle can never disagree with the control that set it. */ +function setChartsWindow(card) { + const chip = card.querySelector('[data-k="charts-window"]'); + if (chip) chip.textContent = state.rangeKey; +} + /** `200 / 253 GiB` — the percentage is the verdict, the bytes are the detail. */ function memorySub(mem) { const used = Dash.fmtBytes(mem.used_bytes, 1); @@ -102,26 +134,39 @@ function memorySub(mem) { return `${bytes(mem.used_bytes, 1)} of ${bytes(mem.total_bytes, 0)}`; } -/** Model, VRAM used / total, and the honest APU pool view: a Strix Halo's model lives in - * GTT, so "VRAM 3%" and "pool 78%" are both true and the card shows both, not whichever - * flatters the machine. */ +/** Model, VRAM used / total, the honest APU pool view, and the card's temperature: + * `Radeon AI PRO R9700 · VRAM 30.5 GB / 31.9 GB · 43 °C`. + * + * A Strix Halo's model lives in GTT, so "VRAM 3%" and "pool 78%" are both true and the card + * shows both, not whichever flatters the machine. + * + * The temperature rides here because it is the one number worth glancing at *while* a card + * is busy (phase 06: "Pull GPU temp for the system resources tab where you can"). It is + * `gpus[].temp_c`, the agent's own max over the card's hwmon labels — no new field, no new + * metric name, and the per-label table in "More detail" still carries the breakdown. + * A card whose agent cannot read a sensor (an APU build with no hwmon input) prints no + * temperature at all: `n/a` is for numbers the card is supposed to have, and `0 °C` would + * be a cool GPU that does not exist. */ function gpuSub(gpu) { const bits = [gpu.model || 'GPU']; bits.push(gpu.vram_used_bytes == null || gpu.vram_total_bytes == null ? 'VRAM n/a' : `VRAM ${bytes(gpu.vram_used_bytes, 1)} / ${bytes(gpu.vram_total_bytes, 1)}`); if (gpu.mem_used_pct != null) bits.push(`pool ${num(gpu.mem_used_pct, 0)}%`); + if (gpu.temp_c != null) bits.push(`${num(gpu.temp_c, 0)} °C`); return bits.join(' · '); } /** The GPU slot of the hero row: one hero per card, or the honest no-GPU state. A box - * with no card is not a box at 0% busy, and gateway must not look broken for it. */ + * with no card is not a box at 0% busy, and gateway must not look broken for it. + * No `severity` and no `bar` on purpose (phase 06): utilisation and VRAM carry no verdict + * on an LLM server, so the number renders uncoloured at 0 % and at 100 % alike. */ function renderGpuSlot(cell, gpus) { if (cell.dataset.count !== String(gpus.length)) { cell.innerHTML = ''; for (let i = 0; i < Math.max(gpus.length, 1); i += 1) { const row = document.createElement('div'); - row.className = 'hero hero-bar'; + row.className = 'hero'; cell.appendChild(row); } cell.dataset.count = String(gpus.length); @@ -135,13 +180,13 @@ function renderGpuSlot(cell, gpus) { Dash.hero(rows[i], { label: gpus.length > 1 ? `GPU ${i}` : 'GPU', value: gpu.busy_pct, unit: '%', digits: 0, - severity: (gpu.severity || {}).busy_pct, bar: true, sub: gpuSub(gpu), }); }); } -/** What the GPU hero left out, in "More detail": power, temperature, both memory pools. */ +/** What the GPU hero left out, in "More detail": power and both memory pools, plus the + * per-label temperature breakdown behind the single number in the hero's footnote. */ function renderGpuExtra(cell, gpus) { if (!gpus.length) { cell.innerHTML = ''; return; } cell.innerHTML = ` @@ -265,6 +310,12 @@ function stalenessText(row) { // // The charts live in a Map keyed by host name and are torn down with the card: a panel for // a host that has left the payload would otherwise keep drawing its last series forever. +// +// They are also **built** lazily, the first time a panel's toggle is opened. A collapsed +// `
` renders nothing, so a `LineChart` constructed inside one measures 0 px wide, +// and building 3 charts x N hosts on load is the work this page does for plots nobody is +// looking at. The history response is still fetched every refresh and kept per host, so the +// first open paints from data already in hand instead of waiting up to one poll. const CHART_HEIGHT = 120; const UTIL_SERIES = [ ['cpu.usage_pct', 'CPU'], @@ -367,23 +418,69 @@ function drawHostChart(chart, card, key, series, format) { DashCharts.renderLegend(card.querySelector(`[data-legend="${key}"]`), series, format, true); } -/** One host's charts, from one request. Called for every host on every refresh. */ -async function renderHostCharts(card, host) { +/** Is this panel's history stack open right now? */ +function chartsOpen(card) { + const box = card.querySelector('[data-k="charts"]'); + return !!(box && box.open); +} + +/** + * The operator opened (or closed) one panel's charts. + * + * Opening paints immediately from the last history response for that window, so the first + * expand is a chart and not an empty box until the next poll. If the cached response belongs + * to a different window (the chips changed while the stack was shut) it is re-fetched — one + * request for a click, never a plot of the wrong range under a label for another one. + */ +function chartsToggled(card, name) { + if (!chartsOpen(card)) return; + const host = state.payloads.get(name); + if (!host) return; // nothing polled this host yet; the next pass paints + const cached = state.history.get(name); + if (cached && cached.range === state.range) drawHostCharts(card, host, cached.response); + else renderHostCharts(card, host); +} + +/** + * The paint pass — and the only place a panel's `LineChart` instances are created. + * + * A closed toggle returns before anything is built or drawn: no instances, no SVG, no + * pixels. The one place that decides *whether* to paint, so the poll path and the toggle + * path cannot drift into drawing a collapsed panel by different routes. + */ +function drawHostCharts(card, host, response) { + if (!chartsOpen(card)) return; const charts = hostChartSet(card, host); + drawHostChart(charts.util, card, 'util', seriesFor(response, UTIL_SERIES), PCT_FORMAT); + drawHostChart(charts.net, card, 'net', seriesFor(response, NET_SERIES), RATE_FORMAT); + if (charts.gpu) { + const gpu = gpuStreams(host).map((s) => [s.metric, s.label]); + drawHostChart(charts.gpu, card, 'gpu', seriesFor(response, gpu), PCT_FORMAT); + } +} + +/** + * One host's history, from one request, on every refresh whether or not its charts are open. + * + * The request discipline is deliberately unchanged by the collapse: the query still names + * every series the panel could draw (utilisation, network, one GPU stream per card), so a + * host's XHR budget stays one per refresh and opening a toggle never has to discover series + * it should have asked for. What the collapse defers is the painting. + */ +async function renderHostCharts(card, host) { + const range = state.range; // the window this request actually asks for const gpu = gpuStreams(host); const wanted = UTIL_SERIES.concat(NET_SERIES, gpu.map((s) => [s.metric, s.label])); const query = `/api/system/history?hosts=${encodeURIComponent(host.name)}` + `&metrics=${encodeURIComponent(wanted.map(([metric]) => metric).join(','))}` + - `&range=${state.range}`; + `&range=${range}`; const response = await fetchJSON(query).catch(() => null); - if (!response || !state.charts.has(host.name)) return; // the panel went away mid-flight - - drawHostChart(charts.util, card, 'util', seriesFor(response, UTIL_SERIES), PCT_FORMAT); - drawHostChart(charts.net, card, 'net', seriesFor(response, NET_SERIES), RATE_FORMAT); - if (charts.gpu) { - drawHostChart(charts.gpu, card, 'gpu', - seriesFor(response, gpu.map((s) => [s.metric, s.label])), PCT_FORMAT); - } + if (!response || !state.hosts.has(host.name)) return; // the panel went away mid-flight + state.history.set(host.name, { range, response }); + // A window change re-runs this pass for every host, so a response that came back for the + // window the operator just left is filed and skipped rather than painted under the new + // label: an hour of samples drawn across a 30 d axis is a chart of the wrong thing. + if (range === state.range) drawHostCharts(card, host, response); } /** @@ -404,13 +501,22 @@ function windowLabel(values, fallback) { : `Δ${lo.toFixed(1)}–${hi.toFixed(1)}s`; } -/** Drop cards (and charts) for hosts that are no longer in the payload. */ +/** + * Drop cards for hosts that are no longer in the payload. + * + * Removing the card takes the collapsed `
` toggle and the chart stack with it (they + * are children of the panel, which is the point of building them per host), so a dead host + * cannot leave an orphan toggle in the grid. The three per-host Maps go the same way: a + * cached response or chart instance that outlives its panel is a machine still being drawn. + */ function pruneHosts(living) { Array.from(state.hosts.keys()).forEach((name) => { if (living.includes(name)) return; state.hosts.get(name).remove(); state.hosts.delete(name); state.charts.delete(name); + state.payloads.delete(name); + state.history.delete(name); }); } @@ -437,6 +543,7 @@ async function refresh() { state.hosts.set(host.name, card); $('#hosts').appendChild(card); } + state.payloads.set(host.name, host); // so opening a toggle can paint without re-fetching renderHostCard(card, host); // one history request per host, all of its charts fed from the same response chartJobs.push(renderHostCharts(card, host)); @@ -488,8 +595,11 @@ document.addEventListener('DOMContentLoaded', async () => { state.pollMs = pollMs; // the window chips re-fetch every panel's history: the range is per page, not per chart - Dash.rangeControl($('#range'), state.range, (seconds) => { + Dash.rangeControl($('#range'), state.range, (seconds, key) => { state.range = seconds; + state.rangeKey = key; + // every panel's toggle says which window it will draw, open or not + state.hosts.forEach((card) => setChartsWindow(card)); refresh(); }); diff --git a/tests/card_contract.py b/tests/card_contract.py index efcebe3..0fafd52 100644 --- a/tests/card_contract.py +++ b/tests/card_contract.py @@ -16,7 +16,7 @@ cpu % ``cpu.usage_pct`` (+ ``cpu.severity.usage_pct``) mem % ``memory.used_pct`` (+ ``memory.severity.used_pct``) cpu temp ``cpu.temp_c`` (+ ``cpu.severity.temp_c``) disk usage % ``disks.worst_used_pct`` + ``disks.worst_mount`` (+ verdict) -graphics card ``gpus[].model/busy_pct/vram_*/mem_used_pct`` (+ verdict) +graphics card ``gpus[].model/busy_pct/vram_*/mem_used_pct/temp_c`` (no verdict) network down/up ``network.rx_bytes_per_s`` / ``tx_bytes_per_s`` (no verdict) =========================== ========================================================== @@ -27,7 +27,16 @@ into its fleet row, so the card and the overview cannot drift apart. Deliberately *not* card metrics (legibility, not missing data — the series stay in SQLite): per-core usage (``cpu.usage_pct@coreN``), swap, the process table, the raw sensor list, and per-interface rows. Network carries no ``severity`` on purpose; throughput has no good/bad, -so ``severity.THRESHOLDS`` has no ``net.*`` row. +so ``severity.THRESHOLDS`` has no ``net.*`` row. GPU rows carry no ``severity`` either +(phase 06): these are LLM servers, a 95 %-busy card is the job and not a fault, so +``THRESHOLDS`` has no ``gpu.*`` row and the card prints its numbers uncoloured at every +utilisation — the *absence* of the dict is pinned below, the way ``network``'s is. + +``gpus[].temp_c`` is on the card because the number worth reading *while* a card is busy is +how hot it is (phase 06, TODO.md L9). It is the agent's **max over that card's hwmon +labels** — the same max the agent already computes and the same value stored as +``gpu.temp_c@N`` — so the hero prints a measurement rather than a new field, and the value +may honestly be ``None`` on a box whose driver exposes no temperature input. """ from __future__ import annotations @@ -54,8 +63,9 @@ GPU_METRICS: tuple[str, ...] = ( "vram_used_pct", # the honest APU view: vram + GTT, because a Strix Halo keeps the model in GTT "mem_used_pct", - "severity.busy_pct", - "severity.mem_used_pct", + # the hero's footnote: the agent's max over this card's hwmon labels (None is legal — + # a driver with no temperature input must not be rendered as 0 °C) + "temp_c", ) #: `network` must never grow a `severity` dict (see the module docstring). @@ -95,6 +105,10 @@ def assert_card_contract(payload: Any) -> None: for gpu in gpus: for path in GPU_METRICS: dig(gpu, path) + assert "severity" not in gpu, ( + "gpus[].severity would grade utilisation on a fleet built to be busy — " + "severity.THRESHOLDS has no gpu.* row and a busy GPU is the job, not a fault" + ) for section in SEVERITY_FREE_SECTIONS: block = payload.get(section) assert isinstance(block, dict), f"{section} must be a dict" diff --git a/tests/test_alerts.py b/tests/test_alerts.py index 0e88794..8c92c0d 100644 --- a/tests/test_alerts.py +++ b/tests/test_alerts.py @@ -216,21 +216,28 @@ def test_a_number_the_upstream_does_not_publish_is_never_a_breach( assert alerts.build(SETTINGS, [state()], [payload]) == [] -def test_a_gpu_breach_names_the_card_and_dedupes_per_metric( - host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: +def test_a_saturated_gpu_is_not_an_alert(host_scrape: Any, + agent_snapshot: dict[str, Any]) -> None: + """Phase 06: these are LLM servers — a fully-worked GPU is the job, not a fault. + + The host is otherwise all-clear, so a card at 100 % busy with both memory pools + filled must produce **no** alert at all: `severity.THRESHOLDS` has no `gpu.*` row, + the payload carries no GPU verdict, and this module owns no threshold of its own + from which to resurrect one. + """ def fill_the_card(snap: dict[str, Any]) -> None: gpu = snap["gpus"][0] - gpu["busy_pct"] = 97.0 + gpu["busy_pct"] = 100.0 for used, total in (("vram_used_bytes", "vram_total_bytes"), ("gtt_used_bytes", "gtt_total_bytes")): if gpu.get(total) is not None: gpu[used] = gpu[total] payload = host_scrape(agent_snapshot, cpu_busy=10.0, mutate=fill_the_card) - messages = {item["id"]: item["message"] for item in alerts.build(SETTINGS, [state()], [payload])} - assert any("GPU utilisation 97% on Radeon AI PRO R9700" in text - for text in messages.values()), messages - assert any("GPU memory" in text and "R9700" in text for text in messages.values()) + assert payload["gpus"][0]["busy_pct"] == 100.0, "the card really is saturated here" + assert alerts.build(SETTINGS, [state()], [payload]) == [], ( + "a busy GPU produced an alert — GPU verdicts were removed in phase 06" + ) def test_a_degraded_agent_warns_and_names_the_sections_that_failed( diff --git a/tests/test_api.py b/tests/test_api.py index 22ed6a5..5979dc0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -161,10 +161,11 @@ def test_summary_carries_the_verdict_for_every_column_the_page_colours(data_clie row = body["system"]["hosts"][0] assert row["severity"] == {"cpu_usage_pct": "ok", "mem_used_pct": "fail", "disk_worst_used_pct": "fail", "temp_c": "fail"} - # the fleet card is judged by the same table, so amber means the same thing at both sizes + # the fleet card is judged by the same table, so amber means the same thing at both + # sizes — and no GPU verdict, at fleet size or host size (phase 06: busy is the job) assert body["system"]["fleet"]["severity"] == { "cpu_usage_pct_avg": "ok", "memory_used_pct": "fail", - "disk_worst_used_pct": "fail", "gpu_busy_pct_avg": "ok", + "disk_worst_used_pct": "fail", } for ups in body["ups"]["ups"]: assert set(ups["severity"]) == {"charge_pct", "load_pct", "runtime_seconds"} @@ -311,7 +312,9 @@ def test_snapshot_payloads_carry_their_own_severity_sections(data_client) -> Non assert set(host["cpu"]["severity"]) == {"usage_pct", "temp_c"} assert set(host["memory"]["severity"]) == {"used_pct"} assert set(host["disks"]["severity"]) == {"worst_used_pct"} - assert set(host["gpus"][0]["severity"]) == {"busy_pct", "mem_used_pct"} + assert "severity" not in host["gpus"][0], ( + "a GPU verdict came back on /api/system/snapshot — phase 06 removed it" + ) ups = data_client.get("/api/ups/snapshot").json() for card in ups["ups"]: assert set(card["severity"]) == {"charge_pct", "load_pct", "battery_runtime"} @@ -603,17 +606,48 @@ def test_system_page_colours_from_the_payload_not_from_js_thresholds(client) -> Phase 03 task 02's promise: every colour on a host card comes from a `severity` dict `severity.py` produced (`cpu.severity.usage_pct`, `memory.severity.used_pct`, - `disks.severity.worst_used_pct`, `gpus[].severity.busy_pct`, `cpu.severity.temp_c`). + `disks.severity.worst_used_pct`, `cpu.severity.temp_c`). A threshold written here could disagree with the overview's alerts; a hero with no verdict prints `n/a` uncoloured, which is `Dash.hero()`'s contract. + + The GPU slot names no verdict on purpose (phase 06): `severity.THRESHOLDS` has no + `gpu.*` row, the payload carries no `gpus[].severity`, and the hero is built without + `severity` or `bar` — a bar under a verdictless number would be a verdict by shape. """ js = client.get("/static/page-system.js").text assert "severityFor(" not in js, "the system page re-invented a threshold" assert "Dash.hero(" in js for hook in ("cpu.severity.usage_pct", "memory.severity.used_pct", - "disks.severity.worst_used_pct", "gpus[].severity.busy_pct", - "cpu.severity.temp_c"): + "disks.severity.worst_used_pct", "cpu.severity.temp_c"): assert hook in js, f"the card no longer names {hook} as a colour source" + # and the GPU hero grades nothing: no verdict hook, no utilisation bar + assert "gpus[].severity" not in js, "the GPU slot invented a colour source again" + gpu_slot = js[js.index("function renderGpuSlot("):js.index("function renderGpuExtra(")] + assert "severity:" not in gpu_slot, "the GPU hero passes a severity option again" + assert "bar:" not in gpu_slot, "the GPU hero grew a bar back (a verdict by shape)" + assert "hero-bar" not in gpu_slot, "the GPU hero rebuilt the bar variant of .hero" + + +def test_gpu_hero_footnote_prints_the_temperature_the_agent_read(client) -> None: + """TODO.md L9: the GPU temperature is glanceable on the hero row, not only in the table. + + `gpus[].temp_c` is the agent's max over that card's hwmon labels (`sources/host.py`, + pinned by `tests/test_system_contract.py`), so the hero needed no new field and the + `gpu.temp_c@N` history series kept its name. The null guard *is* the behaviour, so that + is what the assertion pins (there is no JS test runner — a locked anchor): a card whose + driver exposes no temperature prints none, because `n/a` is for numbers the card is + supposed to have and a `0 °C` would be a cool GPU that does not exist. + """ + js = client.get("/static/page-system.js").text + sub = js[js.index("function gpuSub("):js.index("function renderGpuSlot(")] + assert "gpu.temp_c" in sub, "the GPU hero stopped printing the temperature" + assert "if (gpu.temp_c != null)" in sub, "the temperature lost its null guard" + assert sub.count("°C") == 1, ( + "a second °C in gpuSub is a branch printing a temperature nobody measured" + ) + # "More detail" keeps the per-label breakdown: a surface change, not a de-duplication + extra = js[js.index("function renderGpuExtra("):js.index("function renderHostCard(")] + assert "Temp" in extra and "gpu.temp_c" in extra, "GPU detail lost its temperature column" def test_system_host_card_is_six_heroes_without_per_core(client) -> None: @@ -709,6 +743,112 @@ def test_every_system_chart_is_drawn_inside_its_own_host_panel(client) -> None: "a host that left the payload keeps a live panel and drawing charts" +def test_system_history_charts_are_collapsed_behind_a_toggle(client) -> None: + """TODO.md L4: "The historical charts on the system resources page should only be visible + if expanded. I only want to see current stats to keep things clean." + + The default `/system` view is the hero row. Pinned on the served script because the card + markup is generated in JS and there is no JS test runner (a locked anchor), and the two + ways this regresses are invisible otherwise: a `
` (charts visible by + default, i.e. the task un-done but the page still pretty) and a paint pass that runs + while the stack is shut, which builds three `LineChart`s per host inside a + `display:none` container — measured 0 px wide, never drawn, and rebuilt every poll. + """ + js = client.get("/static/page-system.js").text + card = js[js.index("function hostCard("):js.index("function memorySub(")] + + # the stack is *wrapped*, not merely preceded by a heading: it lives inside the
+ box = card.index('
') + stack = card.index('
') + closed = card.index("
") + assert box < stack < closed, "the chart stack is not inside the toggle" + assert closed < card.index("More detail"), "the charts toggle swallowed the detail row too" + assert not re.search(r"]*\bopen\b", card), \ + "a panel ships its charts open — the default view is not current stats" + assert re.search(r"]*>\s*Charts", card), "the toggle no longer reads Charts" + # it also says which window the expanded stack draws (the chips stay in the page head) + assert 'data-k="charts-window"' in card, "the toggle does not carry the window" + # …and the toggle is what turns the paint pass on + assert "addEventListener('toggle'" in card and "chartsToggled(" in card + + setter = js[js.index("function setChartsWindow("):js.index("function memorySub(")] + assert "state.rangeKey" in setter, \ + "the toggle re-derives its window label instead of using the chips' own" + handler = js[js.index("Dash.rangeControl("):js.index("state.poller = poll(")] + assert "state.rangeKey = key" in handler and "setChartsWindow(" in handler, \ + "changing the window leaves every toggle labelling the old one" + + # nothing is built or drawn while the stack is shut, in either path that could paint + paint = js[js.index("function drawHostCharts("):js.index("async function renderHostCharts(")] + assert "if (!chartsOpen(card)) return" in paint, "a closed panel gets painted anyway" + assert "hostChartSet(card, host)" in paint, \ + "the LineChart instances are no longer built by the open pass" + toggle = js[js.index("function chartsToggled("):js.index("function drawHostCharts(")] + assert "if (!chartsOpen(card)) return" in toggle, "closing the stack still paints" + assert "renderHostCharts(card, host)" in toggle and "state.range" in toggle, \ + "a cached response from another window is drawn under this window's label" + + # the request discipline is unchanged: one per host per refresh, every series named + fetch = js[js.index("async function renderHostCharts("):js.index("function windowLabel(")] + assert "hostChartSet(" not in fetch, "the poll path still builds charts for closed panels" + assert "state.history.set(" in fetch, \ + "the response is not kept, so the first open is an empty box until the next poll" + assert "if (range === state.range) drawHostCharts(" in fetch, \ + "a response for a window the operator just left is painted under the new label" + assert "drawHostCharts(" in fetch, "an open panel is no longer repainted on refresh" + + +def test_a_pruned_host_takes_its_charts_toggle_with_it(client) -> None: + """A host that leaves the payload must leave no orphan control in the grid. + + The toggle is a child of the panel, which is why `card.remove()` is enough — but that is + only true while the markup is built inside `hostCard()`, so the placement is pinned here + along with the three per-host Maps the panel owns. + """ + js = client.get("/static/page-system.js").text + card = js[js.index("function hostCard("):js.index("function memorySub(")] + assert 'data-k="charts"' in card, "the toggle is no longer part of the panel" + pruner = js[js.index("function pruneHosts("):js.index("// ---- snapshot")] + assert "state.hosts.get(name).remove()" in pruner + for owned in ("state.charts", "state.payloads", "state.history"): + assert f"{owned}.delete(name)" in pruner, f"{owned} outlives the panel it belongs to" + + +def test_the_charts_toggle_is_a_quiet_disclosure_not_a_new_component(client) -> None: + """The collapse adds no palette and no second vocabulary (UI conventions). + + Open/closed is carried by a caret *shape* as well as a colour step, the summary is muted + rather than accented, and it introduces no `.sev-*` family — the same rule the hero + numbers live by, checked here because a `` is the first control on the page a + designer might be tempted to grade. + """ + css = client.get("/static/dash.css").text + block = css[css.index("details.charts-box { "):css.index(".host-charts {")] + assert "details.charts-box > summary::before" in block, "the toggle has no caret shape" + assert 'details.charts-box[open] > summary::before' in block, \ + "open and closed differ only in colour" + assert "var(--text-faint)" in block, "the closed toggle is not muted" + assert ".sev-" not in block and "var(--accent)" not in block, "the toggle invented a palette" + assert 'class="charts-box"' in client.get("/static/page-system.js").text, \ + "the stylesheet styles a class the page no longer renders" + + +def test_an_expanded_chart_stack_does_not_grow_its_collapsed_siblings(client) -> None: + """One host open must not stretch the hosts beside it into blank cards. + + `/system` is the same shape `/llm` had to fix: an expanded panel beside a collapsed one + grows the collapsed card to match, and a blank panel reads as "no data" (the misreading + the unconfigured rules exist to prevent). `.grid.items-start` already exists for exactly + this, so the page uses it rather than growing a rule of its own. + """ + html = _code(client.get("/system").text) + hosts_class = re.findall(r'
', html) + assert hosts_class == ["grid auto items-start"], \ + f"the host grid stretches its cards again: {hosts_class}" + assert ".grid.items-start" in client.get("/static/dash.css").text, \ + "the rule the host grid relies on is gone" + + def test_every_system_chart_names_itself_for_assistive_tech(client) -> None: """A 120px SVG is silence to a screen reader unless the page names it. diff --git a/tests/test_host_source.py b/tests/test_host_source.py index 75f6b34..1d44f66 100644 --- a/tests/test_host_source.py +++ b/tests/test_host_source.py @@ -248,12 +248,8 @@ def test_payloads_carry_a_severity_beside_every_rated_value(agent_snapshot) -> N assert payload["cpu"]["severity"]["temp_c"] == "ok" # 68 °C in the capture assert payload["memory"]["severity"]["used_pct"] == "ok" # 68 % of 128 GB assert payload["disks"]["severity"]["worst_used_pct"] == "ok" # 74.6 % worst mount - gpu = payload["gpus"][0] - assert gpu["severity"] == { - "busy_pct": classify("gpu.busy_pct", gpu["busy_pct"]), - "mem_used_pct": classify("gpu.mem_used_pct", gpu["mem_used_pct"]), - } - assert gpu["severity"]["busy_pct"] == "ok" + # GPU rows carry no verdict at all (phase 06): busy is the job on an LLM server + assert "severity" not in payload["gpus"][0] def test_severity_trips_on_the_ratified_boundaries(agent_snapshot) -> None: @@ -263,14 +259,14 @@ def test_severity_trips_on_the_ratified_boundaries(agent_snapshot) -> None: snap["cpu_temperature_celsius"] = 90.0 # cpu.temp_c danger for fs in snap["filesystems"]: fs["used_pct"] = 80.0 # disk.used_pct warn - snap["gpus"][0]["busy_pct"] = 95.0 # gpu.busy_pct danger - snap["gpus"][0]["gtt_used_bytes"] = 65e9 # whole-pool usage well past danger + snap["gpus"][0]["busy_pct"] = 100.0 # no gpu.* row exists to trip, at any level + snap["gpus"][0]["gtt_used_bytes"] = 65e9 # whole-pool usage well past the old bounds payload = parse_host_snapshot("driveripper", snap, make_collector(), now=1_000.0).payload assert payload["memory"]["severity"]["used_pct"] == "fail" assert payload["cpu"]["severity"]["temp_c"] == "fail" assert payload["disks"]["severity"]["worst_used_pct"] == "warn" - assert payload["gpus"][0]["severity"]["busy_pct"] == "fail" - assert payload["gpus"][0]["severity"]["mem_used_pct"] == "fail" + assert payload["gpus"][0]["busy_pct"] == 100.0, "the numbers stay; only the verdict left" + assert "severity" not in payload["gpus"][0], "a GPU verdict row came back" def test_a_value_the_agent_did_not_report_gets_no_colour(agent_snapshot) -> None: @@ -282,7 +278,7 @@ def test_a_value_the_agent_did_not_report_gets_no_colour(agent_snapshot) -> None payload = parse_host_snapshot("driveripper", snap, make_collector(), now=1_000.0).payload assert payload["cpu"]["severity"]["temp_c"] is None assert payload["cpu"]["severity"]["usage_pct"] is None, "one scrape is not a rate" - assert payload["gpus"][0]["severity"]["busy_pct"] is None + assert payload["gpus"][0]["busy_pct"] is None assert payload["disks"]["severity"]["worst_used_pct"] is None # the numbers themselves are unchanged by the verdict sitting next to them assert payload["memory"]["used_pct"] == pytest.approx( diff --git a/tests/test_severity.py b/tests/test_severity.py index aaa1c9c..ae8f956 100644 --- a/tests/test_severity.py +++ b/tests/test_severity.py @@ -23,8 +23,6 @@ RATIFIED = { "mem.used_pct": (80.0, 92.0), "disk.used_pct": (80.0, 90.0), "cpu.temp_c": (75.0, 90.0), - "gpu.busy_pct": (85.0, 95.0), - "gpu.mem_used_pct": (85.0, 95.0), "llm.kv_cache_usage_pct": (75.0, 90.0), "ups.charge_pct": (50.0, 20.0), "ups.battery_runtime": (600.0, 300.0), @@ -109,6 +107,22 @@ def test_unknown_metric_is_none_rather_than_an_exception() -> None: assert classify("", 100) is None +def test_gpu_usage_carries_no_verdict_at_any_level() -> None: + """Phase 06: these are LLM servers — a busy GPU is the job, not a fault. + + The table has **no `gpu.*` row at all** (not a raised ceiling): every utilisation + level classifies to `None`, so the card renders uncoloured rather than inventing + amber for the workload the machines exist to run. Deleting the row is not deleting + the data — the `gpu.busy_pct@N` series are still stored and charted. + """ + assert not any(metric.startswith("gpu.") for metric in THRESHOLDS) + assert classify("gpu.busy_pct", 0) is None + assert classify("gpu.busy_pct", 99) is None + assert classify("gpu.mem_used_pct", 99) is None + assert classify("gpu.mem_used_pct", 100) is None + assert classify("gpu.busy_pct@0", 95) is None, "streams inherit the absent row: nothing" + + @pytest.mark.parametrize("metric", sorted(THRESHOLDS)) def test_none_is_never_ok(metric: str) -> None: assert classify(metric, None) is None @@ -123,10 +137,8 @@ def test_non_finite_values_are_treated_as_missing(metric: str, value: float) -> def test_per_device_streams_inherit_their_base_metric() -> None: assert classify("disk.used_pct@/srv", 95) == "fail" assert classify("disk.used_pct@/srv", 10) == "ok" - assert classify("gpu.busy_pct@0", 84) == "ok" - assert classify("gpu.busy_pct@0", 90) == "warn" - assert classify("gpu.busy_pct@1", 95) == "fail" assert classify("cpu.usage_pct@core7", 80) == "warn" + assert classify("mem.used_pct@any", 95) == "fail" def test_vocabulary_is_exactly_ok_warn_fail_or_none() -> None: diff --git a/tests/test_system_contract.py b/tests/test_system_contract.py index 0b67f40..74ed87c 100644 --- a/tests/test_system_contract.py +++ b/tests/test_system_contract.py @@ -143,6 +143,9 @@ def test_the_discrete_gpu_capture_supplies_real_numbers( assert gpu["busy_pct"] == pytest.approx(snapshot["gpus"][0]["busy_pct"]) assert gpu["vram_used_pct"] == pytest.approx( gpu["vram_used_bytes"] / gpu["vram_total_bytes"] * 100.0) + # the hero's footnote is the agent's own max over this card's hwmon labels (phase 06), + # so the card and the `gpu.temp_c@N` history series can never disagree + assert gpu["temp_c"] == max(snapshot["gpus"][0]["temps"].values()) # one scrape is not a rate: usage and throughput are honestly n/a until the second assert payload["cpu"]["usage_pct"] is None @@ -177,11 +180,34 @@ def test_each_card_number_travels_with_its_own_verdict(host_scrape: Any, "mem.used_pct", payload["memory"]["used_pct"]) assert payload["disks"]["severity"]["worst_used_pct"] == classify( "disk.used_pct", payload["disks"]["worst_used_pct"]) + + +def test_the_gpu_card_carries_no_verdict_at_all(host_scrape: Any, + agent_snapshot: dict[str, Any]) -> None: + """Phase 06: a fully-worked GPU is the job on an LLM server, not a fault. + + The card prints busy % / VRAM with no colour class at any level, so the payload + carries no `severity` dict to colour from — the absence is the contract (pushed past + every old bound in the fixture, so a resurrected threshold row would trip and fail). + """ + def work_the_card(snap: dict[str, Any]) -> None: + gpu = snap["gpus"][0] + gpu["busy_pct"] = 100.0 + for used, total in (("vram_used_bytes", "vram_total_bytes"), + ("gtt_used_bytes", "gtt_total_bytes")): + if gpu.get(total) is not None: + gpu[used] = gpu[total] + + payload = host_scrape(agent_snapshot, cpu_busy=33.0, mutate=work_the_card) gpu = payload["gpus"][0] - assert gpu["severity"] == { - "busy_pct": classify("gpu.busy_pct", gpu["busy_pct"]), - "mem_used_pct": classify("gpu.mem_used_pct", gpu["mem_used_pct"]), - } + assert gpu["busy_pct"] == 100.0, "the numbers themselves stay on the card" + assert gpu["mem_used_pct"] > 90.0, "the whole pool is worked too" + assert "severity" not in gpu, ( + "a GPU verdict came back — classify('gpu.busy_pct', …) must be None and the row " + "must not carry a dict for the page to colour from" + ) + assert classify("gpu.busy_pct", gpu["busy_pct"]) is None + assert classify("gpu.mem_used_pct", gpu["mem_used_pct"]) is None # --------------------------------------------------------------------------- # @@ -208,6 +234,26 @@ def test_the_apu_reports_vram_and_the_whole_addressable_pool( assert gpu["model"], "the card needs a name to put beside the numbers" +def test_a_gpu_with_no_temperature_input_reports_none_not_zero(agent_snapshot: dict[str, Any], + ) -> None: + """Phase 06 put `gpus[].temp_c` on the hero footnote, so its absence must stay honest. + + A driver exposing no hwmon temperature has to come back as `None` (the page prints no + temperature at all) rather than `0 °C`, the same rule + `tests/test_host_source.py::test_missing_sensors_stay_missing` holds for the flat sensor + list — and no `gpu.temp_c@N` sample may be written for a number nobody measured. + """ + snapshot = copy.deepcopy(agent_snapshot) + snapshot["gpus"][0]["temps"] = {} + result = parse_host_snapshot("driveripper", snapshot, make_collector(), now=1_000.0) + gpu = result.payload["gpus"][0] + assert gpu["temp_c"] is None, "an unreadable sensor is not a cold card" + assert gpu["temps"] == {} + assert_card_contract(result.payload) + assert not [s for s in result.samples if s.metric.startswith("gpu.temp_c@")], ( + "a None temperature must not reach the history as a number") + + def test_paths_survive_a_host_that_reports_almost_nothing(agent_snapshot: dict[str, Any]) -> None: """Keys present, values None: the card must render `n/a` without a KeyError.""" snapshot = copy.deepcopy(agent_snapshot) @@ -229,9 +275,10 @@ def test_paths_survive_a_host_that_reports_almost_nothing(agent_snapshot: dict[s assert payload["disks"]["severity"]["worst_used_pct"] is None assert payload["network"]["rx_bytes_per_s"] is None gpu = payload["gpus"][0] - assert gpu["busy_pct"] is None and gpu["severity"]["busy_pct"] is None + assert gpu["busy_pct"] is None assert gpu["vram_used_pct"] is None, "no total, so no percentage — not 0 %" assert gpu["mem_used_pct"] is None + assert gpu["temp_c"] is not None, "the card still has its temperature to print" # --------------------------------------------------------------------------- #