From fefbeb8227aad40e5c6e804d9833de87b0e83223 Mon Sep 17 00:00:00 2001 From: ducoterra Date: Sun, 18 Oct 2026 00:45:01 -0400 Subject: [PATCH] phase: 09_disks_smartctl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Phase 09 — final verification pass (all 4 task files already in `complete/`; nothing re-implemented)** - Verified `GET /api/disks/snapshot` live: 3 hosts / 18 whole-`/sys/block` rows, `smartctl_available` three-way, per-row `severity`, API-summed `fleet`; 404 `{"configured": false}` proven *without dialling* (spies on `httpx.AsyncClient.get` + `collectors.fetch_snapshot`). - Verified `/api/disks/history` live (`disk.temp_c@dev`, unknown names → empty series) + `test_store.py`'s 300 s rollup/prune buckets; `/api/metrics?source=disks` answers the `disks` family. - Browser against live data: 18 drive cards (never a partition/mount), collapsed per-host `Charts` with `aria-label`, 3 `.stale` grids + one `.banner.info` why-line and **0 `.fail` classes / green `disks` dot** → criteria 3 and 4 hold; new shots `data/logs/09-verify-disks-live-dimmed.png`, `09-verify-overview-live.png` (13 total for the phase). - Real-hardware criterion: read 4 driveripper drives through `ops`' `sudo -n smartctl -i -A -H` and ran the live output through `host_agent.parse_smart_output` → `sources/disks.parse_disks_snapshot`: `sda` 35 °C / 30 423 h, `nvme1n1` 58 % wear, `nvme5n1` 13 uncorrectable → `alarm_disks: 1`, temps graded `warn`; consistent with the 16 committed fixtures. - `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov` → **769 passed, 11 skipped, coverage 98.33%** (floor 94); `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean; hygiene + dependency advisories clean. `uv run pytest -m live --live` → 10 passed, 1 skipped (agents still predate the SMART pass). Dev server healthy: `/api/health` `ok`, 9/9 sources. - Docs fixed: AGENTS.md test count 698→769 and coverage 98.2→98.33 %; `disks` added to the event dashboard-name list and to the "every feed renders `Dash.eventText`" sentence; README `POLL_MS_*` row now states `/disks`' deliberate fixed 30 s; new Known gap 4 records the dev-DB `database is locked` writes (reported/recovered per invariant 4, ~590 MB DB / ~70 MB WAL, not reproduced by a controlled 25 s external read txn). - Deviations: **no live-infra change** — granting the agent user the `disk` group and redeploying `host_agent.py` on driveripper/deskwork stay the documented paired operator action (redeploying without the grant yields 16 permission-denied lines; a sudo-spawning agent would break the confirmed "direct read, no sudo" assumption). Store-lock issue left to its own phase rather than patched speculatively. - Next pending phase: `10_dev_direct_dial`. --- .../09_disks_smartctl/00_phase.md | 1 + .../01_agent_smartctl_collection.md | 0 .../02_disks_source_poller_store.md | 0 .../03_disks_api_and_config.md | 0 .../04_disks_page_and_overview.md | 0 .../09_disks_smartctl__00_phase.a1.err | 0 .../09_disks_smartctl__00_phase.a1.md | 10 + .../09_disks_smartctl__00_phase.a1.md.status | 1 + .../09_disks_smartctl__00_phase.a1.validate | 51 ++ ...rtctl__01_agent_smartctl_collection.a1.err | 0 ...artctl__01_agent_smartctl_collection.a1.md | 16 + ..._01_agent_smartctl_collection.a1.md.status | 1 + ...__01_agent_smartctl_collection.a1.validate | 49 ++ ...rtctl__02_disks_source_poller_store.a1.err | 0 ...artctl__02_disks_source_poller_store.a1.md | 18 + ..._02_disks_source_poller_store.a1.md.status | 1 + ...__02_disks_source_poller_store.a1.validate | 50 ++ ...s_smartctl__03_disks_api_and_config.a1.err | 0 ...ks_smartctl__03_disks_api_and_config.a1.md | 12 + ...tctl__03_disks_api_and_config.a1.md.status | 1 + ...rtctl__03_disks_api_and_config.a1.validate | 51 ++ ...martctl__04_disks_page_and_overview.a1.err | 0 ...smartctl__04_disks_page_and_overview.a1.md | 27 + ...l__04_disks_page_and_overview.a1.md.status | 1 + ...tl__04_disks_page_and_overview.a1.validate | 51 ++ .env.dev.example | 13 + .env.example | 11 + AGENTS.md | 233 +++++- README.md | 87 ++- TODO.md | 3 + agent/host_agent.py | 314 +++++++- deploy/install-agent.sh | 35 +- scripts/capture-fixtures.sh | 87 +++ src/dashboards/alerts.py | 52 +- src/dashboards/api.py | 141 +++- src/dashboards/app.py | 82 +- src/dashboards/collectors.py | 76 +- src/dashboards/config.py | 9 + src/dashboards/poller.py | 4 +- src/dashboards/severity.py | 10 + src/dashboards/sources/__init__.py | 1 + src/dashboards/sources/disks.py | 337 +++++++++ src/dashboards/store.py | 7 + src/dashboards/web/pages/disks.html | 101 +++ src/dashboards/web/pages/index.html | 1 + src/dashboards/web/pages/llm.html | 1 + src/dashboards/web/pages/system.html | 1 + src/dashboards/web/pages/ups.html | 1 + src/dashboards/web/static/app.js | 5 + src/dashboards/web/static/dash.css | 39 + src/dashboards/web/static/page-disks.js | 561 ++++++++++++++ src/dashboards/web/static/page-index.js | 4 +- tests/conftest.py | 106 +++ .../fixtures/smartctl_driveripper_nvme0n1.txt | 52 ++ .../fixtures/smartctl_driveripper_nvme1n1.txt | 48 ++ .../fixtures/smartctl_driveripper_nvme2n1.txt | 45 ++ .../fixtures/smartctl_driveripper_nvme3n1.txt | 45 ++ .../fixtures/smartctl_driveripper_nvme4n1.txt | 45 ++ .../fixtures/smartctl_driveripper_nvme5n1.txt | 48 ++ .../fixtures/smartctl_driveripper_nvme6n1.txt | 45 ++ .../fixtures/smartctl_driveripper_nvme7n1.txt | 48 ++ tests/fixtures/smartctl_driveripper_sda.txt | 53 ++ tests/fixtures/smartctl_driveripper_sdb.txt | 51 ++ tests/fixtures/smartctl_driveripper_sdc.txt | 56 ++ tests/fixtures/smartctl_driveripper_sdd.txt | 51 ++ tests/fixtures/smartctl_driveripper_sde.txt | 48 ++ tests/fixtures/smartctl_driveripper_sdf.txt | 56 ++ tests/fixtures/smartctl_driveripper_sdg.txt | 48 ++ tests/fixtures/smartctl_driveripper_sdh.txt | 57 ++ tests/fixtures/smartctl_reference_ata_hdd.txt | 42 ++ tests/fixtures/smartctl_reference_nvme.txt | 55 ++ tests/test_agent.py | 433 ++++++++++- tests/test_alerts.py | 92 +++ tests/test_api.py | 702 +++++++++++++++++- tests/test_collectors.py | 145 ++++ tests/test_config.py | 22 + tests/test_disks_source.py | 474 ++++++++++++ tests/test_live_fleet.py | 85 +++ tests/test_poller.py | 34 + tests/test_severity.py | 42 ++ tests/test_store.py | 74 ++ 81 files changed, 5525 insertions(+), 134 deletions(-) rename .agents/phases/{todo => complete}/09_disks_smartctl/00_phase.md (97%) rename .agents/phases/{todo => complete}/09_disks_smartctl/01_agent_smartctl_collection.md (100%) rename .agents/phases/{todo => complete}/09_disks_smartctl/02_disks_source_poller_store.md (100%) rename .agents/phases/{todo => complete}/09_disks_smartctl/03_disks_api_and_config.md (100%) rename .agents/phases/{todo => complete}/09_disks_smartctl/04_disks_page_and_overview.md (100%) create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.err create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.md create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.md.status create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.validate create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.err create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.md create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.md.status create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.validate create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.err create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.md create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.md.status create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.validate create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.err create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.md create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.md.status create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.validate create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.err create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.md create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.md.status create mode 100644 .agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.validate create mode 100644 src/dashboards/sources/disks.py create mode 100644 src/dashboards/web/pages/disks.html create mode 100644 src/dashboards/web/static/page-disks.js create mode 100644 tests/fixtures/smartctl_driveripper_nvme0n1.txt create mode 100644 tests/fixtures/smartctl_driveripper_nvme1n1.txt create mode 100644 tests/fixtures/smartctl_driveripper_nvme2n1.txt create mode 100644 tests/fixtures/smartctl_driveripper_nvme3n1.txt create mode 100644 tests/fixtures/smartctl_driveripper_nvme4n1.txt create mode 100644 tests/fixtures/smartctl_driveripper_nvme5n1.txt create mode 100644 tests/fixtures/smartctl_driveripper_nvme6n1.txt create mode 100644 tests/fixtures/smartctl_driveripper_nvme7n1.txt create mode 100644 tests/fixtures/smartctl_driveripper_sda.txt create mode 100644 tests/fixtures/smartctl_driveripper_sdb.txt create mode 100644 tests/fixtures/smartctl_driveripper_sdc.txt create mode 100644 tests/fixtures/smartctl_driveripper_sdd.txt create mode 100644 tests/fixtures/smartctl_driveripper_sde.txt create mode 100644 tests/fixtures/smartctl_driveripper_sdf.txt create mode 100644 tests/fixtures/smartctl_driveripper_sdg.txt create mode 100644 tests/fixtures/smartctl_driveripper_sdh.txt create mode 100644 tests/fixtures/smartctl_reference_ata_hdd.txt create mode 100644 tests/fixtures/smartctl_reference_nvme.txt create mode 100644 tests/test_disks_source.py diff --git a/.agents/phases/todo/09_disks_smartctl/00_phase.md b/.agents/phases/complete/09_disks_smartctl/00_phase.md similarity index 97% rename from .agents/phases/todo/09_disks_smartctl/00_phase.md rename to .agents/phases/complete/09_disks_smartctl/00_phase.md index 713e42d..0cfed76 100644 --- a/.agents/phases/todo/09_disks_smartctl/00_phase.md +++ b/.agents/phases/complete/09_disks_smartctl/00_phase.md @@ -50,3 +50,4 @@ reads `n/a` for what `smartctl` cannot see — never red for a missing binary. - [ ] the `/disks` page renders one card per physical disk (never per-partition) with health, temp, power-on hours and non-zero alarm chips - [ ] a host without `smartctl` (or a denied read) shows dimmed `n/a` cards, never a red page - [ ] `bash .agents/validate.sh` green; screenshots in `data/logs/` +- [ ] you have verified with real smartctl data from driveripper. The `ops` user has `sudo smartctl*` access via sudoers. diff --git a/.agents/phases/todo/09_disks_smartctl/01_agent_smartctl_collection.md b/.agents/phases/complete/09_disks_smartctl/01_agent_smartctl_collection.md similarity index 100% rename from .agents/phases/todo/09_disks_smartctl/01_agent_smartctl_collection.md rename to .agents/phases/complete/09_disks_smartctl/01_agent_smartctl_collection.md diff --git a/.agents/phases/todo/09_disks_smartctl/02_disks_source_poller_store.md b/.agents/phases/complete/09_disks_smartctl/02_disks_source_poller_store.md similarity index 100% rename from .agents/phases/todo/09_disks_smartctl/02_disks_source_poller_store.md rename to .agents/phases/complete/09_disks_smartctl/02_disks_source_poller_store.md diff --git a/.agents/phases/todo/09_disks_smartctl/03_disks_api_and_config.md b/.agents/phases/complete/09_disks_smartctl/03_disks_api_and_config.md similarity index 100% rename from .agents/phases/todo/09_disks_smartctl/03_disks_api_and_config.md rename to .agents/phases/complete/09_disks_smartctl/03_disks_api_and_config.md diff --git a/.agents/phases/todo/09_disks_smartctl/04_disks_page_and_overview.md b/.agents/phases/complete/09_disks_smartctl/04_disks_page_and_overview.md similarity index 100% rename from .agents/phases/todo/09_disks_smartctl/04_disks_page_and_overview.md rename to .agents/phases/complete/09_disks_smartctl/04_disks_page_and_overview.md diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.err b/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.md b/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.md new file mode 100644 index 0000000..974854a --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.md @@ -0,0 +1,10 @@ +**Phase 09 — final verification pass (all 4 task files already in `complete/`; nothing re-implemented)** + +- Verified `GET /api/disks/snapshot` live: 3 hosts / 18 whole-`/sys/block` rows, `smartctl_available` three-way, per-row `severity`, API-summed `fleet`; 404 `{"configured": false}` proven *without dialling* (spies on `httpx.AsyncClient.get` + `collectors.fetch_snapshot`). +- Verified `/api/disks/history` live (`disk.temp_c@dev`, unknown names → empty series) + `test_store.py`'s 300 s rollup/prune buckets; `/api/metrics?source=disks` answers the `disks` family. +- Browser against live data: 18 drive cards (never a partition/mount), collapsed per-host `Charts` with `aria-label`, 3 `.stale` grids + one `.banner.info` why-line and **0 `.fail` classes / green `disks` dot** → criteria 3 and 4 hold; new shots `data/logs/09-verify-disks-live-dimmed.png`, `09-verify-overview-live.png` (13 total for the phase). +- Real-hardware criterion: read 4 driveripper drives through `ops`' `sudo -n smartctl -i -A -H` and ran the live output through `host_agent.parse_smart_output` → `sources/disks.parse_disks_snapshot`: `sda` 35 °C / 30 423 h, `nvme1n1` 58 % wear, `nvme5n1` 13 uncorrectable → `alarm_disks: 1`, temps graded `warn`; consistent with the 16 committed fixtures. +- `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov` → **769 passed, 11 skipped, coverage 98.33%** (floor 94); `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean; hygiene + dependency advisories clean. `uv run pytest -m live --live` → 10 passed, 1 skipped (agents still predate the SMART pass). Dev server healthy: `/api/health` `ok`, 9/9 sources. +- Docs fixed: AGENTS.md test count 698→769 and coverage 98.2→98.33 %; `disks` added to the event dashboard-name list and to the "every feed renders `Dash.eventText`" sentence; README `POLL_MS_*` row now states `/disks`' deliberate fixed 30 s; new Known gap 4 records the dev-DB `database is locked` writes (reported/recovered per invariant 4, ~590 MB DB / ~70 MB WAL, not reproduced by a controlled 25 s external read txn). +- Deviations: **no live-infra change** — granting the agent user the `disk` group and redeploying `host_agent.py` on driveripper/deskwork stay the documented paired operator action (redeploying without the grant yields 16 permission-denied lines; a sudo-spawning agent would break the confirmed "direct read, no sudo" assumption). Store-lock issue left to its own phase rather than patched speculatively. +- Next pending phase: `10_dev_direct_dial`. diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.md.status b/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.validate b/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.validate new file mode 100644 index 0000000..627a2c4 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__00_phase.a1.validate @@ -0,0 +1,51 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 9%] +........................................................................ [ 18%] +........................................................................ [ 27%] +........................................................................ [ 36%] +........................................................................ [ 46%] +........................................................................ [ 55%] +...........sssssssssss.................................................. [ 64%] +........................................................................ [ 73%] +........................................................................ [ 83%] +........................................................................ [ 92%] +............................................................ [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 221 0 100% +src/dashboards/app.py 101 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 115 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 250-251 +------------------------------------------------------------------ +TOTAL 2335 39 98% +Required test coverage of 94% reached. Total coverage: 98.33% +769 passed, 11 skipped, 1 warning in 58.25s +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.err b/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.md b/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.md new file mode 100644 index 0000000..3dc3228 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.md @@ -0,0 +1,16 @@ +Everything is green. Final state: + +**Implemented (task 09/01 — agent smartctl collection)** +- `agent/host_agent.py`: new `smart()` pass — `smartctl -i -A -H` per whole `/sys/block` disk (virtual/`dm-*` skipped), stdlib `subprocess`, 5 s/device, 20 s/pass, 240 s cache; `parse_smart_output()` covers ATA+NVMe (health verdict word, temp, power-on hours, reallocated/pending/offline-uncorrectable sectors, wear %; NVMe media-errors mapped; hex-packed raws refused); `smart: {…}` on each `disk_io.devices[]` row + SCHEMA/docstring; `--smartctl`/`--no-smart` flags; failure semantics: missing binary → **one** `errors` line, denied/wedged device → only that disk `null` + its own line, `/health` untouched; fixed a real-box-caused defect (error note quoted the version banner instead of the failure line). +- Fixtures `tests/fixtures/smartctl_reference_{nvme,ata_hdd}.txt` — labels verbatim from the deployed smartmontools 7.5 binary (fleet runs the same build), values synthetic, honestly labelled **reference shapes** (see deviation below). +- `scripts/capture-fixtures.sh`: `SMART_HOSTS=` pass records `smartctl__.txt` per disk (never overwrites good fixtures with permission-denied output); scrubber now also redacts smartctl serials/WWNs and normalizes capture timestamps. `.env.dev.example` documents it. +- `collectors.py`: agent `errors` → payload `degraded` **excludes** `smart:` add-on lines (invariant 3: no smartmontools must not hold `/system` amber); they stay in `agent.errors` for the `/disks` page. 2 new collector tests pin both directions. +- `deploy/install-agent.sh`: grants `disk` group to the agent user (idempotent), notes missing smartmontools, unit replaces `PrivateDevices` with `DevicePolicy=strict` + device-allowed disk nodes. +- Docs: README (agent section + capture commands), AGENTS.md (agent/collectors bullets, fleet facts, capture command). + +**Tests/lint**: `bash .agents/validate.sh` → **EXIT=0** — `pytest --cov --cov-fail-under=94`: **646 passed, 10 skipped, coverage 97.92%**; `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean. Agent file coverage 92% (new SMART code paths fully covered). `uv run pytest -m live --live`: 9 passed, my new smart live test skips with reason “no agent redeployed with the smart pass yet”. `uv pip list --outdated` empty. Dev server restarted; `/api/health` all sources ok. + +**Deviations / live-infra note** +- Fixtures are labelled reference shapes, not field captures: **no reachable environment can open a block device** — fleet `ops` gets `Permission denied` (no smartctl sudo rule, not in `disk`), and the dev box is an unprivileged toolbox container. I made **no live changes** (no group grants, no agent redeploy — a redeploy without the grant would emit 12 permission-denied lines on driveripper). Operator follow-up (documented in AGENTS.md): `sudo usermod -aG disk ` on deskwork+driveripper **together with** an agent redeploy, then `SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh` to replace the reference fixtures with hardware truth (the live shape test then activates automatically). Installed `smartmontools` in the local dev toolbox to verify format strings; harmless, no service runs there. + +**Next pending task**: `09_disks_smartctl/02_disks_source_poller_store.md`. diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.md.status b/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.validate b/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.validate new file mode 100644 index 0000000..33a10c6 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__01_agent_smartctl_collection.a1.validate @@ -0,0 +1,49 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 10%] +........................................................................ [ 21%] +........................................................................ [ 32%] +........................................................................ [ 43%] +.................................................ssssssssss............. [ 54%] +........................................................................ [ 65%] +........................................................................ [ 76%] +........................................................................ [ 87%] +........................................................................ [ 98%] +........ [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 166 0 100% +src/dashboards/api.py 193 0 100% +src/dashboards/app.py 107 6 94% 50-51, 56, 58, 60, 127 +src/dashboards/collectors.py 145 5 97% 105-106, 110-111, 128 +src/dashboards/config.py 113 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 243-244 +------------------------------------------------------------------ +TOTAL 2161 45 98% +Required test coverage of 94% reached. Total coverage: 97.92% +646 passed, 10 skipped, 1 warning in 59.09s +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.err b/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.md b/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.md new file mode 100644 index 0000000..26cf08b --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.md @@ -0,0 +1,18 @@ +**Implemented (task 09/02 — `disks` source, poller job, store)** +- `src/dashboards/sources/disks.py` (new): one row per whole-`/sys/block` disk (never partition/mount), agent `smart` verbatim, `health` = drive's word normalised `ok/failed/None`, three-way `smartctl_available` (yes / "install smartmontools" / unknown) + `smart_errors`; samples `disk.temp_c|power_on_hours|reallocated|pending|uncorrectable|percentage_used@` in a new `disks` sample family; counters stored as reported (no `RateEstimator`, no window); health-transition events under dashboard name `disks`. +- `config.py`: `DISKS_INTERVAL_SECONDS` (300) + `DISKS_TIMEOUT_SECONDS` (30). +- `collectors.py`: shared `fetch_snapshot()` (system+disks, redacted messages, per-request timeout → httpx sentinel, since literal `None` disables timeouts), `fetch_disk_snapshot()`, `disks_jobs()` (`disks:`; no targets ⇒ no jobs ⇒ never dials). +- `app.py`: one `build_hub()` for both wiring paths (production + injected-store tests) now registers the disks jobs. +- `alerts.py` + `store.py`: interval row for `disks`, disks-echo suppression, dashboard-filtered `_state_for`; rollup docstring for a 300 s source (no code change needed). +- **Defect fixed in task 01's work**: an expired SMART cache now refreshes on a background thread (single-flight, keeps the old cache) — an inline 20 s wedged-disk pass blew `HOST_TIMEOUT_SECONDS` and turned `/system` red every 4 minutes. +- Docs: README (env rows, diagram, poller bullet, agent SMART), AGENTS.md (source/poller/alerts/metric family/agent/gaps/counts), `.env.example`, `.env.dev.example`. + +**Tests/lint** — `bash .agents/validate.sh` → **EXIT=0**: `pytest --cov --cov-fail-under=94` **698 passed, 11 skipped, coverage 98.25%** (floor 94; `sources/disks.py` 100%); `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean; `uv pip list --outdated` empty. New `tests/test_disks_source.py` (33) + additions in test_store (300 s rollup buckets/pruning), test_poller (failed disks poll ages, not zeroes), test_collectors (fetch + timeout), test_alerts (echo + interval), test_config, test_api (health record per host), test_agent (background refresh, `not found` wording pin). `pytest -m live --live` (with `.env.dev`): 10 passed, 1 skip (agent SMART — fleet agents not redeployed). Dev server restarted: 9 sources ok incl. `disks:driveripper/deskwork/gateway`. + +**Decisions / deviations** +- Task text assumed `alerts.py` needed *no* change; it did — without a `disks` interval the job ambered `/` every 5 min, and one dead agent printed two identical red rows. Both fixed with tests. +- One job **per host** (`disks:`), not one fleet job: per-host staleness and the stub-per-job shape `/api/system/snapshot` already uses. +- The `404 {"configured": false}` route is task 03; the job half (no targets ⇒ no task, no dial) is pinned here. +- **No live changes** (no agent redeploy, no `disk`-group grant) — that stays the operator action documented in AGENTS.md; live work was read-only. + +**Next pending task**: `09_disks_smartctl/03_disks_api_and_config.md` — and it should add `alerts.HREF["disks"] = "/disks"` together with the `/disks` page. diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.md.status b/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.validate b/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.validate new file mode 100644 index 0000000..c5d6d3c --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__02_disks_source_poller_store.a1.validate @@ -0,0 +1,50 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 10%] +........................................................................ [ 20%] +........................................................................ [ 30%] +........................................................................ [ 40%] +........................................................................ [ 50%] +........................sssssssssss..................................... [ 60%] +........................................................................ [ 71%] +........................................................................ [ 81%] +........................................................................ [ 91%] +............................................................. [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 193 0 100% +src/dashboards/app.py 100 1 99% 139 +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 115 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 104 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 250-251 +------------------------------------------------------------------ +TOTAL 2280 40 98% +Required test coverage of 94% reached. Total coverage: 98.25% +698 passed, 11 skipped, 1 warning in 55.89s +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.err b/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.md b/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.md new file mode 100644 index 0000000..3571c7d --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.md @@ -0,0 +1,12 @@ +**Implemented (task 09/03 — disks API, config wiring, severity rows)** +- `api.py`: new `disks_router()` — `GET /api/disks/snapshot` (per host: disk rows + verbatim `smart` + `health` + per-row `severity` + three-way `smartctl_available` + `smart_errors` + `summary`, plus a `fleet` block summed by the API, and a configured-but-never-answered host as an `unreachable` stub whose counts are `None`, not `0`) and `GET /api/disks/history` (`disk.*@dev` through the existing history mechanism; unknown names → empty series; `hosts` defaults to every configured target). `require_hosts()` gate runs first: `404 {"configured": false}` with no dial. `/api/config` gained `disks {enabled, interval_seconds, timeout_seconds}`. No POST anywhere on the surface. +- `severity.py`: `disk.temp_c` (50/70) and `disk.percentage_used` (80/100); the sector counters and `disk.power_on_hours` deliberately have **no row** (presence chips, never green). +- `sources/disks.py`: `row_severity()` + `SEVERITY_FIELDS` / `UNGRADED_FIELDS` / `HEALTH_SEVERITY` (drive's own word → `ok|fail`, not a threshold) and `fleet_disk_summary()` (dead host counted, never summed). +- `app.py`: `disks_router()` included; `alerts.py` **needed no change** — its existing `disks` interval + echo-suppression + unconfigured rules already cover stale/dead/unconfigured (added a test for the unconfigured case to prove it). +- Docs: README (new `api.py` architecture bullet with the two 404/empty-series rules; severity bullet with the disk rows and the ungraded counters), AGENTS.md (`api.py`, `sources/disks.py`, `severity.py` bullets). + +**Tests/lint** — `bash .agents/validate.sh` → **EXIT=0**: `pytest --cov --cov-fail-under=94` **734 passed, 11 skipped, coverage 98.33%** (`api.py` 100%, `sources/disks.py` 100%, `severity.py` 100%, `app.py` 100% after covering the `UpstreamError` route handler); `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean; `uv pip list --outdated` empty. New pins: 10 disks API tests (payload paths, stub-of-`n/a`s, verdict-equals-`classify` per row, fleet sum, GET-only + route-verb enumeration, 404-without-dialling with spies on `httpx.AsyncClient.get` and `collectors.fetch_snapshot` plus `hub.jobs == {}`, history known/unknown/default-hosts, `/api/metrics?source=disks`, `/api/config` disks section), 9 disks-source tests (severity + fleet summary), 3 severity tests, 1 alerts test; the token-leak test now also asserts the `disks:driveripper` error string is redacted. `pytest -m live --live` (`.env.dev` sourced): 10 passed, 1 skipped, no writes. + +**Notable decisions** — `/api/disks/history` follows its siblings (empty answers, not 404, since it reads only SQLite); `HREF["disks"] = "/disks"` left for task 04 so an alert never links to a route that does not exist. Live-verified against the real fleet (read-only): `/api/disks/snapshot` → 3 hosts / 18 physical disks, `smart: null` + `smartctl_available: null` (deployed agents predate the SMART pass — the designed `n/a`, no red), `/api/health` all 9 sources ok. No live infrastructure change made; no new env vars. + +**Next pending task**: `09_disks_smartctl/04_disks_page_and_overview.md` (the `/disks` page, nav tab, overview entry, header dot, and `alerts.HREF["disks"]`). diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.md.status b/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.validate b/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.validate new file mode 100644 index 0000000..cd298e6 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__03_disks_api_and_config.a1.validate @@ -0,0 +1,51 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 9%] +........................................................................ [ 19%] +........................................................................ [ 28%] +........................................................................ [ 38%] +........................................................................ [ 48%] +................................................sssssssssss............. [ 57%] +........................................................................ [ 67%] +........................................................................ [ 77%] +........................................................................ [ 86%] +........................................................................ [ 96%] +......................... [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 220 0 100% +src/dashboards/app.py 101 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 115 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 250-251 +------------------------------------------------------------------ +TOTAL 2334 39 98% +Required test coverage of 94% reached. Total coverage: 98.33% +734 passed, 11 skipped, 1 warning in 58.36s +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.err b/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.err new file mode 100644 index 0000000..e69de29 diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.md b/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.md new file mode 100644 index 0000000..b146b08 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.md @@ -0,0 +1,27 @@ +Task 04 is complete and the repo is green. + +## `/disks` — the fourth dashboard + +**New:** `src/dashboards/web/pages/disks.html`, `src/dashboards/web/static/page-disks.js`. **Touched:** `app.py` (route), `app.js` (masthead `disks` dot), `dash.css` (drive-panel section), `page-index.js` (footer cadence), nav tab added to all five pages, `api.py` (`/api/summary` gained an additive `disks` block: fleet counts + per-host `health_failed`/`alarm_disks`/`smartctl_available`/`failed_devices`), `alerts.py` (disk-only SMART failure links to `/disks`). + +- One card per **whole `/sys/block` drive** — device path, the drive's own verdict word (`OK`/`FAILED`/`unknown`), model·size·SSD/HDD, then Temp / Power-on / Wear. Colour only from the payload `severity`; power-on hours uncoloured; sector counters are `.chip.fail` **only while non-zero**. +- Missing/denied `smartctl`: host dot stays green, the grid dims, one `.banner.info` why-line (`smartctl not available on — install smartmontools…` / "No SMART data from … and redeploy the agent" / the agent's own sentence). Never red. No partition or mount rows. +- Per-host `Charts` `
`: built and painted only while open, **one** `/api/disks/history` request per host per refresh naming every `disk.temp_c@dev` / `disk.percentage_used@dev`, cached response, `pruneHosts` drops card+charts+maps together. + +## Real-hardware verification (the phase's acceptance line) + +`ops` on driveripper has `NOPASSWD /usr/sbin/smartctl*` (not the `disk` group; deskwork grants neither), so `scripts/capture-fixtures.sh` now tries plain `smartctl` first, falls back to `sudo -n`, scrubs Serial / LU WWN / namespace EUI-64, and writes `tests/fixtures/smartctl_driveripper_.txt` — **all 16 real drives captured**. + +That capture found a real parser bug, now fixed in `agent/host_agent.py`: attribute rows were read from the **end** of the line, but this smartmontools annotates RAW_VALUE — `35 (Min/Max 34/44)` on every Seagate, `43894h+43m+32.519s` on one Western Digital — so **all 8 HDDs reported no temperature and one had no power-on count**. Rows now read the RAW_VALUE column (`_smart_raw_number`, hex raws still refused). New tests: `test_smart_attribute_rows_are_read_from_the_raw_column_not_the_line_end`, `test_the_real_drives_answer_every_number_the_disks_page_prints` (every real capture), scrub test extended to EUI-64. Real findings the page now shows: `nvme5n1` has **13 media-and-data-integrity errors**, `nvme1n1` is at 58 % wear, NVMe temps 48–62 °C render amber. + +## Verification + +- `bash .agents/validate.sh` → **EXIT=0**: 769 passed / 11 skipped (+18 tests), coverage **98.33 %** (floor 94), ruff clean, pyright 0, `node --check` clean, repo hygiene clean. +- `uv run pytest -m live --live` (with `.env.dev`): 10 passed, 1 skipped (`no agent redeployed with the smart pass yet`). +- Dev server restarted; `/api/health` ok with `disks:driveripper/deskwork/gateway` all green. Nothing staged. +- Screenshots in `data/logs/`: `09-disks-real-data.png` (16 real drives, amber temps, `13 uncorrectable` chip, `ALARM DRIVES 1`), `09-disks-real-alarm-drive.png`, `09-disks-real-charts.png`, `09-disks-smart-cards.png` / `-charts-expanded.png` / `-no-smartctl-host.png` (reference shapes + missing-binary state), `09-disks-live-fleet.png` / `-live-charts-open.png` / `-live-overview.png` (live fleet: 3 hosts answering, 18 drives, dimmed `n/a` + why-line; `Disks` tab, `disks` dot, `disks 300s` footer). No page errors on any page. +- `README.md` + `AGENTS.md` updated: four dashboards, `/disks` row + UI conventions, the RAW_VALUE rule, corrected fleet SMART facts, real captures, and the two operator actions. + +## Left for you (deliberately not done) + +Live SMART needs **both**: `sudo usermod -aG disk ` **and** a redeploy of `agent/host_agent.py` — the agent shells `smartctl` as itself and never calls `sudo`, so driveripper's sudoers rule alone only feeds the capture script. I did not add a sudo-spawning path to the agent unprompted; say the word if you'd rather that route exist (opt-in flag) than grant the group. deskwork has neither grant; gateway stays `n/a` permanently (VM). I did not modify any phase file — the `00_phase.md` diff in `git status` is your acceptance line. diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.md.status b/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.md.status new file mode 100644 index 0000000..9766475 --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.md.status @@ -0,0 +1 @@ +ok diff --git a/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.validate b/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.validate new file mode 100644 index 0000000..d0da49a --- /dev/null +++ b/.agents/reports/09_disks_smartctl/09_disks_smartctl__04_disks_page_and_overview.a1.validate @@ -0,0 +1,51 @@ +== uv run pytest --cov (hermetic: no network, live tests skip) == +........................................................................ [ 9%] +........................................................................ [ 18%] +........................................................................ [ 27%] +........................................................................ [ 36%] +........................................................................ [ 46%] +........................................................................ [ 55%] +...........sssssssssss.................................................. [ 64%] +........................................................................ [ 73%] +........................................................................ [ 83%] +........................................................................ [ 92%] +............................................................ [100%] +=============================== warnings summary =============================== +.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1 + /var/home/ducoterra/Projects/Personal/dashboards/.venv/lib64/python3.14/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +================================ tests coverage ================================ +_______________ coverage: platform linux, python 3.14.4-final-0 ________________ + +Name Stmts Miss Cover Missing +------------------------------------------------------------------ +src/dashboards/__init__.py 3 0 100% +src/dashboards/__main__.py 14 1 93% 28 +src/dashboards/alerts.py 176 0 100% +src/dashboards/api.py 221 0 100% +src/dashboards/app.py 101 0 100% +src/dashboards/collectors.py 155 5 97% 141-142, 146-147, 164 +src/dashboards/config.py 115 0 100% +src/dashboards/debug.py 14 0 100% +src/dashboards/poller.py 165 3 98% 192-194, 279 +src/dashboards/prom.py 105 5 95% 107, 144, 158, 161, 164 +src/dashboards/rates.py 37 1 97% 51 +src/dashboards/severity.py 49 0 100% +src/dashboards/sources/__init__.py 0 0 100% +src/dashboards/sources/disks.py 130 0 100% +src/dashboards/sources/host.py 199 1 99% 147 +src/dashboards/sources/llm.py 150 1 99% 228 +src/dashboards/sources/nut.py 389 20 95% 128, 142, 148, 160, 166, 178, 186, 199, 270-271, 273, 489, 501, 536, 569, 573, 630-631, 636, 648 +src/dashboards/sources/nut_sim.py 134 0 100% +src/dashboards/store.py 178 2 99% 250-251 +------------------------------------------------------------------ +TOTAL 2335 39 98% +Required test coverage of 94% reached. Total coverage: 98.33% +769 passed, 11 skipped, 1 warning in 60.88s (0:01:00) +== uv run ruff check src tests agent == +All checks passed! +== uv run pyright == +0 errors, 0 warnings, 0 informations +== node --check src/dashboards/web/static/*.js == diff --git a/.env.dev.example b/.env.dev.example index 50c187b..4c64549 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -14,6 +14,19 @@ export DEV_TUNNELS="9121:driveripper-ops:9120 9122:deskwork-ops:9120 9123:gatewa # this fleet's driveripper answers on 3051, so check before copying the number. export DEV_FORWARDS="3494:driveripper-ops:3051" +# Physical-disk health (phase 09): scripts/capture-fixtures.sh records one +# `smartctl -i -A -H` per whole disk per host into tests/fixtures/smartctl__.txt. +# The capturing user must be able to READ the block devices: install smartmontools and +# `sudo usermod -aG disk ` (an unprivileged run says Permission denied, and the +# script says so instead of recording junk). Unset means the SMART captures are skipped. +# export SMART_HOSTS="deskwork-ops driveripper-ops" + +# The /disks source polls each HOST_TARGETS agent for SMART on its own slow cadence +# (job keys `disks:` in /api/health). Lower the interval while developing the page; +# the timeout must stay above the agent's 20 s smartctl pass budget. +# export DISKS_INTERVAL_SECONDS=60 +# export DISKS_TIMEOUT_SECONDS=30 + # Do not assume upsd is on 3493 — nonstandard ports are common and a refused connection then # reads as "upsd is broken" while the service is fine. Ask the host what actually speaks NUT: # ss -ltn # what is listening diff --git a/.env.example b/.env.example index fde6b9b..9afb5e7 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,17 @@ HOST_TARGETS= HOST_INTERVAL_SECONDS=10 HOST_TIMEOUT_SECONDS=4 +# --------------------------------------------------------------------------- # +# Physical-disk dashboard (/disks): the same agents, polled slowly +# SMART is a 5-minute signal, so this source reads the same /snapshot the system page +# reads, on its own cadence and with its own health record. It has no targets of its own: +# no HOST_TARGETS means no disks job at all, so an unconfigured page is never red. +# --------------------------------------------------------------------------- # +DISKS_INTERVAL_SECONDS=300 +# The agent runs smartctl inside its snapshot handler (bounded 20 s per pass), so this +# fetch has to outlast HOST_TIMEOUT_SECONDS rather than inherit it. +DISKS_TIMEOUT_SECONDS=30 + # --------------------------------------------------------------------------- # # LLM dashboard: either engine is optional — unset means "not deployed here" # and its half of the page disappears instead of showing a dead panel. diff --git a/AGENTS.md b/AGENTS.md index 85c5391..af33a49 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,13 +19,14 @@ keep its numbers honest, and the state of the fleet it watches. ## What this is -Three dashboards for one homelab, behind one domain, in one process, backed by one SQLite -file — plus an overview page that says which of the three you need: +Four dashboards for one homelab, behind one domain, in one process, backed by one SQLite +file — plus an overview page that says which of the four you need: | Route | Question it answers | Source | | --------- | ------------------------------------------------------------------------------ | ------------------------------------------------- | -| `/` | Is anything wrong, and where do I look first? | all three, in one request | +| `/` | Is anything wrong, and where do I look first? | all four, in one request | | `/system` | What are my machines doing — CPU, memory, disk, GPU, network, and the history | `agent/host_agent.py`, our own stdlib agent/host | +| `/disks` | Are any of my physical disks failing, and which one — health, temp, hours, wear | the same agents, `smartctl -i -A -H` per whole drive (never a partition) | | `/llm` | Tokens/sec, active sessions, queue depth, KV pressure, latency percentiles | gufo `/metrics` + `/slots`, vLLM `/metrics` | | `/ups` | Battery health, runtime left, power events, battery-test buttons | NUT `upsd` (our own wire-protocol client; 3051 here, not 3493) | @@ -101,7 +102,7 @@ agent/host_agent.py (each host) gufo / vLLM upsd (NUT) collectors.py ── fetch, one UpstreamError type, no writes except run_ups_command ──┐ │ parse + Δcounter/Δt │ ▼ ▼ -sources/host.py · sources/llm.py · sources/nut.py poller.py (task per source) +sources/host.py · disks.py · llm.py · nut.py poller.py (task per source) │ Result(payload, samples, events) │ ▼ ▼ store.py ── SQLite (WAL): sample(source, host, metric, res, ts) + event ──► api.py ──► web/ @@ -114,20 +115,69 @@ Responsibilities, one line each: HTML status page at `GET /`. Refuses to bind off-loopback without a token (exit 2). Counters stay counters: the dashboard computes rates so the agent never remembers who polled it last. + SMART (phase 09): `smart()` runs `smartctl -i -A -H` per whole `/sys/block` disk + (`parse_smart_output` covers ATA + NVMe; NVMe media errors map to + `offline_uncorrectable`, hex-packed vendor raws are never decoded, and an attribute row is + read from its **RAW_VALUE column** rather than the end of the line because this binary + annotates that column — `35 (Min/Max 34/44)`, `43894h+43m+32.519s` — so its last token is + not the value), 5 s per device / + 20 s per pass / 240 s cache, attached as `disk_io.devices[].smart`. Missing binary → + **one** `errors` line for the pass (the word `not found` in it is what + `sources/disks.py` reads to answer "is smartctl installed", and + `tests/test_agent.py` pins that the line still says it); one denied/wedged device → that + disk's `smart` is `null` with its own line; `--no-smart` opts out silently. Never fails + the snapshot. **An expired cache refreshes on a background thread while the request is + served the previous result** — only the first pass after startup runs inline — because 20 s + of wedged disk on the request path blows `HOST_TIMEOUT_SECONDS` and would turn the + *system* source red every four minutes over an add-on that page never draws. The refresher + is single-flight and keeps the old cache on failure, so a bad pass retries on the next + expiry instead of hammering the disk. - `config.py` — env only, no config file format, no CLI flags. `.env` is a *seed*; a real env var always wins. Defaults are safe except the sources, which default to unset. -- `poller.py` — job loop, health records, stale/last-good semantics, transition events. The +- `poller.py` — job loop, health records, stale/last-good semantics, transition events. + `/disks` is a **source of its own** (`disks:`, `DISKS_INTERVAL_SECONDS` 300 s, + `DISKS_TIMEOUT_SECONDS` 30 s, events under the dashboard name `disks`) reading the same + agents the `system` jobs read: the separate task is what keeps a SMART poll out of the + 10-second utilisation loop and lets `/disks` age on its own instead of blinking `/system`. + The job list is built once (`app.build_hub()`) because it is wired twice — production and + the injected-store test path — and two hand-written copies drifting is how a source ends up + polling in tests and not in the container. The success path records (payload → samples → events) and only *then* moves `last_ok`, so a refused SQLite write ages the source and alerts instead of leaving it current with charts that have silently stopped drawing. - Events are stored under the **dashboard** name (`system`/`llm`/`ups`), not the sample + Events are stored under the **dashboard** name (`system`/`disks`/`llm`/`ups`), not the sample source (`host`) — a page filtering `/api/events` by `host` gets an empty feed forever (found on `/system` during phase 03 browser verification). - `collectors.py` — the fetch half. `NutMonitor` owns per-UPS transition bookkeeping. Failure messages name the **redacted** URL (`strip_userinfo` / `Target.display_url`), never the one - that was dialled, because they become health records and event rows. + that was dialled, because they become health records and event rows. `fetch_snapshot()` is + the one agent `/snapshot` read that `system` and `disks` share, so a dead host says the + same sentence on both pages and only the `timeout` differs (the disks poll has to outlast + the agent's 20 s smartctl pass, and `None` is translated into httpx's sentinel because a + literal `None` means *no timeout at all*). Agent `errors` + become the payload's `degraded` **except** the `smart:` add-on lines — a host without + smartmontools or device read access is unconfigured, not broken (invariant 3); they stay + in `agent.errors` for the `/disks` page to explain. - `sources/host.py` — snapshot → payload + samples; reboot detection; GPU series per card (`gpu.busy_pct@`). +- `sources/disks.py` — the same snapshot projected onto **physical** drives for `/disks` + (phase 09): one row per whole-`/sys/block` disk (never a partition, never a mount), the + agent's `smart` block carried through verbatim, `health` the drive's own word normalised to + `ok`/`failed`/`None`, and `smartctl_available` three-way on purpose — `True` (a disk + answered), `False` (the agent said the binary is not installed, so the page can say *install + smartmontools*), `None` (an agent predating the pass, or `--no-smart`: nobody knows, so + `n/a` plus the agent's own `smart_errors` line and never a guessed verdict). **Nothing here + is a rate**: power-on hours and the sector counters are stored counters-as-reported, so the + source takes no `RateEstimator` and its payload carries no `window_seconds` (rule 2 — a rate + with no estimator, no `Δ` label and no `source` string is an invented number). + `disk.health_ok@dev` is not a series either: a verdict is not a number, so the drive's word + travels in the payload only. Each row does carry a `severity` dict (`SEVERITY_FIELDS` = the + two rated numbers, `temp_c`/`percentage_used`, plus `health` through `HEALTH_SEVERITY`, + which translates the drive's own word instead of grading it), and `UNGRADED_FIELDS` names + the numbers that must never be graded. `fleet_disk_summary()` sums the per-host `summary` + counts for the page header and drops a host with no payload from the sums while still + counting it in `hosts_total` — the `/llm` fleet-block rule, so the page never does + arithmetic and a dead agent never shrinks a fleet total. - `sources/llm.py` — gufo + vLLM field maps (`GUFO_SERIES`, `VLLM_SERIES`, `VLLM_HISTOGRAMS`), measured Δ rates, prefill speed as `mean(prompt tokens)/mean(prefill time)`, session list from gufo `/slots`. @@ -170,6 +220,12 @@ Responsibilities, one line each: a busy GPU is the job at every level, so the GPU card and the fleet GPU column render uncoloured — like `net.*`, whose colourlessness predates it. Removing the verdict is not removing the data: `gpu.busy_pct@N` / `gpu.mem_used_pct@N` are still stored and charted. + The `/disks` rows (phase 09) are `disk.temp_c` (50/70) and `disk.percentage_used` (80/100) + — an SSD at 100 % of its rated wear is a replacement, not a warning — attached per row by + `sources/disks.py`; the sector counters and `disk.power_on_hours` deliberately have **no + row**, because a non-zero counter is an alarm chip on presence and zero is not an + achievement (the `/llm` preemption-chip rule), so `classify("disk.reallocated", 1)` is + `None` and the page shows a chip instead of a colour. - `alerts.py` — `build(settings, job_states, host_payloads, engine_payloads, ups_payload, recent_events)` → the "what is wrong right now" list `/api/summary` serves as `alerts` (`{id, severity, source, message, href, at}`, fail first then warn, then by source). A @@ -177,39 +233,74 @@ Responsibilities, one line each: reachability, the `severity` dicts for breaches, `health.checks[]` for UPS conditions. Rules: an **unconfigured** source never alerts (invariant 3); a source that is ok but older than 3× its own poll interval is `warn` while a source that stopped answering is - `fail`; a dead source gets exactly one line (its stale payload adds nothing) that names it - once — the poller's `"UpstreamError: "` wrapper and the duplicated host name are stripped, - because a message that says `gateway` twice reads as two problems; `unknown` and missing - numbers are silence; one alert per (source, metric) and at most 12, the last being a marker - that says how many it hid. This is why the overview can be two panels. + `fail` (the interval is the job's own dashboard's, so `disks` goes amber after 15 quiet + minutes rather than 30 seconds); a dead source gets exactly one line (its stale payload + adds nothing) that names it once — the poller's `"UpstreamError: "` wrapper and the + duplicated host name are stripped, because a message that says `gateway` twice reads as two + problems, and the same rule silences the **`disks` echo** of a dead agent (the disk job + dials the same `/snapshot`, so its failure carries no new information — while a `disks` job + failing *while its host answers*, SMART wedged inside the handler, is the only voice that + fault has); `unknown` and missing numbers are silence; one alert per (source, metric) and at most 12, the last being a marker + that says how many it hid. This is why the overview can be two panels. There is **no + disk-health rule and none is wanted**: a drive the hardware itself called `FAILED` is a red + chip on its `/disks` card and a line in that page's event feed, not an overview alert — + `severity.py` rates no sector counter and this module may not have a threshold of its own, so + grading a drive here would be the bug, not the fix (phase 09). Its `HREF` table does carry + `/disks`, so the one alert a disk job *can* raise — SMART failing while its host answers — + links to the page that names the drive. - `store.py` — `sample(source,host,metric,res,ts)` + `event` + `meta`, rollups, pruning. - `api.py` — GETs everywhere plus the one POST. `/api/config` is redaction boundary. + `/api/disks/*` (phase 09) is the drive-health pair: `/snapshot` answers one entry per + configured host (disk rows + `smart` + `smartctl_available` + per-row `severity` + a + `fleet` block summed by the API, with a configured-but-never-answered host as an + `unreachable` stub whose counts are `None`, not `0`), and `/history` serves the + `disk.*@dev` series with unknown names answering as empty series like the other two + history endpoints. The configured gate runs **before** anything that could reach the + network — `HOST_TARGETS` empty is `404 {"configured": false}` with no client, no fetch and + no `disks` job in the hub at all (invariant 3), and `tests/test_api.py` proves it with + spies on `httpx.AsyncClient.get` and `collectors.fetch_snapshot`. The new surface grows no + write of its own — a fourth dashboard must not quietly become a second one, so the app's + only POST stays `/api/ups/command` (invariant 1, pinned by enumerating every served route's + verbs). `/api/summary` is the overview's single call and the cheapest whole-fleet probe there is: - the page renders only `alerts` plus `system.fleet`/`system.hosts`, while `llm.engines`, - `ups.ups`, the `states` maps and `recent_events` stay for programmatic consumers (live - tests, an external check, a notifier). `system`/`llm`/`ups`/`recent_events` are unchanged - and `alerts` is additive — `tests/test_api.py` pins every one of those key paths, with and - without targets configured, so retiring a panel can never shrink the payload. A configured + the page renders only `alerts` plus `system.fleet`/`system.hosts`, while `disks.hosts`, + `llm.engines`, `ups.ups`, the `states` maps and `recent_events` stay for programmatic + consumers (live tests, an external check, a notifier). `system`/`llm`/`ups`/`recent_events` + are unchanged and `alerts` (phase 02) and `disks` (phase 09) are additive — the `disks` + block is `fleet_disk_summary` verbatim plus one row per host (`disks`/`with_smart`/ + `health_failed`/`alarm_disks`/`smartctl_available`/`failed_devices`), so a notifier learns + *which* drive the hardware called `FAILED` without a second call, and + `tests/test_api.py` pins every one of those key paths, with and without targets configured, + so retiring a panel can never shrink the payload. A configured job with no payload yet is listed as a stub (`unreachable: true`, every value `null`) in `system.hosts`/`llm.engines` exactly as `/api/system/snapshot` does, so `hosts_total` counts the machines there are and `hosts_live` the ones answering. - `app.py` — assembly; `create_app()` takes injectable settings/store so tests build an app with an in-memory DB and no targets. -- `web/` — four static pages, one stylesheet, `charts.js`. No framework. `app.js` owns the +- `web/` — five static pages (`/`, `/system`, `/disks`, `/llm`, `/ups`), one stylesheet, + `charts.js`. No framework. `app.js`'s masthead status dots are one per dashboard job — + including `disks`, because the same agent can answer the 10-second utilisation poll and + have a wedged SMART pass, and one dot for both would be right about neither. `app.js` owns the colour vocabulary: `Dash.severityFrom(cellOrSeverity)` turns a payload verdict into an `ok|warn|fail` class (`''` for anything unrated, so unknown never looks green) and `Dash.hero(el, {label, value, unit, digits, severity, sub, bar})` renders/patches a `.hero` big number (plus an optional `.bar`) from that class — it never compares a number to a threshold. `Dash.severityFor(value, warnAt, failAt)` is the documented fallback for values the API does not rate, and **no page script calls it any more** — `/` was converted - in phase 02, `/system` in phase 03, `/llm` in phase 04 and `/ups` in phase 05, so every - colour on all four pages comes from a `severity` verdict the API sent + in phase 02, `/system` in phase 03, `/llm` in phase 04, `/ups` in phase 05 and `/disks` + shipped that way from its first commit, so every colour on all five pages comes from a + `severity` verdict the API sent (`tests/test_api.py` fails if a page script names it again). A value the API should rate gets a row in `severity.py`, never a bound in JS. Metric naming: dotted (`cpu.usage_pct`, `mem.used_pct`, `disk.used_pct@/srv`, `gpu.busy_pct@0`, `llm.decode_tokens_per_second`, `ups.charge_pct`), `@stream` for per-device -series — `/api/metrics` answers with exactly these names. +series — `/api/metrics` answers with exactly these names. The `sample.source` column is the +dashboard family — `host`, `llm`, `ups`, and `disks` for the SMART series +(`disk.temp_c@nvme0n1`, `disk.power_on_hours@…`, `disk.reallocated@…`, `disk.pending@…`, +`disk.uncorrectable@…`, `disk.percentage_used@…`). Those keep their own family instead of +sitting beside `host`'s `disk.used_pct@/mount` because the cadences differ thirty-fold and +`GET /api/metrics?source=disks` has to answer "what does *this page* have history for". That scheme is the history API's key — treat it as a public interface. ## The fleet (real environment, verified 2026-10) @@ -243,6 +334,42 @@ That scheme is the history API's key — treat it as a public interface. - Host access is via ssh keys as `ops`. Passwordless sudo exists but is command-scoped; the set we rely on is documented in `deploy/nut-sudoers` (additions are listed there for the work that is still blocked). +- **SMART is captured from real hardware; the live agents still cannot read it.** `smartctl` + 7.5 is installed on deskwork and driveripper. The `ops` account (which runs the dev agents) + is in no `disk` group; driveripper grants it `(root) NOPASSWD: /usr/sbin/smartctl*` and + deskwork grants nothing, so the agent's own unprivileged `smartctl -i -A -H` — it shells the + binary and never calls sudo — answers *Permission denied* (exit 2) on every host and `smart` + stays `null` end to end: the designed degradation, not a bug. The three deployed agents also + predate the SMART pass, so the `disks` source today yields one row per physical disk (16 on + driveripper, 1 on deskwork, `vda` on gateway) with `smart: null`, `smart_errors: []` and + `smartctl_available: null` — "nobody told us either way", which is why that flag is + three-way and not a boolean. + **Real data is in the repo now:** `tests/fixtures/smartctl_driveripper_.txt` is all 16 + drives (8 NVMe, 8 spinning) read through that sudo rule, and `tests/test_agent.py` parses + every one — which is what found the parser's one real bug: this binary annotates the + attribute table's RAW_VALUE column (`35 (Min/Max 34/44)` on every Seagate, and one Western + Digital writes its hours as `43894h+43m+32.519s`), so reading the *end* of the row lost the + temperature of all eight HDDs and one power-on count. The captures are also the first + honest look at this fleet's drives: `nvme5n1` reports 13 media-and-data-integrity errors and + `nvme1n1` is at 58 % of its rated wear, so the alarm chip and the amber wear bar on `/disks` + are real features with real data behind them, not demo furniture. + **Lighting up the live page is two operator actions that travel together:** give the agent + user device read access — the `disk` group (`sudo usermod -aG disk `, which is + what `deploy/install-agent.sh` does on install), *not* a sudo rule, because the agent must + read the device itself — and redeploy `agent/host_agent.py` (never redeploy without the + grant: 16 permission-denied lines would age nothing but are pure noise, while with the grant + they are 16 drives' worth of temperature, hours and sector counters). Until then `/disks` + renders the designed degradation: one dimmed card per physical drive, every number `n/a`, and + the one-line why — which is also what gateway shows permanently, being a VM with no physical + disk to read and no smartmontools at all. + `tests/fixtures/smartctl_reference_*.txt` stay as what they are: **reference shapes** (labels + verbatim from the deployed binary, values synthetic), kept beside the real captures so the + synthetic edge cases stay pinned. Re-record the real ones with + `SMART_HOSTS="driveripper-ops" ./scripts/capture-fixtures.sh`: it tries plain `smartctl` + first (so a fixture can never read healthier than the agent does), falls back to `sudo -n`, + refuses to overwrite good fixtures with unreadable output, and scrubs + Serial/WWN/EUI-64 (`test_smartctl_captures_carry_no_unscrubbed_serials` fails the day it + stops). ## Commands @@ -256,11 +383,14 @@ cp .env.dev.example .env.dev # dev-only: ssh tunnels + live targets ./scripts/dev-server.sh status # per-source ok/DOWN ./scripts/dev-server.sh logs -uv run pytest # hermetic: 624 passed + 9 live-skipped, no network -uv run pytest --cov # coverage gate (97.9%; floor 94%) +uv run pytest # hermetic: 769 passed + 11 live-skipped, no network +uv run pytest --cov # coverage gate (98.33%; floor 94%) uv run pytest -m live --live # opt-in: real fleet / engines / upsd uv run ruff check src tests agent && uv run pyright ./scripts/capture-fixtures.sh # re-record ground truth (agents + UPS + gufo /slots) +SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh + # ... plus smartctl per physical disk (needs device read + # access: plain smartctl, else `sudo -n`; scrubs serials) bash .agents/validate.sh # the phase gate: the four checks above (pytest # --cov --cov-fail-under=94, ruff, pyright, plus a @@ -284,7 +414,11 @@ branch `main`); there is no CI on it, so `bash .agents/validate.sh` is the only - Anything that must match an upstream — `/proc` layouts, vLLM series names, NUT framing — is asserted against a verbatim capture in `tests/fixtures/`. If the schema drifts, a - fixture test fails before a dashboard does. + fixture test fails before a dashboard does. `smartctl_driveripper_.txt` is the whole + 16-drive fleet read through a `smartctl` sudo rule (Serial/WWN/EUI-64 scrubbed), and + `test_the_real_drives_answer_every_number_the_disks_page_prints` asserts every number + `/disks` prints comes out of real drive output — that test is what caught the parser + reading the end of an attribute row instead of its annotated RAW_VALUE column. - Anything that needs a failure mode — upsd refusing a command, an engine 500ing, a host rebooting mid-poll, a counter reset — is a fake in `tests/conftest.py`. - **Page payloads that a rewrite must not break are pinned as a table.** @@ -337,9 +471,11 @@ measured rates, `n/a` for the unknown, `SIMULATED` badges on modelled data, per- dots in the header. Every chart names itself for assistive tech: pages pass `ariaLabel` to `LineChart`, which puts `role="img"` + `aria-label` on the `` (an unlabelled plot is silence, and the legend is a sibling that labels nothing). Every `LineChart` on every page has -one — `/system`'s three per-host slots, `/llm`'s four trend plots, `/ups`'s four battery -plots — each pinned by `tests/test_api.py`, so a chart that ships anonymous fails a test. -Window chips (5m…30d on `/system` and `/llm`, 1h…90d on `/ups`; the API accepts 5m…90d). A +one — `/system`'s three per-host slots, `/disks`'s two per-host plots, `/llm`'s four trend +plots, `/ups`'s four battery plots — each pinned by `tests/test_api.py`, so a chart that ships +anonymous fails a test. +Window chips (5m…30d on `/system` and `/llm`, 1h…90d on `/ups` and `/disks`; the API accepts +5m…90d — SMART lands every 300 s, so a 5-minute window on `/disks` is one dot). A panel that cannot get data says why in the panel — the operator should never have to open devtools to learn that a source is unconfigured. Escape anything upstream-provided through `DashCharts.escapeHtml`. @@ -386,6 +522,34 @@ muted unit, mono `.sub` footnote (`Δ10.0s`, "at current load"), coloured by the family and no second palette, and `tests/test_api.py` fails if either appears or if a CSS rule reaches for a token that `:root` does not declare. +**The `/disks` card is one panel per *physical* drive, led by identity and verdict, not by a +hero** (phase 09): a drive has no one-number story the way a UPS has charge %, so the card opens +with the device path (`/dev/nvme0n1`) and the drive's own word (`OK` / `FAILED` / `unknown`, a +`.chip` coloured only by `severity.health` — an unreadable drive gets grey `unknown`, never a +green pass), then the identity line (`model · size · SSD|HDD`, nothing invented where a field is +absent) and three equal `.metric` cells: **Temp**, **Power-on**, **Wear**. Colour comes from +`severity.temp_c` / `severity.percentage_used` and nowhere else; `power_on_hours` is an odometer, +written with a `null` verdict, so it cannot read green at 100 h or red at 60 000 h. The sector +counters are **`.chip.fail` alarm chips that exist only while non-zero** (the `/llm` preemption +rule, and why `severity.py` rates no counter): a drive at zero draws no chip row at all, because a +permanent "0 pending sectors" line trains the eye to ignore it and a green one would call a quiet +drive an achievement. Per host the three-way `smartctl_available` decides one `.banner.info` +why-line and nothing else — `false` says *install smartmontools and grant the disk group* (the +agent said so, so the page may), `null` prints the agent's own `smart_errors` line or names both +fixes without claiming to know which half is missing, and an agent that is not answering says +that instead — and in every case the drive grid takes the shared `.stale` dim while the host card +stays uncoloured (invariant 3: a missing binary is unconfigured, not broken, and a red panel here +would train the operator to ignore red on the drives that do report). Nothing on the page is per +partition or per mount: the rows are the agent's whole-`/sys/block` list, and a filesystem field +naming its way into `page-disks.js` fails a test. The fleet header renders +`fleet_disk_summary()` — hidden below two hosts (one box's sums duplicate its own cards) and +coloured nowhere, because a count of drives is not a graded number — and the per-host `Charts` +disclosure follows `/system` exactly: collapsed, lazy-built, **one** `/api/disks/history` request +per host per refresh naming `disk.temp_c@dev` and `disk.percentage_used@dev` for every drive (the +endpoint answers unknown names as empty series on purpose), the wear plot created only once some +drive publishes the field, legends naming drives because the card heading already says who, and +`pruneHosts` taking the cards, the toggle and the plots off the page together. + A cell-scale trend is `DashCharts.sparkline()` — the `llm_monitor` `drawSpark` idiom, so a percent metric is plotted against `fixedMax` (**never** autoscaled: an 8 % KV window that once held a 92 % spike must not read flat, and the axis gets top/bottom scale marks so "a third of @@ -442,7 +606,7 @@ repaints only when charge / verdict / state change (`drawGlyph()` keys a `data-d charge float jitters every `NUT_INTERVAL_SECONDS`, and rebuilding the `` on that re-announces an accessible name for a difference no eye can see. -An event row names its subject **once**. Every feed (`/system`, `/llm`, `/ups`) renders the +An event row names its subject **once**. Every feed (`/system`, `/disks`, `/llm`, `/ups`) renders the message through `Dash.eventText(event)`, which drops the `UpstreamError: ` wrapper and a *leading* duplicated subject the way `alerts._clean_error()` does for the overview — `poller.py` messages start with their own label, and each row already prints that label as a chip, so @@ -452,7 +616,8 @@ This is presentation only — the stored event and `/api/events` keep every word error string still surfaces in `/api/health`. An event feed is a **window**, not a printout (`Dash.eventsPager(el, events, opts)` in -`app.js`, one helper with three call sites — a pager that works on one page is the bug this +`app.js`, one helper with four call sites (`/system`, `/disks`, `/llm`, `/ups`) — a pager that +works on one page is the bug this shape prevents). `ul.events` is a bounded 320px scroll container (`overscroll-behavior: contain`) and the pager puts `EVENT_PAGE_SIZE` = **50** events in it, with a `.events-pager` strip under the list: `1–50 of 200` between `‹ prev` and `next ›`, both `aria-label`led for @@ -658,6 +823,16 @@ memory plot; the `cpu.usage_pct@coreN` series is still stored and still queryabl field map is written. 3. **UPS history depth.** Events and the numeric series are stored; long-window battery health (capacity fade over months) has the rollups but no dedicated view. +4. **The dev DB is big enough to refuse a write now and then.** The dev store (~590 MB, WAL + grown to ~70 MB) logs `store-error: OperationalError: database is locked` a few times an + hour across `system`/`llm`/`ups`; the poller handles it exactly as invariant 4 requires — + the source goes red with the real reason, ages, and writes `recording` when the next poll + lands — so nothing freezes, but those polls' samples are lost. The one cause this repo can + prove is two servers on one file, which is why `scripts/dev-server.sh stop` waits for the + old pid to die (its comment records the incident). A controlled 25 s external read + transaction during this pass produced none, so checkpoint starvation is not yet shown. + Retention/`wal_checkpoint` tuning for a long-running store is its own phase, not a + phase-09 follow-up. ## Locked decisions (and why) diff --git a/README.md b/README.md index 9d164b2..4ccb558 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # Homelab Dashboards -Three dashboards, one domain, one process, one SQLite file: +Four dashboards, one domain, one process, one SQLite file: | Page | What it answers | Source | | ----------- | ------------------------------------------------------------------------------- | -------------------------------------------------- | -| `/` | Is anything wrong, and which dashboard do I need? | fleet + alerts in one request (`/api/summary`) | +| `/` | Is anything wrong, and which dashboard do I need? | all four, in one request (`/api/summary`) | | `/system` | What are my machines doing — CPU, memory, disk, GPU, network, and the history of all of it | `agent/host_agent.py`, our own stdlib agent per host | +| `/disks` | Are any of my **physical** disks failing, and which one — health, temperature, power-on hours, wear, sector alarms | the same agents, running `smartctl -i -A -H` per whole drive (never a partition or a mount) | | `/llm` | Tokens/sec, active sessions, queue depth, KV-cache pressure, latency percentiles | llama.cpp-family (gufo) `/metrics` + `/slots`, vLLM `/metrics` — read-only | | `/ups` | Battery health, runtime left, power events, and battery-test buttons | NUT `upsd`, on whatever port it really listens on (its own wire-protocol client) | @@ -71,6 +72,22 @@ It is read-only by construction: there is no code path that writes, and the HTTP only answers `GET`. Counters are reported as counters — the dashboard computes rates over a labelled window, so the agent never has to remember who polled it last. +Per-disk `smart` health is a best-effort add-on on the whole-disk `/sys/block` list (no +partitions): the agent runs `smartctl -i -A -H` per physical disk — 5 s bound per device, +20 s per pass, results cached 4 minutes — and parses health, temperature, power-on hours, +the sector alarms and SSD wear into `disk_io.devices[].smart`. Refreshing an expired cache +happens on a background thread while the request is served the previous result, so a wedged +disk can never cost a poll more than its own timeout (only the very first pass after the +agent starts runs inline). A host without +`smartmontools`, or whose agent user cannot read the block devices, is *not broken*: every +disk reports `smart: null` with one explanatory `errors` line, and that line never makes +the system page amber. Enable SMART with `smartmontools` plus the `disk` group (which +`deploy/install-agent.sh` grants when it exists) — the agent shells `smartctl` as itself and +never calls `sudo`, so a sudoers rule alone is not enough — and **redeploy the agent**, since +one installed before phase 09 has no SMART pass to run. Opt out silently with `--no-smart`. +The drive truth this is all developed against lives in `tests/fixtures/smartctl_driveripper_*.txt`: +16 real drives, recorded with serials scrubbed, each one parsed by a test. + Binding off-loopback without a token is refused (exit 2) rather than warned about. Prefer `--token-file` over `--token`: argv is world-readable through `ps`, and this token guards hostname, mounts, disk and GPU models, and process names. `deploy/install-agent.sh` creates @@ -94,7 +111,9 @@ documentation and is never loaded, and `scripts/dev-server.sh` exports `.env.dev | `RATE_WINDOW_SECONDS` | the Δ window behind every measured rate — tokens/sec, bytes/sec, jiffies→CPU % (default 10); each value carries the window it actually got, and `/system`'s footnote prints that, not this default | | `DASH_HOST` / `DASH_PORT` | bind address for the dashboard itself (default `127.0.0.1:8090`; the container sets `0.0.0.0`) | | `HOST_INTERVAL_SECONDS` / `LLM_INTERVAL_SECONDS` / `NUT_INTERVAL_SECONDS` | how often each source is polled (10 / 2 / 5 s) | +| `DISKS_INTERVAL_SECONDS` | how often the disk-health source re-reads each agent for SMART (default 300 s — SMART is not a per-second signal). No targets of its own: the `disks:` jobs exist exactly when `HOST_TARGETS` does | | `HOST_TIMEOUT_SECONDS` / `LLM_TIMEOUT_SECONDS` / `NUT_TIMEOUT_SECONDS` | per-source fetch timeout — one slow source never holds up the others | +| `DISKS_TIMEOUT_SECONDS` | the disks fetch's own budget (default 30 s), because the agent runs `smartctl` inside its snapshot handler and 20 s of wedged disk must not read as a dead host | | `GUFO_API_KEY` / `VLLM_API_KEY` | only if an engine is behind a bearer token; they never leave the server | | `NUT_HOST` / `NUT_PORT` | upsd address — **whatever port it really listens on**, not always 3493. Reads are unauthenticated in stock NUT | | `NUT_USER` / `NUT_PASSWORD` | only needed for instant commands (battery tests) | @@ -102,7 +121,7 @@ documentation and is never loaded, and `scripts/dev-server.sh` exports `.env.dev | `NUT_SIMULATE` | `1` replaces upsd with a **labelled** simulator (development only) | | `DASH_READ_ONLY` | `1` disables every mutating endpoint regardless of other flags | | `DASH_DB` | SQLite path (default `data/dashboards.sqlite3`) | -| `POLL_MS_SYSTEM` / `POLL_MS_LLM` / `POLL_MS_UPS` | how often the *browser* refetches each page (10 s / 2 s / 5 s) — shown in the page footer | +| `POLL_MS_SYSTEM` / `POLL_MS_LLM` / `POLL_MS_UPS` | how often the *browser* refetches each page (10 s / 2 s / 5 s) — shown in the page footer. `/disks` has no `POLL_MS_*` of its own and polls at a fixed 30 s: its payload only moves when the `disks` job re-reads the agents | | `DEBUGPY` / `DEBUGPY_PORT` | `DEBUGPY=1` opens a debugpy listener (default 5678) without blocking startup | | `RETAIN_RAW_HOURS` / `RETAIN_MINUTE_DAYS` / `RETAIN_HOUR_DAYS` | history rollup and pruning | @@ -120,12 +139,12 @@ unreachable host writes its failure message into the event table and `/api/event ``` agent/host_agent.py (each host) gufo / vLLM upsd (NUT) │ GET /snapshot │ GET /metrics │ LIST UPS / LIST VAR / LIST CMD - │ │ + /slots (gufo) │ + │ (twice: system 10s, disks 5m) │ + /slots (gufo) │ ▼ ▼ ▼ src/dashboards/collectors.py ────────────────────────────────────────────┐ │ parse + Δcounter/Δt (rates.py) │ ▼ ▼ -sources/host.py · sources/llm.py · sources/nut.py poller.py (one task per source) +sources/host.py · disks.py · llm.py · nut.py poller.py (one task per source) │ samples events │ ▼ ▼ store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── api.py → web/ @@ -133,7 +152,11 @@ store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── * `poller.py` — every source gets its own asyncio task, its own health record, and its own error handling. A dead host keeps its last good values and is labelled stale; it never - blanks or blocks another source. And it never quietly *stops*: the task is that source's only + blanks or blocks another source. Disk health is a source of its own (`disks:` in + `/api/health`, 5-minute cadence, its own fetch timeout) reading the *same* agent + `/snapshot` the system page reads — a separate task is what keeps a SMART poll out of the + 10-second utilisation poll and lets drive history go stale without blinking `/system`, and + `alerts.py` is why one dead agent is still one line on the overview rather than two. And it never quietly *stops*: the task is that source's only poller, so a SQLite write that gets refused is reported (`ok: false`, the real reason, the numbers aging, one line in the event log) instead of freezing the source under a green dot. * `store.py` — raw samples roll up to 1-minute and 1-hour averages and the raw rows are @@ -147,7 +170,12 @@ store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── published as a 0-1 ratio is rescaled *for the verdict only* — the number printed stays what the upstream published. There is no `gpu.*` row: these are LLM servers, a busy GPU is the job, so the GPU card (and the fleet GPU column) renders uncoloured at every - utilisation — the `gpu.*` history series are still stored and charted. + utilisation — the `gpu.*` history series are still stored and charted. Drive health gets two + rows: `disk.temp_c` (amber 50 °C, red 70 °C) and `disk.percentage_used` (amber 80 %, red + 100 % — an SSD that has spent its rated wear is a replacement). The sector counters + (`disk.reallocated`, `disk.pending`, `disk.uncorrectable`) and `disk.power_on_hours` have + **no row on purpose**: a non-zero counter is an alarm chip on presence, zero is the absence + of a chip and never a green "healthy drive", and an odometer has no bad value. * `alerts.py` — "what is wrong right now", as data. `alerts.build(settings, job_states, host_payloads, engine_payloads, ups_payload, recent_events)` is a pure projection of the state the poller already keeps plus the verdicts `severity.py` and the UPS health checks @@ -157,7 +185,27 @@ store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── is `warn`; an **unconfigured** source never alerts; a dead source is one line rather than one per stale metric; the list dedupes per (source, metric) and is capped at 12 with a marker that says how many it hid. -* `web/` — four static pages, one stylesheet, hand-rolled SVG charts (`charts.js`). No +* `api.py` — the JSON every page reads: `/api/config`, `/api/health`, `/api/events`, + `/api/history`, `/api/metrics`, `/api/summary`, and one `snapshot` + `history` pair per + dashboard (`/api/system/*`, `/api/disks/*`, `/api/llm/*`, `/api/ups/*`). GETs everywhere + except the single `POST /api/ups/command`. Two rules make the pages legible: a source with + no env config answers `404 {"configured": false}` **without dialling** (so a page says + "not configured" instead of going red), and a history query for a series that does not + exist answers an empty series instead of an error (so a card asks for every plot it could + draw and reads `n/a` for the ones the drive never reported). `/api/disks/snapshot` is the + drive-health surface: one entry per host with its physical disk rows, the agent's `smart` + block verbatim, `smartctl_available` (`true` / `false` = install smartmontools / `null` = + the agent never said), a `severity` beside each row's rated numbers, and a `fleet` block + summed here so the page does no arithmetic — a host that has not answered yet is listed as + an `unreachable` stub of `n/a`s and counted, never summed. `/api/config` carries + `disks.interval_seconds`, which is why a drive card four minutes old is behaving, not + broken. `/api/summary` grew a `disks` block with that page (the same `fleet` sums plus one row + per host — counts, `smartctl_available`, and `failed_devices`, so a notifier learns *which* + drive the hardware called `FAILED` without a second call); it is additive, exactly as `alerts` + was, and the key paths of both are pinned with and without targets configured. +* `web/` — five static pages (`/`, `/system`, `/disks`, `/llm`, `/ups`), one stylesheet, hand-rolled + SVG charts (`charts.js`), and a masthead status dot per dashboard job (`disks` has its own: + the same agent can answer the 10-second utilisation poll and have a wedged SMART pass). No framework, no CDN, no bundler, and no narration: a page never explains its own pipeline, so the words on screen are the ones you act on — a unit, a `Δ10.0s` provenance tag, a `SIMULATED` badge, or the sentence beside an empty panel. The overview (`/`) is exactly two panels — a @@ -196,7 +244,23 @@ store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── processes — folds into a collapsed "More detail" disclosure. Each panel pulls its series in **one** `/api/system/history` request (`hosts=&metrics=…&range=…`) every refresh, stack open or not, and a host that stops - reporting takes its panel, its Charts toggle and its charts off the page. `/llm` opens with + reporting takes its panel, its Charts toggle and its charts off the page. + `/disks` is the drive-health page: **one card per physical drive** (the agent's + whole-`/sys/block` list — never a partition, never a mount), led by the device path and the + drive's own verdict word (`OK` / `FAILED` / `unknown`, coloured only by `severity.health`), then + three equal cells — **Temp**, **Power-on** hours, **Wear** — coloured by `disk.temp_c` and + `disk.percentage_used` from the API, with the odometer deliberately uncoloured because + `severity.py` rates no power-on hours. The sector counters appear as **red chips that exist only + while non-zero**, so a healthy drive shows no alarm row at all rather than a wall of zeroes. + A host whose SMART cannot be read keeps its cards, dimmed with every number `n/a` and one + `.banner` line saying why — `smartctl not available on ` when the agent reported the binary + missing, its own sentence when it reported something else, and "not answering" when the agent is + down — never a red panel, because a missing binary is unconfigured, not broken. The header's + drive counts are the API's `fleet` block (hidden below two hosts, so one box's sums never + duplicate its own cards), and each host's **Charts** disclosure behaves exactly like `/system`'s: + collapsed, built only while open, and fed by one `/api/disks/history` request per host per + refresh naming `disk.temp_c@dev` and `disk.percentage_used@dev` for every drive it has. + `/llm` opens with the **fleet**: one block above the panels — `ALL ENGINES`, an `N of M engines` tag, the note *sums only engines answering now*, then **running / waiting / prompt tok/s / decode tok/s / total tokens** — and every number on it is `fleet_totals()`, so the page sums nothing itself. @@ -240,7 +304,7 @@ store.py ── SQLite (WAL): sample(source,host,metric,res,ts) + event ── battery °C) kept below them in smaller muted type because they are what you read *while* the UPS is on battery. Health checks, the battery-test buttons (with their `DANGER` confirmation) and the NUT variable dump are unchanged, just second. - The three event feeds (`/system`, `/llm`, `/ups`) are a **paged window** over the `event` + The four event feeds (`/system`, `/disks`, `/llm`, `/ups`) are a **paged window** over the `event` table rather than a printout of it: the list is a bounded box that scrolls, 50 events to a page, with `1–50 of 200` between `‹ prev` and `next ›` (disabled at the ends) underneath. One helper, `Dash.eventsPager()`, renders every feed and each keeps its own row markup; the fetch @@ -258,6 +322,9 @@ uv run pytest -m live --live # optional: real fleet / engines uv run ruff check src tests agent && uv run pyright # lint + types ./scripts/capture-fixtures.sh # re-record ground truth: host agents, UPS replies, gufo /slots ./scripts/capture-fixtures.sh --with-llm # ... and the engines' /metrics while they are up +SMART_HOSTS="host-a-ops host-b-ops" ./scripts/capture-fixtures.sh + # ... and smartctl per physical disk: plain smartctl, + # `sudo -n` if that is denied; scrubs serials/WWNs bash .agents/validate.sh # all of the above + `node --check` on every page # script + repo hygiene, with the 94% coverage floor ``` diff --git a/TODO.md b/TODO.md index b47c328..2e7ac81 100644 --- a/TODO.md +++ b/TODO.md @@ -1,3 +1,6 @@ # TODO - [ ] "Last self-test: Done and passed" still shows up as a warning in the overview and UPS section +- [ ] In the LLM engines page the "fleet" section has no spacing between it and the engine summaries +- [ ] Decode tok/s sometimes jumps unrealistically high, could be a calculation error +- [ ] For privileged actions (UPS commands) the exporter should be installed with password protection and that password must be used to register it with the collector. Effectively the exporters become protected web servers. Additionally, the collector should generate and manage a root certificate that needs to be installed with each exporter. In fact, just use the certs as auth rather than a password. \ No newline at end of file diff --git a/agent/host_agent.py b/agent/host_agent.py index 0db10b9..771d991 100644 --- a/agent/host_agent.py +++ b/agent/host_agent.py @@ -21,8 +21,19 @@ Design rules monitoring agent is not allowed to make. * **Every read is guarded.** A vanished /proc entry (process exited mid-scan) skips that item; the snapshot still succeeds. The poll is degraded, never failed. -* **The only state** is a small pid->cpu-time cache so `processes.top_cpu` can answer a - question /proc cannot (CPU needs two samples). It is bounded and cleared when stale. +* **SMART is a best-effort add-on.** The per-disk `smart` block runs `smartctl -i -A -H` + in a subprocess: bounded 5 s per device and 20 s per pass, cached 240 s so a fast + dashboard poll cannot re-issue ATA commands every few seconds. Refreshing an expired + cache happens on a background thread while the request is served the previous result — + 20 s of wedged disk must never cost a poll more than its own timeout. No binary, a denied + device or a wedged disk degrades *that disk's* `smart` to `null` plus one `errors` + entry (a missing binary produces exactly one entry for the whole pass) and never + fails the snapshot or `/health`. Absence is a documented state, not a red panel: + install `smartmontools` and give the agent user read access to the block devices + (typically the `disk` group), or pass `--no-smart` to opt out silently. +* **The state** is a small pid->cpu-time cache so `processes.top_cpu` can answer a + question /proc cannot (CPU needs two samples), bounded and cleared when stale, plus + the short-TTL SMART result cache above. SCHEMA (GET /snapshot) ---------------------- @@ -46,8 +57,17 @@ SCHEMA (GET /snapshot) disk_io {devices: [{name, model, rotational, size_bytes, reads_total, writes_total, read_bytes_total, write_bytes_total, in_flight, io_time_ms_total, weighted_io_ms_total, - discard_bytes_total}], + discard_bytes_total, + smart: {model, serial, overall_health, temp_c, + power_on_hours, reallocated_sector_ct, + current_pending_sector_ct, offline_uncorrectable, + percentage_used} | null}], device_names: [str]} # whole disks from /sys/block, no partitions + # `smart` is `smartctl -i -A -H` parsed; null with an errors[] entry + # when the binary is missing or that device cannot be read. Fields + # the drive does not publish stay null (e.g. NVMe has no + # reallocated_sector_ct), and NVMe `offline_uncorrectable` carries + # the health log's "Media and Data Integrity Errors" counter. network {interfaces: [{name, rx_bytes_total, tx_bytes_total, rx_packets_total, tx_packets_total, rx_errors_total, tx_errors_total, rx_dropped_total, tx_dropped_total, speed_mbps}]} @@ -70,6 +90,7 @@ import json import os import re import socket +import subprocess import threading import time from collections.abc import Callable @@ -100,6 +121,41 @@ VIRTUAL_NET_PREFIXES = ("lo", "veth", "docker", "br-", "virbr", "br", "tap", "vn # or LUKS volume is carved out of a disk we already count, so charting both double-counts. VIRTUAL_DISK_PREFIXES = ("loop", "ram", "zram", "sr", "dm-", "md127") +# SMART collection bounds. 5 s per device is generous for a healthy drive and caps how +# long one wedged device can hold a snapshot hostage; 20 s caps the whole pass so even a +# shelf of bad disks cannot stall the HTTP handler; 240 s is below the dashboard's +# 5-minute disks cadence, so the slow disks job gets fresh data while the fast system +# poll never re-runs smartctl every 10 s. +SMARTCTL_TIMEOUT_SECONDS = 5.0 +SMARTCTL_BUDGET_SECONDS = 20.0 +SMARTCTL_CACHE_SECONDS = 240.0 + +# The smart block's field set — the shape the `/disks` page (phase 09) consumes. A field +# the drive does not publish stays None; the page renders `n/a`, never 0. +SMART_FIELDS = ("model", "serial", "overall_health", "temp_c", "power_on_hours", + "reallocated_sector_ct", "current_pending_sector_ct", + "offline_uncorrectable", "percentage_used") + +_SMART_HEALTH_RE = re.compile(r"^SMART overall-health self-assessment test result:\s*(\S+)") +# The number a SMART attribute RAW_VALUE column leads with. Captured from the deployed +# smartmontools 7.5 on this fleet: a Seagate IronWolf writes `35 (Min/Max 34/44)`, a +# Western Digital writes its packed hours as `43894h+43m+32.519s`, and most drives write a +# bare `30423`. In every shape the drive's own figure is the leading decimal group. +_SMART_RAW_NUMBER_RE = re.compile(r"^\d[\d,]*(?:\.\d+)?") +# ATA SMART attribute table rows, keyed by the exact attribute names the deployed +# smartmontools 7.5 prints (verified against its own drive database, not from memory). +_SMART_ATTR_FIELDS = { + "Temperature_Celsius": "temp_c", + "Airflow_Temperature_Cel": "temp_c", + "Power_On_Hours": "power_on_hours", + "Reallocated_Sector_Ct": "reallocated_sector_ct", + "Current_Pending_Sector": "current_pending_sector_ct", + "Offline_Uncorrectable": "offline_uncorrectable", + "Percent_Lifetime_Used": "percentage_used", + "Percent_Life_Used": "percentage_used", + "Percentage_Lifetime_Use": "percentage_used", +} + # pci slot -> human model, filled on first look (lspci output cannot change at runtime). _GPU_MODEL_CACHE: dict[str, str | None] = {} @@ -167,10 +223,20 @@ class Collector: of fixture files instead of a live machine.""" def __init__(self, root: str | os.PathLike[str] = "/", *, top_processes: int = 5, - net_all: bool = False) -> None: + net_all: bool = False, smartctl: str = "smartctl", + smart: bool = True) -> None: self.root = Path(root) self.top_processes = top_processes self.net_all = net_all + self.smartctl = smartctl + self.smart_enabled = smart + self.smart_timeout_seconds = SMARTCTL_TIMEOUT_SECONDS + self.smart_budget_seconds = SMARTCTL_BUDGET_SECONDS + self.smart_cache_seconds = SMARTCTL_CACHE_SECONDS + self._smart_lock = threading.Lock() # ThreadingHTTPServer may poll while a pass runs + self._smart_cache: tuple[float, dict[str, dict[str, Any] | None], + list[str]] | None = None + self._smart_refreshing = False # at most one background refresh at a time self._lock = threading.Lock() # one HTTP handler at a time may touch the pid cache self._pid_cpu: dict[int, tuple[float, int, str]] = {} # pid -> (ts, jiffies, name) self._boot_ts: float | None = None @@ -443,6 +509,129 @@ class Collector: }) return {"devices": devices, "device_names": names} + # -- smart (smartctl) ------------------------------------------------------ # + def smart(self, device_names: list[str]) -> tuple[dict[str, dict[str, Any] | None], + list[str]]: + """Best-effort `smartctl -i -A -H ` per whole disk; never raises. + + Returns (per-device parsed dict or None, error lines). Failure semantics (phase + 09 task 01): the binary missing produces **one** error entry for the whole pass + and every disk `None`; one denied/wedged device produces its own error entry and + nulls only that disk. Results are cached for `smart_cache_seconds` so the fast + system poll cannot re-issue ATA commands every few seconds; the cache replays the + same error lines, so a snapshot always states why `smart` is null. + + An **expired** cache is refreshed on a background thread and the request is served + the previous result, because the pass budget (20 s) is five times the dashboard's + `HOST_TIMEOUT_SECONDS` (4 s): run inline on the request path, one wedged disk would + turn the *system* source red every `smart_cache_seconds` for a fault in an add-on + (AGENTS.md invariant 4 — SMART must not blink the page that does not use it). The + very first pass still runs inline, so a freshly started agent answers `/snapshot` + with real SMART fields instead of a page of `n/a` until someone polls it twice. + """ + now = time.monotonic() + with self._smart_lock: + cached = self._smart_cache + if cached is not None: + if now - cached[0] < self.smart_cache_seconds or self._smart_refreshing: + return dict(cached[1]), list(cached[2]) + self._smart_refreshing = True + threading.Thread(target=self._refresh_smart, args=(list(device_names),), + name="smart-refresh", daemon=True).start() + return dict(cached[1]), list(cached[2]) + return self._smart_pass(device_names) + + def _refresh_smart(self, device_names: list[str]) -> None: + """The expired-cache pass, off the request path (see `smart()`). + + A failure here keeps the previous cache (and its age) rather than clearing it: the + snapshot keeps answering with the last truth it had, and the retry comes on the next + expiry instead of hammering a disk that just failed. + """ + try: + self._smart_pass(device_names) + except Exception as exc: # noqa: BLE001 - the thread must always clear its own flag + with self._smart_lock: + previous = self._smart_cache + results = dict(previous[1]) if previous else dict.fromkeys(device_names) + errors = list(previous[2]) if previous else [] + self._smart_cache = (time.monotonic(), results, + [*errors, + f"smart: background refresh failed: " + f"{type(exc).__name__}: {exc}"]) + finally: + with self._smart_lock: + self._smart_refreshing = False + + def _smart_pass(self, device_names: list[str]) -> tuple[dict[str, dict[str, Any] | None], + list[str]]: + """One `smartctl` pass over the whole disks, and the cache write that goes with it.""" + now = time.monotonic() + results: dict[str, dict[str, Any] | None] = {} + errors: list[str] = [] + deadline = now + self.smart_budget_seconds + for name in device_names: + left = deadline - time.monotonic() + if left <= 0: + remaining = [n for n in device_names if n not in results] + for skipped_name in remaining: + results[skipped_name] = None + errors.append(f"smart: pass budget of {self.smart_budget_seconds:.0f}s " + f"exhausted, {len(remaining)} disk(s) skipped") + break + try: + proc = subprocess.run([self.smartctl, "-i", "-A", "-H", f"/dev/{name}"], + capture_output=True, text=True, + timeout=min(left, self.smart_timeout_seconds), + check=False) + except FileNotFoundError: + # one entry for the pass, not one per disk: a missing binary is one fact + errors.append(f"smart: {self.smartctl} not found — install smartmontools " + "and give this user device read access (disk group) for " + "SMART data") + for missing_name in device_names: + results.setdefault(missing_name, None) + break + except subprocess.TimeoutExpired: + results[name] = None + errors.append(f"smart: {name}: smartctl timed out after " + f"{self.smart_timeout_seconds:.0f}s (device wedged?)") + continue + except OSError as exc: + results[name] = None + errors.append(f"smart: {name}: smartctl could not run: {exc}") + continue + parsed = parse_smart_output(proc.stdout) + if any(value is not None for value in parsed.values()): + results[name] = parsed # partial output is still what the drive published + else: + results[name] = None + errors.append(f"smart: {name}: " + f"{smart_failure_line(proc.returncode, proc.stdout, proc.stderr)}") + with self._smart_lock: + self._smart_cache = (time.monotonic(), results, errors) + return dict(results), list(errors) + + def attach_smart(self, disk_io: dict[str, Any], errors: list[str]) -> None: + """Put `smart` on every disk row; `--no-smart` leaves them all null silently.""" + devices = disk_io.get("devices") or [] + for device in devices: + device["smart"] = None + if not devices or not self.smart_enabled: + return + results, smart_errors = self.smart([str(device.get("name")) for device in devices]) + errors.extend(smart_errors) + for device in devices: + parsed = results.get(str(device.get("name"))) + if not parsed: + continue + smart = dict(parsed) + if not smart.get("model"): + # smartctl's identity block can be absent (some bridges refuse -i while + # answering the health log); the agent's sysfs model is the fallback. + smart["model"] = device.get("model") + device["smart"] = smart + # -- network ------------------------------------------------------------- # def network(self) -> dict[str, Any]: interfaces: list[dict[str, Any]] = [] @@ -787,6 +976,10 @@ class Collector: processes = self._safe(self.processes, "processes", errors) or {"total": None, "top_rss": [], "top_cpu": []} + disk_io = self._safe(self.disk_io, "disk_io", errors) or {"devices": [], "device_names": []} + # best-effort, after the cheap reads: a wedged disk can cost the pass budget, not + # the snapshot (see the module docstring and smart() for the bounds) + self._safe(lambda: self.attach_smart(disk_io, errors), "smart", errors) return { "schema": SCHEMA_VERSION, "agent_version": AGENT_VERSION, @@ -796,7 +989,7 @@ class Collector: "cpu": self._safe(self.cpu, "cpu", errors) or {}, "memory": self._safe(self.memory, "memory", errors) or {}, "filesystems": self._safe(self.filesystems, "filesystems", errors) or [], - "disk_io": self._safe(self.disk_io, "disk_io", errors) or {"devices": [], "device_names": []}, + "disk_io": disk_io, "network": self._safe(self.network, "network", errors) or {"interfaces": []}, "gpus": self._safe(self.gpus, "gpus", errors) or [], "temperatures": temps, @@ -820,6 +1013,108 @@ MEMORY_KEYS = ("total_bytes", "free_bytes", "available_bytes", "used_bytes", "bu "cached_bytes", "dirty_bytes", "swap_total_bytes", "swap_used_bytes", "committed_bytes") +_SMART_FAILURE_HINT_RE = re.compile(r"failed|denied|error|unable|unsupported|not supported", re.I) + + +def smart_failure_line(returncode: int, stdout: str | None, stderr: str | None) -> str: + """The sentence that says *why* smartctl answered nothing. + + Never its first line: smartctl opens every run with its version banner, and errors + land on stdout, on stderr, or both. Take the first line that reads like a failure, + else the last non-empty line, else the exit status — an error entry that quotes the + copyright line is worse than no error entry. + """ + blobs = [stderr or "", stdout or ""] + for blob in blobs: + for line in blob.splitlines(): + if line.strip() and _SMART_FAILURE_HINT_RE.search(line): + return line.strip() + for blob in blobs: # nothing that reads like a failure: last real line of stderr first + lines = [line.strip() for line in blob.splitlines() if line.strip()] + if lines: + return lines[-1] + return f"exit status {returncode}" + + +def _smart_number(text: str) -> float | None: + """`8,760` and `38` are numbers; a hex-packed vendor raw is not one we can decode + without a drive-specific table, so it stays None — an invented sector count is the + exact failure mode this agent exists to avoid.""" + return _to_float(text.strip().replace(",", "")) + + +def _smart_raw_number(raw: str) -> float | None: + """The value of a SMART attribute row's RAW_VALUE column (column 9, `raw`). + + smartctl prints one bare number for most attributes and the shapes above for the rest, + all of which are read here; `0x...` is a packed raw, and a packed raw read as its + leading `0` would report a healthy-looking zero for a value we never decoded, so that + one answers None.""" + token = raw.strip().split(" ")[0] + if not token or token.lower().startswith("0x"): + return None + match = _SMART_RAW_NUMBER_RE.match(token) + return _smart_number(match.group(0)) if match else None + + +def parse_smart_output(text: str | None) -> dict[str, Any]: + """Parse `smartctl -i -A -H` stdout (ATA, NVMe and SCSI share this entry point). + + Only labels the deployed smartmontools 7.5 actually prints are matched (they were + verified against that binary's own format strings and drive database, and are pinned + by tests/fixtures/smartctl_*.txt). Values the drive or the parser cannot supply stay + None. `overall_health` keeps smartctl's own verdict word (`PASSED`, `FAILED`, + `UNKNOWN`, trailing `!` stripped); NVMe `offline_uncorrectable` carries the health + log's `Media and Data Integrity Errors` counter — same meaning, different name, and + the ATA `Offline_Uncorrectable` attribute when both exist wins because they are the + drive's own words for it. ATA rows are read from the RAW_VALUE column rather than the + end of the line, because this binary annotates that column — `35 (Min/Max 34/44)` on a + Seagate, `43894h+43m+32.519s` on a Western Digital — and the last token of such a row + is not the value. + """ + out: dict[str, Any] = dict.fromkeys(SMART_FIELDS) + if not text: + return out + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + continue + label, sep, value = stripped.partition(":") + if sep: + label, value = label.strip(), value.strip() + if label in ("Device Model", "Model Number"): + if value: + out["model"] = value + elif label == "Serial Number": + if value: + out["serial"] = value + elif label == "Temperature": # NVMe health log: `38 Celsius` + number = _smart_number(value.split()[0]) if value.split() else None + if number is not None: + out["temp_c"] = number + elif label == "Power On Hours": + out["power_on_hours"] = _smart_number(value) + elif label == "Percentage Used": + out["percentage_used"] = _smart_number(value.rstrip("%")) + elif label == "Media and Data Integrity Errors" and out["offline_uncorrectable"] is None: + out["offline_uncorrectable"] = _smart_number(value) + health = _SMART_HEALTH_RE.match(stripped) + if health: + out["overall_health"] = health.group(1).rstrip("!") + continue + fields = stripped.split() + if fields and fields[0].isdigit() and len(fields) >= 6: + attribute = _SMART_ATTR_FIELDS.get(fields[1]) + if attribute is not None: + # ID NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE — + # the raw is column 9, and reading the *last* token instead loses every + # drive that annotates it: `35 (Min/Max 34/44)` ends in `44)`. + number = _smart_raw_number(fields[9] if len(fields) > 9 else fields[-1]) + if number is not None: + out[attribute] = number + return out + + # /proc//stat comm is in parens and may itself contain spaces and parens. _STAT_COMM_RE = re.compile(r"^\d+ \((?P.*)\) (?P.*)$", re.S) @@ -969,10 +1264,17 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--root", default="/", help="filesystem root to read (for tests)") parser.add_argument("--top", type=int, default=5, help="processes per top-N list") parser.add_argument("--net-all", action="store_true", help="include veth/bridge interfaces") + parser.add_argument("--smartctl", default="smartctl", + help="smartctl binary; missing is a documented degradation " + "(every disk's smart is null, one errors entry)") + parser.add_argument("--no-smart", action="store_true", + help="skip the smartctl pass entirely (disks report smart: null, " + "no error entries)") parser.add_argument("--once", action="store_true", help="print one snapshot as JSON and exit") args = parser.parse_args(argv) - collector = Collector(args.root, top_processes=args.top, net_all=args.net_all) + collector = Collector(args.root, top_processes=args.top, net_all=args.net_all, + smartctl=args.smartctl, smart=not args.no_smart) if args.once: print(json.dumps({"ok": True, **collector.snapshot()}, indent=2, default=str)) return 0 diff --git a/deploy/install-agent.sh b/deploy/install-agent.sh index 4a84ad1..05987bb 100755 --- a/deploy/install-agent.sh +++ b/deploy/install-agent.sh @@ -14,6 +14,14 @@ # The agent is read-only by construction (it answers GET and writes nothing), runs as its # own system user, and gets the usual systemd lockdown. It still needs to read /proc, /sys # and statvfs(), which the hardening below leaves alone. +# +# Disk health (`smartctl -i -A -H`, the `/disks` page) is the one add-on that needs more +# than /proc and /sys: smartctl must open the disk device nodes. The installer therefore +# adds the agent user to the `disk` group when one exists, and the unit device-allows the +# disk nodes instead of PrivateDevices. This is raw *read* access to the disks — the +# standard arrangement for SMART monitoring, granted deliberately — and its absence is a +# documented degradation, never an install failure: no smartmontools (or no group) simply +# means every disk reports `smart: null` with an explanatory error entry. set -euo pipefail SERVICE="dashboards-agent" @@ -94,6 +102,22 @@ install() { useradd --system --home-dir / --shell /usr/sbin/nologin "$AGENT_USER" fi + # SMART (phase 09): the usual distro answer for reading drive health is the `disk` + # group — no sudoers entry, no root. Re-runs are idempotent. + if getent group disk >/dev/null 2>&1; then + if id -nG "$AGENT_USER" 2>/dev/null | tr ' ' '\n' | grep -qx disk; then + echo " $AGENT_USER is already in the disk group (SMART reads enabled)" + else + usermod -aG disk "$AGENT_USER" + echo " added $AGENT_USER to the disk group — raw device reads, which is what smartctl needs" + fi + else + echo " no disk group on this host — SMART data will read null (harmless; see README)" + fi + if ! command -v smartctl >/dev/null 2>&1; then + echo " note: smartctl is not installed — every disk reports smart: null until smartmontools is" + fi + install -d -m 0755 "$AGENT_DIR" install -m 0755 "$AGENT_SOURCE" "$AGENT_TARGET" @@ -124,10 +148,17 @@ RestartSec=5 Environment=PYTHONUNBUFFERED=1 SyslogIdentifier=$SERVICE # The agent reads /proc, /sys and statvfs() and writes nothing, so everything writable can -# be taken away. PrivateDevices keeps /dev closed; the snapshot never needs it. +# be taken away. smartctl is the one thing that needs a real /dev node, so instead of +# PrivateDevices the unit closes all devices and reopens only the disk nodes, read/write +# at the cgroup layer while the code (and smartctl -i -A -H) stays read-only by design. NoNewPrivileges=true PrivateTmp=true -PrivateDevices=true +PrivateDevices=false +DevicePolicy=strict +DeviceAllow=/dev/nvme* rw +DeviceAllow=/dev/sd* rw +DeviceAllow=/dev/vd* rw +DeviceAllow=/dev/sg* rw ProtectSystem=strict ProtectHome=read-only ProtectKernelTunables=true diff --git a/scripts/capture-fixtures.sh b/scripts/capture-fixtures.sh index 6b3a544..ef9dca6 100755 --- a/scripts/capture-fixtures.sh +++ b/scripts/capture-fixtures.sh @@ -4,9 +4,15 @@ # ./scripts/capture-fixtures.sh # needs the tunnels up: scripts/dev-tunnels.sh start # ./scripts/capture-fixtures.sh --with-llm # also scrape the live engines' /metrics # # (gufo /slots is captured without the flag) +# SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh +# # also capture smartctl per physical disk # # Fixtures are verbatim, not hand-written: a fixture you typed is a test of your memory. # When a host's kernel or an engine's metric names change, re-run this and read the diff. +# Exception on record: tests/fixtures/smartctl_reference_*.txt are reference shapes, not +# field captures (no host could reach a block device when they were written). The SMART_HOSTS +# pass below writes hardware truth beside them as smartctl__.txt, so the reference +# shapes stay what they are: synthetic edge cases with verbatim labels. set -uo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" @@ -106,6 +112,67 @@ else echo " (skipped UPS captures — NUT_HOST unset; see .env.dev.example)" fi +# --- SMART (smartctl -i -A -H per physical disk) -------------------------------- +# The phase-09 `/disks` page parses these. smartctl must be able to READ the block +# devices, and a fleet grants that one of two ways: the agent user in the `disk` group +# (`sudo usermod -aG disk `, what deploy/install-agent.sh does) or a sudoers rule +# for smartctl alone (what this fleet has). Plain smartctl is tried first because that is +# exactly what agent/host_agent.py shells — if the capture needs sudo and the agent does +# not have it, the fixtures would read healthier than the fleet does. When plain smartctl +# answers "Permission denied" the capture retries under `sudo -n`. A drive that answers +# neither way is deliberately NOT recorded (a good fixture is never replaced by a dead +# reader's output). +# Identifiers never go in the repo: Serial Number, LU WWN Device Id and the NVMe namespace +# EUI-64 (which encodes the serial) are replaced with SCRUBBED* markers; everything else is +# verbatim, and tests/test_agent.py fails the day this stops happening. +# Skipped when SMART_HOSTS is unset, so a laptop without fleet ssh still re-records the +# rest. Whole disks only, the same virtual-prefix skip agent/host_agent.py applies — +# partitions and dm-* would double-capture the same platter. +if [ -n "${SMART_HOSTS:-}" ]; then + for host in $SMART_HOSTS; do + name="${host%-ops}" # fixtures are named for the host, not the ssh alias, like agent_*.json + devs="$(ssh -o BatchMode=yes -o ConnectTimeout=8 "$host" \ + 'for d in /sys/block/*; do n=${d##*/}; case "$n" in loop*|ram*|zram*|sr*|dm-*|md*) ;; *) echo "$n";; esac; done' 2>/dev/null)" + if [ -z "$devs" ]; then + echo " ✗ smartctl_${name}_* — ssh did not answer (is $host an ssh alias?)" + failed=1 + continue + fi + for dev in $devs; do + tmp="$FIX/.smartctl_${name}_${dev}.txt.tmp" + ssh -o BatchMode=yes -o ConnectTimeout=8 "$host" "smartctl -i -A -H /dev/$dev" >"$tmp" 2>&1 + rc=$? + if grep -qi "Permission denied" "$tmp"; then + ssh -o BatchMode=yes -o ConnectTimeout=8 "$host" "sudo -n smartctl -i -A -H /dev/$dev" >"$tmp" 2>&1 + rc=$? + fi + # A drive that cannot answer health at all (no SMART support, virtual HBA tricks) + # is the same fixture contract as an unreadable one: the parse tests require model + + # health in every recorded capture, so do not record what cannot be parsed. + if [ ! -s "$tmp" ] || grep -qiE "Permission denied|failed to open|not found|does not support SMART|SMART support is: Unavailable|not smart-capable" "$tmp"; then + rm -f "$tmp" + echo " ✗ smartctl_${name}_${dev} — no readable SMART (rc $rc): install smartmontools and grant this user the disk group or a smartctl sudo rule" + failed=1 + else + sed -Ei \ + -e 's/^([[:space:]]*Serial [Nn]umber:)[[:space:]]*.*/\1 SCRUBBEDSERIAL/' \ + -e 's/^([[:space:]]*LU WWN Device Id:)[[:space:]]*.*/\1 SCRUBBEDWWN/' \ + -e 's/^([[:space:]]*(Namespace [0-9]+ )?IEEE EUI-?64:)[[:space:]]*.*/\1 SCRUBBEDEUI64/' \ + "$tmp" + { echo "# Verbatim smartctl -i -A -H /dev/$dev from $name, recorded by" + echo "# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED," + echo "# every other byte is what the drive answered. Volatile rows (the" + echo "# Temperature raws, Local Time) move; the parse tests read shape." + cat "$tmp"; } > "$FIX/smartctl_${name}_${dev}.txt" + rm -f "$tmp" + echo " ✓ smartctl_${name}_${dev}.txt" + fi + done + done +else + echo " (skipped SMART captures — SMART_HOSTS unset; see .env.dev.example)" +fi + # --- gufo /slots (per-slot rows) --------------------------------------------- # `tests/slots_contract.py` pins the shape of the `/llm` slot grid (task 04) against # `tests/fixtures/slots_gufo.json`, so this capture is NOT behind `--with-llm`: it is one @@ -176,6 +243,13 @@ changed = [] # upsd quotes a field only when it has to, so both `VAR "apc" "ups.serial" "x"` and # `VAR apc ups.serial "x"` are real wire shapes; the value is always quoted. serial_re = re.compile(r'(^VAR\s+\S+\s+"?(?:device|ups)\.serial"?\s+")([^"\n]*)(")', re.M) +# smartctl identities: the serial identifies the physical unit and the LU WWN encodes +# the vendor's serial — both scrubbed, same deterministic markers as the UPS serials. +# "Local Time is:" is capture time, not device truth: normalised so re-captures do not +# churn the diff. +smart_serial_re = re.compile(r"(?im)^(Serial Number:\s+)(\S.*?)(\s*)$") +smart_wwn_re = re.compile(r"(?im)^(LU WWN Device Id:\s+)(\S.*?)(\s*)$") +smart_time_re = re.compile(r"(?im)^Local Time is:.*$") for path in sorted(fix.iterdir()): if path.suffix not in (".json", ".txt", ".vars"): continue @@ -191,6 +265,7 @@ for path in sorted(fix.iterdir()): seen[value] = f"00000000-0000-4000-8000-{len(seen) + 1:012d}" return seen[value] + seen_wwns: dict[str, str] = {} text = uuid_re.sub(swap, text) # "hostname": "gw.lab.example.com" -> "gw": the dashboard's host key is the target name # from HOST_TARGETS, never whatever the box calls itself @@ -210,6 +285,18 @@ for path in sorted(fix.iterdir()): return match.group(1) + serials[body] + pad + match.group(3) text = serial_re.sub(scrub_serial, text) + if path.name.startswith("smartctl_"): + def scrub_field(match: re.Match[str], prefix: str, store: dict[str, str]) -> str: + body = match.group(2).strip() + if body.startswith(prefix): + return match.group(0) + if body not in store: + store[body] = f"{prefix}{len(store) + 1}" + return match.group(1) + store[body] + match.group(3) + + text = smart_serial_re.sub(lambda m: scrub_field(m, "SCRUBBEDSERIAL", serials), text) + text = smart_wwn_re.sub(lambda m: scrub_field(m, "SCRUBBEDWWN", seen_wwns), text) + text = smart_time_re.sub("Local Time is: (time of capture removed by scripts/capture-fixtures.sh)", text) if text != path.read_text(): path.write_text(text) changed.append(path.name) diff --git a/src/dashboards/alerts.py b/src/dashboards/alerts.py index c49ac06..8081b2d 100644 --- a/src/dashboards/alerts.py +++ b/src/dashboards/alerts.py @@ -56,8 +56,10 @@ WARN = "warn" #: could be said — `AGENTS.md` rule 2, no measurement, no verdict, no colour. _RANK = {FAIL: 0, WARN: 1} -#: Where each dashboard lives, so every alert can say where to look. -HREF = {"system": "/system", "llm": "/llm", "ups": "/ups"} +#: Where each dashboard lives, so every alert can say where to look. `disks` joined with the +#: `/disks` page itself (phase 09 task 04): before that route existed a disk alert pointed at +#: `/`, because a link to a page that answers 404 is worse than no link. +HREF = {"system": "/system", "disks": "/disks", "llm": "/llm", "ups": "/ups"} DEFAULT_HREF = "/" #: A source whose last good value is older than this many of its own poll intervals is @@ -160,9 +162,31 @@ def _interval(settings: Settings, dashboard: str) -> float: return float(settings.nut_interval) if dashboard == "system": return float(settings.host_interval) + if dashboard == "disks": + # 300 s by default, so "stale" here means 15 minutes of no SMART poll. Reading the + # wrong interval would amber the landing page every 5 minutes for a source that is + # behaving exactly as configured. + return float(settings.disks_interval) return 60.0 +def _system_partner_outage(states: Sequence[Mapping[str, Any]], key: str) -> bool: + """Is the `system` job for the same host down too? (used for `disks` job keys) + + The disk-health job reads the same agent `/snapshot` the system dashboard does, so its + own failure is the *consequence* of an agent being down, never a second fault. The + system alert already names the machine once (`AGENTS.md`: one dead source, one line), + so this suppresses the echo and nothing else: a `disks` job that fails while its host + answers — SMART wedged inside the agent's snapshot handler, say — is still one alert, + because that failure has no other voice. + """ + host = key.split(":", 1)[1] if ":" in key else key + partner = next((state for state in states + if state.get("dashboard") == "system" + and str(state.get("key") or "") == f"host:{host}"), None) + return partner is not None and not bool(partner.get("ok")) + + # --------------------------------------------------------------------------- # # the sink: dedupe, bound, sort # --------------------------------------------------------------------------- # @@ -225,21 +249,31 @@ class _Sink: # --------------------------------------------------------------------------- # # the rules # --------------------------------------------------------------------------- # -def _state_for(states: Sequence[Mapping[str, Any]], name: str) -> Mapping[str, Any] | None: - """The health record for the job that produced `name` (a host or engine label).""" +def _state_for(states: Sequence[Mapping[str, Any]], name: str, + dashboard: str | None = None) -> Mapping[str, Any] | None: + """The health record for the job that produced `name` (a host or engine label). + + `dashboard` is not a nicety: a host is polled by two jobs (`host:` for the + system page and `disks:` for the drive-health page) that both end in the host + name and both carry it as their label, so an unfiltered lookup answers "is that host + answering?" with whichever job registered first. + """ for state in states: + if dashboard is not None and state.get("dashboard") != dashboard: + continue if state.get("label") == name or str(state.get("key") or "").endswith(f":{name}"): return state return None -def _answering(states: Sequence[Mapping[str, Any]], name: str) -> bool: +def _answering(states: Sequence[Mapping[str, Any]], name: str, + dashboard: str | None = None) -> bool: """Is that source currently answering? Unknown (no state at all) counts as yes. A payload that exists is data; inventing a reachability verdict for a source with no health record would be this module's opinion, not the poller's. """ - state = _state_for(states, name) + state = _state_for(states, name, dashboard) return True if state is None else bool(state.get("ok")) @@ -273,6 +307,8 @@ def _source_alerts(sink: _Sink, settings: Settings, states: Sequence[Mapping[str href = HREF.get(dashboard, DEFAULT_HREF) key = str(state.get("key") or label) if not state.get("ok"): + if dashboard == "disks" and _system_partner_outage(states, key): + continue # the host agent is already named by its own alert; one line # the alert starts with the name, so the name inside the message is removed — # "driveripper: driveripper: ..." reads like two problems instead of one error = _clean_error(label, str(state.get("error") or "")) or "is not answering" @@ -291,7 +327,7 @@ def _source_alerts(sink: _Sink, settings: Settings, states: Sequence[Mapping[str def _host_alerts(sink: _Sink, host: Mapping[str, Any], states: Sequence[Mapping[str, Any]]) -> None: name = str(host.get("name") or "host") - if host.get("unreachable") or not _answering(states, name): + if host.get("unreachable") or not _answering(states, name, "system"): return # the job-level alert already names this host; one line, not four at = host.get("ts") if isinstance(host.get("ts"), (int, float)) else None for section, field, metric in _HOST_CELLS: @@ -317,7 +353,7 @@ def _host_alerts(sink: _Sink, host: Mapping[str, Any], def _engine_alerts(sink: _Sink, engine: Mapping[str, Any], states: Sequence[Mapping[str, Any]]) -> None: name = str(engine.get("engine") or "engine") - if engine.get("unreachable") or not _answering(states, name): + if engine.get("unreachable") or not _answering(states, name, "llm"): return at = engine.get("scraped_at") if isinstance(engine.get("scraped_at"), (int, float)) else None metrics = engine.get("metrics") or {} diff --git a/src/dashboards/api.py b/src/dashboards/api.py index 9c1508d..53e64ba 100644 --- a/src/dashboards/api.py +++ b/src/dashboards/api.py @@ -22,9 +22,12 @@ from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request from . import alerts as alert_rules from .collectors import NutMonitor from .config import Settings, strip_userinfo -from .poller import Hub +from .poller import Hub, SourceState from .severity import classify from .sources import nut +from .sources.disks import DASHBOARD as DISKS_DASHBOARD +from .sources.disks import SOURCE as DISKS_SOURCE +from .sources.disks import fleet_disk_summary from .sources.host import fleet_summary from .sources.llm import fleet_totals from .store import Event, SampleStore @@ -148,6 +151,17 @@ def common_router() -> APIRouter: "poll_ms": settings.poll_ms_llm, "rate_window_seconds": settings.rate_window, }, + "disks": { + # The drive-health page has no upstream of its own: it reads the same agent + # `/snapshot` the system page does, so it is configured exactly when + # `HOST_TARGETS` is, and says "not configured" with no targets (invariant 3). + # Its cadence is its own and worth printing: a card reading data from four + # minutes ago is behaving, not broken, and the interval is the only fact that + # makes that legible. + "enabled": bool(settings.host_targets), + "interval_seconds": settings.disks_interval, + "timeout_seconds": settings.disks_timeout, + }, "ups": { "enabled": settings.nut_enabled, "server": f"{settings.nut_host}:{settings.nut_port}" if settings.nut_enabled else None, @@ -204,8 +218,10 @@ def common_router() -> APIRouter: Everything else here is for **programmatic use**: this is the cheapest whole-fleet probe that exists, so the live tests, an external check and a notifier later read `llm.engines`, `ups.ups` and the `states`/`recent_events` no panel paints any more. - Keys are additive-only — `alerts` was added, nothing was renamed, dropped, or given - a new meaning. `tests/test_api.py` pins each path below against a future cleanup. + Keys are additive-only — `alerts` was added in phase 02 and `disks` in phase 09; the + `disks` block is for programmatic consumers too (which drives are down to `n/a`, which + ones the drive itself called `FAILED`), and nothing was renamed, dropped, or given a new + meaning. `tests/test_api.py` pins each path below against a future cleanup. """ # A configured job that has never answered gets the same stub `/api/*/snapshot` # hands out: a host in `HOST_TARGETS` with no payload yet is still in the fleet, and @@ -222,6 +238,13 @@ def common_router() -> APIRouter: "unreachable": True} for job in ctx.hub.jobs.values() if job.dashboard == "llm" ] + # the fourth dashboard's own jobs, not the system ones: a host can answer the + # 10-second utilisation poll and still have a wedged SMART pass, and `/api/summary` + # is the one call an external probe makes, so it has to be able to tell those apart + disks_hosts: list[dict[str, Any]] = [ + job.state.payload or _disks_stub(job.label, job.state) + for job in ctx.hub.jobs.values() if job.dashboard == DISKS_DASHBOARD + ] ups_payload = None nut_job = ctx.hub.jobs.get("nut") if nut_job and nut_job.state.payload: @@ -265,6 +288,25 @@ def common_router() -> APIRouter: "unreachable": bool(e.get("unreachable"))} for e in engines], "states": ctx.hub.states("llm")}, + # additive in phase 09: the drive-health view of the same fleet. Counts come from + # each host's own `summary` and the fleet sum, never recomputed here, and a host + # that has not answered is a row of `None`s — "unknown" is not "no drives". + # Nothing in this block carries a `severity`: a count of drives is not a graded + # number and a non-zero sector counter is an alarm on presence (`severity.py`). + "disks": {"enabled": bool(ctx.settings.host_targets), + "fleet": fleet_disk_summary(disks_hosts) if disks_hosts else None, + "hosts": [{"name": h["name"], + "disks": (h.get("summary") or {}).get("disks"), + "with_smart": (h.get("summary") or {}).get("with_smart"), + "health_failed": (h.get("summary") or {}).get("health_failed"), + "alarm_disks": (h.get("summary") or {}).get("alarm_disks"), + "smartctl_available": h.get("smartctl_available"), + "failed_devices": [row.get("device") + for row in h.get("disks") or [] + if row.get("health") == "failed"], + "unreachable": bool(h.get("unreachable"))} + for h in disks_hosts], + "states": ctx.hub.states(DISKS_DASHBOARD)}, "ups": {"enabled": ctx.settings.nut_enabled, "simulated": bool(ups_payload and ups_payload.get("simulated")), "ups": [{"name": card["name"], "overall": card["health"]["overall"], @@ -328,6 +370,99 @@ def system_router() -> APIRouter: return router +# --------------------------------------------------------------------------- # +# Disks dashboard (physical drive health) +# --------------------------------------------------------------------------- # +def _disks_stub(label: str, state: SourceState) -> dict[str, Any]: + """A configured host agent that has never answered, in the `/disks` payload's shape. + + The same statement `/api/system/snapshot` makes with `unreachable: true`: the machine is + still in the fleet (a dead agent must not read as "never configured"), it has no rows to + draw, and its counts are `None` rather than `0` — "unknown" and "no drives" are different + sentences (rule 2), and the page prints `n/a` for what it cannot know. + """ + return { + "name": label, + "ts": state.last_attempt, + "unreachable": True, + "smartctl_available": None, + "smart_errors": [], + "disks": [], + "summary": {"disks": None, "with_smart": None, "health_failed": None, + "alarm_disks": None}, + } + + +def disks_router() -> APIRouter: + """`/api/disks/*` — GETs only, like every other dashboard's surface. + + There is deliberately no POST here: the app's one write stays `POST /api/ups/command` + (invariant 1). Nothing about reading a drive's SMART attributes justifies a mutation, and + `smartctl -t` (a self-test) would put real load on a disk something is plugged into. + """ + router = APIRouter(prefix="/api/disks", tags=["disks"]) + + def require_hosts(ctx: AppContext) -> None: + """Configured-ness first, before anything that could reach the network. + + `disks` has no targets of its own — it reads the agent snapshots the system page + reads — so unconfigured means `HOST_TARGETS` is empty, and the answer is a 404 that + dials nothing (invariant 3). With no targets there is no `disks:` job in the hub + either, so there is no fetch to age out and no red dot for a page nobody installed. + """ + if not ctx.settings.host_targets: + raise HTTPException(404, detail={"configured": False, + "detail": "no hosts configured — set HOST_TARGETS"}) + + @router.get("/snapshot") + async def snapshot(ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + """One entry per host: its physical disks, their `smart` block, and why either is missing. + + Every host in `HOST_TARGETS` is a row, answering or not (invariant 4). A disk row is + one whole-`/sys/block` device — never a partition, never a mount — carrying the + agent's `smartctl` parse verbatim in `smart`, that parse's verdict normalised to + `health`, a `severity` dict for the two rated numbers, and `None` for whatever the + drive does not publish (the page prints `n/a`, rule 2). Per host, + `smartctl_available` says whether SMART is readable at all — `False` means "install + smartmontools", `None` means the agent never said — and `smart_errors` carries the + agent's own sentence. Nothing here is a rate: the payload has no `window_seconds`, + because this source measures nothing over a window. + """ + require_hosts(ctx) + hosts: list[dict[str, Any]] = [] + for job in ctx.hub.jobs.values(): + if job.dashboard != DISKS_DASHBOARD: + continue + hosts.append(job.state.payload or _disks_stub(job.label, job.state)) + hosts.sort(key=lambda item: item.get("name") or "") + return {"ts": time.time(), "hosts": hosts, "fleet": fleet_disk_summary(hosts), + "states": ctx.hub.states(DISKS_DASHBOARD)} + + @router.get("/history") + def history(metrics: str = Query("disk.temp_c"), hosts: str = Query(""), + range: str = Query("24h"), points: int = Query(600, ge=10, le=4000), + ctx: AppContext = Depends(get_context)) -> dict[str, Any]: + """The `disk.*@dev` series, through the same history mechanism as every other page. + + `metrics` are full series names (`disk.temp_c@nvme0n1`, comma-separated) and an + unknown one answers as an **empty series**, not a 404 — the contract + `/api/system/history` and `/api/llm/history` already have, so a card asks for every + series it could draw in one request and a drive with no temperature sensor draws an + empty plot instead of breaking the call. `hosts` is comma-separated or empty for every + configured target. History reads only SQLite, so nothing dials here whichever way the + config goes; with no targets the answer is no hosts and no series. + """ + span = parse_range(range) + wanted_metrics = [item.strip() for item in metrics.split(",") if item.strip()] + wanted_hosts = [item.strip() for item in hosts.split(",") if item.strip()] or [ + target.name for target in ctx.settings.host_targets] + return {"hosts": wanted_hosts, "range_seconds": span, + "series": [_history(ctx.store, DISKS_SOURCE, host, metric, span, points) + for host in wanted_hosts for metric in wanted_metrics]} + + return router + + # --------------------------------------------------------------------------- # # LLM dashboard # --------------------------------------------------------------------------- # diff --git a/src/dashboards/app.py b/src/dashboards/app.py index ac802cf..7ea4dfe 100644 --- a/src/dashboards/app.py +++ b/src/dashboards/app.py @@ -1,4 +1,4 @@ -"""Application assembly: one FastAPI app, four pages, one SQLite file, N poll loops. +"""Application assembly: one FastAPI app, the dashboard pages, one SQLite file, N poll loops. `create_app()` takes injectable settings and store so tests can build an app with an in-memory DB and no targets — the same code path production uses, without the network. @@ -19,21 +19,61 @@ from fastapi.responses import FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles from starlette.exceptions import HTTPException as StarletteHTTPException -from .api import NO_STORE, AppContext, common_router, llm_router, system_router, ups_router -from .collectors import NutMonitor, UpstreamError, host_jobs, llm_jobs, nut_job +from .api import NO_STORE, AppContext, common_router, disks_router, llm_router, system_router, ups_router +from .collectors import ( + NutMonitor, + UpstreamError, + disks_jobs, + host_jobs, + llm_jobs, + nut_job, +) from .config import SETTINGS as DEFAULT_SETTINGS from .config import Settings from .poller import Hub from .rates import RateEstimator +from .sources.disks import DisksCollector from .sources.host import HostCollector from .sources.nut_sim import from_env as simulator_from_env from .store import SampleStore WEB = Path(__file__).resolve().parent / "web" -PAGES = {"/": "index.html", "/system": "system.html", "/llm": "llm.html", "/ups": "ups.html"} +PAGES = {"/": "index.html", "/system": "system.html", "/disks": "disks.html", + "/llm": "llm.html", "/ups": "ups.html"} VERSION = "0.1.0" +def build_hub(settings: Settings, store: SampleStore, + client: httpx.AsyncClient) -> tuple[Hub, NutMonitor | None]: + """Every poll loop the settings describe, on one hub. + + One function because the assembly is wired twice: `build_context()` for production, and + the injected-store path in `lifespan` for tests. Two hand-written copies drift, and the + one that drifts is the one production does not run — which is how a source can end up + polling in tests and not in the container. + + Order is host → disks → llm → ups so far only for readability: two jobs carry the same + host label (`host:gw`, `disks:gw`), so anything that looks a job up by label alone gets + the system record — `alerts.py` filters by dashboard instead and does not depend on it. + """ + rates = RateEstimator(span=settings.rate_window) + hub = Hub(store) + for job in host_jobs(list(settings.host_targets), client, HostCollector(rates), + settings.host_interval): + hub.add(job) + for job in disks_jobs(list(settings.host_targets), client, DisksCollector(), + settings.disks_interval, timeout=settings.disks_timeout): + hub.add(job) + for job in llm_jobs(settings, client, rates): + hub.add(job) + nut_monitor: NutMonitor | None = None + if settings.nut_enabled: + simulator = simulator_from_env(os.environ.get("NUT_SIMULATE")) + nut_monitor = NutMonitor(settings, simulator=simulator) + hub.add(nut_job(settings, nut_monitor, client)) + return hub, nut_monitor + + def build_context(settings: Settings, client: httpx.AsyncClient) -> tuple[ AppContext, SampleStore, Callable[[], None]]: """Create the store, the collectors, and the hub's jobs. Returns (ctx, store, start).""" @@ -43,22 +83,7 @@ def build_context(settings: Settings, client: httpx.AsyncClient) -> tuple[ minute_keep_days=settings.minute_keep_days, hour_keep_days=settings.hour_keep_days, ) - rates = RateEstimator(span=settings.rate_window) - host_collector = HostCollector(rates) - nut_monitor: NutMonitor | None = None - if settings.nut_enabled: - simulator = simulator_from_env(os.environ.get("NUT_SIMULATE")) - nut_monitor = NutMonitor(settings, simulator=simulator) - - hub = Hub(store) - for job in host_jobs(list(settings.host_targets), client, host_collector, - settings.host_interval): - hub.add(job) - for job in llm_jobs(settings, client, rates): - hub.add(job) - if nut_monitor is not None: - hub.add(nut_job(settings, nut_monitor, client)) - + hub, nut_monitor = build_hub(settings, store, client) ctx = AppContext(settings=settings, store=store, hub=hub, nut_monitor=nut_monitor, started_at=time.time()) return ctx, store, hub.start @@ -79,20 +104,8 @@ def create_app(settings: Settings | None = None, store: SampleStore | None = Non ) try: if store is not None: - # injected: rebuild everything except the DB - rates = RateEstimator(span=settings.rate_window) - hub = Hub(store) - nut_monitor = None - if settings.nut_enabled: - simulator = simulator_from_env(os.environ.get("NUT_SIMULATE")) - nut_monitor = NutMonitor(settings, simulator=simulator) - for job in host_jobs(list(settings.host_targets), client, - HostCollector(rates), settings.host_interval): - hub.add(job) - for job in llm_jobs(settings, client, rates): - hub.add(job) - if nut_monitor is not None: - hub.add(nut_job(settings, nut_monitor, client)) + # injected: rebuild everything except the DB, with the production wiring + hub, nut_monitor = build_hub(settings, store, client) ctx = AppContext(settings=settings, store=store, hub=hub, nut_monitor=nut_monitor, started_at=time.time()) start = hub.start @@ -119,6 +132,7 @@ def create_app(settings: Settings | None = None, store: SampleStore | None = Non app.include_router(common_router()) app.include_router(system_router()) + app.include_router(disks_router()) app.include_router(llm_router()) app.include_router(ups_router()) diff --git a/src/dashboards/collectors.py b/src/dashboards/collectors.py index 04aea5c..18deb15 100644 --- a/src/dashboards/collectors.py +++ b/src/dashboards/collectors.py @@ -16,6 +16,7 @@ import httpx from .config import Target, strip_userinfo from .poller import Job, Result from .rates import RateEstimator +from .sources import disks as disks_source from .sources import llm as llm_source from .sources import nut from .sources.host import HostCollector, parse_host_snapshot @@ -34,15 +35,27 @@ class UpstreamError(Exception): # --------------------------------------------------------------------------- # # System hosts # --------------------------------------------------------------------------- # -async def fetch_host_snapshot(client: httpx.AsyncClient, target: Target, - host_collector: HostCollector) -> Result: +async def fetch_snapshot(client: httpx.AsyncClient, target: Target, + *, timeout: float | None = None) -> dict[str, Any]: + """One agent `/snapshot`, fetched and checked. Shared by every dashboard that reads one. + + `system` and `disks` dial the same URL on different cadences, so the failure sentences + come from one place too: a host that is down has to say it the same way on both pages. + Every message uses the **redacted** URL — an `UpstreamError` string is kept in the job's + health record and written to the event table, both of which the browser reads, and + `target.url` may carry the agent's token as userinfo (invariant 6). + + `timeout` overrides the client default for this one request (`None` = keep the + client's). The SMART pass lives inside the agent's snapshot handler, so the slow + `disks` poll needs a budget a 4-second system poll cannot afford — and must not + inherit one. `None` is translated to httpx's own sentinel because passing a literal + `None` to `get()` means *no timeout at all*, which would hang the loop forever. + """ + request_timeout: Any = httpx.USE_CLIENT_DEFAULT if timeout is None else timeout url = f"{target.url}/snapshot" - # Every message below uses the redacted URL on purpose: an UpstreamError string is kept - # in the job's health record and written to the event table, both of which the browser - # reads, and `target.url` may carry the agent's token as userinfo. shown = f"{target.display_url}/snapshot" try: - response = await client.get(url, headers=target.headers) + response = await client.get(url, headers=target.headers, timeout=request_timeout) except httpx.TimeoutException as exc: raise UpstreamError(f"{target.name}: {shown} timed out", status=504) from exc except httpx.HTTPError as exc: @@ -59,14 +72,45 @@ async def fetch_host_snapshot(client: httpx.AsyncClient, target: Target, f"({response.text[:80]!r})") from exc if not body.get("ok", False): raise UpstreamError(f"{target.name}: agent said not ok: {body.get('detail')}") + return body + + +async def fetch_host_snapshot(client: httpx.AsyncClient, target: Target, + host_collector: HostCollector) -> Result: + body = await fetch_snapshot(client, target) result = parse_host_snapshot(target.name, body, host_collector) if body.get("errors"): # The agent degraded a section (a /proc read failed) but still answered. Say so - # in the payload rather than presenting a partial snapshot as a complete one. - result.payload["degraded"] = body["errors"] + # in the payload rather than presenting a partial snapshot as a complete one — + # except the SMART add-on: a host without smartmontools or without device read + # access is *unconfigured, not broken* (invariant 3), and its `smart:` lines must + # not hold the system page amber forever. They stay in the snapshot payload + # (`agent.errors`) where the /disks page explains them in its own panel. + errors = body["errors"] + core = ([line for line in errors if not str(line).startswith("smart")] + if isinstance(errors, list) else errors) + if core: + result.payload["degraded"] = core return result +# --------------------------------------------------------------------------- # +# Physical disks (the /disks dashboard) +# --------------------------------------------------------------------------- # +async def fetch_disk_snapshot(client: httpx.AsyncClient, target: Target, + disks_collector: disks_source.DisksCollector, + *, timeout: float | None = None) -> Result: + """The same `/snapshot`, projected onto physical drives and their SMART health. + + Deliberately a second read of an endpoint `system` already polls: the two dashboards + want different cadences (SMART is a 5-minute signal, utilisation is a 10-second one), + and sharing one fetch would either chart stale SMART or run `smartctl` every 10 + seconds. Separate jobs also means a dead disk poll ages `/disks` alone (invariant 4). + """ + body = await fetch_snapshot(client, target, timeout=timeout) + return disks_source.parse_disks_snapshot(target.name, body, disks_collector) + + # --------------------------------------------------------------------------- # # LLM engines # --------------------------------------------------------------------------- # @@ -231,6 +275,22 @@ def host_jobs(targets: list[Target], client: httpx.AsyncClient, host_collector: ] +def disks_jobs(targets: list[Target], client: httpx.AsyncClient, + disks_collector: disks_source.DisksCollector, interval: float, + *, timeout: float | None = None) -> list[Job]: + """One job per host agent — the same target list `system` polls, at its own cadence. + + No targets, no jobs: `disks` has no upstream of its own, so "unconfigured" means the + same thing it means for `system` (invariant 3), and `/api/disks/*` 404s without ever + dialling (task 03). + """ + return [ + Job(f"disks:{target.name}", disks_source.DASHBOARD, target.name, interval, + lambda t=target: fetch_disk_snapshot(client, t, disks_collector, timeout=timeout)) + for target in targets + ] + + def llm_jobs(settings: Any, client: httpx.AsyncClient, rates: RateEstimator) -> list[Job]: jobs: list[Job] = [] if settings.gufo_url: diff --git a/src/dashboards/config.py b/src/dashboards/config.py index eef9aca..300691c 100644 --- a/src/dashboards/config.py +++ b/src/dashboards/config.py @@ -156,6 +156,15 @@ class Settings: host_interval: float = _num("HOST_INTERVAL_SECONDS", 10) host_timeout: float = _num("HOST_TIMEOUT_SECONDS", 4) + # --- disks dashboard --------------------------------------------------------- + # SMART is not a per-second signal, so the `/disks` source reads the *same* agent + # snapshots on its own slow cadence with its own health record. The timeout is its own + # too, and deliberately generous: a smartctl pass inside the agent costs up to its + # 20 s budget on a wedged disk, and the answer must not be a red `disks` source. + # No targets of their own — `disks` is configured exactly when `HOST_TARGETS` is. + disks_interval: float = _num("DISKS_INTERVAL_SECONDS", 300) + disks_timeout: float = _num("DISKS_TIMEOUT_SECONDS", 30) + # --- llm dashboard ------------------------------------------------------------ gufo_url: str = _base_url("GUFO_BASE_URL") gufo_api_key: str = _env("GUFO_API_KEY") diff --git a/src/dashboards/poller.py b/src/dashboards/poller.py index 12655e3..9c4359e 100644 --- a/src/dashboards/poller.py +++ b/src/dashboards/poller.py @@ -41,8 +41,8 @@ class Result: @dataclass class SourceState: - key: str # "host:driveripper", "llm:gufo", "nut" - dashboard: str # "system" | "llm" | "ups" + key: str # "host:driveripper", "disks:driveripper", "llm:gufo", "nut" + dashboard: str # "system" | "disks" | "llm" | "ups" label: str configured: bool = True ok: bool = False diff --git a/src/dashboards/severity.py b/src/dashboards/severity.py index 1562240..33e10b9 100644 --- a/src/dashboards/severity.py +++ b/src/dashboards/severity.py @@ -47,10 +47,20 @@ FAIL = "fail" #: card is the job, not a fault. A busy-at-every-level metric has no honest amber, so the #: table simply does not judge it — ``classify`` returns ``None`` and the page renders the #: number uncoloured (the same rule ``net.*`` follows for throughput). +# +#: The two ``disk.*`` rows added in phase 09 are the drive-health ones, and they are the +#: only disk numbers with a row: a spinning disk at 70 °C is a fire and an SSD at 100 % of +#: its rated wear is a replacement, so both deserve red rather than amber. The sector +#: counters (``disk.reallocated``, ``disk.pending``, ``disk.uncorrectable``) and +#: ``disk.power_on_hours`` get **no row on purpose** — a non-zero sector count is an +#: incident, not a graded number, so it renders as an alarm chip on presence (the ``/llm`` +#: preemption-chip rule) and zero is the absence of a chip, never a green "good drive". THRESHOLDS: dict[str, tuple[float, float]] = { "cpu.usage_pct": (75.0, 90.0), "mem.used_pct": (80.0, 92.0), "disk.used_pct": (80.0, 90.0), + "disk.temp_c": (50.0, 70.0), + "disk.percentage_used": (80.0, 100.0), "cpu.temp_c": (75.0, 90.0), "llm.kv_cache_usage_pct": (75.0, 90.0), "ups.charge_pct": (50.0, 20.0), diff --git a/src/dashboards/sources/__init__.py b/src/dashboards/sources/__init__.py index 8c7c1c8..ddd58ec 100644 --- a/src/dashboards/sources/__init__.py +++ b/src/dashboards/sources/__init__.py @@ -1,6 +1,7 @@ """Data sources. Each module turns one upstream's raw payload into our cell shape. host -> our own stdlib agent (JSON) +disks -> the same agent snapshot, projected onto physical drives + smartctl llm -> gufo / vLLM Prometheus expositions (parsed by our own Prom class) nut -> the upsd wire protocol (reads + instant commands) nut_sim-> a labelled simulator for development, never a substitute for real hardware diff --git a/src/dashboards/sources/disks.py b/src/dashboards/sources/disks.py new file mode 100644 index 0000000..4a99ac8 --- /dev/null +++ b/src/dashboards/sources/disks.py @@ -0,0 +1,337 @@ +"""Disks source: the physical-drive view of the same agent snapshot `/system` reads. + +`/system` answers "is the filesystem nearly full". This answers the other storage +question, the one a filesystem can never tell you: **is the drive itself healthy**. It +reads the agent's whole-`/sys/block` enumeration (never partitions — one card per +physical disk) and the `smart` block task 09/01 added to it. + +What this module owns, and what it deliberately does not: + +* **It adds no numbers.** The `smart` block is the agent's own `smartctl -i -A -H` parse, + carried through verbatim. A field the drive does not publish stays `None` and renders + `n/a` (`AGENTS.md` rule 2) — a 0 reallocated-sector count and "this drive does not + report one" are different sentences, and on a 90-day chart they look identical. +* **Nothing here is a rate.** `power_on_hours` and the sector counters are + counters-as-reported: monotonic by nature, plotted as they are. A dashboard-side + `Δpower_on_hours/Δt` would be a measurement with no estimator, no `Δ` label and + no `source` string, which is exactly what rule 2 forbids. So this source needs no + `RateEstimator` and its payload carries no window. +* **It is not a verdict either** — except where `severity.py` says so. `health` below is + the drive's own word (`smartctl`'s `PASSED`/`FAILED`) normalised to the app's + `ok`/`failed` vocabulary, and every row carries the `severity` dict the other payloads + carry: the two rated numbers (`temp_c`, `percentage_used`) through `severity.classify`, + and `health` through `HEALTH_SEVERITY` because it is the drive's own word rather than a + threshold. The sector counters get no verdict at all (`UNGRADED_FIELDS`): a non-zero one + is an alarm chip on presence, zero is the absence of a chip, never a green "good drive". + +Metric names are the history namespace, `@stream` per device, and are a public interface: +`disk.temp_c@nvme0n1`, `disk.power_on_hours@nvme0n1`, `disk.reallocated@nvme0n1`, +`disk.pending@nvme0n1`, `disk.uncorrectable@nvme0n1`, `disk.percentage_used@nvme0n1` — +all under the `disks` sample family, so `GET /api/metrics?source=disks` answers "what does +this page actually have" and a 5-minute SMART series never mixes with the 10-second +system ones. `disk.health_ok@dev` is **not** a series: a verdict is not a number, so the +drive's word travels in the payload only. + +Samples carry the snapshot's timestamp: that is when the dashboard read the value, not +when `smartctl` ran (the agent caches a pass for `SMARTCTL_CACHE_SECONDS`). The value is +the drive's; only the x-axis position is the poll's. +""" + +from __future__ import annotations + +import math +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from ..poller import Result +from ..severity import classify +from ..store import Event, Sample + +#: The sample family and the dashboard name. They are the same word here on purpose: +#: `/api/events?source=disks` and `hub.states("disks")` must agree, which is the phase-03 +#: lesson (events filed under the sample source `host` gave `/system` an empty feed). +SOURCE = "disks" +DASHBOARD = "disks" + +#: (agent `smart` field -> dotted metric base). The `@` stream suffix is added per +#: disk. Names are kept short because they are chart legends and `/api/metrics` output; +#: the long form stays in the payload's `smart` block, which is verbatim from the agent. +SMART_SERIES: tuple[tuple[str, str], ...] = ( + ("temp_c", "disk.temp_c"), + ("power_on_hours", "disk.power_on_hours"), + ("reallocated_sector_ct", "disk.reallocated"), + ("current_pending_sector_ct", "disk.pending"), + ("offline_uncorrectable", "disk.uncorrectable"), + ("percentage_used", "disk.percentage_used"), +) + +#: `smartctl`'s own verdict words. `OK` is what some ATA drives answer where the spec says +#: `PASSED`; anything else (`UNKNOWN`, an empty string, a driver's shrug) is *no verdict*, +#: which is `None` here and `n/a` on the page — never a green pass for an unknown. +HEALTH_OK_WORDS = frozenset({"PASSED", "OK"}) +HEALTH_FAILED_WORDS = frozenset({"FAILED"}) + +#: Sector/media-error counters, the ones that mean "the drive is losing sectors" when they +#: are not zero. `percentage_used` is wear, not an alarm, and stays out of this set. +ALARM_FIELDS = ("reallocated_sector_ct", "current_pending_sector_ct", "offline_uncorrectable") + +#: The rated `smart` fields, as (`agent field`, `severity metric`): the only place a disk +#: number is matched to a threshold row, mirroring `sources/llm.py`'s `SEVERITY_CELLS`. A +#: new rated number is a row in `severity.THRESHOLDS` plus a row here, and nothing in a page. +SEVERITY_FIELDS: tuple[tuple[str, str], ...] = ( + ("temp_c", "disk.temp_c"), + ("percentage_used", "disk.percentage_used"), +) + +#: The numbers deliberately left ungraded, so a test can name them and a future "helpful" +#: threshold on a counter fails instead of colouring a drive: `reallocated_sector_ct`, +#: `current_pending_sector_ct` and `offline_uncorrectable` are alarms on **presence** +#: (non-zero is an incident; zero is not an achievement), and `power_on_hours` is an +#: odometer with no bad value. `health` is a word, so it goes through `HEALTH_SEVERITY` +#: rather than a numeric row. +UNGRADED_FIELDS: tuple[str, ...] = ("reallocated_sector_ct", "current_pending_sector_ct", + "offline_uncorrectable", "power_on_hours") + +#: The drive's normalised verdict -> the app's colour. Not a threshold: this translates a +#: verdict `smartctl` published itself, the way `sources/nut.py`'s `assess()` reports its +#: own checks. No word, no colour: `normalize_health()` answers `None` and so does this. +HEALTH_SEVERITY: dict[str, str] = {"ok": "ok", "failed": "fail"} + +#: The word `host_agent.Collector._smart_pass()` writes when the binary is not there. The match is on this +#: word, and `tests/test_agent.py` pins that the agent's own line still contains it: a +#: rewording fails a test instead of quietly turning "install smartmontools" into "no +#: SMART data, reason unknown". +MISSING_BINARY_WORD = "not found" + + +def normalize_health(word: Any) -> str | None: + """`smartctl`'s verdict word -> `ok` / `failed` / ``None`` (no verdict). + + Case-insensitive and `!`-tolerant because the field is a human-readable log line, not + an enum: smartctl shouts `FAILED!`, the agent strips the bang, and an agent that + stopped stripping it must still read red rather than "no verdict". + """ + if not isinstance(word, str): + return None + text = word.strip().rstrip("!").upper() + if text in HEALTH_OK_WORDS: + return "ok" + if text in HEALTH_FAILED_WORDS: + return "failed" + return None + + +def smart_error_lines(errors: Sequence[Any]) -> list[str]: + """The agent's SMART lines out of `errors` — the page's *why this card is `n/a`*. + + The agent prefixes them (`smart: …`, `smart: sdb: …`) and `collectors.py` keeps them + out of the system payload's `degraded` list, because a host without smartmontools is + unconfigured, not broken (invariant 3). + """ + lines: list[str] = [] + for line in errors or (): + text = str(line) + if text.strip().lower().startswith("smart"): + lines.append(text) + return lines + + +def smartctl_available(disks: Sequence[Mapping[str, Any]], + errors: Sequence[Any]) -> bool | None: + """Can this host show SMART data at all? Three answers, because there are three facts. + + * ``True`` — the agent returned a `smart` block for at least one disk. + * ``False`` — the agent said its own way that `smartctl` is not installed, so the page + can say "install smartmontools" instead of leaving an empty card a mystery. + * ``None`` — nothing to go on: an agent that predates the SMART pass, one started with + `--no-smart` (which is silent by design), or a pass where every device was denied. + `smartctl` may well be installed; the dashboard does not know, so it prints `n/a` + plus the agent's own line rather than a verdict it did not measure (rule 2). + """ + if any(disk.get("smart") is not None for disk in disks): + return True + for line in smart_error_lines(errors): + if MISSING_BINARY_WORD in line.lower(): + return False + return None + + +def row_severity(row: Mapping[str, Any]) -> dict[str, str | None]: + """The verdicts a `/disks` card may paint, beside the numbers they judge. + + Same rule as every other payload: colour comes from `severity.classify` (or, for + `health`, from the drive's own word) and a page never compares a disk number to a + bound. A drive with no `smart` block still gets a dict of `None`s rather than no key, so + the card prints every field `n/a` and uncoloured instead of branching on a missing key. + """ + smart = row.get("smart") + values = smart if isinstance(smart, Mapping) else {} + verdicts: dict[str, str | None] = { + field: classify(metric, _number(values.get(field))) + for field, metric in SEVERITY_FIELDS + } + verdicts["health"] = HEALTH_SEVERITY.get(str(row.get("health") or "")) + return verdicts + + +def disk_rows(snapshot: Mapping[str, Any]) -> list[dict[str, Any]]: + """The agent's whole-disk rows, projected to what a `/disks` card needs. + + `model` prefers `smartctl`'s identity block and falls back to the agent's sysfs one; + the agent already does that fallback inside `smart`, so preferring it here keeps the + card's title the same string in both the healthy and the identity-less case. + """ + rows: list[dict[str, Any]] = [] + devices = (snapshot.get("disk_io") or {}).get("devices") or [] + for device in devices: + if not isinstance(device, Mapping): + continue + name = str(device.get("name") or "").strip() + if not name: + continue + raw = device.get("smart") + smart = dict(raw) if isinstance(raw, Mapping) and raw else None + row: dict[str, Any] = { + "device": name, + "path": f"/dev/{name}", + "model": (smart or {}).get("model") or device.get("model"), + "serial": (smart or {}).get("serial"), + "size_bytes": device.get("size_bytes"), + "rotational": device.get("rotational"), + "health": normalize_health((smart or {}).get("overall_health")), + "smart": smart, + } + row["severity"] = row_severity(row) + rows.append(row) + return rows + + +def fleet_disk_summary(hosts: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Cross-host counts for the `/disks` header, summed by the API and not by the page. + + A host with no payload yet is **counted and not summed**: `hosts_total` is how many + agents the deployment configures, `hosts_live` how many answered, and the disk counts + cover only the answering ones — `fleet_totals()`'s rule for `/llm`, so a dead agent + cannot silently shrink a fleet total while the page looks current. There is no + `severity` here on purpose: these are counts of drives and of alarms, and alarms are + presence, not a graded number (`UNGRADED_FIELDS`). + """ + fleet: dict[str, Any] = {"hosts_total": len(hosts), "hosts_live": 0, "disks": 0, + "with_smart": 0, "health_failed": 0, "alarm_disks": 0} + for host in hosts: + if host.get("unreachable"): + continue + fleet["hosts_live"] += 1 + summary = host.get("summary") + if not isinstance(summary, Mapping): + continue # answered with something uncountable: do not invent a contribution + for key in ("disks", "with_smart", "health_failed", "alarm_disks"): + value = summary.get(key) + if isinstance(value, int) and not isinstance(value, bool): + fleet[key] += value + return fleet + + +def disk_summary(rows: Sequence[Mapping[str, Any]]) -> dict[str, int]: + """What a host card counts. Unknown is never counted as good *or* bad (rule 2). + + `alarm_disks` is the number of drives publishing a non-zero sector or media-error + counter; `health_failed` is the drives whose own verdict is not OK. A drive with no + `smart` block is in `disks` and in neither of the other two — it is unreadable, which + is a different fact from unhealthy. + """ + summary = {"disks": len(rows), "with_smart": 0, "health_failed": 0, "alarm_disks": 0} + for row in rows: + smart = row.get("smart") + if not isinstance(smart, Mapping): + continue + summary["with_smart"] += 1 + if row.get("health") == "failed": + summary["health_failed"] += 1 + if any(_number(smart.get(key)) not in (None, 0.0) for key in ALARM_FIELDS): + summary["alarm_disks"] += 1 + return summary + + +def _number(value: Any) -> float | None: + """A finite number, or ``None`` — bools included, because a bool is not a measurement.""" + if isinstance(value, bool) or not isinstance(value, int | float): + return None + number = float(value) + return number if math.isfinite(number) else None + + +@dataclass +class DisksCollector: + """Per-host memory across polls: only what a *comparison* needs. + + That is one thing — the last SMART health verdict per device — so a drive going + `PASSED → FAILED` is one event instead of one event per 5-minute poll (or, the other + way round, silence). + """ + + hosts: dict[str, dict[str, str | None]] = field(default_factory=dict) + + def previous_health(self, host: str) -> dict[str, str | None]: + return self.hosts.get(host, {}) + + def record_health(self, host: str, health: Mapping[str, str | None]) -> None: + """Replace, never merge: a disk that left the host must not linger as history.""" + self.hosts[host] = dict(health) + + +def health_events(collector: DisksCollector, host: str, rows: Sequence[Mapping[str, Any]], + now: float) -> list[Event]: + """One event per drive whose own health verdict *changed*. + + The first sighting of a drive is not news (that would print the whole fleet at boot, + the mistake `sources/nut.py`'s transitions already avoid). Landing in `failed` is + `error`; leaving it is `info`, because the drive is no longer the problem it was. + """ + previous = collector.previous_health(host) + events: list[Event] = [] + for row in rows: + device = str(row.get("device") or "") + verdict = row.get("health") + before = previous.get(device) + if before is None or before == verdict: + continue + severity = "error" if verdict == "failed" else "info" + events.append(Event(now, DASHBOARD, host, "smart-health", severity, + f"{host}: {device} SMART health {before} → {verdict or 'unknown'}", + {"device": device, "from": before, "to": verdict})) + collector.record_health(host, {str(row.get("device")): row.get("health") + for row in rows}) + return events + + +def parse_disks_snapshot(name: str, snapshot: Mapping[str, Any], collector: DisksCollector, + *, now: float | None = None) -> Result: + """One agent snapshot -> (`/disks` payload, history samples, transition events).""" + now = time.time() if now is None else now + ts = float(snapshot.get("ts") or now) + errors = snapshot.get("errors") or [] + rows = disk_rows(snapshot) + samples: list[Sample] = [] + for row in rows: + smart = row.get("smart") + if not isinstance(smart, Mapping): + continue # nothing was read: no sample, and the page prints `n/a` + for field_name, metric in SMART_SERIES: + value = _number(smart.get(field_name)) + if value is not None: + samples.append(Sample(SOURCE, name, f"{metric}@{row['device']}", value, ts)) + payload = { + "name": name, + "ts": ts, + # three-way on purpose: "yes", "no, install smartmontools", and "the agent does + # not say" are different sentences for the operator (see `smartctl_available`). + "smartctl_available": smartctl_available(rows, errors), + "smart_errors": smart_error_lines(errors), + "disks": rows, + "summary": disk_summary(rows), + } + return Result(payload=payload, samples=samples, + events=health_events(collector, name, rows, now)) diff --git a/src/dashboards/store.py b/src/dashboards/store.py index 24bfb4a..4dc191c 100644 --- a/src/dashboards/store.py +++ b/src/dashboards/store.py @@ -14,6 +14,13 @@ addressed by `(source, host, metric, res, ts)`: the file stays bounded (a 3-host / 3-engine / 2-UPS fleet is a few hundred MB a year, not a disk emergency). +The rollups do not care how often a source writes. A slow one (`disks` samples SMART every +300 s) puts one raw row in every fifth 1-minute bucket and *nothing* in the other four: +an empty bucket produces no rollup row rather than a fabricated one, and a bucket with a +sample in it rolls up that sample's average, so the "latest wins" shape a fast source +gets is the same arithmetic. `history()` then falls back to `raw` while the rollups lag, +which is what keeps a five-minute series drawable in its first hour. + The connection is guarded by a lock and every method is synchronous: async callers wrap them in `anyio.to_thread.run_sync`, sync FastAPI handlers run in the threadpool already. """ diff --git a/src/dashboards/web/pages/disks.html b/src/dashboards/web/pages/disks.html new file mode 100644 index 0000000..ee17b80 --- /dev/null +++ b/src/dashboards/web/pages/disks.html @@ -0,0 +1,101 @@ + + + + + +Homelab · disks + + + + +
+ ◆ Homelab + +
+ + --:--:-- + + +
+
+ +
+
+
+

Physical disks

+
reading drives…
+
+
+
window
+
+ + + + +
+

Per host

+ +
+ +
+ +
+

Drive events

+
  • loading…
+
+ +
+ +
+
+ + + + + + diff --git a/src/dashboards/web/pages/index.html b/src/dashboards/web/pages/index.html index e87ee38..6c9a168 100644 --- a/src/dashboards/web/pages/index.html +++ b/src/dashboards/web/pages/index.html @@ -13,6 +13,7 @@ diff --git a/src/dashboards/web/pages/llm.html b/src/dashboards/web/pages/llm.html index ae73396..ca9ca6b 100644 --- a/src/dashboards/web/pages/llm.html +++ b/src/dashboards/web/pages/llm.html @@ -13,6 +13,7 @@ diff --git a/src/dashboards/web/pages/system.html b/src/dashboards/web/pages/system.html index 9ca2bf3..1e532e4 100644 --- a/src/dashboards/web/pages/system.html +++ b/src/dashboards/web/pages/system.html @@ -13,6 +13,7 @@ diff --git a/src/dashboards/web/pages/ups.html b/src/dashboards/web/pages/ups.html index 164e789..dea0c81 100644 --- a/src/dashboards/web/pages/ups.html +++ b/src/dashboards/web/pages/ups.html @@ -13,6 +13,7 @@ diff --git a/src/dashboards/web/static/app.js b/src/dashboards/web/static/app.js index e36ae9a..e6c35f8 100644 --- a/src/dashboards/web/static/app.js +++ b/src/dashboards/web/static/app.js @@ -511,8 +511,13 @@ const Dash = (() => { const nav = $('#nav-status'); if (nav) { + // One dot per dashboard's own jobs (`hub.states()` keys them by dashboard), so a + // source nobody configured stays grey instead of red — and the `disks` job gets its + // own dot rather than hiding inside `hosts`: the same agent can answer the 10-second + // utilisation poll and have a wedged SMART pass, and those are different sentences. const jobs = [ { key: 'system', label: 'hosts', href: '/system' }, + { key: 'disks', label: 'disks', href: '/disks' }, { key: 'llm', label: 'engines', href: '/llm' }, { key: 'ups', label: 'UPS', href: '/ups' }, ]; diff --git a/src/dashboards/web/static/dash.css b/src/dashboards/web/static/dash.css index ce036fe..507c174 100644 --- a/src/dashboards/web/static/dash.css +++ b/src/dashboards/web/static/dash.css @@ -409,6 +409,45 @@ main { padding: 1.15rem 1.25rem 3rem; max-width: 1680px; margin: 0 auto; } } .fleet .metrics { margin-top: .55rem; } +/* ---- drive panels ----------------------------------------------------------- + * `/disks`: one `.disk` mini-panel per **physical** drive inside its host's `.card`, in the + * `.slot` shape (bordered mini-panel, `.chip` for the drive's own word) because a drive is an + * item in a grid, not a page-level panel. `.metrics`/`.metric` and `.alarms` are the existing + * vocabulary; the three columns are fixed so a 16-drive host reads as rows of equal cards. + * + * Colour is the payload's verdict and nothing else: `--warn`/`--fail` on the two rated numbers + * (`disk.temp_c`, `disk.percentage_used`), `--fail` on a `FAILED` health chip via `.chip.fail`, + * and *no* colour on the power-on odometer or on a drive that published nothing (`.value.is-null` + * keeps printing faint `n/a`). A host whose SMART cannot be read is dimmed with the shared + * `.stale` on `.disks` — an unreadable drive is unconfigured, not broken (invariant 3) — so + * nothing in this section paints a red panel for a missing binary. + */ +.disks { + display: grid; gap: .5rem; + grid-template-columns: repeat(auto-fill, minmax(min(230px, 100%), 1fr)); +} +.disk { + border: 1px solid var(--border); border-radius: var(--radius-sm); + padding: .5rem .6rem; background: var(--bg-sunken); min-width: 0; +} +.disk-head { display: flex; align-items: center; gap: .45rem; } +.disk-dev { + font-size: 12px; color: var(--text); min-width: 0; + overflow: hidden; text-overflow: ellipsis; white-space: nowrap; +} +.disk-head .chip { margin-inline-start: auto; } +.disk-model { + font-size: 10.5px; margin-bottom: .45rem; + overflow: hidden; text-overflow: ellipsis; white-space: nowrap; +} +.disk .metrics { grid-template-columns: repeat(3, minmax(0, 1fr)); gap: .4rem .5rem; } +.disk .metric .label { font-size: 9.5px; letter-spacing: .04em; } +.disk .alarms { margin-top: .45rem; } +/* the two rated numbers, coloured by the `severity` the API sent — `ok` stays quiet ink, the + fleet table's rule, so a wall of healthy drives is not a wall of green */ +.disk .metric.warn .value { color: var(--warn); } +.disk .metric.fail .value { color: var(--fail); } + /* ---- hero numbers -------------------------------------------------------- * The big glanceable number: label, value + unit, and optional bar and footnote. * Colour comes from the *same* `ok|warn|fail` classes the rest of the stylesheet uses diff --git a/src/dashboards/web/static/page-disks.js b/src/dashboards/web/static/page-disks.js new file mode 100644 index 0000000..5f4c9af --- /dev/null +++ b/src/dashboards/web/static/page-disks.js @@ -0,0 +1,561 @@ +/* Drive-health page: one card per physical disk, never per partition or mount. + * + * `/system` answers "is the filesystem nearly full". This answers the question a + * filesystem can never tell you — **is the drive itself healthy** — from + * `/api/disks/snapshot`: the agent's whole-`/sys/block` rows plus the `smartctl -i -A -H` + * block it parsed for each one. + * + * The page owns no thresholds. Colour comes from the row's own `severity` + * (`disk.temp_c`, `disk.percentage_used`, and the drive's `PASSED`/`FAILED` word through + * `HEALTH_SEVERITY`), so `severity.py` stays the only place a bound lives, and a number the + * drive does not publish prints `n/a` with no colour at all (rule 2). + * + * The sector counters are **chips that exist only while non-zero** — the `/llm` preemption + * idiom. A permanent "0 pending sectors" row trains the eye to ignore it, and a green one + * would call a quiet drive an achievement; `severity.py` deliberately rates no counter, so + * presence is the whole signal. + * + * Charts follow the `/system` discipline: one collapsed `
` per host, built and + * painted only while it is open, fed by **one** `/api/disks/history` request per host per + * refresh naming every series that host could draw. A host that leaves the payload loses its + * cards, its toggle and its plots together. + */ +'use strict'; + +const { $, fetchJSON, setTxt, setDot, bytes, num, duration, poll, markStale } = Dash; + +const state = { + range: 86400, + rangeKey: '24h', // the window chips' own label, echoed in each host's toggle + smartEvery: 300, // DISKS_INTERVAL_SECONDS, from /api/config — never guessed here + hosts: new Map(), // name -> host card element + disks: new Map(), // name -> Map(device -> disk element): patched in place, not rebuilt + payloads: new Map(), // name -> last host payload (so a toggle paints without fetching) + charts: new Map(), // name -> { temp, wear|null } — built on first expand only + history: new Map(), // name -> { range, response } from the last snapshot pass + poller: null, + /* Refetched every 30 s whatever the source's cadence: the payload only changes when the + * `disks` job re-reads the agents (300 s by default), and an age stamp refreshed once a + * minute is the difference between "SMART is slow" and "the page is stuck". There is no + * `POLL_MS_DISKS`: one browser cadence per page, and this page has no faster signal. */ + pollMs: 30000, +}; + +// ---- disk cards ------------------------------------------------------------ +// +// A drive has no one-number story the way a UPS has charge %, so the card leads with its +// identity (device path, model, size, spindle or silicon) and its own verdict word, then +// three equal cells: temperature, power-on hours, wear. The first and last are rated by the +// payload; power-on hours is an odometer with no bad value and never colours. +const ALARM_FIELDS = [ + ['reallocated_sector_ct', 'reallocated', 'sectors the drive has remapped'], + ['current_pending_sector_ct', 'pending', 'sectors waiting to be remapped'], + ['offline_uncorrectable', 'uncorrectable', 'sectors the drive cannot read or rewrite'], +]; + +/** The non-zero counters as chips — `[]` when the drive has nothing to report. */ +function alarmChips(smart) { + const chips = []; + for (const [field, word, why] of ALARM_FIELDS) { + const value = smart ? smart[field] : null; + if (value == null || !isFinite(value) || value <= 0) continue; + chips.push({ text: `${num(value, 0)} ${word}`, + title: `${field}: ${num(value, 0)} — ${why}` }); + } + return chips; +} + +/** `HDD` or `SSD` from the agent's rotation flag, and nothing when it could not read one. */ +function kindOf(rotational) { + if (rotational === true) return 'HDD'; + if (rotational === false) return 'SSD'; + return null; +} + +/** The identity line: `Samsung SSD 980 PRO 2TB · 1.8 TiB · SSD`. */ +function diskIdentity(row) { + const bits = []; + if (row.model) bits.push(row.model); + if (row.size_bytes != null) bits.push(bytes(row.size_bytes, 0)); + const kind = kindOf(row.rotational); + if (kind) bits.push(kind); + return bits.join(' · '); +} + +/** + * One drive's panel, built once and patched in place afterwards. + * + * The health word is the drive's own (`OK` / `FAILED` / `unknown`) and its colour comes from + * `severity.health` alone: a drive that answered nothing gets a grey `unknown` chip, never a + * green pass (`severity.classify` returns `None` for an unknown, and `Dash.severityFrom` + * turns that into no class at all). + */ +function diskCard(device) { + const el = document.createElement('article'); + el.className = 'disk'; + el.dataset.device = device; + el.innerHTML = ` +
+ + +
+
+
+
+
Temp
n/a
+
+
+
Power-on
n/a
+
+
+
Wear
n/a
+
+
+ `; + return el; +} + +/** Write one `.metric` cell with the verdict the API sent for it — no JS threshold. */ +function setMetric(card, key, text, severity) { + const cell = card.querySelector(`[data-metric="${key}"]`); + if (!cell) return; + setTxt(cell.querySelector('.value'), text); + const kind = Dash.severityFrom(severity); + cell.classList.remove('ok', 'warn', 'fail'); + if (kind) cell.classList.add(kind); +} + +function renderDiskCard(card, row) { + const smart = row.smart || null; + const severity = row.severity || {}; + setTxt(card.querySelector('.disk-dev'), row.path || `/dev/${row.device}`); + const healthChip = card.querySelector('.health'); + const word = row.health === 'ok' ? 'OK' : row.health === 'failed' ? 'FAILED' : 'unknown'; + setTxt(healthChip, word); + healthChip.className = `chip health${Dash.severityFrom(severity.health) + ? ` ${Dash.severityFrom(severity.health)}` : ''}`; + assign(card.querySelector('.disk-model'), diskIdentity(row)); + + setMetric(card, 'temp', smart && smart.temp_c != null + ? `${num(smart.temp_c, 0)} °C` : null, severity.temp_c); + // an odometer: the drive's own counter, uncoloured, never a rate (`Δpower_on_hours/Δt` + // would be a measurement with no estimator and no window label — rule 2) + setMetric(card, 'hours', smart && smart.power_on_hours != null + ? `${num(smart.power_on_hours, 0)} h` : null, null); + setMetric(card, 'wear', smart && smart.percentage_used != null + ? `${num(smart.percentage_used, 0)}%` : null, severity.percentage_used); + + const alarms = card.querySelector('.alarms'); + const chips = alarmChips(smart); + alarms.hidden = chips.length === 0; + const markup = chips.map((chip) => `${DashCharts.escapeHtml(chip.text)}`).join(''); + if (alarms.dataset.markup !== markup) { + alarms.dataset.markup = markup; + alarms.innerHTML = markup; + } +} + +/** Text that prints nothing when there is nothing to say (unlike `setTxt`'s `n/a`). */ +function assign(el, text) { + const value = text == null ? '' : String(text); + if (el.textContent !== value) el.textContent = value; +} + +// ---- host cards ------------------------------------------------------------ +// +// One card per host in `HOST_TARGETS`, answering or not: a machine whose agent went quiet +// keeps its card (dimmed, `unreachable`) rather than vanishing, because dropping the row +// would turn a dead agent into "never configured" (invariants 3 and 4). +function hostCard(name) { + const card = document.createElement('article'); + card.className = 'card'; + card.dataset.host = name; + card.innerHTML = ` +
+ +

+ + + +
+
+ +
+
+ Charts +
+
+
+
+
+
+
+
`; + card.querySelector('[data-k="charts"]').addEventListener('toggle', + () => chartsToggled(card, name)); + setChartsWindow(card); + return card; +} + +/** The window the expanded stack draws, echoed inside the toggle (same as `/system`). */ +function setChartsWindow(card) { + const chip = card.querySelector('[data-k="charts-window"]'); + if (chip) chip.textContent = state.rangeKey; +} + +/** + * Why this host's drive numbers are missing, in one line the operator can act on. + * + * Three different facts, and `smartctl_available` is three-way precisely so they stay apart: + * the agent is not answering at all; `smartctl` is not installed (the agent said so, so this + * page may say "install smartmontools"); or nobody ever told the dashboard either way, in + * which case the fix is named without claiming to know which half is missing. `true` says + * nothing — nothing is wrong. + */ +function whyLine(host) { + if (host.unreachable) return 'This agent is not answering — no drive data was read.'; + if (host.smartctl_available === true) return null; + if (host.smartctl_available === false) { + return `smartctl not available on ${host.name} — install smartmontools and give the ` + + "agent's user read access to the block devices (disk group)."; + } + const lines = (host.smart_errors || []).join('; '); + if (lines) return lines; + return `No SMART data from ${host.name} — install smartmontools, add the agent's user to ` + + 'the disk group, and redeploy the agent.'; +} + +/** + * Patch one host card from its payload. + * + * Disk panels are keyed by the agent's device name and patched in place, so a 16-drive host + * does not rebuild 16 nodes every refresh (that would drop hover and re-announce the grid to + * assistive tech for a difference of one degree); a drive that left the host leaves the DOM. + */ +function renderHostCard(card, host) { + const stateRow = host.state || {}; + card.querySelector('h3').textContent = host.name; + const slow = 3 * state.smartEvery; + setDot(card.querySelector('.dot'), stateRow.ok === false ? 'fail' + : (Dash.ageSeconds(stateRow.last_ok) > slow ? 'warn' : 'ok')); + setTxt(card.querySelector('.updated'), stalenessText(stateRow)); + markStale(card, stateRow.ok === false || Dash.ageSeconds(stateRow.last_ok) > 5 * state.smartEvery); + + const rows = host.disks || []; + const counts = card.querySelector('.counts'); + if (host.unreachable) { + counts.hidden = true; + } else { + const alarms = (host.summary && host.summary.alarm_disks) || 0; + setTxt(counts, alarms ? `${rows.length} drives · ${alarms} alarming` + : `${rows.length} drives`); + counts.hidden = false; + } + + const why = card.querySelector('.why'); + const reason = whyLine(host); + why.hidden = reason == null; + assign(why, reason); + + const grid = card.querySelector('.disks'); + const seen = state.disks.get(host.name) || new Map(); + state.disks.set(host.name, seen); + rows.forEach((row) => { + let panel = seen.get(row.device); + if (!panel) { + panel = diskCard(row.device); + seen.set(row.device, panel); + grid.appendChild(panel); + } + renderDiskCard(panel, row); + }); + Array.from(seen.keys()).forEach((device) => { + if (rows.some((row) => row.device === device)) return; + seen.get(device).remove(); + seen.delete(device); + }); + // "no SMART to show" dims the drive grid without reddening the host: a box that cannot + // read SMART is unconfigured, not broken (invariant 3), and red here would train the + // operator to ignore red on the drives that do report. + grid.classList.toggle('stale', rows.length > 0 && host.smartctl_available !== true); +} + +function stalenessText(row) { + if (!row || !row.configured) return 'not configured'; + if (row.last_ok == null) return row.error || 'never reached'; + const seconds = Dash.ageSeconds(row.last_ok); + return seconds > state.smartEvery ? `stale ${duration(seconds)}` + : `updated ${seconds < 2 ? 'now' : duration(seconds)}`; +} + +// ---- per-host charts -------------------------------------------------------- +// +// Two plots per host, each naming drives rather than hosts (the card heading already says who): +// +// temp disk.temp_c@ per drive zero-based, never clipped +// wear disk.percentage_used@ per drive 0-100 % axis, built only once a drive +// publishes the field (an HDD may not) +// +// Both come from **one** request per host per refresh that asks for every series the panel +// could draw — `/api/disks/history` answers an unknown name as an empty series on purpose, so +// asking for what a drive has never reported costs a legend entry and nothing else. +const CHART_HEIGHT = 120; +const TEMP_FORMAT = (v) => `${num(v, 0)} °C`; +const WEAR_FORMAT = (v) => `${num(v, 0)}%`; +const TEMP_SERIES = 'disk.temp_c'; +const WEAR_SERIES = 'disk.percentage_used'; + +/** Every `disk.*@dev` name this host's cards could plot — the request's whole metric list. */ +function diskStreams(host) { + const devices = (host.disks || []).map((row) => row.device); + return { + temp: devices.map((device) => ({ metric: `${TEMP_SERIES}@${device}`, label: device })), + wear: devices.map((device) => ({ metric: `${WEAR_SERIES}@${device}`, label: device })), + }; +} + +/** + * The chart objects inside one host card, built once and reused every poll. + * + * The wear plot is created the first time a series for it has points and removed when it + * stops having any — a chart of a field no drive publishes is a plot of nothing. + */ +function hostChartSet(card, host) { + let set = state.charts.get(host.name); + if (!set) { + const common = { height: CHART_HEIGHT, xSpan: state.range }; + set = { + temp: new DashCharts.LineChart(card.querySelector('[data-chart="temp"]'), + Object.assign({}, common, { yMin: 0, yFormat: TEMP_FORMAT, + ariaLabel: `${host.name} — drive temperature in Celsius over the selected window, ` + + 'one line per physical disk' })), + wear: null, + }; + state.charts.set(host.name, set); + } + const wantsWear = (host.disks || []).some((row) => row.smart + && row.smart.percentage_used != null); + if (wantsWear && !set.wear) { + const slot = document.createElement('div'); + slot.className = 'host-chart'; + slot.dataset.chartSlot = 'wear'; + slot.innerHTML = '
' + + '
'; + card.querySelector('.host-charts').appendChild(slot); + set.wear = new DashCharts.LineChart(slot.querySelector('[data-chart="wear"]'), { + height: CHART_HEIGHT, xSpan: state.range, yMin: 0, yMax: 100, yFormat: WEAR_FORMAT, + ariaLabel: `${host.name} — SSD percentage of rated wear used, over the selected window` }); + } + if (!wantsWear && set.wear) { + card.querySelectorAll('[data-chart-slot="wear"]').forEach((node) => node.remove()); + set.wear = null; + } + return set; +} + +/** `{metric,label}[]` -> LineChart series, from one host's history response. */ +function seriesFor(response, definitions) { + return definitions.map((definition) => { + const found = (response.series || []).find((s) => s.metric === definition.metric); + return { name: definition.label, + points: ((found || {}).points || []).map((p) => ({ ts: p.ts, value: p.value })) }; + }); +} + +/** Plot one series set and repaint its legend; the fill stays off (many lines, one axis). */ +function drawChart(chart, card, key, series, format) { + chart.opts.xSpan = state.range; + chart.opts.area = false; + chart.setData(series); + DashCharts.renderLegend(card.querySelector(`[data-legend="${key}"]`), series, format, false); +} + +/** Is this host's history stack open right now? */ +function chartsOpen(card) { + const box = card.querySelector('[data-k="charts"]'); + return !!(box && box.open); +} + +/** + * The operator opened (or closed) one host's charts. + * + * Opening paints from the last response for this window, so the first expand is a chart and + * not an empty box until the next poll; a cached response for a different window is + * re-fetched rather than drawn under the wrong label. + */ +function chartsToggled(card, name) { + if (!chartsOpen(card)) return; + const host = state.payloads.get(name); + if (!host) return; + const cached = state.history.get(name); + if (cached && cached.range === state.range) drawHostCharts(card, host, cached.response); + else renderHostCharts(card, host); +} + +/** + * The paint pass, and the only place this page's `LineChart` instances are created. + * + * A closed toggle returns before anything is built: no instances, no SVG, no pixels. One + * place decides *whether* to paint, so the poll path and the toggle path cannot drift. + */ +function drawHostCharts(card, host, response) { + if (!chartsOpen(card)) return; + const charts = hostChartSet(card, host); + const streams = diskStreams(host); + drawChart(charts.temp, card, 'temp', seriesFor(response, streams.temp), TEMP_FORMAT); + if (charts.wear) { + drawChart(charts.wear, card, 'wear', seriesFor(response, streams.wear), WEAR_FORMAT); + } +} + +/** + * One host's history, one request, every refresh — open or closed (`/system`'s rule). + * + * The query names every `disk.*@dev` series the panel could draw, so the XHR budget stays + * one per host per refresh and opening a toggle never has to discover series it should have + * asked for. What the collapse defers is the painting. + */ +async function renderHostCharts(card, host) { + const range = state.range; + const streams = diskStreams(host); + const wanted = streams.temp.concat(streams.wear); + if (!wanted.length) return; + const query = `/api/disks/history?hosts=${encodeURIComponent(host.name)}` + + `&metrics=${encodeURIComponent(wanted.map((s) => s.metric).join(','))}` + + `&range=${range}`; + const response = await fetchJSON(query).catch(() => null); + if (!response || !state.hosts.has(host.name)) return; // the card went away mid-flight + state.history.set(host.name, { range, response }); + if (range === state.range) drawHostCharts(card, host, response); +} + +/** + * Drop cards for hosts that left the payload, charts and disk panels included. + * + * The toggle and the plots are children of the card, so removing it takes them; the Maps go + * the same way — a cached response or chart instance that outlives its card is a machine + * still being drawn. + */ +function pruneHosts(living) { + Array.from(state.hosts.keys()).forEach((name) => { + if (living.includes(name)) return; + state.hosts.get(name).remove(); + state.hosts.delete(name); + state.disks.delete(name); + state.charts.delete(name); + state.payloads.delete(name); + state.history.delete(name); + }); +} + +// ---- fleet header ----------------------------------------------------------- +// +// Every number here is the API's `fleet` block (`fleet_disk_summary()`), which counts a +// host that has not answered without summing it — so these totals never quietly get smaller. +// No cell is coloured: `severity.py` rates no drive count, and `alarm drives` is the presence +// of incidents per drive, which is already painted by the chips on those drives. +function renderFleet(fleet, hosts) { + const block = $('#fleet'); + const total = hosts.length; + if (total < 2) { + block.hidden = true; + return; + } + const live = fleet ? fleet.hosts_live : 0; + block.hidden = false; + setDot($('#fleet-dot'), live === 0 ? 'fail' : live < total ? 'warn' : 'ok'); + const count = $('#fleet-count'); + setTxt(count, `${live} of ${total} hosts`); + count.title = live < total + ? `${total - live} agent${total - live === 1 ? '' : 's'} not answering — their drives are ` + + 'left out of these counts' + : 'every configured agent answered'; + setTxt($('#fleet-disks'), fleet == null ? null : fleet.disks); + setTxt($('#fleet-smart'), fleet == null ? null : fleet.with_smart); + setTxt($('#fleet-failed'), fleet == null ? null : fleet.health_failed); + setTxt($('#fleet-alarms'), fleet == null ? null : fleet.alarm_disks); +} + +// ---- snapshot --------------------------------------------------------------- +async function refresh() { + let snapshot; + try { + snapshot = await fetchJSON('/api/disks/snapshot'); + } catch (err) { + $('#sub').textContent = err.message; + return; + } + const states = {}; + (snapshot.states || []).forEach((row) => { states[row.key.split(':')[1]] = row; }); + const hosts = snapshot.hosts || []; + const names = []; + const chartJobs = []; + hosts.forEach((host) => { + names.push(host.name); + host.state = states[host.name]; + let card = state.hosts.get(host.name); + if (!card) { + card = hostCard(host.name); + state.hosts.set(host.name, card); + $('#hosts').appendChild(card); + } + state.payloads.set(host.name, host); + renderHostCard(card, host); + chartJobs.push(renderHostCharts(card, host)); + }); + pruneHosts(names); + $('#hosts-empty').hidden = names.length > 0; + renderFleet(snapshot.fleet, hosts); + + const fleet = snapshot.fleet || {}; + const live = fleet.hosts_live == null ? hosts.length : fleet.hosts_live; + $('#sub').textContent = `${live}/${hosts.length} hosts answering · ` + + `window ${duration(state.range)}`; + $('#foot-note').textContent = `smartctl re-read by each agent every ` + + `${state.smartEvery}s · page poll ${Math.round(state.pollMs / 1000)}s`; + await Promise.all(chartJobs); +} + +async function refreshEvents() { + try { + // `disks`, the dashboard name the poller files SMART health transitions under — not the + // sample family a page would have to guess at (`poller.py` writes `Event(ts, dashboard)`). + const data = await fetchJSON(`/api/events?limit=${Dash.EVENT_FEED_LIMIT}&source=disks`); + Dash.eventsPager($('#events'), data.events, { + label: 'Drive events', + empty: '
  • No drive transitions ' + + 'recorded yet — this log fills when a disk\'s SMART health verdict changes.
    • ', + row: (event) => `${ + DashCharts.escapeHtml(event.host)} ${ + DashCharts.escapeHtml(Dash.eventText(event))}`, + }); + } catch (err) { /* the next poll will try again */ } +} + +document.addEventListener('DOMContentLoaded', async () => { + try { + const config = await fetchJSON('/api/config'); + if (Number.isFinite(config.disks && config.disks.interval_seconds) + && config.disks.interval_seconds > 0) { + state.smartEvery = config.disks.interval_seconds; + } + if (!config.disks.enabled) { + $('#sub').textContent = 'no hosts configured — set HOST_TARGETS'; + } + } catch (err) { /* the defaults are the shipped ones */ } + + Dash.rangeControl($('#range'), state.range, (seconds, key) => { + state.range = seconds; + state.rangeKey = key; + state.hosts.forEach((card) => setChartsWindow(card)); + refresh(); + }, [['1h', 3600], ['6h', 21600], ['24h', 86400], ['7d', 604800], ['30d', 2592000], + ['90d', 7776000]]); + + state.poller = poll(async () => { await refresh(); await refreshEvents(); }, state.pollMs); + Dash.bindPause(state.poller); +}); diff --git a/src/dashboards/web/static/page-index.js b/src/dashboards/web/static/page-index.js index fa675f0..0991d51 100644 --- a/src/dashboards/web/static/page-index.js +++ b/src/dashboards/web/static/page-index.js @@ -44,9 +44,11 @@ async function loadConfig() { const host = $('#banners'); host.innerHTML = banners.map(([kind, html]) => ``).join(''); + // every source's own cadence, so a drive card four minutes old reads as behaving $('#foot-versions').textContent = `dashboards ${config.app.version} · poll system ${config.system.interval_seconds}s · ` + - `llm ${config.llm.interval_seconds}s · ups ${config.ups.interval_seconds}s · ` + + `disks ${config.disks.interval_seconds}s · llm ${config.llm.interval_seconds}s · ` + + `ups ${config.ups.interval_seconds}s · ` + `uptime ${duration(config.app.started_at ? Date.now() / 1000 - config.app.started_at : null)}` + (config.app.read_only ? ' · read-only' : ''); return config; diff --git a/tests/conftest.py b/tests/conftest.py index b6277c4..9696265 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,6 +107,112 @@ def host_scrape() -> Any: return scrape +# ---- phase 09: physical disks (the /disks source and page) ----------------- # + +def _load_host_agent() -> Any: + """The stdlib agent module, loaded by path: it is not an installed package (and must + not become one — it ships as a single file to every host). Same loader as + tests/test_agent.py, kept private here because only the disk fixtures need it.""" + import importlib.util # noqa: PLC0415 - only the disk fixtures load the agent + + path = FIXTURES.parent.parent / "agent" / "host_agent.py" + spec = importlib.util.spec_from_file_location("host_agent", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules.setdefault("host_agent", module) + spec.loader.exec_module(module) + return module + + +#: The two `smartctl` shapes the fleet has: NVMe (health-log fields) and ATA (the vendor +#: attribute table). `tests/fixtures/smartctl_reference_*.txt` — labels verbatim from the +#: deployed smartmontools 7.5, values synthetic (see the header of each file). +SMART_SHAPES = ("nvme", "ata_hdd") + +#: The agent's own words for the way SMART goes missing that the page must name: the +#: binary is not installed. Mirrors `host_agent.Collector._smart_pass()`; the agent test +#: `test_missing_smartctl_binary_is_one_error_for_the_whole_pass` pins that the line it +#: really writes still contains the word `sources.disks.MISSING_BINARY_WORD` looks for, so +#: a rewording fails a test instead of turning "install smartmontools" into "unknown". +SMART_MISSING_BINARY = ("smart: /usr/sbin/smartctl not found — install smartmontools and " + "give this user device read access (disk group) for SMART data") + + +@pytest.fixture +def smart_shape() -> Any: + """`smart_shape('nvme')` / `smart_shape('ata_hdd')` -> the capture, parsed by the agent. + + Fixture-driven on purpose (AGENTS.md rule 7): the `smart` blocks a `/disks` test hands + to the source are the recorded `smartctl` text run through the agent's own parser, so a + field rename on either side of the wire fails a test instead of printing a dash. + """ + agent = _load_host_agent() + + def shape(kind: str = "nvme") -> dict[str, Any]: + text = (FIXTURES / f"smartctl_reference_{kind}.txt").read_text() + return agent.parse_smart_output(text) + + return shape + + +@pytest.fixture +def disk_snapshot(agent_snapshots: dict[str, dict[str, Any]], smart_shape: Any) -> Any: + """`build(host="driveripper", mode="healthy")` — a real capture with SMART attached. + + The fleet captures predate the agent's SMART pass, so there is no single capture with + both halves. This assembles the two that exist: the physical disk rows come from the + host capture verbatim, the `smart` blocks from the recorded smartmontools shapes, and + the failure modes are the ones `host_agent.smart()` actually produces. + + Modes: + + * `healthy` — every disk reads, alternating the NVMe and ATA shapes + * `no_binary` — every `smart` is null plus the agent's single "not found" line + * `denied` — the last disk is null with its own permission line (SMART is installed) + * `silent` — every `smart` is null with no error line (`--no-smart`, or an agent that + predates the pass: the dashboard cannot tell why, and must not guess) + * `failed` — one drive answering `FAILED`, the rest healthy + """ + def build(host: str = "driveripper", *, mode: str = "healthy", + mutate: Any = None) -> dict[str, Any]: + import copy # noqa: PLC0415 - every build starts from the untouched capture + + snapshot = copy.deepcopy(agent_snapshots[host]) + devices = (snapshot.get("disk_io") or {}).get("devices") or [] + errors: list[str] = list(snapshot.get("errors") or []) + for index, device in enumerate(devices): + if mode == "no_binary": + device["smart"] = None + continue + if mode == "silent": + device["smart"] = None + continue + if mode == "denied" and index == len(devices) - 1: + device["smart"] = None + continue + shape = smart_shape(SMART_SHAPES[index % len(SMART_SHAPES)]) + block = dict(shape) + # the identity in the capture is the disk the row is about; the shape files + # describe one drive each, so the row keeps its own model and the counters + # come from the shape + block["model"] = block.get("model") or device.get("model") + if mode == "failed" and index == 0: + block["overall_health"] = "FAILED" + device["smart"] = block + if mode == "no_binary": + errors.append(SMART_MISSING_BINARY) + if mode == "denied" and devices: + name = str(devices[-1].get("name")) + errors.append(f"smart: {name}: Smartctl open device: /dev/{name} failed: " + "Permission denied") + snapshot["errors"] = errors + if mutate is not None: + mutate(snapshot) + return snapshot + + return build + + @pytest.fixture def metrics_text() -> Any: """metrics_text('gufo') / metrics_text('vllm') -> the captured exposition verbatim.""" diff --git a/tests/fixtures/smartctl_driveripper_nvme0n1.txt b/tests/fixtures/smartctl_driveripper_nvme0n1.txt new file mode 100644 index 0000000..9011f33 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_nvme0n1.txt @@ -0,0 +1,52 @@ +# Verbatim smartctl -i -A -H /dev/nvme0n1 from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: Samsung SSD 980 PRO 2TB +Serial Number: SCRUBBEDSERIAL +Firmware Version: 5B2QGXA7 +PCI Vendor/Subsystem ID: 0x144d +IEEE OUI Identifier: 0x002538 +Total NVM Capacity: 2,000,398,934,016 [2.00 TB] +Unallocated NVM Capacity: 0 +Controller ID: 6 +NVMe Version: 1.3 +Number of Namespaces: 1 +Namespace 1 Size/Capacity: 2,000,398,934,016 [2.00 TB] +Namespace 1 Utilization: 1,616,151,379,968 [1.61 TB] +Namespace 1 Formatted LBA Size: 512 +Namespace 1 IEEE EUI-64: SCRUBBEDEUI64 +Local Time is: Sat Oct 17 23:55:46 2026 EDT + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0x1) +Critical Warning: 0x00 +Temperature: 59 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 14% +Data Units Read: 281,647,984 [144 TB] +Data Units Written: 323,868,857 [165 TB] +Host Read Commands: 1,597,979,105 +Host Write Commands: 9,090,416,691 +Controller Busy Time: 146,032 +Power Cycles: 2,339 +Power On Hours: 6,301 +Unsafe Shutdowns: 336 +Media and Data Integrity Errors: 0 +Error Information Log Entries: 0 +Warning Comp. Temperature Time: 24 +Critical Comp. Temperature Time: 0 +Temperature Sensor 1: 59 Celsius +Temperature Sensor 2: 73 Celsius +Thermal Temp. 1 Transition Count: 87 +Thermal Temp. 2 Transition Count: 54 +Thermal Temp. 1 Total Time: 3708 +Thermal Temp. 2 Total Time: 960 + diff --git a/tests/fixtures/smartctl_driveripper_nvme1n1.txt b/tests/fixtures/smartctl_driveripper_nvme1n1.txt new file mode 100644 index 0000000..623f624 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_nvme1n1.txt @@ -0,0 +1,48 @@ +# Verbatim smartctl -i -A -H /dev/nvme1n1 from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: Samsung SSD 970 EVO Plus 250GB +Serial Number: SCRUBBEDSERIAL +Firmware Version: 2B2QEXM7 +PCI Vendor/Subsystem ID: 0x144d +IEEE OUI Identifier: 0x002538 +Total NVM Capacity: 250,059,350,016 [250 GB] +Unallocated NVM Capacity: 0 +Controller ID: 4 +NVMe Version: 1.3 +Number of Namespaces: 1 +Namespace 1 Size/Capacity: 250,059,350,016 [250 GB] +Namespace 1 Utilization: 30,404,874,240 [30.4 GB] +Namespace 1 Formatted LBA Size: 512 +Namespace 1 IEEE EUI-64: SCRUBBEDEUI64 +Local Time is: Sat Oct 17 23:55:47 2026 EDT + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0x1) +Critical Warning: 0x00 +Temperature: 53 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 58% +Data Units Read: 43,599,098 [22.3 TB] +Data Units Written: 92,033,256 [47.1 TB] +Host Read Commands: 500,908,623 +Host Write Commands: 2,587,884,718 +Controller Busy Time: 8,424 +Power Cycles: 319 +Power On Hours: 13,161 +Unsafe Shutdowns: 177 +Media and Data Integrity Errors: 0 +Error Information Log Entries: 285 +Warning Comp. Temperature Time: 0 +Critical Comp. Temperature Time: 0 +Temperature Sensor 1: 53 Celsius +Temperature Sensor 2: 55 Celsius + diff --git a/tests/fixtures/smartctl_driveripper_nvme2n1.txt b/tests/fixtures/smartctl_driveripper_nvme2n1.txt new file mode 100644 index 0000000..ee4d555 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_nvme2n1.txt @@ -0,0 +1,45 @@ +# Verbatim smartctl -i -A -H /dev/nvme2n1 from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: WDS100T1X0E-00AFY0 +Serial Number: SCRUBBEDSERIAL +Firmware Version: 613200WD +PCI Vendor/Subsystem ID: 0x15b7 +IEEE OUI Identifier: 0x001b44 +Total NVM Capacity: 1,000,204,886,016 [1.00 TB] +Unallocated NVM Capacity: 0 +Controller ID: 8224 +NVMe Version: 1.4 +Number of Namespaces: 1 +Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB] +Namespace 1 Formatted LBA Size: 512 +Namespace 1 IEEE EUI-64: SCRUBBEDEUI64 +Local Time is: Sat Oct 17 23:55:47 2026 EDT + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff) +Critical Warning: 0x00 +Temperature: 49 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 16% +Data Units Read: 99,711,973 [51.0 TB] +Data Units Written: 493,282,793 [252 TB] +Host Read Commands: 943,634,622 +Host Write Commands: 8,794,186,017 +Controller Busy Time: 10,776 +Power Cycles: 180 +Power On Hours: 37,984 +Unsafe Shutdowns: 90 +Media and Data Integrity Errors: 0 +Error Information Log Entries: 6 +Warning Comp. Temperature Time: 0 +Critical Comp. Temperature Time: 0 + diff --git a/tests/fixtures/smartctl_driveripper_nvme3n1.txt b/tests/fixtures/smartctl_driveripper_nvme3n1.txt new file mode 100644 index 0000000..85a41e1 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_nvme3n1.txt @@ -0,0 +1,45 @@ +# Verbatim smartctl -i -A -H /dev/nvme3n1 from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: WDBRPG0010BNC-WRSN +Serial Number: SCRUBBEDSERIAL +Firmware Version: 102000WD +PCI Vendor/Subsystem ID: 0x15b7 +IEEE OUI Identifier: 0x001b44 +Total NVM Capacity: 1,000,204,886,016 [1.00 TB] +Unallocated NVM Capacity: 0 +Controller ID: 8215 +NVMe Version: 1.3 +Number of Namespaces: 1 +Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB] +Namespace 1 Formatted LBA Size: 512 +Namespace 1 IEEE EUI-64: SCRUBBEDEUI64 +Local Time is: Sat Oct 17 23:55:48 2026 EDT + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff) +Critical Warning: 0x00 +Temperature: 49 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 0% +Data Units Read: 140,935,676 [72.1 TB] +Data Units Written: 737,387,949 [377 TB] +Host Read Commands: 3,673,573,884 +Host Write Commands: 15,230,962,004 +Controller Busy Time: 30,285 +Power Cycles: 296 +Power On Hours: 52,249 +Unsafe Shutdowns: 133 +Media and Data Integrity Errors: 0 +Error Information Log Entries: 2,038 +Warning Comp. Temperature Time: 0 +Critical Comp. Temperature Time: 0 + diff --git a/tests/fixtures/smartctl_driveripper_nvme4n1.txt b/tests/fixtures/smartctl_driveripper_nvme4n1.txt new file mode 100644 index 0000000..cc2f432 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_nvme4n1.txt @@ -0,0 +1,45 @@ +# Verbatim smartctl -i -A -H /dev/nvme4n1 from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: WDBRPG0010BNC-WRSN +Serial Number: SCRUBBEDSERIAL +Firmware Version: 102000WD +PCI Vendor/Subsystem ID: 0x15b7 +IEEE OUI Identifier: 0x001b44 +Total NVM Capacity: 1,000,204,886,016 [1.00 TB] +Unallocated NVM Capacity: 0 +Controller ID: 8215 +NVMe Version: 1.3 +Number of Namespaces: 1 +Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB] +Namespace 1 Formatted LBA Size: 512 +Namespace 1 IEEE EUI-64: SCRUBBEDEUI64 +Local Time is: Sat Oct 17 23:55:48 2026 EDT + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff) +Critical Warning: 0x00 +Temperature: 49 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 0% +Data Units Read: 131,673,036 [67.4 TB] +Data Units Written: 737,398,022 [377 TB] +Host Read Commands: 2,970,582,194 +Host Write Commands: 15,268,475,455 +Controller Busy Time: 29,500 +Power Cycles: 288 +Power On Hours: 52,397 +Unsafe Shutdowns: 134 +Media and Data Integrity Errors: 0 +Error Information Log Entries: 2,050 +Warning Comp. Temperature Time: 0 +Critical Comp. Temperature Time: 0 + diff --git a/tests/fixtures/smartctl_driveripper_nvme5n1.txt b/tests/fixtures/smartctl_driveripper_nvme5n1.txt new file mode 100644 index 0000000..6a41e2a --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_nvme5n1.txt @@ -0,0 +1,48 @@ +# Verbatim smartctl -i -A -H /dev/nvme5n1 from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: Samsung SSD 980 PRO 1TB +Serial Number: SCRUBBEDSERIAL +Firmware Version: 2B2QGXA7 +PCI Vendor/Subsystem ID: 0x144d +IEEE OUI Identifier: 0x002538 +Total NVM Capacity: 1,000,204,886,016 [1.00 TB] +Unallocated NVM Capacity: 0 +Controller ID: 6 +NVMe Version: 1.3 +Number of Namespaces: 1 +Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB] +Namespace 1 Utilization: 699,798,638,592 [699 GB] +Namespace 1 Formatted LBA Size: 512 +Namespace 1 IEEE EUI-64: SCRUBBEDEUI64 +Local Time is: Sat Oct 17 23:55:49 2026 EDT + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0x1) +Critical Warning: 0x00 +Temperature: 49 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 23% +Data Units Read: 134,905,771 [69.0 TB] +Data Units Written: 340,234,345 [174 TB] +Host Read Commands: 1,296,968,853 +Host Write Commands: 6,485,156,779 +Controller Busy Time: 88,424 +Power Cycles: 716 +Power On Hours: 26,278 +Unsafe Shutdowns: 328 +Media and Data Integrity Errors: 13 +Error Information Log Entries: 13 +Warning Comp. Temperature Time: 0 +Critical Comp. Temperature Time: 0 +Temperature Sensor 1: 49 Celsius +Temperature Sensor 2: 54 Celsius + diff --git a/tests/fixtures/smartctl_driveripper_nvme6n1.txt b/tests/fixtures/smartctl_driveripper_nvme6n1.txt new file mode 100644 index 0000000..4541a7c --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_nvme6n1.txt @@ -0,0 +1,45 @@ +# Verbatim smartctl -i -A -H /dev/nvme6n1 from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: WDBRPG0010BNC-WRSN +Serial Number: SCRUBBEDSERIAL +Firmware Version: 102000WD +PCI Vendor/Subsystem ID: 0x15b7 +IEEE OUI Identifier: 0x001b44 +Total NVM Capacity: 1,000,204,886,016 [1.00 TB] +Unallocated NVM Capacity: 0 +Controller ID: 8215 +NVMe Version: 1.3 +Number of Namespaces: 1 +Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB] +Namespace 1 Formatted LBA Size: 512 +Namespace 1 IEEE EUI-64: SCRUBBEDEUI64 +Local Time is: Sat Oct 17 23:55:49 2026 EDT + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff) +Critical Warning: 0x00 +Temperature: 50 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 0% +Data Units Read: 132,700,971 [67.9 TB] +Data Units Written: 721,481,575 [369 TB] +Host Read Commands: 3,035,612,517 +Host Write Commands: 14,384,001,247 +Controller Busy Time: 29,604 +Power Cycles: 263 +Power On Hours: 52,894 +Unsafe Shutdowns: 126 +Media and Data Integrity Errors: 0 +Error Information Log Entries: 1,957 +Warning Comp. Temperature Time: 0 +Critical Comp. Temperature Time: 0 + diff --git a/tests/fixtures/smartctl_driveripper_nvme7n1.txt b/tests/fixtures/smartctl_driveripper_nvme7n1.txt new file mode 100644 index 0000000..f94cc31 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_nvme7n1.txt @@ -0,0 +1,48 @@ +# Verbatim smartctl -i -A -H /dev/nvme7n1 from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: Samsung SSD 970 EVO 1TB +Serial Number: SCRUBBEDSERIAL +Firmware Version: 2B2QEXE7 +PCI Vendor/Subsystem ID: 0x144d +IEEE OUI Identifier: 0x002538 +Total NVM Capacity: 1,000,204,886,016 [1.00 TB] +Unallocated NVM Capacity: 0 +Controller ID: 4 +NVMe Version: 1.3 +Number of Namespaces: 1 +Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB] +Namespace 1 Utilization: 694,159,712,256 [694 GB] +Namespace 1 Formatted LBA Size: 512 +Namespace 1 IEEE EUI-64: SCRUBBEDEUI64 +Local Time is: Sat Oct 17 23:55:50 2026 EDT + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0x1) +Critical Warning: 0x00 +Temperature: 60 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 11% +Data Units Read: 131,460,130 [67.3 TB] +Data Units Written: 390,323,927 [199 TB] +Host Read Commands: 845,847,299 +Host Write Commands: 6,716,124,714 +Controller Busy Time: 50,013 +Power Cycles: 1,012 +Power On Hours: 30,083 +Unsafe Shutdowns: 492 +Media and Data Integrity Errors: 0 +Error Information Log Entries: 5,817 +Warning Comp. Temperature Time: 0 +Critical Comp. Temperature Time: 0 +Temperature Sensor 1: 60 Celsius +Temperature Sensor 2: 77 Celsius + diff --git a/tests/fixtures/smartctl_driveripper_sda.txt b/tests/fixtures/smartctl_driveripper_sda.txt new file mode 100644 index 0000000..cd373f6 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_sda.txt @@ -0,0 +1,53 @@ +# Verbatim smartctl -i -A -H /dev/sda from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Seagate IronWolf +Device Model: ST8000VN004-2M2101 +Serial Number: SCRUBBEDSERIAL +LU WWN Device Id: SCRUBBEDWWN +Firmware Version: SC60 +User Capacity: 8,001,563,222,016 bytes [8.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 7200 rpm +Form Factor: 3.5 inches +Device is: In smartctl database 7.5/6204 +ATA Version is: ACS-4 (minor revision not indicated) +SATA Version is: SATA 3.3, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Sat Oct 17 23:55:50 2026 EDT +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART Attributes Data Structure revision number: 10 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x000f 083 064 044 Pre-fail Always - 0/208845750 + 3 Spin_Up_Time 0x0003 082 081 000 Pre-fail Always - 0 + 4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 127 + 5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x000f 086 060 045 Pre-fail Always - 0/368951419 + 9 Power_On_Hours 0x0032 066 066 000 Old_age Always - 30423 + 10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0 + 12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 127 + 18 Head_Health 0x000b 100 100 050 Pre-fail Always - 0 +187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0 +188 Command_Timeout 0x0032 100 100 000 Old_age Always - 0 0 0 +190 Airflow_Temperature_Cel 0x0022 065 049 040 Old_age Always - 35 (Min/Max 34/44) +192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 37 +193 Load_Cycle_Count 0x0032 009 009 000 Old_age Always - 183890 +194 Temperature_Celsius 0x0022 035 046 000 Old_age Always - 35 (0 13 0 0 0) +195 Hardware_ECC_Recovered 0x001a 083 064 000 Old_age Always - 0/208845750 +197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 0 +240 Head_Flying_Hours 0x0000 100 253 000 Old_age Offline - 4365h+15m+41.227s +241 Total_LBAs_Written 0x0000 100 253 000 Old_age Offline - 88499415587 +242 Total_LBAs_Read 0x0000 100 253 000 Old_age Offline - 485213567179 + diff --git a/tests/fixtures/smartctl_driveripper_sdb.txt b/tests/fixtures/smartctl_driveripper_sdb.txt new file mode 100644 index 0000000..c164a90 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_sdb.txt @@ -0,0 +1,51 @@ +# Verbatim smartctl -i -A -H /dev/sdb from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Seagate IronWolf +Device Model: ST2000VN004-2E4164 +Serial Number: SCRUBBEDSERIAL +LU WWN Device Id: SCRUBBEDWWN +Firmware Version: SC60 +User Capacity: 2,000,398,934,016 bytes [2.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 5900 rpm +Form Factor: 3.5 inches +Device is: In smartctl database 7.5/6204 +ATA Version is: ACS-2, ACS-3 T13/2161-D revision 3b +SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Sat Oct 17 23:55:51 2026 EDT +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART Attributes Data Structure revision number: 10 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x000f 114 099 006 Pre-fail Always - 0/69728504 + 3 Spin_Up_Time 0x0003 095 095 000 Pre-fail Always - 0 + 4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 255 + 5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x000f 086 060 030 Pre-fail Always - 1/440523587 + 9 Power_On_Hours 0x0032 044 044 000 Old_age Always - 49075 + 10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0 + 12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 256 +184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0 +187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0 +188 Command_Timeout 0x0032 100 100 000 Old_age Always - 1 1 1 +189 High_Fly_Writes 0x003a 001 001 000 Old_age Always - 292 +190 Airflow_Temperature_Cel 0x0022 069 060 045 Old_age Always - 31 (Min/Max 30/39) +191 G-Sense_Error_Rate 0x0032 100 100 000 Old_age Always - 0 +192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 147 +193 Load_Cycle_Count 0x0032 100 100 000 Old_age Always - 360 +194 Temperature_Celsius 0x0022 031 040 000 Old_age Always - 31 (0 8 0 0 0) +197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 1 + diff --git a/tests/fixtures/smartctl_driveripper_sdc.txt b/tests/fixtures/smartctl_driveripper_sdc.txt new file mode 100644 index 0000000..d630d4b --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_sdc.txt @@ -0,0 +1,56 @@ +# Verbatim smartctl -i -A -H /dev/sdc from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Seagate IronWolf +Device Model: ST6000VN001-2BB186 +Serial Number: SCRUBBEDSERIAL +LU WWN Device Id: SCRUBBEDWWN +Firmware Version: SC60 +User Capacity: 6,001,175,126,016 bytes [6.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 5425 rpm +Form Factor: 3.5 inches +Device is: In smartctl database 7.5/6204 +ATA Version is: ACS-3 T13/2161-D revision 5 +SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Sat Oct 17 23:55:52 2026 EDT +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART Attributes Data Structure revision number: 10 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x000f 081 064 006 Pre-fail Always - 0/125715512 + 3 Spin_Up_Time 0x0003 091 091 000 Pre-fail Always - 0 + 4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 177 + 5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x000f 086 060 045 Pre-fail Always - 0/367065832 + 9 Power_On_Hours 0x0032 061 061 000 Old_age Always - 35014 + 10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0 + 12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 177 +183 Runtime_Bad_Block 0x0032 099 099 000 Old_age Always - 1 +184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0 +187 Reported_Uncorrect 0x0032 095 095 000 Old_age Always - 5 +188 Command_Timeout 0x0032 100 099 000 Old_age Always - 1 1 1 +189 High_Fly_Writes 0x003a 100 100 000 Old_age Always - 0 +190 Airflow_Temperature_Cel 0x0022 064 056 040 Old_age Always - 36 (Min/Max 34/43) +191 G-Sense_Error_Rate 0x0032 100 100 000 Old_age Always - 0 +192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 1281 +193 Load_Cycle_Count 0x0032 097 097 000 Old_age Always - 7402 +194 Temperature_Celsius 0x0022 036 044 000 Old_age Always - 36 (0 15 0 0 0) +195 Hardware_ECC_Recovered 0x001a 081 064 000 Old_age Always - 0/125715512 +197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 1 +240 Head_Flying_Hours 0x0000 100 253 000 Old_age Offline - 33995h+31m+30.750s +241 Total_LBAs_Written 0x0000 100 253 000 Old_age Offline - 89359900052 +242 Total_LBAs_Read 0x0000 100 253 000 Old_age Offline - 466542530444 + diff --git a/tests/fixtures/smartctl_driveripper_sdd.txt b/tests/fixtures/smartctl_driveripper_sdd.txt new file mode 100644 index 0000000..77c95e8 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_sdd.txt @@ -0,0 +1,51 @@ +# Verbatim smartctl -i -A -H /dev/sdd from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Seagate IronWolf +Device Model: ST2000VN004-2E4164 +Serial Number: SCRUBBEDSERIAL +LU WWN Device Id: SCRUBBEDWWN +Firmware Version: SC60 +User Capacity: 2,000,398,934,016 bytes [2.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 5900 rpm +Form Factor: 3.5 inches +Device is: In smartctl database 7.5/6204 +ATA Version is: ACS-2, ACS-3 T13/2161-D revision 3b +SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Sat Oct 17 23:55:52 2026 EDT +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART Attributes Data Structure revision number: 10 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x000f 120 099 006 Pre-fail Always - 0/242964512 + 3 Spin_Up_Time 0x0003 095 095 000 Pre-fail Always - 0 + 4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 262 + 5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x000f 066 056 030 Pre-fail Always - 146/471494654 + 9 Power_On_Hours 0x0032 044 044 000 Old_age Always - 49216 + 10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0 + 12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 262 +184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0 +187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0 +188 Command_Timeout 0x0032 100 100 000 Old_age Always - 1 1 1 +189 High_Fly_Writes 0x003a 001 001 000 Old_age Always - 337 +190 Airflow_Temperature_Cel 0x0022 069 060 045 Old_age Always - 31 (Min/Max 30/40) +191 G-Sense_Error_Rate 0x0032 100 100 000 Old_age Always - 0 +192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 152 +193 Load_Cycle_Count 0x0032 100 100 000 Old_age Always - 367 +194 Temperature_Celsius 0x0022 031 040 000 Old_age Always - 31 (0 10 0 0 0) +197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 0 + diff --git a/tests/fixtures/smartctl_driveripper_sde.txt b/tests/fixtures/smartctl_driveripper_sde.txt new file mode 100644 index 0000000..b9e068e --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_sde.txt @@ -0,0 +1,48 @@ +# Verbatim smartctl -i -A -H /dev/sde from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Device Model: WDC WD180EDGZ-11B2DA0 +Serial Number: SCRUBBEDSERIAL +LU WWN Device Id: SCRUBBEDWWN +Firmware Version: 85.00A85 +User Capacity: 18,000,207,937,536 bytes [18.0 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 7200 rpm +Form Factor: 3.5 inches +Device is: Not in smartctl database 7.5/6204 +ATA Version is: ACS-4 published, ANSI INCITS 529-2018 +SATA Version is: SATA 3.3, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Sat Oct 17 23:55:53 2026 EDT +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART Attributes Data Structure revision number: 16 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x000b 100 100 001 Pre-fail Always - 0 + 2 Throughput_Performance 0x0004 133 133 054 Old_age Offline - 108 + 3 Spin_Up_Time 0x0007 082 082 001 Pre-fail Always - 360 (Average 360) + 4 Start_Stop_Count 0x0012 100 100 000 Old_age Always - 253 + 5 Reallocated_Sector_Ct 0x0033 100 100 001 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x000a 100 100 001 Old_age Always - 0 + 8 Seek_Time_Performance 0x0004 140 140 020 Old_age Offline - 15 + 9 Power_On_Hours 0x0012 095 095 000 Old_age Always - 40014 + 10 Spin_Retry_Count 0x0012 100 100 001 Old_age Always - 0 + 12 Power_Cycle_Count 0x0032 097 097 000 Old_age Always - 253 + 22 Helium_Level 0x0023 100 100 025 Pre-fail Always - 100 +192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 2210 +193 Load_Cycle_Count 0x0012 100 100 000 Old_age Always - 2210 +194 Temperature_Celsius 0x0002 047 047 000 Old_age Always - 35 (Min/Max 13/56) +196 Reallocated_Event_Count 0x0032 100 100 000 Old_age Always - 0 +197 Current_Pending_Sector 0x0022 100 100 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0008 100 100 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x000a 100 100 000 Old_age Always - 0 + diff --git a/tests/fixtures/smartctl_driveripper_sdf.txt b/tests/fixtures/smartctl_driveripper_sdf.txt new file mode 100644 index 0000000..130a174 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_sdf.txt @@ -0,0 +1,56 @@ +# Verbatim smartctl -i -A -H /dev/sdf from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Seagate IronWolf +Device Model: ST6000VN001-2BB186 +Serial Number: SCRUBBEDSERIAL +LU WWN Device Id: SCRUBBEDWWN +Firmware Version: SC60 +User Capacity: 6,001,175,126,016 bytes [6.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 5425 rpm +Form Factor: 3.5 inches +Device is: In smartctl database 7.5/6204 +ATA Version is: ACS-3 T13/2161-D revision 5 +SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Sat Oct 17 23:55:53 2026 EDT +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART Attributes Data Structure revision number: 10 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x000f 071 064 006 Pre-fail Always - 0/12209952 + 3 Spin_Up_Time 0x0003 091 091 000 Pre-fail Always - 0 + 4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 117 + 5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x000f 085 060 045 Pre-fail Always - 0/283506067 + 9 Power_On_Hours 0x0032 066 066 000 Old_age Always - 30228 + 10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0 + 12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 117 +183 Runtime_Bad_Block 0x0032 100 100 000 Old_age Always - 0 +184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0 +187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0 +188 Command_Timeout 0x0032 100 100 000 Old_age Always - 0 0 0 +189 High_Fly_Writes 0x003a 100 100 000 Old_age Always - 0 +190 Airflow_Temperature_Cel 0x0022 064 056 040 Old_age Always - 36 (Min/Max 34/43) +191 G-Sense_Error_Rate 0x0032 100 100 000 Old_age Always - 0 +192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 1094 +193 Load_Cycle_Count 0x0032 097 097 000 Old_age Always - 7121 +194 Temperature_Celsius 0x0022 036 044 000 Old_age Always - 36 (0 15 0 0 0) +195 Hardware_ECC_Recovered 0x001a 071 064 000 Old_age Always - 0/12209952 +197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 0 +240 Head_Flying_Hours 0x0000 100 253 000 Old_age Offline - 29304h+52m+38.336s +241 Total_LBAs_Written 0x0000 100 253 000 Old_age Offline - 61796587979 +242 Total_LBAs_Read 0x0000 100 253 000 Old_age Offline - 430008992227 + diff --git a/tests/fixtures/smartctl_driveripper_sdg.txt b/tests/fixtures/smartctl_driveripper_sdg.txt new file mode 100644 index 0000000..645e9f8 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_sdg.txt @@ -0,0 +1,48 @@ +# Verbatim smartctl -i -A -H /dev/sdg from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Western Digital Blue (CMR) +Device Model: WDC WD40EZRZ-00GXCB0 +Serial Number: SCRUBBEDSERIAL +LU WWN Device Id: SCRUBBEDWWN +Firmware Version: 80.00A80 +User Capacity: 4,000,787,030,016 bytes [4.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 5400 rpm +Form Factor: 3.5 inches +Device is: In smartctl database 7.5/6204 +ATA Version is: ACS-3 T13/2161-D revision 5 +SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Sat Oct 17 23:55:54 2026 EDT +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART Attributes Data Structure revision number: 16 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x002f 200 200 051 Pre-fail Always - 0 + 3 Spin_Up_Time 0x0027 166 164 021 Pre-fail Always - 6691 + 4 Start_Stop_Count 0x0032 095 095 000 Old_age Always - 5256 + 5 Reallocated_Sector_Ct 0x0033 200 200 140 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x002e 100 253 000 Old_age Always - 0 + 9 Power_On_Hours 0x0032 088 088 000 Old_age Always - 9195 + 10 Spin_Retry_Count 0x0032 100 100 000 Old_age Always - 0 + 11 Calibration_Retry_Count 0x0032 100 100 000 Old_age Always - 0 + 12 Power_Cycle_Count 0x0032 095 095 000 Old_age Always - 5097 +192 Power-Off_Retract_Count 0x0032 200 200 000 Old_age Always - 24 +193 Load_Cycle_Count 0x0032 145 145 000 Old_age Always - 166790 +194 Temperature_Celsius 0x0022 119 099 000 Old_age Always - 31 +196 Reallocated_Event_Count 0x0032 200 200 000 Old_age Always - 0 +197 Current_Pending_Sector 0x0032 200 200 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0030 200 200 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x0032 200 200 000 Old_age Always - 0 +200 Multi_Zone_Error_Rate 0x0008 200 200 000 Old_age Offline - 4 + diff --git a/tests/fixtures/smartctl_driveripper_sdh.txt b/tests/fixtures/smartctl_driveripper_sdh.txt new file mode 100644 index 0000000..ba4b968 --- /dev/null +++ b/tests/fixtures/smartctl_driveripper_sdh.txt @@ -0,0 +1,57 @@ +# Verbatim smartctl -i -A -H /dev/sdh from driveripper, recorded by +# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED, +# every other byte is what the drive answered. Volatile rows (the +# Temperature raws, Local Time) move; the parse tests read shape. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Family: Seagate BarraCuda Pro +Device Model: ST10000DM0004-1ZC101 +Serial Number: SCRUBBEDSERIAL +LU WWN Device Id: SCRUBBEDWWN +Firmware Version: DN01 +User Capacity: 10,000,831,348,736 bytes [10.0 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 7200 rpm +Form Factor: 3.5 inches +Device is: In smartctl database 7.5/6204 +ATA Version is: ACS-3 T13/2161-D revision 5 +SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s) +Local Time is: Sat Oct 17 23:55:54 2026 EDT +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED +See vendor-specific Attribute list for marginal Attributes. + +SMART Attributes Data Structure revision number: 10 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x000f 078 064 044 Pre-fail Always - 65117974 + 3 Spin_Up_Time 0x0003 089 089 000 Pre-fail Always - 0 + 4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 246 + 5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0 + 7 Seek_Error_Rate 0x000f 093 060 045 Pre-fail Always - 1869403165 + 9 Power_On_Hours 0x0032 050 050 000 Old_age Always - 43894h+52m+24.646s + 10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0 + 12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 246 +184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0 +187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0 +188 Command_Timeout 0x0032 100 099 000 Old_age Always - 1 1 2 +189 High_Fly_Writes 0x003a 001 001 000 Old_age Always - 270 +190 Airflow_Temperature_Cel 0x0022 067 040 040 Old_age Always In_the_past 33 (Min/Max 31/41) +191 G-Sense_Error_Rate 0x0032 095 095 000 Old_age Always - 11928 +192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 81 +193 Load_Cycle_Count 0x0032 035 035 000 Old_age Always - 131758 +194 Temperature_Celsius 0x0022 033 060 000 Old_age Always - 33 (0 13 0 0 0) +195 Hardware_ECC_Recovered 0x001a 006 001 000 Old_age Always - 65117974 +197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0 +198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0 +199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 0 +200 Pressure_Limit 0x0023 100 100 001 Pre-fail Always - 0 +240 Head_Flying_Hours 0x0000 100 253 000 Old_age Offline - 6541h+31m+23.388s +241 Total_LBAs_Written 0x0000 100 253 000 Old_age Offline - 222471819122 +242 Total_LBAs_Read 0x0000 100 253 000 Old_age Offline - 628009900310 + diff --git a/tests/fixtures/smartctl_reference_ata_hdd.txt b/tests/fixtures/smartctl_reference_ata_hdd.txt new file mode 100644 index 0000000..00fcbec --- /dev/null +++ b/tests/fixtures/smartctl_reference_ata_hdd.txt @@ -0,0 +1,42 @@ +# REFERENCE capture, not a field capture — see smartctl_reference_nvme.txt for the +# provenance rules (labels verbatim from smartmontools 7.5, values synthetic). This file +# is the ATA shape: identity from IDENTIFY, health from the self-assessment line, and +# the numbers from the vendor SMART attribute table (raw value is the last column). +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Device Model: EXAMPLE-ST8000VN004 +Serial Number: SCRUBBEDSERIAL2 +LU WWN Device Id: SCRUBBEDWWN1 +Firmware Version: SC60 +User Capacity: 8,001,563,222,016 bytes [8.00 TB] +Sector Sizes: 512 bytes logical, 4096 bytes physical +Rotation Rate: 7200 rpm +Form Factor: 3.5 inches +Device is: Not in smartctl database [7.5-20250430] +Local Time is: (time of capture removed by scripts/capture-fixtures.sh) +SMART support is: Available - device has SMART capability. +SMART support is: Enabled + +=== START OF READ SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART Attributes Data Structure revision number: 10 +Vendor Specific SMART Attributes with Thresholds: +ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE + 1 Raw_Read_Error_Rate 0x000f 079 066 006 Pre-fail ALWAYS - 219967840 + 5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail ALWAYS - 8 + 7 Seek_Error_Rate 0x000f 084 060 045 Pre-fail ALWAYS - 238386372 + 9 Power_On_Hours 0x0032 076 076 000 Old_age ALWAYS - 17,520 +187 Reported_Uncorrect 0x0032 100 100 000 Old_age ALWAYS - 0 +188 Command_Timeout 0x0012 100 099 000 Old_age ALWAYS - 6 +193 Load_Cycle_Count 0x0032 100 100 000 Old_age ALWAYS - 4242 +194 Temperature_Celsius 0x0022 031 055 000 Old_age ALWAYS - 31 +195 Hardware_ECC_Recovered 0x001a 196 195 000 Old_age ALWAYS - 219967840 +197 Current_Pending_Sector 0x0012 100 100 000 Old_age ALWAYS - 3 +198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 2 +199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age ALWAYS - 0 +240 Head_Flying_Hours 0x0025 070 100 000 Pre-fail ALWAYS - 14624 +241 Total_LBAs_Written 0x0032 100 253 000 Old_age ALWAYS - 117210553521 +242 Total_LBAs_Read 0x0032 100 253 000 Old_age ALWAYS - 96652971088 diff --git a/tests/fixtures/smartctl_reference_nvme.txt b/tests/fixtures/smartctl_reference_nvme.txt new file mode 100644 index 0000000..1513e99 --- /dev/null +++ b/tests/fixtures/smartctl_reference_nvme.txt @@ -0,0 +1,55 @@ +# REFERENCE capture, not a field capture. Every LABEL here is the verbatim format string +# of the deployed smartmontools 7.5 binary (verified against its own strings and drive +# database on 2026-10-18); the values are synthetic, because no host reachable at +# reference-writing time could open a block device (fleet agents need the `disk` group — +# see README). Replace these files with hardware truth: +# SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh +# which records smartctl__.txt per physical disk with serials scrubbed. +# The parse tests treat this file as the NVMe shape: Temperature/Percentage Used/ +# Power On Hours/Media and Data Integrity Errors come from the health log, and the ATA +# sector attributes stay absent — n/a on the page, never 0. +smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build) +Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org + +=== START OF INFORMATION SECTION === +Model Number: EXAMPLE NVMe SSD 2TB +Serial Number: SCRUBBEDSERIAL1 +Firmware Version: FITMVT04 +IEEE OUI Identifier: 0x000000 +Total NVM Capacity: 2,000,398,934,016 (2.00 TB) +Unallocated NVM Capacity: 0 +Controller ID: 0 +NVMe Version: 1.4 +Number of Namespaces: 1 +Namespace 1 Size: 2,000,398,934,016 (2.00 TB) +Namespace 1 Capacity: 2,000,398,934,016 (2.00 TB) +Namespace 1 Utilization: 1,000,000,000,000 (1.00 TB) +Namespace 1 Formatted LBA Size: 512 +Local Time is: (time of capture removed by scripts/capture-fixtures.sh) + +=== START OF SMART DATA SECTION === +SMART overall-health self-assessment test result: PASSED + +SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff) +Critical Warning: 0x00 +Temperature: 38 Celsius +Available Spare: 100% +Available Spare Threshold: 10% +Percentage Used: 1% +Data Units Read: 120,000,000 [61.4 TB] +Data Units Written: 90,000,000 [46.0 TB] +Host Read Commands: 1,000,000,000 +Host Write Commands: 800,000,000 +Controller Busy Time: 100 +Power Cycles: 57 +Power On Hours: 8,760 +Unsafe Shutdowns: 12 +Media and Data Integrity Errors: 0 +Error Information Log Entries: 0 +Warning Comp. Temperature Time: 0 +Critical Comp. Temperature Time: 0 +Temperature Sensor 1: 42 Celsius +Temperature Sensor 2: 45 Celsius + +Error Information (NVMe Log 0x01, 16 of 64 entries) +No Errors Logged diff --git a/tests/test_agent.py b/tests/test_agent.py index ee886c2..ee558d5 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -117,6 +117,418 @@ def test_cpu_modes_total_excludes_guest_ticks(tmp_path: Path) -> None: assert sample["total"] == pytest.approx(100 + 10 + 20 + 500 + 5 + 3 + 2 + 1) +# ---- SMART (smartctl) ------------------------------------------------------ # + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +NVME_CAPTURE = (FIXTURES / "smartctl_reference_nvme.txt").read_text() +ATA_HDD_CAPTURE = (FIXTURES / "smartctl_reference_ata_hdd.txt").read_text() + + +def read_fixture(text: str) -> str: + """Strip the `#` provenance header the capture script writes above a fixture.""" + return "\n".join(line for line in text.splitlines() if not line.startswith("#")) + + +@pytest.mark.parametrize("path", sorted(FIXTURES.glob("smartctl_*.txt"))) +def test_every_smartctl_capture_parses_without_inventing_numbers(path: Path) -> None: + """A capture the parser cannot read must fail here, not on a `/disks` card. A serial + in a fixture is a serial in the browser, so every captured serial is scrubbed.""" + parsed = host_agent.parse_smart_output(read_fixture(path.read_text())) + assert parsed["model"], f"{path.name}: no model identity parsed" + # a recorded FAILED drive is a real drive failing, not a test to loosen: replace the + # drive, or capture its truth into a purpose-built fixture and say why here. + assert parsed["overall_health"] == "PASSED", f"{path.name}: health is {parsed['overall_health']!r}" + assert parsed["power_on_hours"] is not None, f"{path.name}: no power-on hours parsed" + assert parsed["serial"] is None or parsed["serial"].startswith("SCRUBBEDSERIAL"), path.name + for key in host_agent.SMART_FIELDS: + assert key in parsed, f"{path.name}: {key} missing from the parsed shape" + + +def test_smart_output_parser_reads_the_nvme_health_log() -> None: + parsed = host_agent.parse_smart_output(read_fixture(NVME_CAPTURE)) + assert parsed["model"] == "EXAMPLE NVMe SSD 2TB" + assert parsed["serial"] == "SCRUBBEDSERIAL1" + assert parsed["temp_c"] == 38.0 + assert parsed["power_on_hours"] == 8760.0, "the capture writes 8,760 — commas are notation" + assert parsed["percentage_used"] == 1.0 + # NVMe's media-error counter IS this drive family's offline-uncorrectable fact + assert parsed["offline_uncorrectable"] == 0.0 + # and what an NVMe does not publish stays None — reallocated/pending are ATA attributes + assert parsed["reallocated_sector_ct"] is None + assert parsed["current_pending_sector_ct"] is None + + +def test_smart_output_parser_reads_the_ata_attribute_table() -> None: + parsed = host_agent.parse_smart_output(read_fixture(ATA_HDD_CAPTURE)) + assert parsed["model"] == "EXAMPLE-ST8000VN004" + assert parsed["overall_health"] == "PASSED" + assert parsed["temp_c"] == 31.0, "raw value (last column), not the normalised VALUE column" + assert parsed["power_on_hours"] == 17520.0 + assert parsed["reallocated_sector_ct"] == 8.0 + assert parsed["current_pending_sector_ct"] == 3.0 + assert parsed["offline_uncorrectable"] == 2.0 + assert parsed["percentage_used"] is None, "this HDD publishes no wear percentage" + + +def test_smart_output_parser_keeps_the_drives_own_verdict_words() -> None: + assert host_agent.parse_smart_output( + "SMART overall-health self-assessment test result: FAILED!\n")["overall_health"] == "FAILED" + assert host_agent.parse_smart_output( + "SMART overall-health self-assessment test result: UNKNOWN!\n")["overall_health"] == "UNKNOWN" + + +def test_smart_failure_note_quotes_the_error_not_the_version_banner() -> None: + """Caught on a real box: smartctl prints its banner first and the failure later, so + 'first line of output' reported the copyright instead of Permission denied.""" + banner = ("smartctl 7.5 2025-04-30 r5714 [x86_64-linux] (local build)\n" + "Copyright (C) 2002-25, Bruce Allen, Christian Franke\n\n") + denied = "Smartctl open device: /dev/nvme0 failed: Permission denied\n" + assert "Permission denied" in host_agent.smart_failure_line(2, banner + denied, "") + assert host_agent.smart_failure_line(7, "", "") == "exit status 7" + assert host_agent.smart_failure_line(4, "some odd note\n", "") == "some odd note" + # stderr counts too — smartctl puts some failures there and nowhere else + assert "scsi hangup" in host_agent.smart_failure_line(2, banner, "scsi hangup") + + +def test_smart_output_parser_returns_all_none_for_output_it_cannot_read() -> None: + junk = ("smartctl 7.5 2025-04-30 r5714 (local build)\n" + "Smartctl open device: /dev/sdb failed: Permission denied\n") + parsed = host_agent.parse_smart_output(junk) + assert set(parsed) == set(host_agent.SMART_FIELDS) + assert all(value is None for value in parsed.values()), "a denied read is n/a, never 0" + for empty in (None, ""): + assert all(value is None for value in host_agent.parse_smart_output(empty).values()) + + +def test_smart_output_parser_refuses_to_decode_vendor_hex_packed_raws() -> None: + """Some vendors pack temperature into a hex raw whose byte layout is drive-specific. + Decoding it from memory would be an invented number, so it stays None.""" + parsed = host_agent.parse_smart_output( + "194 Temperature_Celsius 0x0022 073 059 000 Old_age ALWAYS - 0x001f003c3430\n") + assert parsed["temp_c"] is None + + +def test_smart_attribute_rows_are_read_from_the_raw_column_not_the_line_end() -> None: + """Every shape the deployed binary writes into RAW_VALUE, captured off real drives. + + Reading the last token of the row — the obvious way to read a column table — loses the + temperature of every spinning disk on driveripper, because a Seagate ends its row with + `35 (Min/Max 34/44)` and the token there is `44)`. A Western Digital packs its hours as + `43894h+43m+32.519s`, which is hours first, minutes and seconds after: the drive's own + number is the leading group, and the value the page prints is that number.""" + parsed = host_agent.parse_smart_output( + "190 Airflow_Temperature_Cel 0x0022 065 049 040 Old_age Always" + " In_the_past 35 (Min/Max 34/44)\n" + "194 Temperature_Celsius 0x0022 035 046 000 Old_age Always" + " - 35 (0 13 0 0 0)\n" + " 9 Power_On_Hours 0x0032 055 055 000 Old_age Always" + " - 43894h+43m+32.519s\n" + "197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always" + " - 2 (of 4 failed when written)\n") + assert parsed["temp_c"] == 35.0 + assert parsed["power_on_hours"] == 43894.0 + assert parsed["current_pending_sector_ct"] == 2.0 + + +def test_the_real_drives_answer_every_number_the_disks_page_prints() -> None: + """`smartctl_driveripper_*.txt` is hardware truth, 16 drives, both shapes. + + The values move on every re-capture, so this pins that each number `/disks` prints is + there and plausible rather than a particular figure — the check that a fleet which looks + healthy in a synthetic fixture is also healthy when read from a real device. A drive that + stops answering health fails here, and the honest response is to replace the drive or + re-capture, never to loosen the assertion.""" + real = sorted(FIXTURES.glob("smartctl_driveripper_*.txt")) + assert real, "no real captures on disk: SMART_HOSTS= ./scripts/capture-fixtures.sh" + for path in real: + parsed = host_agent.parse_smart_output(read_fixture(path.read_text())) + assert parsed["model"], f"{path.name}: no identity" + hours = parsed["power_on_hours"] + assert hours is not None and hours > 0 and hours == int(hours), ( + f"{path.name}: power-on hours read as {hours!r}") + temp = parsed["temp_c"] + assert temp is not None and 10 <= temp <= 85, ( + f"{path.name}: temperature read as {temp!r}") + # The two shapes disagree the way the page renders them: no ATA drive on this fleet + # publishes a wear percentage, every NVMe does, and neither answer is a 0. + if "_sd" in path.name: + assert parsed["percentage_used"] is None, f"{path.name}: an HDD has no wear percent" + assert parsed["reallocated_sector_ct"] is not None, f"{path.name}: no ATA counters" + else: + assert parsed["percentage_used"] is not None, f"{path.name}: an NVMe must have wear" + assert parsed["offline_uncorrectable"] is not None, ( + f"{path.name}: the media-error log is an NVMe's uncorrectable fact") + + +def make_fake_smartctl(tmp_path: Path, outputs: dict[str, str], + exit_codes: dict[str, int] | None = None, + slow: tuple[str, ...] = ()) -> str: + """A stdlib fake `smartctl`: per-device stdout/exit code, plus an audit log of which + devices were asked — so 'virtual disks are skipped' is an assertion, not a hope.""" + data = tmp_path / "smart-out" + data.mkdir(exist_ok=True) + for device, text in outputs.items(): + (data / device).write_text(text) + for device, code in (exit_codes or {}).items(): + (data / f"{device}.rc").write_text(str(code)) + script = tmp_path / "bin" / "smartctl" + script.parent.mkdir(exist_ok=True) + script.write_text(f"""#!{sys.executable} +import os, sys, time +device = sys.argv[-1].rsplit("/", 1)[-1] +data = os.environ["FAKE_SMARTCTL_DIR"] +with open(os.path.join(data, "_called"), "a") as handle: + handle.write(device + "\\n") +if device in {list(slow)!r}: + time.sleep(30) +rc_path = os.path.join(data, device + ".rc") +code = int(open(rc_path).read()) if os.path.exists(rc_path) else 0 +out_path = os.path.join(data, device) +text = open(out_path).read() if os.path.exists(out_path) else "" +if code and not text: + # real smartctl shape: banner first, failure later — the error note must quote + # the failure, never the banner + text = (f"smartctl 7.5 2025-04-30 r5714 [x86_64-linux] (local build)\\n" + f"Copyright (C) 2002-25, Bruce Allen, Christian Franke\\n\\n" + f"Smartctl open device: /dev/{{device}} failed: Permission denied\\n") +sys.stdout.write(text) +sys.exit(code) +""") + script.chmod(0o755) + return str(script) + + +def write_fake_disk(root: Path, name: str, model: str | None = None) -> None: + write_file(root, f"sys/block/{name}/stat", "1 2 3 4 5 6 7 8 9 10 11") + write_file(root, f"sys/block/{name}/queue/rotational", "0\n") + write_file(root, f"sys/block/{name}/size", "2048\n") + if model: + write_file(root, f"sys/block/{name}/device/model", f"{model}\n") + + +def fake_calls(tmp_path: Path) -> list[str]: + called = tmp_path / "smart-out" / "_called" + return called.read_text().split() if called.exists() else [] + + +def test_smart_pass_attaches_parsed_health_per_disk(tmp_path, monkeypatch) -> None: + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1", model="SYSFS nvme0n1") + write_fake_disk(root, "sdb", model="SYSFS sdb") + write_fake_disk(root, "loop0") # virtual: must never be asked + write_fake_disk(root, "dm-0") + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector(str(root), + smartctl=make_fake_smartctl(tmp_path, { + "nvme0n1": read_fixture(NVME_CAPTURE), + "sdb": read_fixture(ATA_HDD_CAPTURE), + })) + snap = collector.snapshot() + assert snap["errors"] == [], snap["errors"] + by_name = {device["name"]: device for device in snap["disk_io"]["devices"]} + assert by_name["nvme0n1"]["smart"]["model"] == "EXAMPLE NVMe SSD 2TB" + assert by_name["nvme0n1"]["smart"]["power_on_hours"] == 8760.0 + assert by_name["sdb"]["smart"]["reallocated_sector_ct"] == 8.0 + called = fake_calls(tmp_path) + assert "loop0" not in called and "dm-0" not in called, "virtual entries are not physical disks" + assert sorted(called) == ["nvme0n1", "sdb"] + + +def test_smart_identity_falls_back_to_the_agents_own_disk_model(tmp_path, monkeypatch) -> None: + """Some bridges refuse the identity block while still answering the health log.""" + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1", model="SYSFS DISK MODEL") + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector( + str(root), + smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": "Power On Hours: 42\n"})) + device = collector.snapshot()["disk_io"]["devices"][0] + assert device["smart"]["model"] == "SYSFS DISK MODEL" + assert device["smart"]["power_on_hours"] == 42.0 + assert device["smart"]["overall_health"] is None + + +def test_missing_smartctl_binary_is_one_error_for_the_whole_pass(tmp_path, monkeypatch) -> None: + root = tmp_path / "root" + for device in ("nvme0n1", "sdb"): + write_fake_disk(root, device) + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl") + snap = collector.snapshot() + smart_errors = [e for e in snap["errors"] if e.startswith("smart:")] + assert len(smart_errors) == 1, "one missing binary is one fact, not one per disk" + assert "smartmontools" in smart_errors[0] + # cross-module contract: `sources/disks.py` reads this word to answer + # "is smartctl installed?" (three-way: yes / no / the agent does not say). Reword the + # line here and that test fails, instead of the /disks page losing its explanation. + assert "not found" in smart_errors[0] + assert all(device["smart"] is None for device in snap["disk_io"]["devices"]) + # the rest of the snapshot is intact — SMART is an add-on, not a dependency + assert snap["disk_io"]["device_names"] == ["nvme0n1", "sdb"] + + +def test_one_denied_device_nulls_only_that_disk(tmp_path, monkeypatch) -> None: + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1") + write_fake_disk(root, "sdb") + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector( + str(root), + smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": read_fixture(NVME_CAPTURE)}, + exit_codes={"sdb": 2})) + snap = collector.snapshot() + by_name = {device["name"]: device for device in snap["disk_io"]["devices"]} + assert by_name["nvme0n1"]["smart"] is not None + assert by_name["sdb"]["smart"] is None + smart_errors = [e for e in snap["errors"] if e.startswith("smart:")] + assert len(smart_errors) == 1 and "sdb" in smart_errors[0] and "Permission denied" in smart_errors[0] + + +def test_a_wedged_device_costs_its_timeout_not_the_snapshot(tmp_path, monkeypatch) -> None: + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1") + write_fake_disk(root, "sdb") + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector( + str(root), + smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": read_fixture(NVME_CAPTURE)}, + slow=("sdb",))) + collector.smart_timeout_seconds = 0.5 + snap = collector.snapshot() + by_name = {device["name"]: device for device in snap["disk_io"]["devices"]} + assert by_name["nvme0n1"]["smart"] is not None + assert by_name["sdb"]["smart"] is None + assert any("timed out" in e for e in snap["errors"]) + + +def test_the_pass_budget_skips_the_rest_with_one_error(tmp_path, monkeypatch) -> None: + root = tmp_path / "root" + for device in ("nvme0n1", "sdb", "sdc"): + write_fake_disk(root, device) + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector( + str(root), smartctl=make_fake_smartctl(tmp_path, {})) + collector.smart_budget_seconds = 0.0 + snap = collector.snapshot() + assert all(device["smart"] is None for device in snap["disk_io"]["devices"]) + assert any("budget" in e and "3 disk(s) skipped" in e for e in snap["errors"]) + + +def test_unexecutable_smartctl_degrades_like_a_missing_one(tmp_path, monkeypatch) -> None: + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1") + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + blocked = tmp_path / "not-executable" + blocked.write_text("not a program\n") + collector = host_agent.Collector(str(root), smartctl=str(blocked)) + snap = collector.snapshot() + assert all(device["smart"] is None for device in snap["disk_io"]["devices"]) + assert any(e.startswith("smart:") for e in snap["errors"]) + + +def wait_for_calls(tmp_path, expected: int, *, timeout: float = 5.0) -> list[str]: + """Wait for the background refresher to have run `expected` passes.""" + deadline = time.time() + timeout + while time.time() < deadline: + calls = fake_calls(tmp_path) + if len(calls) >= expected: + return calls + time.sleep(0.02) + return fake_calls(tmp_path) + + +def test_smart_results_are_cached_and_an_expired_cache_refreshes_off_the_request_path( + tmp_path, monkeypatch) -> None: + """The system page polls every few seconds; smartctl must not re-run every poll. + + And when the cache *does* expire, the refresh runs on a background thread while the + snapshot answers from the previous result: `SMARTCTL_BUDGET_SECONDS` (20) is five + times the dashboard's `HOST_TIMEOUT_SECONDS` (4), so an inline refresh would turn the + whole `system` source red every four minutes because of an add-on it does not use. + """ + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1") + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector( + str(root), smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": read_fixture(NVME_CAPTURE)})) + first = collector.snapshot() + second = collector.snapshot() + assert first["disk_io"]["devices"][0]["smart"]["temp_c"] == 38.0 + assert second["disk_io"]["devices"][0]["smart"] == first["disk_io"]["devices"][0]["smart"] + assert fake_calls(tmp_path) == ["nvme0n1"], "one pass, then the cache" + + collector.smart_cache_seconds = 0.0 # an expired cache re-runs the pass… + served = collector.snapshot() + assert served["disk_io"]["devices"][0]["smart"] == first["disk_io"]["devices"][0]["smart"] + assert wait_for_calls(tmp_path, 2) == ["nvme0n1", "nvme0n1"], "…in the background" + + +def test_an_expired_cache_never_stalls_the_snapshot_on_a_wedged_disk(tmp_path, + monkeypatch) -> None: + """The regression that matters: a disk that takes 30 s must cost a poll nothing. + + Asserted on wall-clock because the whole point is the request's latency — the agent + answers with the last SMART it had while the refresher sits in `smartctl`'s timeout. + """ + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1") + write_fake_disk(root, "sdb") + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector( + str(root), + smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": read_fixture(NVME_CAPTURE), + "sdb": "Power On Hours: 1\n"}, + slow=("sdb",))) + collector.smart_timeout_seconds = 1.0 # the slow device costs ~1 s per pass + first = collector.snapshot() + assert any("timed out" in line for line in first["errors"]) + temp = first["disk_io"]["devices"][0]["smart"]["temp_c"] + + collector.smart_cache_seconds = 0.0 + started = time.perf_counter() + served = collector.snapshot() + elapsed = time.perf_counter() - started + assert served["disk_io"]["devices"][0]["smart"]["temp_c"] == temp + assert elapsed < 0.5, f"a wedged disk cost the snapshot {elapsed:.2f}s" + # the refresher is single-flight: a second poll while it runs gets the cache too + started = time.perf_counter() + collector.snapshot() + assert time.perf_counter() - started < 0.5 + wait_for_calls(tmp_path, 4) # let the background pass finish before teardown + + +def test_no_smart_flag_stays_silent_and_null(tmp_path, monkeypatch) -> None: + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1") + monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out")) + collector = host_agent.Collector(str(root), + smartctl=make_fake_smartctl(tmp_path, {}), smart=False) + snap = collector.snapshot() + assert snap["errors"] == [] + assert snap["disk_io"]["devices"][0]["smart"] is None + assert fake_calls(tmp_path) == [] + + +def test_an_unexpected_smart_crash_degrades_the_addon_not_the_snapshot(tmp_path, monkeypatch) -> None: + collector = host_agent.Collector("/", smart=False) + monkeypatch.setattr(collector, "smart_enabled", True) + monkeypatch.setattr(collector, "smart", + lambda names: (_ for _ in ()).throw(RuntimeError("boom"))) + snap = collector.snapshot() + assert any(e.startswith("smart:") and "boom" in e for e in snap["errors"]) + assert snap["disk_io"]["devices"], "the disk rows themselves come from /sys, not smartctl" + + +def test_smartctl_captures_carry_no_unscrubbed_serials() -> None: + """The capture script scrubs; this fails the day it stops doing its job. The NVMe + namespace EUI-64 is on the list because it embeds the serial.""" + for path in sorted(FIXTURES.glob("smartctl_*.txt")): + for line in path.read_text().splitlines(): + lowered = line.lower() + if "serial number" in lowered or "wwn" in lowered or "eui" in lowered: + assert "SCRUBBED" in line, f"{path.name}: unscrubbed identifier: {line!r}" + + # ---- the real collector --------------------------------------------------- # @linux_only @@ -133,7 +545,10 @@ def test_snapshot_against_the_live_system_is_complete() -> None: assert len(snap["cpu"]["cores"]) >= 1 assert snap["memory"]["total_bytes"] > 0 assert snap["filesystems"], "the box it runs on has at least one mounted filesystem" - assert snap["errors"] == [], f"degraded sections must be empty on a healthy box: {snap['errors']}" + # SMART notes are allowed here: the box running pytest may have no smartctl or no + # device read access, and both are documented degradations, never snapshot failures. + hard_errors = [e for e in snap["errors"] if not e.startswith("smart")] + assert hard_errors == [], f"degraded sections must be empty on a healthy box: {hard_errors}" def test_captured_fixture_has_the_shape_the_dashboard_depends_on(agent_snapshot) -> None: @@ -215,6 +630,22 @@ def _get(url: str, headers: dict[str, str] | None = None) -> tuple[int, dict]: return error.code, {"raw": body.decode(errors="replace")} +def test_health_stays_up_while_smart_is_degraded(agent_server, tmp_path) -> None: + """Phase-09 criterion: SMART failures are add-on notes, never an agent outage.""" + root = tmp_path / "root" + write_fake_disk(root, "nvme0n1") + collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl") + base, _port = agent_server() + status, health = _get(base + "/health") + assert status == 200 and health["ok"] is True + handler = host_agent.Handler + handler.collector = collector + status, snap = _get(base + "/snapshot") + assert status == 200 + assert snap["disk_io"]["devices"][0]["smart"] is None + assert any(e.startswith("smart:") for e in snap["errors"]) + + def test_health_info_schema_snapshot_all_answer(agent_server) -> None: base, _port = agent_server() for path in ("/health", "/info", "/schema", "/snapshot"): diff --git a/tests/test_alerts.py b/tests/test_alerts.py index 8c92c0d..9e00972 100644 --- a/tests/test_alerts.py +++ b/tests/test_alerts.py @@ -166,6 +166,98 @@ def test_a_dashboard_the_builder_does_not_know_about_still_gets_a_verdict_and_a_ assert "every 60s" in out[0]["message"] +def test_the_disk_job_is_stale_only_after_three_of_its_own_five_minute_polls() -> None: + """SMART is a 300 s signal: judging it by the 10 s system interval would amber `/` + every five minutes for a source that is behaving exactly as configured.""" + disks = Settings(disks_interval=300.0) + fresh = state(key="disks:driveripper", dashboard="disks", label="driveripper", age=600.0) + assert alerts.build(disks, [fresh]) == [], "600s is two polls, not an outage" + stale = {**fresh, "age_seconds": 1_000.0} + out = alerts.build(disks, [stale]) + assert out[0]["severity"] == "warn" + assert "every 300s" in out[0]["message"], "the message must print the disks interval" + + +def test_a_disk_job_that_failed_because_its_agent_is_down_is_one_line_not_two() -> None: + """`disks` reads the same agent snapshot `system` does. + + A dead host would otherwise print the same sentence twice (`alerts` rule: one dead + source, one line), with only the internal id telling the two apart — which is a lie of + the same shape as a duplicated host name. + """ + out = alerts.build(SETTINGS, [ + state(key="disks:driveripper", dashboard="disks", label="driveripper", + ok=False, error="UpstreamError: driveripper: http://h:9121/snapshot " + "unreachable (ConnectError)"), + state(ok=False, error="UpstreamError: driveripper: http://h:9121/snapshot " + "unreachable (ConnectError)"), + ], now=2_000.0) + assert [item["source"] for item in out] == ["driveripper"] + assert len(out) == 1, "the echo of the same outage must not read as two problems" + assert out[0]["id"].startswith("driveripper:host:") + + +def test_a_disk_job_that_fails_while_its_host_answers_is_the_only_voice_it_has() -> None: + """SMART wedged inside the agent is a `disks`-only fault, and must still be said.""" + out = alerts.build(SETTINGS, [ + state(key="disks:driveripper", dashboard="disks", label="driveripper", + ok=False, error="UpstreamError: driveripper: timed out"), + state(), + ], now=2_000.0) + assert len(out) == 1 + assert out[0]["id"].startswith("driveripper:disks:") + assert "timed out" in out[0]["message"] + + +def test_a_disk_only_alert_links_to_the_drive_page() -> None: + """The overview's row is a link, so it has to point at the page that explains it. + + A wedged SMART pass has no other voice (the host itself is answering), and `/system` + cannot tell the operator which drive stopped reporting — `/disks` can. Before that route + existed this link pointed at `/` on purpose; now it must not point anywhere else. + """ + out = alerts.build(SETTINGS, [ + state(key="disks:driveripper", dashboard="disks", label="driveripper", + ok=False, error="UpstreamError: driveripper: timed out"), + state(), + ], now=2_000.0) + assert len(out) == 1 + assert out[0]["href"] == "/disks" + + +def test_an_unconfigured_disk_source_says_nothing_even_when_its_job_says_down() -> None: + """No `HOST_TARGETS` means no `disks` job exists; a reported-but-unconfigured one must + still be silence (invariant 3). + + The builder is fed job states rather than settings, so a hub that ever hands over a + `disks` record with `configured: false` — a config that arrives between two polls, a + future target list of its own — must not amber `/` over a page nobody installed. + """ + out = alerts.build(Settings(disks_interval=300.0), [ + state(key="disks:driveripper", dashboard="disks", label="driveripper", + ok=False, configured=False, error="no hosts configured"), + ], now=2_000.0) + assert out == [] + + +def test_a_host_threshold_reads_the_system_job_not_the_disk_job( + host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: + """Two jobs share a host label, so the lookup has to be filtered by dashboard. + + `disks` is registered after `system`, but the builder must not depend on that: if an + unfiltered lookup found the failing disk job first, the host would be called + unreachable and its real threshold breach would vanish. + """ + payload = host_scrape(agent_snapshot, cpu_busy=95.0) # cpu.usage_pct -> fail + out = alerts.build(SETTINGS, [ + state(key="disks:driveripper", dashboard="disks", label="driveripper", + ok=False, error="timed out"), + state(), + ], [payload], [], None, now=2_000.0) + ids = [item["id"] for item in out] + assert "driveripper:cpu.usage_pct" in ids, "the breach is real; the disk job is not it" + + def test_a_failing_source_says_when_the_poller_first_noticed() -> None: events = [{"ts": 1_500.0, "source": "system", "host": "driveripper", "kind": "unreachable", "severity": "error", "message": "ConnectError", "data": None}, diff --git a/tests/test_api.py b/tests/test_api.py index ce550a8..89004d2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -14,6 +14,7 @@ from contextlib import contextmanager from pathlib import Path from typing import Any +import httpx import pytest from fastapi import HTTPException from fastapi.testclient import TestClient @@ -21,9 +22,11 @@ from fastapi.testclient import TestClient from dashboards import alerts from dashboards.api import RANGE_SECONDS, parse_range from dashboards.app import create_app +from dashboards.collectors import UpstreamError from dashboards.config import Settings, Target from dashboards.severity import classify from dashboards.sources import nut +from dashboards.sources.disks import SMART_SERIES, DisksCollector, parse_disks_snapshot from dashboards.sources.llm import ( HISTORY_METRICS, parse_gufo_metrics, @@ -44,6 +47,11 @@ from slots_contract import ( # refused immediately, and never dials out of the test environment DEAD_URL = "http://127.0.0.1:1" +#: Every dashboard route, in nav order. The markup pins below (no CDN, no `.sev-*`, no +#: narration, no orphan `id`) run across this list rather than a hand-written four, so a +#: fifth page cannot join the masthead and skip the checks the others had to pass. +PAGES = ("/", "/system", "/disks", "/llm", "/ups") + @contextmanager def build_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, simulate: bool = False, @@ -137,22 +145,36 @@ def data_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, def healthy_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, agent_snapshot: dict[str, Any], host_scrape: Any) -> Iterator[TestClient]: - """The all-clear app: one host answering, nothing past a threshold, no UPS configured. + """The all-clear app: one host answering on both of its jobs, nothing past a threshold. The job's collector is swapped for one that hands back the injected payload, so the - health record says `ok` truthfully instead of being overridden behind the poller. + health record says `ok` truthfully instead of being overridden behind the poller — and + that has to happen for the `disks` job too, which polls the same dead port: a host is + either answering or it is not, and a fixture that claims all-clear cannot leave one of + its two readers failing. Each reader gets **its own** projection of the same capture + (`parse_disks_snapshot` for the drive page), because a `disks` job holding a `system` + payload would make `/api/summary`'s drive block answer from the wrong shape and read as + a passing test that proved nothing. """ with build_client(tmp_path, monkeypatch, host_targets=(Target("driveripper", DEAD_URL),)) as test_client: from dashboards.poller import Result # noqa: PLC0415 - only this fixture needs it payload = host_scrape(agent_snapshot, cpu_busy=10.0) + disks_payload = parse_disks_snapshot("driveripper", agent_snapshot, + DisksCollector()).payload async def collect() -> Result: return Result(payload=payload, samples=[]) - job = test_client.app.state.ctx.hub.jobs["host:driveripper"] # type: ignore[attr-defined] + async def collect_disks() -> Result: + return Result(payload=disks_payload, samples=[]) + + hub = test_client.app.state.ctx.hub # type: ignore[attr-defined] + job = hub.jobs["host:driveripper"] + disks_job = hub.jobs["disks:driveripper"] job.fn = collect + disks_job.fn = collect_disks # let the poll already in flight against the dead port land first (it fails in # microseconds), then take the health record over: the alerts must describe a host # that is answering, not the half-second of startup where it was not @@ -160,11 +182,14 @@ def healthy_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, while time.time() < deadline and job.state.attempts == 0: time.sleep(0.02) time.sleep(0.05) - job.state.payload = payload - job.state.ok = True - job.state.error = None - job.state.consecutive_failures = 0 - job.state.last_ok = time.time() + for reader in hub.jobs.values(): + if reader.dashboard not in ("system", "disks"): + continue + reader.state.payload = disks_payload if reader is disks_job else payload + reader.state.ok = True + reader.state.error = None + reader.state.consecutive_failures = 0 + reader.state.last_ok = time.time() yield test_client @@ -204,7 +229,8 @@ def test_summary_shape_did_not_shrink(data_client) -> None: assert fleet_keys <= set(body["system"]["fleet"]), fleet_keys - set(body["system"]["fleet"]) assert host_keys <= set(body["system"]["hosts"][0]), host_keys - set(body["system"]["hosts"][0]) assert ups_keys <= set(body["ups"]["ups"][0]), ups_keys - set(body["ups"]["ups"][0]) - assert set(body) == {"ts", "system", "llm", "ups", "recent_events", "alerts"} + # `disks` joined in phase 09 and changed nothing else: the set grows, never shrinks + assert set(body) == {"ts", "system", "disks", "llm", "ups", "recent_events", "alerts"} # ...and the new keys are there beside the old ones, not instead of them assert "severity" in body["system"]["fleet"] assert "severity" in body["system"]["hosts"][0] @@ -215,6 +241,7 @@ def test_summary_shape_did_not_shrink(data_client) -> None: # read by something that is not the overview page: the live suite, an external probe, a # notifier later. Deleting a card must never delete a key. SUMMARY_CONTRACT = ("system.enabled", "system.fleet", "system.hosts", + "disks.enabled", "disks.fleet", "disks.hosts", "llm.engines", "ups.ups", "recent_events", "alerts") @@ -245,6 +272,11 @@ def test_summary_serves_every_path_another_consumer_reads(request, fixture: str) assert (dig(body, "system.fleet") is not None) is populated names = {host["name"] for host in dig(body, "system.hosts")} assert names == ({"driveripper"} if populated else set()) + # the drive-health block is configured exactly when the agents it reads are, and lists the + # same hosts: a `disks` job exists for every `HOST_TARGETS` entry and for no other reason + assert dig(body, "disks.enabled") is populated + assert (dig(body, "disks.fleet") is not None) is populated + assert {host["name"] for host in dig(body, "disks.hosts")} == names batteries = {item["name"] for item in dig(body, "ups.ups")} assert batteries == ({"apc", "cyberpower"} if populated else set()) assert dig(body, "llm.engines") == [], "neither fixture configures an engine" @@ -342,6 +374,9 @@ def test_config_reports_every_source_as_off(client) -> None: body = client.get("/api/config").json() assert body["system"]["enabled"] is False assert body["system"]["targets"] == [] + assert body["disks"] == {"enabled": False, "interval_seconds": 300.0, + "timeout_seconds": 30.0}, ( + "the drive-health page is configured exactly when the host agents are") assert body["llm"]["gufo"]["enabled"] is False assert body["llm"]["vllm"]["enabled"] is False assert body["ups"]["enabled"] is False @@ -357,6 +392,31 @@ def test_health_with_nothing_configured_is_not_a_failure(client) -> None: assert body["sources"] == [] +def test_the_disk_source_has_its_own_health_record_per_host( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """`disks` is a source of its own, not a field of `system` (phase 09). + + Its own record is what gives the page its own dot and its own staleness, and it is + also what makes `alerts.py` need a rule: two jobs read the same agent, so one dead + host must still be one line on the overview (see `test_summary_raises_one_alert...`). + """ + with build_client(tmp_path, monkeypatch, + host_targets=(Target("driveripper", DEAD_URL),)) as test_client: + sources = {row["key"]: row for row in test_client.get("/api/health").json()["sources"]} + assert "disks:driveripper" in sources, "no separate record, no separate dot" + assert sources["disks:driveripper"]["dashboard"] == "disks" + assert sources["disks:driveripper"]["label"] == "driveripper" + assert sources["disks:driveripper"]["configured"] is True + assert sources["host:driveripper"]["dashboard"] == "system", ( + "the system job is still its own source: one slow SMART poll must not " + "decide whether the host looks alive") + + +def test_no_targets_means_no_disk_job_that_could_look_broken(client) -> None: + """No host agents, no `disks` task at all — so nothing red for an unconfigured page.""" + assert client.get("/api/health").json()["sources"] == [] + + @pytest.mark.parametrize("path", ["/api/system/snapshot", "/api/llm/snapshot", "/api/ups/snapshot", "/api/ups/commands"]) def test_unconfigured_source_is_404_configured_false(client, path) -> None: @@ -380,6 +440,30 @@ def test_unknown_route_is_json_not_html(client) -> None: assert response.headers["content-type"].startswith("application/json") assert "no such route" in response.json()["detail"] +def test_an_upstream_failure_escaping_a_route_keeps_its_own_status(tmp_path: Path) -> None: + """`collectors.UpstreamError` is the one error type a route may raise to say "upstream". + + The handler turns it into `{ok: false, detail}` with the collector's status (502, or 504 + for a timeout) instead of a 500 traceback, and the message is the collector's already + redacted sentence. Nothing dials here: the error is raised by the route under test. + """ + db = str(tmp_path / "upstream-error.sqlite3") + store = SampleStore(db) + app = create_app(Settings(db_path=db), store) + + @app.get("/api/_boom") + async def boom() -> None: + raise UpstreamError("driveripper: http://127.0.0.1:9121/snapshot timed out", status=504) + + with TestClient(app) as test_client: + response = test_client.get("/api/_boom") + assert response.status_code == 504, "a timeout stays a timeout, not a 500" + assert response.json() == {"ok": False, + "detail": "driveripper: http://127.0.0.1:9121/snapshot " + "timed out"} + assert "no-store" in response.headers["cache-control"] + store.close() + # ---- tokens --------------------------------------------------------------- # @@ -433,12 +517,20 @@ def test_a_dead_host_never_writes_its_token_into_the_feed(tmp_path: Path) -> Non time.sleep(0.05) else: pytest.fail("the dead host never recorded an error") - for path in ("/api/health", "/api/events", "/api/system/snapshot", "/api/summary"): + for path in ("/api/health", "/api/events", "/api/system/snapshot", + "/api/disks/snapshot", "/api/summary"): body = client.get(path).text assert "SEKRIT-AGENT-TOKEN" not in body, f"{path} leaked the agent token" # and the outage is still legible — redaction is not the same as silence assert "driveripper" in client.get("/api/health").text assert "unreachable" in client.get("/api/events").text + # the `disks` job dials the same token-bearing URL on its own cadence, so its + # sentence is a second copy of the same risk: `fetch_snapshot` is the one place + # either dashboard builds a failure message, and it uses the redacted URL + disks_state = next(row for row in client.get("/api/health").json()["sources"] + if row["key"] == "disks:driveripper") + assert disks_state["error"] and "SEKRIT-AGENT-TOKEN" not in disks_state["error"] + assert "127.0.0.1:1/snapshot" in disks_state["error"], "the failure still names where" store.close() @@ -478,7 +570,8 @@ def test_target_problems_are_surfaced_not_swallowed(client) -> None: # ---- pages ---------------------------------------------------------------- # @pytest.mark.parametrize("path,title", [ - ("/", "overview"), ("/system", "system"), ("/llm", "llm"), ("/ups", "ups"), + ("/", "overview"), ("/system", "system"), ("/disks", "disks"), ("/llm", "llm"), + ("/ups", "ups"), ]) def test_pages_serve_and_declare_themselves(client, path, title) -> None: response = client.get(path) @@ -489,16 +582,14 @@ def test_pages_serve_and_declare_themselves(client, path, title) -> None: def test_no_page_loads_anything_from_the_internet(client) -> None: """No CDN, no webfonts: this dashboard has to work with no route to the internet.""" - for path in ("/", "/system", "/llm", "/ups"): + for path in PAGES: html = client.get(path).text for fragment in ("//cdn.", "https://fonts.", "//unpkg.com", "//cdnjs"): assert fragment not in html, f"{path} references {fragment}" def test_static_assets_are_served(client) -> None: - for asset in ("/static/dash.css", "/static/app.js", "/static/charts.js", - "/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js", - "/static/page-index.js"): + for asset in ASSETS: assert client.get(asset).status_code == 200, asset @@ -511,14 +602,26 @@ def test_static_assets_are_served(client) -> None: # comparing the number to a threshold in JavaScript. ASSETS = ("/static/dash.css", "/static/app.js", "/static/charts.js", - "/static/page-index.js", "/static/page-system.js", "/static/page-llm.js", - "/static/page-ups.js") + "/static/page-index.js", "/static/page-disks.js", "/static/page-system.js", + "/static/page-llm.js", "/static/page-ups.js") def _css(client: TestClient) -> str: return client.get("/static/dash.css").text +def _css_section(css: str, name: str) -> str: + """One `/* ---- name … */` section of the stylesheet, up to the **next** section header. + + Slicing to a named neighbour worked while the sheet had five sections. Every page has since + added a component section, and a slice that runs across someone else's rules accuses the + fleet block of colouring numbers it never coloured — so the boundary is structural: the end + of *this* section, whoever comes next. + """ + start = css.index(f"/* ---- {name}") + return css[start:css.index("/* ----", start + 10)] + + def _hero_css(css: str) -> str: """Just the `hero numbers` section of the stylesheet.""" return css.split("/* ---- hero numbers", 1)[1].split("/* ---- bars", 1)[0] @@ -599,7 +702,7 @@ def _fleet_field_metrics(js: str) -> list[str]: def test_no_parallel_severity_vocabulary_exists_anywhere(client) -> None: """One word per colour: `ok|warn|fail`, as in severity.py and the existing components.""" - for asset in (*ASSETS, "/", "/system", "/llm", "/ups"): + for asset in (*ASSETS, *PAGES): body = _code(client.get(asset).text) assert ".sev-" not in body, f"{asset} invented a .sev-* class" assert not re.search(r"classList\.(?:add|toggle)\(\s*[\"'](?:danger|good|warning)[\"']", @@ -639,7 +742,7 @@ def test_severity_for_is_kept_as_the_documented_fallback(client) -> None: js = client.get("/static/app.js").text assert "FALLBACK ONLY" in js assert "severityFor" in js, "the documented fallback itself was deleted" - for page in ("page-index.js", "page-system.js", "page-llm.js", "page-ups.js"): + for page in PAGE_SCRIPTS.values(): assert "severityFor" not in client.get(f"/static/{page}").text, \ f"{page} colours something from a threshold of its own" @@ -924,7 +1027,7 @@ def test_every_system_chart_names_itself_for_assistive_tech(client) -> None: # ---- event feeds: a bounded, paginated window ----------------------------- # # -# The three `ul.events` feeds are one component (`Dash.eventsPager`) because a pager that +# Every `ul.events` feed is one component (`Dash.eventsPager`) because a pager that # works on only one page is the bug this shape prevents: the `event` table grows for as long # as the fleet runs, and an unbounded feed turns `/system` into a scroll to nowhere # (`TODO.md` L6 — "All the events tables need to be scrollable and paginated"). There is no @@ -935,6 +1038,7 @@ def test_every_system_chart_names_itself_for_assistive_tech(client) -> None: # page -> (its script, the `source=` the poller stores that page's events under) EVENT_FEEDS = { "/system": ("page-system.js", "system"), + "/disks": ("page-disks.js", "disks"), "/llm": ("page-llm.js", "llm"), "/ups": ("page-ups.js", "ups"), } @@ -953,11 +1057,12 @@ def _feed_fetch_url(js: str) -> str: return found.group(1) -def test_the_event_feeds_are_one_shared_pager_not_three(client) -> None: - """`Dash.eventsPager()` renders all three feeds, and no page grows its own pager. +def test_the_event_feeds_are_one_shared_pager_not_one_per_page(client) -> None: + """`Dash.eventsPager()` renders every feed, and no page grows its own pager. - Three copies of a pager is three chances for one page to keep the old behaviour — and - `/system` is the one whose feed was already too tall when this was written down. + One copy per page is one chance per page to keep the old behaviour — and `/system` is the + one whose feed was already too tall when this was written down. `/disks` joined as a + fourth call site of the same helper, not as a fifth implementation of it. """ app_js = _code(client.get("/static/app.js").text) assert app_js.count("function eventsPager(") == 1 @@ -1913,7 +2018,7 @@ def test_llm_fleet_block_colours_no_number(client) -> None: "a dead engine no longer dims the tag or darkens the block's dot" css = client.get("/static/dash.css").text - section = css[css.index("/* ---- fleet totals"):css.index("/* ---- hero numbers")] + section = _css_section(css, "fleet totals") for token in ("var(--ok)", "var(--warn)", "var(--fail)"): assert token not in section, f"the fleet block coloured a number with {token}" assert ".fleet-head" in section and ".fleet-note" in section, \ @@ -2004,7 +2109,7 @@ def test_llm_slot_grid_style_is_a_marker_and_not_a_verdict(client) -> None: the marker survives monochrome, which is the battery glyph's rule applied here. """ css = client.get("/static/dash.css").text - section = css[css.index("/* ---- engine slot grid"):css.index("/* ---- fleet totals")] + section = _css_section(css, "engine slot grid") for selector in (".slots {", ".slot {", ".slot.busy", ".slot-top", ".slot-id", ".slot-model", ".slot dt", ".slot dd", ".slots-empty", ".slots-note", ".section-label"): @@ -2292,10 +2397,11 @@ def test_overview_dims_the_fleet_row_of_a_host_that_stopped_answering(client) -> # Which script owns which page, for the orphan check below. `charts.js` and `app.js` are -# loaded by all four pages, so an id either page code or shared chrome may read. +# loaded by every page, so an id either page code or shared chrome may read. PAGE_SCRIPTS = { "/": "page-index.js", "/system": "page-system.js", + "/disks": "page-disks.js", "/llm": "page-llm.js", "/ups": "page-ups.js", } @@ -2366,7 +2472,7 @@ def test_no_page_narrates_its_own_pipeline(client, fragment: str) -> None: the same sentence migrating into another page's card is the same slop, and a pin that only guards its original file would let that through. """ - for path in ("/", "/system", "/llm", "/ups", *ASSETS): + for path in (*PAGES, *ASSETS): body = client.get(path).text assert fragment not in body, f"{path} narrates the app again: {fragment!r}" @@ -2378,7 +2484,7 @@ def test_pages_leave_no_empty_narration_container(client) -> None: renders a gap where the sentence used to be. Elements the scripts fill (`#sub`, `#alerts-count`) carry an `id` and are not in this category. """ - for path in ("/", "/system", "/llm", "/ups"): + for path in PAGES: html = _code(client.get(path).text) for cls in ("sub", "faint", "mono"): assert f'' not in html, \ @@ -2393,12 +2499,16 @@ def test_the_copy_the_operator_acts_on_survives_the_audit(client) -> None: is this number a measurement, is this battery real, is this host's data last-good — and each one is the reason the audit deletes sentences and not labels. """ - pages = {path: client.get(path).text for path in ("/", "/system", "/llm", "/ups")} + pages = {path: client.get(path).text for path in PAGES} scripts = {name: client.get(f"/static/{name}").text for name in PAGE_SCRIPTS.values()} # an empty panel says why, in terms the operator can act on assert "No hosts configured" in pages["/"] assert "HOST_TARGETS" in pages["/system"] + # …including the fourth page's: a `n/a` drive card is only actionable if the page names + # the two things that make SMART readable (the binary and the device-read grant). + assert "smartmontools" in pages["/disks"] + scripts["page-disks.js"] + assert "disk group" in scripts["page-disks.js"] assert "No engines configured" in pages["/llm"] assert "No UPS reporting" in pages["/ups"] assert "instcmds" in pages["/ups"], "the UPS setup note stopped naming the grant that matters" @@ -2612,6 +2722,538 @@ def test_metrics_endpoint_lists_what_actually_has_history(client) -> None: assert body["metrics"] == [] +# ---- the `/disks` API (phase 09 task 03) ---------------------------------- # +# +# The drive-health surface answers like every other dashboard's: GETs only, the same +# stale/stub semantics for a host that is not answering, the same history mechanism for the +# `disk.*@dev` series, and the same 404-`configured: false` sentence when no agent is +# configured at all — the last one *without dialling*, because "not deployed here" must not +# cost a connect timeout or a red dot (invariant 3). + +@pytest.fixture +def disks_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + disk_snapshot: Any) -> Iterator[TestClient]: + """Two hosts configured; one of them answering SMART, through the production parser. + + The payload and the stored samples are what `parse_disks_snapshot` makes of the + fixture-shaped capture (the agent's whole-disk rows plus the recorded `smartctl` shapes), + so every path a `/disks` card will read is a path the source really emits. The other host + keeps its `unreachable` stub, which is the shape a dead agent must keep (invariant 4). + Both targets point at a dead port, so nothing dials. + """ + db = str(tmp_path / "disks.sqlite3") + store = SampleStore(db) + app = create_app(Settings(db_path=db, host_targets=(Target("driveripper", DEAD_URL), + Target("deskwork", DEAD_URL))), store) + with TestClient(app) as test_client: + now = time.time() + snapshot = disk_snapshot("driveripper") + snapshot["ts"] = now # "the agent answered just now", so charts are in range + result = parse_disks_snapshot("driveripper", snapshot, DisksCollector(), now=now) + store.add_many(result.samples) + app.state.ctx.hub.jobs["disks:driveripper"].state.payload = result.payload + yield test_client + store.close() + + +#: (`smart` field, severity metric) for the two drive numbers the table rates — the same +#: pair `sources/disks.py:SEVERITY_FIELDS` declares, restated here so a rename that only +#: touches one side fails rather than colouring nothing. +RATED_CELLS: tuple[tuple[str, str], ...] = ( + ("temp_c", "disk.temp_c"), + ("percentage_used", "disk.percentage_used"), +) + + +def disk_rows(body: dict[str, Any], host: str = "driveripper") -> list[dict[str, Any]]: + """The disk rows of one host out of a `/api/disks/snapshot` body.""" + return next(row for row in body["hosts"] if row["name"] == host)["disks"] + + +def flat_routes(app: Any) -> list[Any]: + """Every route the app serves, with `include_router()` results flattened to their routes. + + The FastAPI this repo pins keeps an included router as an `_IncludedRouter` wrapper with + no `path` of its own, so walking `app.routes` alone sees five anonymous objects instead of + the API surface — and the verb assertion below is only as good as this walk. + """ + out: list[Any] = [] + for route in app.routes: + inner = getattr(route, "original_router", None) + out.extend(inner.routes if inner is not None else [route]) + return out + + +def series_source(body: dict[str, Any]) -> set[str]: + """Which sample family a history answer came from: `disks`, never the 10-second `host`.""" + return {series["source"] for series in body["series"]} + + +def test_disks_snapshot_answers_one_entry_per_host_with_its_physical_disks( + disks_client) -> None: + """`GET /api/disks/snapshot` is per host: rows + `smart` + `smartctl_available`.""" + body = disks_client.get("/api/disks/snapshot") + assert response_ok(body) + payload = body.json() + assert [host["name"] for host in payload["hosts"]] == ["deskwork", "driveripper"], ( + "every configured host is a row, in name order, answering or not") + rows = disk_rows(payload) + assert rows, "the capture has 16 whole-disk devices, so the page has 16 cards" + assert all("/" not in row["device"] for row in rows), "a mount leaked into the disk list" + first = rows[0] + assert first["path"] == f"/dev/{first['device']}" + assert set(first) >= {"device", "path", "model", "size_bytes", "rotational", + "health", "smart", "severity"} + assert first["smart"]["power_on_hours"] > 0, "the drive's own counters travel verbatim" + live = next(host for host in payload["hosts"] if host["name"] == "driveripper") + assert live["smartctl_available"] is True + assert live["smart_errors"] == [] + assert live["summary"]["disks"] == len(rows) + assert "unreachable" not in live, "answering is the absence of the flag, as on /system" + assert [state["key"] for state in payload["states"]] == ["disks:driveripper", + "disks:deskwork"], ( + "the page's dots come from the disks jobs, never the system ones") + + +def test_disks_snapshot_carries_the_verdicts_the_page_will_paint(disks_client) -> None: + """Colour is in the payload: each row's `severity` is what `severity.classify` says. + + This is the pin that keeps the threshold out of the page — `page-disks.js` reads these + and never compares a drive temperature to a bound. + """ + rows = disk_rows(disks_client.get("/api/disks/snapshot").json()) + assert "ok" in {row["severity"]["temp_c"] for row in rows}, "the capture is healthy" + for row in rows: + smart = row["smart"] or {} + assert set(row["severity"]) == {field for field, _metric in RATED_CELLS} | {"health"} + for field, metric in RATED_CELLS: + assert row["severity"][field] == classify(metric, smart.get(field)), ( + f"{row['device']}.{field} is coloured by something other than the one table") + assert not {key for row in rows for key in row["severity"]} & { + "reallocated_sector_ct", "pending", "uncorrectable", "power_on_hours"}, ( + "a counter is an alarm chip on presence, never a graded cell") + + +def test_a_configured_host_with_no_payload_yet_is_a_row_of_nas(disks_client) -> None: + """The stub shape `/api/system/snapshot` hands out, so `n/a` is never a missing row. + + A host in `HOST_TARGETS` that has not answered is still in the fleet (invariants 3 and + 4) with its counts `None` rather than `0`: "unknown" and "no drives" are different + sentences (rule 2), and the fleet header counts it without summing it. + """ + payload = disks_client.get("/api/disks/snapshot").json() + stub = next(host for host in payload["hosts"] if host["name"] == "deskwork") + assert stub["unreachable"] is True + assert stub["disks"] == [], "nothing to draw, but the host is still listed" + assert stub["smartctl_available"] is None, "an agent that never answered says nothing" + assert stub["smart_errors"] == [] + assert set(stub["summary"]) == {"disks", "with_smart", "health_failed", "alarm_disks"} + assert all(value is None for value in stub["summary"].values()), "no invented zeros" + fleet = payload["fleet"] + assert (fleet["hosts_total"], fleet["hosts_live"]) == (2, 1) + assert fleet["disks"] == len(disk_rows(payload)) + + +def test_the_disks_fleet_header_is_summed_by_the_api_not_the_page(disks_client) -> None: + """`fleet` is the header's only arithmetic source, and it carries no verdict.""" + payload = disks_client.get("/api/disks/snapshot").json() + live = next(host for host in payload["hosts"] if host["name"] == "driveripper") + fleet = payload["fleet"] + assert fleet["disks"] == live["summary"]["disks"] + assert fleet["alarm_disks"] == live["summary"]["alarm_disks"] > 0, ( + "the recorded ATA shape carries reallocated sectors, so the fleet has chips to draw") + assert "severity" not in fleet, "a count of drives is not a graded number" + + +def test_the_disks_surface_is_gets_only_and_the_write_stays_the_ups_one( + disks_client) -> None: + """Invariant 1: the app has exactly one POST, and a fourth dashboard does not add one.""" + routes = flat_routes(disks_client.app) + disks = [route for route in routes + if str(getattr(route, "path", "")).startswith("/api/disks")] + assert {route.path for route in disks} == {"/api/disks/snapshot", "/api/disks/history"} + verbs = {verb for route in disks for verb in (getattr(route, "methods", None) or set())} + assert verbs == {"GET"}, "a write appeared on the drive-health surface" + for verb in ("POST", "PUT", "PATCH", "DELETE"): + response = disks_client.request(verb, "/api/disks/snapshot", json={}) + assert response.status_code == 405, f"{verb} /api/disks/snapshot was accepted" + writes = sorted(str(route.path) for route in routes + if "POST" in (getattr(route, "methods", None) or set())) + assert writes == ["/api/ups/command"], "one write endpoint, four gates, unchanged" + + +def test_config_prints_the_disks_cadence_the_page_explains_itself_with( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A card four minutes old is behaving, not broken — and the interval is the proof. + + `/disks` has no upstream of its own, so `enabled` follows `HOST_TARGETS` and the cadence + is its own. Both numbers come from config rather than a constant in the page, so changing + `DISKS_INTERVAL_SECONDS` moves the sentence and the poller together. + """ + with build_client(tmp_path, monkeypatch, host_targets=(Target("driveripper", DEAD_URL),), + disks_interval=120.0, disks_timeout=45.0) as test_client: + disks = test_client.get("/api/config").json()["disks"] + assert disks["enabled"] is True + assert disks["interval_seconds"] == 120.0 + assert disks["timeout_seconds"] == 45.0 + + +def test_disks_snapshot_404s_without_dialling_a_host_that_was_never_configured( + client, monkeypatch: pytest.MonkeyPatch) -> None: + """The gate runs before anything that could reach the network (invariant 3). + + Both spies raise, so a route that constructed a client or fetched a snapshot before + checking `HOST_TARGETS` fails here instead of costing a timeout and a red dot. The hub + assertion is the structural half: with no targets there is no `disks` job to dial with. + """ + calls: list[str] = [] + + def spy_get(self: Any, url: Any, **kwargs: Any) -> Any: + calls.append(f"GET {url}") + raise AssertionError(f"the unconfigured disks route dialled {url}") + + def spy_fetch(*args: Any, **kwargs: Any) -> Any: + calls.append("fetch_snapshot") + raise AssertionError("the unconfigured disks route fetched an agent snapshot") + + monkeypatch.setattr(httpx.AsyncClient, "get", spy_get) + monkeypatch.setattr("dashboards.collectors.fetch_snapshot", spy_fetch) + + response = client.get("/api/disks/snapshot") + assert response.status_code == 404 + body = response.json() + assert body["configured"] is False + assert "HOST_TARGETS" in body["detail"], "the sentence names the fix" + assert calls == [], "an unconfigured page must not touch the network" + assert client.app.state.ctx.hub.jobs == {}, ( + "no targets means no disks job, so nothing can age out or go red") + + +def test_disks_history_serves_the_dev_series_the_page_charts(disks_client) -> None: + """One request, every series a host's cards could draw, points from the stored samples.""" + rows = disk_rows(disks_client.get("/api/disks/snapshot").json()) + metrics = [f"{base}@{row['device']}" for row in rows for _field, base in SMART_SERIES] + body = disks_client.get("/api/disks/history", params={ + "hosts": "driveripper", "metrics": ",".join(metrics), "range": "1h"}).json() + assert body["hosts"] == ["driveripper"], "one host, one panel" + assert body["range_seconds"] == 3_600.0 + assert [series["metric"] for series in body["series"]] == metrics + assert series_source(body) == {"disks"}, "the disks family, never the 10-second host one" + points = {(series["metric"]): series["points"] for series in body["series"]} + assert points["disk.temp_c@nvme0n1"][0]["value"] == pytest.approx(38.0) + assert points["disk.power_on_hours@nvme0n1"][0]["value"] == pytest.approx(8_760.0) + + +def test_disks_history_answers_empty_for_a_series_a_drive_does_not_publish( + disks_client) -> None: + """Unknown names are an empty series, not a 404 — the /system and /llm contract. + + The recorded ATA shape reports no `percentage_used`, so that drive's wear plot has + nothing to draw, and a device name the agent never listed behaves the same way. The card + asks for every series it could draw and reads `n/a` where there is nothing, instead of + the request failing. + """ + rows = disk_rows(disks_client.get("/api/disks/snapshot").json()) + unworn = next(row["device"] for row in rows + if (row["smart"] or {}).get("percentage_used") is None) + body = disks_client.get("/api/disks/history", params={ + "hosts": "driveripper", + "metrics": f"disk.percentage_used@{unworn},disk.temp_c@nosuchdisk," + "disk.temp_c@nvme0n1", + "range": "1h"}).json() + by_metric = {series["metric"]: series["points"] for series in body["series"]} + assert by_metric[f"disk.percentage_used@{unworn}"] == [], "no measurement, no line" + assert by_metric["disk.temp_c@nosuchdisk"] == [] + assert by_metric["disk.temp_c@nvme0n1"], "the series that does exist still answers" + + +def test_disks_history_charts_every_configured_host_in_one_request(disks_client) -> None: + """With no `hosts` the answer covers every configured agent, dead ones included (empty).""" + body = disks_client.get("/api/disks/history", + params={"metrics": "disk.temp_c@nvme0n1,disk.temp_c@sda", + "range": "24h"}).json() + assert body["hosts"] == ["driveripper", "deskwork"] + assert body["range_seconds"] == 86_400.0 + assert {(series["host"], series["metric"]) for series in body["series"]} == { + ("driveripper", "disk.temp_c@nvme0n1"), ("driveripper", "disk.temp_c@sda"), + ("deskwork", "disk.temp_c@nvme0n1"), ("deskwork", "disk.temp_c@sda")} + assert [series for series in body["series"] if series["host"] == "deskwork"], ( + "a host with no history is an empty series, never a missing one") + + +def test_the_metrics_endpoint_answers_for_the_disks_family(disks_client) -> None: + """"Why is this chart empty" has an answer per page: /api/metrics?source=disks.""" + body = disks_client.get("/api/metrics", params={"source": "disks"}).json() + assert "disk.temp_c@nvme0n1" in body["metrics"] + assert not [name for name in body["metrics"] if not name.startswith("disk.")], ( + "the disks family holds only disk series") + assert not [name for name in disks_client.get( + "/api/metrics", params={"source": "host"}).json()["metrics"] + if name.startswith("disk.temp_c")], "SMART never mixes into the system family" + + +# ---- the `/disks` page (phase 09 task 04) ---------------------------------- # +# +# The fourth dashboard, asserted the way the other three are: on the served bytes, because a +# JS test runner would be a build step (a locked anchor). What is pinned is not that the page +# exists but that it keeps the four promises the phase wrote down — one card per **physical** +# drive and never one per partition, alarm chips that exist only while a counter is non-zero, +# a host without `smartctl` dimmed rather than reddened, and charts that belong to one host, +# stay collapsed until opened, and cost one history request per host per refresh. + +def _disks_js(client: TestClient) -> str: + return client.get("/static/page-disks.js").text + + +def test_the_disks_tab_is_in_the_masthead_of_every_page(client) -> None: + """Four tabs became five, on every page, with exactly one marking itself current. + + The masthead is the app's only "which dashboard do I need" surface since phase 02 deleted + the overview's table, so a tab missing from one page is a page you cannot get to, and a + tab marked current on the wrong page is a lie about where you are. + """ + for path in PAGES: + html = client.get(path).text + assert '' in html, f"{path} does not mark itself" + assert (html.index('href="/system"') < html.index('href="/disks"') + < html.index('href="/llm"')), "tab order is machine, drive, engine, power" + + +def test_the_header_status_dot_picks_up_the_disks_job(client) -> None: + """`disks` gets its own dot instead of hiding inside `hosts`. + + The same agent can answer the 10-second utilisation poll and have a wedged SMART pass; + one dot for both would have to be right about neither. + """ + app_js = client.get("/static/app.js").text + assert "{ key: 'disks', label: 'disks', href: '/disks' }" in app_js + assert "s.dashboard === job.key" in app_js, "the dot stopped filtering by dashboard" + + +def test_disks_page_draws_one_card_per_physical_drive_from_the_api(client) -> None: + """The card is a drive, keyed by the agent's device name, patched in place. + + A mount or a partition would need a filesystem field, and `/system` already answers "is + the filesystem full"; if one leaks back in here, the page stops being about the drive. + """ + html = client.get("/disks").text + js = _disks_js(client) + assert 'id="hosts"' in html and 'id="hosts-empty"' in html + assert "fetchJSON('/api/disks/snapshot')" in js + assert "row.path" in js and "/dev/" in js, "the card stopped naming the device it is about" + # identity + verdict lead the card: no hero row, no big single number + assert 'class="heroes"' not in html, "the disk card grew a hero row" + assert "Dash.hero(" not in js + for filesystem in ("mountpoint", "fstype", "free_bytes", "worst_mount"): + assert filesystem not in js, f"{filesystem} leaked into the drive page" + # built once per device and patched, so a 16-drive host does not rebuild 16 panels a poll + assert "seen.get(row.device)" in js and "renderDiskCard(panel, row)" in js + assert "pruneHosts(" in js, "a drive that left the host keeps its card forever" + + +def test_disk_cards_print_the_drive_numbers_and_colour_only_rated_ones(client) -> None: + """Temp, power-on hours and wear, coloured by the API's verdicts and nothing else. + + The odometer is the deliberate exception: `severity.THRESHOLDS` has no `disk.power_on_hours` + row, so the cell is written with a `null` verdict and cannot go green at 100 h or red at + 60 000 h. A drive that publishes no wear (the recorded HDD shape) prints `n/a` because + `setTxt` is the only writer. + """ + js = _disks_js(client) + for field in ("smart.temp_c", "smart.power_on_hours", "smart.percentage_used", + "row.health"): + assert field in js, f"{field} is no longer on the card" + for verdict in ("severity.temp_c", "severity.percentage_used", "severity.health"): + assert verdict in js, f"the card stopped rendering {verdict}" + assert "severityFor" not in js, "the drive page wrote a threshold of its own" + body = js[js.index("function renderDiskCard("):js.index("/** Text that prints nothing")] + assert "severity.temp_c" in body and "severity.percentage_used" in body + assert re.search(r"setMetric\(card, 'hours',[^;]*null\)", body), ( + "the odometer is being graded by something severity.py does not have") + # the drive's own word, and `unknown` for a drive that gave none — never a green pass + assert "'FAILED'" in body and "'unknown'" in body + + +def test_disk_alarm_chips_exist_only_while_a_counter_is_non_zero(client) -> None: + """Zero and unknown both draw no chip; only a non-zero counter draws a red one. + + The `/llm` preemption rule: a permanent "0 pending sectors" row trains the eye to ignore + it, and grading a counter green would call a quiet drive an achievement. `severity.py` + rates none of these, so the chip's presence is the entire signal here. + """ + js = _disks_js(client) + for counter in ("reallocated_sector_ct", "current_pending_sector_ct", + "offline_uncorrectable"): + assert counter in js, f"{counter} is no longer an alarm" + assert "value <= 0) continue" in js, "a zero now draws a chip" + assert "alarms.hidden = chips.length === 0" in js, "the empty chip row stays on screen" + assert 'class="chip fail"' in js + css = client.get("/static/dash.css").text + assert ".alarms {" in css and ".chip.fail" in css + + +def test_a_host_without_smartctl_is_dimmed_with_a_one_line_why_and_never_red(client) -> None: + """Unconfigured is not broken (invariant 3), and the three answers stay three. + + `smartctl_available` is `false` (install smartmontools — the agent said so, so the page + may say it), `null` (nobody told us either way, so name the fix without claiming to know + which half is missing), or the agent is not answering at all, which is a different + sentence again. None of the three reddens anything. + """ + js = _disks_js(client) + html = client.get("/disks").text + assert "smartctl not available on" in js + assert "disk group" in js, "the why stopped naming the grant that fixes it" + assert "smartctl_available === false" in js and "smartctl_available === true" in js + assert "host.unreachable" in js, "a dead agent must not be reported as a missing binary" + assert "smart_errors" in js, "the agent's own sentence is no longer printed" + assert "classList.toggle('stale'" in js + assert 'class="banner info why" hidden' in js, "the why-line has no home on the card" + assert "banner fail" not in js and html.count("banner") == 0, ( + "a missing binary reddens the page") + + +def test_disks_charts_are_collapsed_lazy_and_cost_one_history_request_per_host(client) -> None: + """`/system`'s chart discipline, reused: collapsed, painted only while open, one XHR. + + The request still names every series the host could draw (temperature and wear per + drive), because `/api/disks/history` answers an unknown name as an empty series and a + panel should never have to discover series it did not ask for. + """ + js = _disks_js(client) + assert '
    None: + """An unlabelled plot is silence, and the legend is a sibling that labels nothing.""" + js = _code(_disks_js(client)) + labels = re.findall(r"ariaLabel: `([^`]+)`", js) + assert len(labels) == 2, "every drive chart must name itself (temperature, wear)" + assert all(label.startswith("${host.name}") for label in labels), ( + "a chart that does not say which host it plots belongs on no page") + + +def test_the_disks_fleet_header_reads_the_api_sums_and_invents_none(client) -> None: + """The header's arithmetic is the API's (`fleet_disk_summary`), never the page's. + + Hidden below two hosts, because with one box in `HOST_TARGETS` these sums only duplicate + that host's own cards — `/llm`'s fleet-block rule. And no cell is coloured: a count of + drives is not a graded number (`severity.py` has no row for one). + """ + html = client.get("/disks").text + js = _disks_js(client) + for element in ("fleet", "fleet-dot", "fleet-count", "fleet-disks", "fleet-smart", + "fleet-failed", "fleet-alarms"): + assert f'id="{element}"' in html, f"the header lost #{element}" + fleet = js[js.index("function renderFleet("):js.index("// ---- snapshot")] + for path in ("fleet.hosts_live", "fleet.disks", "fleet.with_smart", + "fleet.health_failed", "fleet.alarm_disks"): + assert path in fleet, f"the header no longer reads {path}" + assert "reduce(" not in fleet and "+=" not in fleet, "the page is summing drives itself" + assert "severity" not in fleet, "a drive count grew a verdict" + assert "block.hidden = true" in fleet, "one host's sums duplicate its own cards" + + +def test_the_disk_panel_style_is_existing_vocabulary_with_no_new_palette(client) -> None: + """`.disk` is the `.slot` shape, and the only colours it reaches for are the verdicts.""" + css = _css(client) + block = _css_section(css, "drive panels") + for selector in (".disks {", ".disk {", ".disk-head", ".disk-dev", ".disk-model", + ".disk .metrics", ".disk .alarms", ".disk .metric.warn .value", + ".disk .metric.fail .value"): + assert selector in block, f"dash.css lost {selector}" + for token in ("var(--warn)", "var(--fail)", "var(--border)", "var(--bg-sunken)", + "var(--text)"): + assert token in block, f"the drive panel no longer uses {token}" + assert "--disk" not in css, "a drive-only colour token appeared" + assert ".stale" in css, "the dim that marks an unreadable host is gone" + + +def test_the_overview_prints_the_fourth_source_cadence(client) -> None: + """`/` names every source's interval, so a drive card four minutes old reads as behaving.""" + js = client.get("/static/page-index.js").text + assert "config.disks.interval_seconds" in js + + +@pytest.mark.parametrize("fixture", ["client", "data_client"]) +def test_summary_carries_the_drive_block_for_programmatic_consumers( + request, fixture: str) -> None: + """`disks` is additive: the same configured-ness, the same hosts, no key retired. + + `client` has no targets, so it proves the block answers as `False`/`None`/`[]` instead of + disappearing; `data_client` proves it lists the host the fleet has. + """ + test_client = request.getfixturevalue(fixture) + body = test_client.get("/api/summary").json() + populated = fixture == "data_client" + assert body["disks"]["enabled"] is populated + assert (body["disks"]["fleet"] is not None) is populated + assert body["disks"]["hosts"] == ([] if not populated else [ + {"name": "driveripper", "disks": None, "with_smart": None, "health_failed": None, + "alarm_disks": None, "smartctl_available": None, "failed_devices": [], + "unreachable": True}]), "a configured host with no payload is a row of nones" + assert [row["name"] for row in body["disks"]["hosts"]] == [ + row["name"] for row in body["system"]["hosts"]] + + +def test_summary_serves_the_same_counts_the_disks_page_prints(disks_client) -> None: + """One projection, two consumers: the summary's numbers *are* the snapshot's.""" + body = disks_client.get("/api/summary").json() + snapshot = disks_client.get("/api/disks/snapshot").json() + disks = body["disks"] + assert disks["fleet"] == snapshot["fleet"], "the header would disagree with the page" + rows = {row["name"]: row for row in disks["hosts"]} + live = next(host for host in snapshot["hosts"] if host["name"] == "driveripper") + assert (rows["driveripper"]["disks"], rows["driveripper"]["with_smart"], + rows["driveripper"]["health_failed"], rows["driveripper"]["alarm_disks"]) == ( + live["summary"]["disks"], live["summary"]["with_smart"], + live["summary"]["health_failed"], live["summary"]["alarm_disks"]) + assert rows["driveripper"]["smartctl_available"] is True + assert rows["driveripper"]["unreachable"] is False + assert rows["deskwork"]["unreachable"] is True + assert rows["deskwork"]["disks"] is None, "unknown, never 0 (rule 2)" + assert sorted(state["key"] for state in disks["states"]) == ["disks:deskwork", + "disks:driveripper"] + + +def test_summary_names_the_drive_the_health_verdict_called_failed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, disk_snapshot: Any) -> None: + """Which drive the hardware itself called `FAILED` is part of the fleet answer. + + A notifier reading `/api/summary` should not have to fetch a second endpoint to name the + device, and the count and the list have to agree or one of them is wrong. + """ + db = str(tmp_path / "summary-failed.sqlite3") + store = SampleStore(db) + app = create_app(Settings(db_path=db, + host_targets=(Target("driveripper", DEAD_URL),)), store) + with TestClient(app) as test_client: + now = time.time() + snapshot = disk_snapshot("driveripper", mode="failed") + snapshot["ts"] = now + result = parse_disks_snapshot("driveripper", snapshot, DisksCollector(), now=now) + app.state.ctx.hub.jobs["disks:driveripper"].state.payload = result.payload + row = test_client.get("/api/summary").json()["disks"]["hosts"][0] + assert len(row["failed_devices"]) == 1 + assert row["failed_devices"][0] == result.payload["disks"][0]["device"] + assert row["health_failed"] == result.payload["summary"]["health_failed"] == 1 + store.close() + + # ---- events --------------------------------------------------------------- # def test_events_are_returned_in_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/test_collectors.py b/tests/test_collectors.py index 93d3aa1..ad30532 100644 --- a/tests/test_collectors.py +++ b/tests/test_collectors.py @@ -19,12 +19,15 @@ import pytest from dashboards.collectors import ( NutMonitor, UpstreamError, + fetch_disk_snapshot, fetch_engine, fetch_host_snapshot, + fetch_snapshot, ) from dashboards.config import Target from dashboards.rates import RateEstimator from dashboards.sources import nut +from dashboards.sources.disks import DisksCollector from dashboards.sources.host import HostCollector FIXTURES = Path(__file__).parent / "fixtures" @@ -203,6 +206,148 @@ async def test_a_degraded_agent_still_renders_and_says_what_broke() -> None: assert result.payload["memory"]["total_bytes"] > 0 +async def test_smart_addon_errors_do_not_degrade_the_system_source() -> None: + """A host without smartmontools (or without device read access) is unconfigured, not + broken (invariant 3): its `smart:` lines must never hold the system page amber. They + stay in `agent.errors`, where the /disks page explains them in its own panel. + """ + body = agent_body() + body["errors"] = ["smart: sdb: Smartctl open device: /dev/sdb failed: Permission denied"] + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=body) + + async with client_for(handler) as client: + result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + assert "degraded" not in result.payload + assert any("smart:" in str(e) for e in result.payload["agent"]["errors"]) + + +async def test_real_degradations_still_show_when_smart_noise_is_alongside() -> None: + body = agent_body() + body["errors"] = ["smart: smartctl not found", "temperatures: no hwmon sensors"] + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=body) + + async with client_for(handler) as client: + result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"), + HostCollector(RateEstimator())) + assert result.payload["degraded"] == ["temperatures: no hwmon sensors"] + + +# --------------------------------------------------------------------------- # +# physical disks (the /disks source's fetch half) +# --------------------------------------------------------------------------- # +async def test_the_disk_fetch_reads_the_same_agent_snapshot_as_the_system_one( + disk_snapshot: Any) -> None: + body = disk_snapshot(mode="healthy") + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/snapshot", "disks has no endpoint of its own" + seen.append(request.url.path) + return httpx.Response(200, json=body) + + target = Target("driveripper", "http://agent.test") + async with client_for(handler) as client: + result = await fetch_disk_snapshot(client, target, DisksCollector(), timeout=30) + assert seen == ["/snapshot"] + assert result.payload["name"] == "driveripper" + assert result.payload["disks"], "the physical rows come with the snapshot" + assert any(sample.metric.startswith("disk.temp_c@") for sample in result.samples) + + +async def test_the_disk_fetch_reports_the_agent_failure_like_the_system_one( + monkeypatch: pytest.MonkeyPatch) -> None: + """One dead host must say the same sentence on both pages (shared `fetch_snapshot`).""" + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(500, text="boom") + + target = Target("gw", "http://agent.test") + async with client_for(handler) as client: + with pytest.raises(UpstreamError) as system_error: + await fetch_host_snapshot(client, target, HostCollector(RateEstimator())) + with pytest.raises(UpstreamError) as disk_error: + await fetch_disk_snapshot(client, target, DisksCollector()) + assert disk_error.value.message == system_error.value.message + + +async def test_an_agent_that_is_not_ok_fails_the_disk_poll_too() -> None: + """A degraded agent that says `ok: false` has no trustworthy disk rows to hand over.""" + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"}) + + async with client_for(handler) as client: + with pytest.raises(UpstreamError, match="procfs unreadable"): + await fetch_disk_snapshot(client, Target("gw", "http://agent.test"), + DisksCollector()) + + +class TimeoutRecordingClient(httpx.AsyncClient): + """An `AsyncClient` that remembers the `timeout=` each request was handed. + + A `MockTransport` replaces httpcore's connection pool, so it does not enforce read + timeouts and a slow-handler test would prove nothing. The argument is what matters + anyway: `timeout=None` means *no timeout at all* to httpx, which would hang a poll + loop forever, and the client default means "the system poll's 4 s". + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.timeouts: list[Any] = [] + + async def get(self, url: Any, **kwargs: Any) -> httpx.Response: # type: ignore[override] + self.timeouts.append(kwargs.get("timeout", "")) + return await super().get(url, **kwargs) + + +async def test_the_slow_disk_poll_carries_its_own_timeout_and_never_disables_one() -> None: + """`HOST_TIMEOUT_SECONDS` (4 s) stays `/system`'s budget; `disks` brings its own 30 s. + + The SMART pass runs inside the agent's snapshot handler, so the two dashboards cannot + share one budget: too short and `/disks` reads red on every SMART poll, too long and + `/system` stops noticing a dead host. And the default must stay httpx's sentinel — + passing a literal `None` silently turns timeout protection off altogether. + """ + body = {"ok": True, "ts": 1.0, "disk_io": {"devices": []}, "errors": []} + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=body) + + client = TimeoutRecordingClient(transport=httpx.MockTransport(handler), + timeout=4.0, base_url="http://agent.test") + target = Target("gw", "http://agent.test") + async with client: + await fetch_host_snapshot(client, target, HostCollector(RateEstimator())) + assert client.timeouts == [httpx.USE_CLIENT_DEFAULT] + await fetch_disk_snapshot(client, target, DisksCollector(), timeout=30) + assert client.timeouts[-1] == 30, "the disk poll waits for the SMART pass" + await fetch_snapshot(client, target) + assert client.timeouts[-1] is httpx.USE_CLIENT_DEFAULT + assert all(timeout is not None for timeout in client.timeouts), ( + "`timeout=None` disables httpx's timeouts entirely — translate it, never pass it") + + +async def test_smart_addon_errors_reach_the_disk_payload_verbatim( + disk_snapshot: Any) -> None: + """`collectors` keeps the `smart:` lines out of `/system`'s `degraded`; `/disks` is + where they earn their keep — they are the page's reason for an empty card.""" + body = disk_snapshot(mode="denied") + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=body) + + async with client_for(handler) as client: + disks = await fetch_disk_snapshot(client, Target("driveripper", "http://agent.test"), + DisksCollector()) + system = await fetch_host_snapshot(client, Target("driveripper", "http://agent.test"), + HostCollector(RateEstimator())) + assert disks.payload["smart_errors"], "the /disks page gets the agent's own sentence" + assert "degraded" not in system.payload, "and /system stays unbothered by it" + + # --------------------------------------------------------------------------- # # LLM engines # --------------------------------------------------------------------------- # diff --git a/tests/test_config.py b/tests/test_config.py index d687bb4..cd06411 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -86,6 +86,28 @@ def test_settings_default_to_no_sources_configured() -> None: assert settings.any_source_enabled is False +def test_the_disk_source_defaults_to_a_five_minute_cadence() -> None: + """SMART is not a per-second signal (phase 09), and the fetch waits for the agent's + own 20 s smartctl budget. Both are env-only, like every other interval.""" + settings = Settings() + assert settings.disks_interval == 300.0 + assert settings.disks_timeout == 30.0 + + +def test_the_disk_cadence_is_an_env_var_like_every_other_one(monkeypatch: pytest.MonkeyPatch) -> None: + """`DISKS_INTERVAL_SECONDS=60` is how an operator watches a failing drive closely.""" + monkeypatch.setenv("DISKS_INTERVAL_SECONDS", "60") + monkeypatch.setenv("DISKS_TIMEOUT_SECONDS", "45") + reloaded = importlib.reload(config) + try: + assert reloaded.SETTINGS.disks_interval == 60.0 + assert reloaded.SETTINGS.disks_timeout == 45.0 + finally: + monkeypatch.delenv("DISKS_INTERVAL_SECONDS") + monkeypatch.delenv("DISKS_TIMEOUT_SECONDS") + importlib.reload(config) + + def test_actions_need_the_flag_and_are_dead_under_read_only() -> None: assert Settings(nut_allow_actions=True, read_only=False).actions_allowed is True assert Settings(nut_allow_actions=True, read_only=True).actions_allowed is False diff --git a/tests/test_disks_source.py b/tests/test_disks_source.py new file mode 100644 index 0000000..d77a9c0 --- /dev/null +++ b/tests/test_disks_source.py @@ -0,0 +1,474 @@ +"""The `disks` source: physical drives in, `disk.*@dev` samples out, nothing invented. + +Phase 09 task 02. The behaviours this pins, in the order it hurts to break them: + +* **One row per physical disk, never per partition.** The rows come from the agent's + whole-`/sys/block` list; the filesystems in the same snapshot must not grow a card. +* **A value the drive does not publish stays absent.** Not 0, not a stored sample — a + stored 0 and "this drive reports nothing" are indistinguishable on a 90-day chart, and + one of them is a lie (`AGENTS.md` rule 2). +* **Nothing is a rate.** Power-on hours and the sector counters are stored as reported; + a second poll produces the same number, not a delta. +* **Why a card is empty is a fact, not a mystery.** `smartctl_available` is three-way + (yes / no, install smartmontools / the agent does not say) because those are three + different sentences for the operator, and `smart_errors` carries the agent's own line. +* **The colour comes from the shared table.** A row carries a `severity` dict for the two + rated numbers (`temp_c`, `percentage_used`) and the drive's own `health` word, so the page + paints a class instead of comparing a number to a bound — and the sector counters get no + verdict at all, because a non-zero one is a chip and zero is not an achievement. +* **Events go under the dashboard name** `disks`, once per verdict change — the phase-03 + lesson: events filed under the sample source leave the page's feed empty forever. +* **The job is its own, at a 5-minute cadence**, and no host targets means no job at all, + so an unconfigured deployment never dials (invariant 3). +""" +from __future__ import annotations + +from typing import Any + +import httpx +import pytest + +from dashboards.collectors import disks_jobs +from dashboards.config import Settings, Target +from dashboards.severity import THRESHOLDS +from dashboards.sources.disks import ( + DASHBOARD, + SEVERITY_FIELDS, + SMART_SERIES, + SOURCE, + UNGRADED_FIELDS, + DisksCollector, + disk_summary, + fleet_disk_summary, + normalize_health, + parse_disks_snapshot, + row_severity, + smartctl_available, +) + + +class NeverDialClient(httpx.AsyncClient): # pragma: no cover - reaching it fails a test + """A client that fails the test the moment anything tries to dial it.""" + + async def get(self, *args: Any, **kwargs: Any) -> httpx.Response: + raise AssertionError("a job factory must not dial; only the poll may") + + +def parse(snapshot: dict[str, Any], *, name: str = "driveripper", + collector: DisksCollector | None = None, + now: float = 1_700_000_000.0): + """One poll of the source, with the defaults the tests do not care about filled in.""" + return parse_disks_snapshot(name, snapshot, collector or DisksCollector(), now=now) + + +def metric_names(result: Any) -> set[str]: + return {sample.metric for sample in result.samples} + + +def values(result: Any) -> dict[str, float]: + return {sample.metric: sample.value for sample in result.samples} + + +# ---- the rows: one per physical disk -------------------------------------- # + +def test_every_physical_disk_the_agent_lists_becomes_a_row(disk_snapshot: Any) -> None: + snapshot = disk_snapshot() + result = parse(snapshot) + rows = result.payload["disks"] + assert [row["device"] for row in rows] == [ + device["name"] for device in snapshot["disk_io"]["devices"] + ], "the card set is the agent's whole-disk list, in its order" + agent_rows = {device["name"]: device for device in snapshot["disk_io"]["devices"]} + for row in rows: + source_row = agent_rows[row["device"]] + assert row["path"] == f"/dev/{row['device']}" + assert row["size_bytes"] == source_row["size_bytes"], "capacity is the agent's, not ours" + assert row["rotational"] == source_row["rotational"], "SSD-vs-spinning is a fact" + assert row["smart"] is not None + + +def test_filesystems_never_grow_a_disk_card(disk_snapshot: Any) -> None: + """"Not per-partition" is the phase's one hard UI constraint, so it is asserted, not hoped. + + The snapshot carries a dozen mounts (`/`, `/srv`, `/boot`); a card per mount would + double-count the same platter and bury the drive that is actually failing. + """ + snapshot = disk_snapshot(mutate=lambda snap: snap["filesystems"].append( + {"device": "/dev/mapper/srv", "mountpoint": "/srv", "fstype": "xfs", + "used_pct": 91.0})) + rows = parse(snapshot).payload["disks"] + assert all("/" not in row["device"] for row in rows), "a mountpoint leaked into the cards" + assert {row["device"] for row in rows}.isdisjoint( + {fs["mountpoint"] for fs in snapshot["filesystems"]}) + + +def test_the_smart_block_is_carried_verbatim_and_the_verdict_is_normalised( + disk_snapshot: Any) -> None: + snapshot = disk_snapshot() + rows = parse(snapshot).payload["disks"] + ata = next(row for row in rows if row["smart"] and row["smart"]["reallocated_sector_ct"]) + assert ata["smart"]["reallocated_sector_ct"] == 8.0 + assert ata["smart"]["current_pending_sector_ct"] == 3.0 + assert ata["smart"]["offline_uncorrectable"] == 2.0 + assert ata["smart"]["power_on_hours"] == 17_520.0 + assert ata["smart"]["temp_c"] == 31.0 + # `PASSED` is smartctl's word; the app's colour vocabulary is `ok` + assert ata["smart"]["overall_health"] == "PASSED", "the drive's own word stays verbatim" + assert ata["health"] == "ok" + + +@pytest.mark.parametrize("word,expected", [ + ("PASSED", "ok"), ("OK", "ok"), (" ok ", "ok"), ("passed", "ok"), + ("FAILED", "failed"), ("failed!", "failed"), + ("UNKNOWN", None), ("", None), (None, None), (7, None), +]) +def test_only_the_drives_own_verdict_words_count(word: Any, expected: str | None) -> None: + """`UNKNOWN` is not a pass: no measurement, no colour (`severity.py`'s first rule).""" + assert normalize_health(word) == expected + + +def test_a_host_that_reports_no_disks_answers_with_an_empty_list_not_an_error() -> None: + """gateway is a VM on virtio with nothing behind it; `/disks` renders "no disks".""" + result = parse({"ts": 1.0, "disk_io": {"devices": []}, "errors": []}, name="gateway") + assert result.payload["disks"] == [] + assert result.payload["summary"]["disks"] == 0 + assert result.samples == [] + assert result.payload["smartctl_available"] is None + + +def test_a_malformed_device_row_is_skipped_rather_than_crashing(disk_snapshot: Any) -> None: + """`/disks` must survive an agent that answered something odd. + + A device with no name would render a card called `/dev/`, which is a worse lie than a + missing card; a non-object row is what a truncated JSON body looks like. + """ + clean = parse(disk_snapshot(mode="healthy")).payload["disks"] + + def mangle(snapshot: dict[str, Any]) -> None: + snapshot["disk_io"]["devices"].append("not-a-dict") + snapshot["disk_io"]["devices"].append({"name": " ", "smart": {"temp_c": 44.0}}) + + mangled = parse(disk_snapshot(mode="healthy", mutate=mangle)).payload["disks"] + assert mangled == clean, "only the two junk rows were dropped, in place and unchanged" + assert all(row["device"].strip() for row in mangled) + + +# ---- samples: the history namespace, counters as reported ----------------- # + +def test_the_series_are_the_pinned_disk_names_per_device(disk_snapshot: Any) -> None: + snapshot = disk_snapshot(mode="healthy") + result = parse(snapshot) + devices = [device["name"] for device in snapshot["disk_io"]["devices"]] + # every series this source may write is `@` from the pinned table + allowed = {f"{base}@{device}" for device in devices for _field, base in SMART_SERIES} + assert metric_names(result) <= allowed + assert all(name.startswith("disk.") and "@" in name for name in metric_names(result)), \ + "an unqualified disk name would collide with /system's per-mount series" + # all six bases show up: the ATA shape supplies the sector counters, the NVMe shape + # the wear percentage, so between them the whole table is exercised + assert {name.split("@")[0] for name in metric_names(result)} == { + base for _field, base in SMART_SERIES} + # and the family is the disks one, so /api/metrics?source=disks answers for this page + assert {sample.source for sample in result.samples} == {SOURCE} == {"disks"} + + +# ---- why a card is `n/a` --------------------------------------------------- # + +def test_a_denied_device_nulls_only_that_disk(disk_snapshot: Any) -> None: + snapshot = disk_snapshot(mode="denied") + result = parse(snapshot) + denied = snapshot["disk_io"]["devices"][-1]["name"] + by_device = {row["device"]: row for row in result.payload["disks"]} + assert by_device[denied]["smart"] is None + assert by_device[denied]["health"] is None + assert not [name for name in metric_names(result) if name.endswith(f"@{denied}")] + assert any(row["smart"] for row in result.payload["disks"]), "its neighbours are fine" + assert result.payload["smart_errors"] == [ + line for line in snapshot["errors"] if line.startswith("smart:")] + + +def test_a_missing_binary_is_answered_as_not_installed(disk_snapshot: Any) -> None: + """"smartctl not available" is the sentence the page needs, and it comes from here.""" + result = parse(disk_snapshot(mode="no_binary")) + assert result.payload["smartctl_available"] is False + assert all(row["smart"] is None for row in result.payload["disks"]) + assert len(result.payload["smart_errors"]) == 1, "one missing binary is one fact" + assert result.samples == [], "nothing was read, so nothing is stored" + + +def test_a_silent_opt_out_is_unknown_rather_than_not_installed(disk_snapshot: Any) -> None: + """`--no-smart` and an agent predating the pass say nothing; guessing is worse. + + The page still has an honest state: `None` renders `n/a` with no colour, and the cards + are dimmed rather than red (invariant 3). + """ + result = parse(disk_snapshot(mode="silent")) + assert result.payload["smartctl_available"] is None + assert result.payload["smart_errors"] == [] + + +def test_smartctl_available_is_true_the_moment_any_disk_answers() -> None: + rows = [{"smart": {"temp_c": 31.0}}, {"smart": None}] + assert smartctl_available(rows, ["smart: sdb: Permission denied"]) is True + + +def test_smart_errors_ignore_the_lines_that_are_not_about_smart() -> None: + """`errors` also carries /proc failures, and those belong to `/system`, not here.""" + result = parse({"ts": 1.0, "disk_io": {"devices": []}, + "errors": ["cpu: OSError: /proc/stat", "smart: sdb: denied"]}) + assert result.payload["smart_errors"] == ["smart: sdb: denied"] + + +# ---- the host summary ------------------------------------------------------ # + +def test_summary_counts_reads_alarm_drives_and_failures_apart(disk_snapshot: Any) -> None: + result = parse(disk_snapshot(mode="healthy")) + summary = result.payload["summary"] + devices = result.payload["disks"] + assert summary["disks"] == len(devices) == summary["with_smart"] + assert summary["health_failed"] == 0 + # every ATA shape carries reallocated/pending/uncorrectable sectors; the NVMe shapes + # carry zeros, so the alarm count is exactly the number of ATA-shaped disks + ata_disks = [row for row in devices if row["smart"]["reallocated_sector_ct"]] + assert summary["alarm_disks"] == len(ata_disks) > 0 + + +def test_a_drive_answering_failed_is_counted_and_the_neighbours_are_not( + disk_snapshot: Any) -> None: + result = parse(disk_snapshot(mode="failed")) + assert result.payload["summary"]["health_failed"] == 1 + assert result.payload["disks"][0]["health"] == "failed" + + +def test_an_unreadable_drive_is_not_counted_as_failed(disk_snapshot: Any) -> None: + """"Cannot read it" and "it is failing" are different alarms (rule 2).""" + result = parse(disk_snapshot(mode="no_binary")) + summary = result.payload["summary"] + assert summary["disks"] > 0 + assert summary["with_smart"] == summary["health_failed"] == summary["alarm_disks"] == 0 + + +def test_disk_summary_ignores_a_zero_counter(disk_snapshot: Any) -> None: + rows = parse(disk_snapshot(mode="healthy")).payload["disks"] + clean = [{**row, "smart": {**(row["smart"] or {}), + "reallocated_sector_ct": 0.0, + "current_pending_sector_ct": 0.0, + "offline_uncorrectable": 0.0}} for row in rows] + assert disk_summary(clean)["alarm_disks"] == 0 + + +# ---- events: one per verdict change, filed under the dashboard ------------- # + +def test_a_drive_crossing_into_failed_writes_one_error_event(disk_snapshot: Any) -> None: + collector = DisksCollector() + snapshot = disk_snapshot(mode="healthy") + device = snapshot["disk_io"]["devices"][0]["name"] + assert parse(snapshot, collector=collector).events == [], "first sighting is not news" + assert parse(snapshot, collector=collector).events == [], "an unchanged verdict is quiet" + + degraded = disk_snapshot(mode="failed") + events = parse(degraded, collector=collector).events + assert len(events) == 1 + event = events[0] + assert event.source == DASHBOARD == "disks", ( + "a feed filtered by dashboard must find these — phase 03's empty-feed bug") + assert event.host == "driveripper" and event.kind == "smart-health" + assert event.severity == "error" + assert device in event.message and "failed" in event.message + assert event.data == {"device": device, "from": "ok", "to": "failed"} + + assert parse(degraded, collector=collector).events == [] + + +def test_recovering_from_failed_is_a_quiet_info_event(disk_snapshot: Any) -> None: + collector = DisksCollector() + parse(disk_snapshot(mode="failed"), collector=collector) + events = parse(disk_snapshot(mode="healthy"), collector=collector).events + assert [event.severity for event in events] == ["info"] + + +def test_a_disk_that_left_and_came_back_is_a_first_sighting_again(disk_snapshot: Any) -> None: + """State is replaced per poll, so a removed drive cannot compare against its old self.""" + collector = DisksCollector() + snapshot = disk_snapshot(mode="healthy") + device = snapshot["disk_io"]["devices"][0]["name"] + parse(snapshot, collector=collector) + empty = {"ts": snapshot["ts"], "errors": [], "disk_io": {"devices": []}} + parse(empty, collector=collector) + assert collector.previous_health("driveripper") == {} + assert parse(disk_snapshot(mode="failed"), collector=collector).events == [] + assert collector.previous_health("driveripper")[device] == "failed" + + +def test_two_hosts_do_not_share_verdict_state(disk_snapshot: Any) -> None: + collector = DisksCollector() + parse(disk_snapshot("driveripper"), name="driveripper", collector=collector) + parse(disk_snapshot("deskwork"), name="deskwork", collector=collector) + assert set(collector.hosts) == {"driveripper", "deskwork"} + assert len(collector.hosts["deskwork"]) == 1, "a host compares only against its own disks" + + +# ---- the job: its own, slow, and absent rather than dialling --------------- # + +def test_one_disks_job_per_host_agent_at_the_slow_cadence() -> None: + targets = [Target("driveripper", "http://a:9121"), Target("deskwork", "http://b:9122")] + # the client is never used by the factory, only captured by the job closures + jobs = disks_jobs(targets, NeverDialClient(), DisksCollector(), 300.0, timeout=30.0) + assert [job.key for job in jobs] == ["disks:driveripper", "disks:deskwork"] + assert [job.dashboard for job in jobs] == ["disks", "disks"] + assert [job.label for job in jobs] == ["driveripper", "deskwork"], ( + "the label is the host: it is what the event feed and the alert print") + assert [job.interval for job in jobs] == [300.0, 300.0], "SMART is not a 10 s signal" + + +def test_the_default_cadence_is_five_minutes_with_a_slow_fetch_timeout() -> None: + """The defaults are the phase's decision (300 s) and the agent's SMART budget (30 s).""" + settings = Settings() + assert settings.disks_interval == 300.0 + assert settings.disks_timeout == 30.0, "a wedged smartctl pass costs the agent 20 s" + + +def test_no_host_targets_means_no_disks_job_and_nothing_dialled() -> None: + """`disks` has no upstream of its own, so unconfigured means *no task at all*. + + The 404 `{"configured": false}` half of invariant 3 is task 03's route; this is the + half that matters first: with no targets the app must not open a socket, and + `/api/health` must not carry a `disks` source that is red because nobody installed one. + """ + assert disks_jobs([], NeverDialClient(), DisksCollector(), 300.0) == [] + + +def test_a_host_that_has_never_polled_has_no_previous_verdicts() -> None: + assert DisksCollector().previous_health("never-seen") == {} + + +# ---- the verdicts a card paints (task 03's severity rows) ------------------ # + +def hot_drive(disk_snapshot: Any, **fields: Any) -> dict[str, Any]: + """The healthy capture with one drive's rated numbers moved onto a chosen bound. + + Driven through the fixture's `mutate` so the numbers under test are the ones the agent's + own parser would have produced for a drive running hot and half-worn out. + """ + def tweak(snapshot: dict[str, Any]) -> None: + (snapshot["disk_io"]["devices"][0].get("smart") or {}).update(fields) + + return disk_snapshot(mode="healthy", mutate=tweak) + + +def test_a_row_carries_a_verdict_for_each_rated_number_and_for_the_drives_word( + disk_snapshot: Any) -> None: + """"72 °C, 85 % wear, PASSED" arrives as `fail`/`warn`/`ok`, never as a bound in JS.""" + rows = parse(hot_drive(disk_snapshot, temp_c=72.0, percentage_used=85.0)).payload["disks"] + assert rows[0]["severity"] == {"temp_c": "fail", "percentage_used": "warn", + "health": "ok"} + + +def test_a_drive_answering_failed_is_the_row_that_reads_red(disk_snapshot: Any) -> None: + """`health` is the drive's own verdict, so its colour is a translation, not a threshold.""" + rows = parse(disk_snapshot(mode="failed")).payload["disks"] + assert rows[0]["health"] == "failed" and rows[0]["severity"]["health"] == "fail" + assert rows[1]["severity"]["health"] == "ok", "its neighbours keep their own verdict" + + +def test_an_unreadable_drive_gets_no_verdict_at_all(disk_snapshot: Any) -> None: + """No `smart` block, no colour: the keys are present and every one is `None`. + + The card must be able to read `severity.temp_c` without branching on a missing key, and + must not render green for a drive the agent could not open (`AGENTS.md` rule 2). + """ + rows = parse(disk_snapshot(mode="no_binary")).payload["disks"] + assert rows, "the drive is still listed, dimmed" + for row in rows: + assert row["severity"] == {"temp_c": None, "percentage_used": None, "health": None} + + +def test_a_missing_number_stays_unjudged_rather_than_good(disk_snapshot: Any) -> None: + """An ATA spinning disk publishes no `percentage_used`; that row reads `n/a`, not `ok`.""" + rows = parse(disk_snapshot(mode="healthy")).payload["disks"] + spinning = [row for row in rows if row["smart"]["percentage_used"] is None] + assert spinning, "the recorded ATA shape has no wear field, which is the case here" + for row in spinning: + assert row["severity"]["percentage_used"] is None + assert row["severity"]["temp_c"] == "ok", "the number it does report is still judged" + + +def test_the_sector_counters_are_never_part_of_a_rows_verdict(disk_snapshot: Any) -> None: + """The alarm fields appear on the row and in `summary`, never as an `ok|warn|fail`.""" + rows = parse(disk_snapshot(mode="healthy")).payload["disks"] + alarming = next(row for row in rows if row["smart"]["reallocated_sector_ct"]) + assert set(alarming["severity"]) == {field for field, _metric in SEVERITY_FIELDS} | {"health"} + assert not set(alarming["severity"]) & set(UNGRADED_FIELDS) + assert alarming["smart"]["reallocated_sector_ct"] == 8.0, "the number still travels" + + +def test_the_rated_rows_exist_in_the_one_threshold_table() -> None: + """`SEVERITY_FIELDS` may only name metrics `severity.THRESHOLDS` actually carries. + + A row here without a bound there would silently colour nothing; a bound for a field the + source never reads is the same drift seen from the other side. + """ + for field, metric in SEVERITY_FIELDS: + assert metric in THRESHOLDS, f"{field} is rated by a metric nobody defines" + assert UNGRADED_FIELDS and not set(UNGRADED_FIELDS) & { + field for field, _metric in SEVERITY_FIELDS} + assert {base for _field, base in SMART_SERIES} >= { + metric for _field, metric in SEVERITY_FIELDS}, ( + "a rated number that is not stored cannot be charted in its own colour") + + +def test_row_severity_tolerates_a_row_with_nothing_in_it() -> None: + """The API hands this a dict built from an agent that answered oddly; it must not raise.""" + assert row_severity({}) == {"temp_c": None, "percentage_used": None, "health": None} + assert row_severity({"health": "no-such-word", "smart": {"temp_c": "hot"}}) == { + "temp_c": None, "percentage_used": None, "health": None} + + +# ---- the fleet header the API sums, not the page --------------------------- # + +def test_the_fleet_counts_drives_across_the_hosts_that_answered( + disk_snapshot: Any) -> None: + """`/disks` says how many drives the fleet has and how many it could read.""" + hosts = [parse(disk_snapshot("driveripper"), name="driveripper").payload, + parse(disk_snapshot("deskwork"), name="deskwork").payload] + fleet = fleet_disk_summary(hosts) + disks = sum(host["summary"]["disks"] for host in hosts) + assert fleet["hosts_total"] == fleet["hosts_live"] == 2 + assert fleet["disks"] == disks == len(hosts[0]["disks"]) + len(hosts[1]["disks"]) + assert fleet["with_smart"] == disks + assert fleet["health_failed"] == 0 + # the recorded ATA shape carries reallocated sectors, so the fleet's alarm count is the + # sum of the hosts' own chip counts, not a number this function invented + assert fleet["alarm_disks"] == sum(host["summary"]["alarm_disks"] for host in hosts) > 0 + + +def test_a_host_that_never_answered_is_counted_but_never_summed( + disk_snapshot: Any) -> None: + """A dead agent shrinks `hosts_live`, not the drive totals presented as current. + + Adding a stub's zeros to the sums would halve the fleet's alarm count on the strength of + a host that said nothing — the `/llm` fleet block's rule, same failure shape. + """ + live = parse(disk_snapshot(mode="failed"), name="driveripper").payload + stub = {"name": "deskwork", "ts": None, "unreachable": True, "disks": []} + fleet = fleet_disk_summary([live, stub]) + assert fleet["hosts_total"] == 2 and fleet["hosts_live"] == 1 + assert fleet["disks"] == len(live["disks"]), "only the answering host is in the sum" + assert fleet["health_failed"] == 1 + + +def test_a_payload_with_no_countable_summary_contributes_nothing() -> None: + """"Answered" is not "counted": a summary that is missing or junk adds zero, not a guess.""" + assert fleet_disk_summary([{"name": "gw", "disks": []}])["disks"] == 0 + weird = {"name": "gw", "summary": {"disks": True, "with_smart": "many", + "health_failed": None, "alarm_disks": 2}} + fleet = fleet_disk_summary([weird]) + assert fleet["disks"] == fleet["with_smart"] == fleet["health_failed"] == 0 + assert fleet["alarm_disks"] == 2 + assert fleet["hosts_live"] == 1 + + +def test_an_empty_fleet_answers_with_zeros_rather_than_none() -> None: + """No hosts configured never reaches this route, but an empty list must not crash either.""" + assert fleet_disk_summary([]) == {"hosts_total": 0, "hosts_live": 0, "disks": 0, + "with_smart": 0, "health_failed": 0, "alarm_disks": 0} diff --git a/tests/test_live_fleet.py b/tests/test_live_fleet.py index d9757ed..8fb02b1 100644 --- a/tests/test_live_fleet.py +++ b/tests/test_live_fleet.py @@ -32,6 +32,7 @@ from dashboards.config import Settings, load from dashboards.rates import RateEstimator from dashboards.severity import classify from dashboards.sources import nut +from dashboards.sources.disks import SMART_SERIES, DisksCollector from dashboards.sources.host import HostCollector from slots_contract import assert_slot_shape from ups_contract import assert_ups_hero_contract @@ -95,6 +96,90 @@ async def test_every_host_serves_the_hero_card_contract(settings: Settings, ) +async def test_agent_disk_rows_carry_the_smart_shape(settings: Settings) -> None: + """Phase-09: a redeployed agent answers `smart` per physical disk of `/snapshot`. + + Agents are redeployed by hand, so an agent whose rows carry no `smart` key at all + SKIPS this host rather than failing it. `smart: null` is a documented state (no + smartctl binary, or the agent user lacks device read access) — what must hold is the + shape of a non-null block and that unknowns are null, never 0. Read-only: nothing + here sends `smartctl -t`; a self-test on hardware something is plugged into is the + operator's call, not a unit test. + """ + if not settings.host_targets: + pytest.skip("HOST_TARGETS is unset — start ./scripts/dev-tunnels.sh") + seen_smart = 0 + async with httpx.AsyncClient(timeout=settings.host_timeout) as client: + for target in settings.host_targets: + response = await client.get(f"{target.url}/snapshot", headers=target.headers) + if response.status_code >= 400: + continue # reachability has its own live test + devices = (response.json().get("disk_io") or {}).get("devices") or [] + if devices and not all("smart" in device for device in devices): + continue # agent predates the smart pass — redeploy it to test this + for device in devices: + seen_smart += 1 + smart = device.get("smart") + if smart is None: + continue # degraded by design; the errors[] entry says why + # the literal set is the pin: a rename on the agent must fail here too + assert set(smart) == {"model", "serial", "overall_health", "temp_c", + "power_on_hours", "reallocated_sector_ct", + "current_pending_sector_ct", + "offline_uncorrectable", "percentage_used"}, \ + f"{target.name}/{device['name']}" + if smart["power_on_hours"] is not None: + assert smart["power_on_hours"] > 0 + if smart["temp_c"] is not None: + assert -20.0 < smart["temp_c"] < 125.0 + assert smart["overall_health"] in (None, "PASSED", "FAILED", "UNKNOWN") + if not seen_smart: + pytest.skip("no agent redeployed with the smart pass yet (deploy/install-agent.sh)") + + +async def test_disks_source_charts_the_physical_drives_the_agent_reports( + settings: Settings) -> None: + """Phase 09 task 02: the `disks` source over a real agent snapshot. + + This is the production code path — the fetch the poller runs, not a hand-built dict — + so what fails here is a rename between the agent and the page, not a typo in a fixture. + The assertions hold whether or not this fleet can read SMART yet: one row per whole + disk, `disk.*@` series only for values that exist, and no invented numbers. + """ + if not settings.host_targets: + pytest.skip("HOST_TARGETS is unset — start ./scripts/dev-tunnels.sh") + collector = DisksCollector() + checked = 0 + async with httpx.AsyncClient(timeout=settings.disks_timeout) as client: + for target in settings.host_targets: + try: + result = await collectors.fetch_disk_snapshot(client, target, collector, + timeout=settings.disks_timeout) + except collectors.UpstreamError: + continue # reachability has its own live test + checked += 1 + payload = result.payload + assert payload["name"] == target.name + assert payload["disks"], f"{target.name}: no physical disks at all?" + for row in payload["disks"]: + assert row["path"] == f"/dev/{row['device']}" + assert "/" not in row["device"], f"partition {row['device']} in a disk list" + assert row["health"] in (None, "ok", "failed") + devices = {row["device"] for row in payload["disks"]} + bases = {base for _field, base in SMART_SERIES} + for sample in result.samples: + assert sample.source == "disks", "the wrong family would chart nowhere" + base, _, device = sample.metric.partition("@") + assert base in bases and device in devices, ( + f"{target.name}: {sample.metric} is not a pinned disk series") + assert sample.value == sample.value and abs(sample.value) != float("inf"), \ + "NaN must never reach the store" + if any(row["smart"] for row in payload["disks"]): + assert payload["smartctl_available"] is True + if not checked: + pytest.skip("no host agent answered — start ./scripts/dev-tunnels.sh") + + @pytest.mark.parametrize("engine", ["gufo", "vllm"]) async def test_engine_still_publishes_the_series_we_parse(engine: str, settings: Settings) -> None: diff --git a/tests/test_poller.py b/tests/test_poller.py index 1d7cd00..25c15bf 100644 --- a/tests/test_poller.py +++ b/tests/test_poller.py @@ -113,6 +113,40 @@ async def test_grace_allows_a_blip_without_an_event(store) -> None: assert len(store.recent_events(limit=10)) == 1, "one event per outage, not per poll" +async def test_a_failed_disks_poll_ages_its_cards_instead_of_zeroing_them(store) -> None: + """Invariant 4 on the phase-09 job: SMART goes quiet, the numbers stay and age. + + The `/disks` page is the one a operator reads *because* something looks wrong, so a + failed poll must not turn "last read 12 minutes ago" into a page of zeros — and the + outage must be filed under the `disks` dashboard, which is where that page filters. + """ + payload = {"name": "driveripper", "smartctl_available": True, + "disks": [{"device": "nvme0n1", "health": "ok", + "smart": {"temp_c": 38.0, "power_on_hours": 8760.0}}], + "summary": {"disks": 1, "with_smart": 1, "health_failed": 0, + "alarm_disks": 0}} + + async def good() -> Result: + return Result(payload=payload, + samples=[Sample("disks", "driveripper", "disk.temp_c@nvme0n1", + 38.0, time.time())]) + + job = Job("disks:driveripper", "disks", "driveripper", 300.0, good) + assert job.interval == 300.0, "SMART is not a 10 s signal" + assert await job.run_once(store) is True + assert store.query("disks", "driveripper", "disk.temp_c@nvme0n1", + start=time.time() - 60), "the series is written under `disks`" + + job.fn = failing + assert await job.run_once(store) is False + assert job.state.payload == payload, "the card keeps the last temperature it read" + assert job.state.ok is False + assert job.state.as_json()["age_seconds"] is not None + events = store.recent_events(limit=10) + assert [event.source for event in events] == ["disks"], ( + "the outage must land where /disks filters its feed, not under the sample source") + + async def test_a_slow_poll_never_overlaps_itself(store) -> None: calls = 0 gate = asyncio.Event() diff --git a/tests/test_severity.py b/tests/test_severity.py index ae8f956..00ef2b7 100644 --- a/tests/test_severity.py +++ b/tests/test_severity.py @@ -22,6 +22,8 @@ RATIFIED = { "cpu.usage_pct": (75.0, 90.0), "mem.used_pct": (80.0, 92.0), "disk.used_pct": (80.0, 90.0), + "disk.temp_c": (50.0, 70.0), + "disk.percentage_used": (80.0, 100.0), "cpu.temp_c": (75.0, 90.0), "llm.kv_cache_usage_pct": (75.0, 90.0), "ups.charge_pct": (50.0, 20.0), @@ -134,10 +136,50 @@ def test_non_finite_values_are_treated_as_missing(metric: str, value: float) -> assert classify(metric, value) is None +def test_a_drive_temperature_and_its_wear_are_the_only_disk_verdicts() -> None: + """Phase 09: the two `/disks` numbers worth a colour, with the bounds the phase set. + + A spinning drive at 70 °C is a fire and an SSD that has consumed its rated wear is a + replacement, so both trip red; the amber in between is the "watch it" band. An SSD with + no temperature sensor has **no value**, and therefore no verdict — the absence of a + number is not a healthy reading (`classify(metric, None)` stays `None`). + """ + assert classify("disk.temp_c", 49.9) == OK + assert classify("disk.temp_c", 50) == WARN + assert classify("disk.temp_c", 69.9) == WARN + assert classify("disk.temp_c", 70) == FAIL + assert classify("disk.temp_c", 75) == FAIL + assert classify("disk.temp_c", None) is None + assert classify("disk.percentage_used", 79.9) == OK + assert classify("disk.percentage_used", 80) == WARN + assert classify("disk.percentage_used", 100) == FAIL + assert classify("disk.percentage_used", None) is None + + +def test_sector_counters_and_power_on_hours_are_never_graded() -> None: + """A non-zero sector count is an alarm, not a threshold breach (`/llm`'s chip rule). + + Colouring 0 reallocated sectors green would call a drive healthy because a number is + small, and colouring 1 amber would make the chip redundant: the page shows an alarm on + **presence** and says nothing at all at zero. `power_on_hours` is an odometer with no bad + value. So none of them may ever grow a row in `THRESHOLDS`. + """ + counters = ("disk.reallocated", "disk.pending", "disk.uncorrectable", + "disk.power_on_hours", "disk.health_ok") + assert set(counters).isdisjoint(THRESHOLDS), "a counter is not a graded number" + for metric in counters: + assert classify(metric, 0) is None + assert classify(metric, 1) is None + assert classify(metric, 99_000) is None + assert classify(f"{metric}@sda", 1) is None + + def test_per_device_streams_inherit_their_base_metric() -> None: assert classify("disk.used_pct@/srv", 95) == "fail" assert classify("disk.used_pct@/srv", 10) == "ok" assert classify("cpu.usage_pct@core7", 80) == "warn" + assert classify("disk.temp_c@nvme0n1", 71) == "fail" + assert classify("disk.percentage_used@nvme0n1", 80) == "warn" assert classify("mem.used_pct@any", 95) == "fail" diff --git a/tests/test_store.py b/tests/test_store.py index ec4391d..15acb3a 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -252,3 +252,77 @@ def test_rollup_does_not_average_a_partial_minute(db) -> None: def test_hour_constant_is_used_in_retention(db) -> None: assert db.raw_keep_seconds == pytest.approx(48 * HOUR) + + +# ---- a 300-second source (phase 09: `disks` samples SMART every 5 minutes) -- # +# +# The rollup SQL groups by minute and hour buckets, so a source slower than a minute +# puts one raw row in every fifth bucket and nothing in the other four. Two things had to +# be true and are asserted rather than assumed: an empty bucket produces *no* rollup row +# (inventing a value for a minute nothing was sampled is rule 2), and the hour bucket +# still averages the samples it did get. + +SMART_STEP = 300.0 + + +def test_a_300s_source_rolls_into_minutes_without_inventing_the_empty_ones(db) -> None: + now = time.time() + # five SMART polls at exactly the phase's cadence (-30, -25, -20, -15, -10 minutes), + # so every bucket is fully in the past (+60 s slack) and maintenance may roll all five + start = int(now // MINUTE) * MINUTE - 30 * MINUTE + add_series(db, "disks", "driveripper", "disk.temp_c@nvme0n1", + [38, 39, 38, 40, 37], start=start, step=SMART_STEP) + counts = db.maintain(now=now) + minutes = db.query("disks", "driveripper", "disk.temp_c@nvme0n1", res=RES_MIN, start=0) + assert counts["to_minute"] == 5 + assert [value for _ts, value in minutes] == [38.0, 39.0, 38.0, 40.0, 37.0], ( + "a bucket holding one sample rolls up that sample, unchanged") + assert [int(ts) for ts, _value in minutes] == [ + int(start + index * SMART_STEP // MINUTE * MINUTE) for index in range(5)], ( + "every fifth minute bucket has a row; the empty ones must have none") + assert db.count(res=RES_MIN) == 5, "empty buckets became fabricated rows" + + +def test_a_300s_source_averages_into_the_hour_bucket_it_lands_in(db) -> None: + now = time.time() + # aligned to the epoch, and old enough that the hour bucket is complete: 12 samples + # five minutes apart inside one hour average to their mean, not to a sample count + hour = int(now // HOUR) * HOUR - 3 * HOUR + add_series(db, "disks", "driveripper", "disk.power_on_hours@nvme0n1", + [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 124], + start=hour, step=SMART_STEP) + db.maintain(now=now) + db.maintain(now=now) # the 1h pass reads 1m rows, so the first pass only makes them + hours = db.query("disks", "driveripper", "disk.power_on_hours@nvme0n1", res=RES_HOUR, + start=0) + assert len(hours) == 1 + assert hours[0][0] == hour + assert hours[0][1] == pytest.approx((11 * 100 + 124) / 12) + + +def test_a_300s_series_is_queryable_at_every_resolution(db) -> None: + """A 5-minute series over a 1h window is 12 points; `pick_res` must find them raw.""" + now = time.time() + add_series(db, "disks", "driveripper", "disk.reallocated@nvme0n1", + [8] * 12, start=now - 3600 + 60, step=SMART_STEP) + assert db.pick_res("disks", "driveripper", "disk.reallocated@nvme0n1", 3600, 600) == RES_RAW + points = db.history("disks", "driveripper", "disk.reallocated@nvme0n1", span=3600) + assert len(points) == 12, "the chart is empty while raw history exists" + db.maintain(now=now) + db.maintain(now=now) + # with rollups present, a long window reads the rolled-up series, not 288 raw rows + assert db.pick_res("disks", "driveripper", "disk.reallocated@nvme0n1", + 24 * 3600, 600) in (RES_MIN, RES_RAW) + + +def test_a_300s_source_is_pruned_by_the_existing_retention(db) -> None: + """288 rows a day per series is fine for 48h and not fine forever — rule 9.""" + now = time.time() + expired = now - db.raw_keep_seconds - 6 * SMART_STEP + add_series(db, "disks", "h", "disk.temp_c@sda", [30] * 4, start=expired, step=SMART_STEP) + add_series(db, "disks", "h", "disk.temp_c@sdb", [30], start=now - SMART_STEP, + step=SMART_STEP) + counts = db.maintain(now=now) + assert counts["pruned"] >= 4 + assert db.query("disks", "h", "disk.temp_c@sda", start=0) == [] + assert len(db.query("disks", "h", "disk.temp_c@sdb", start=0)) == 1