phase: 09_disks_smartctl

**Phase 09 — final verification pass (all 4 task files already in `complete/`; nothing re-implemented)**

- Verified `GET /api/disks/snapshot` live: 3 hosts / 18 whole-`/sys/block` rows, `smartctl_available` three-way, per-row `severity`, API-summed `fleet`; 404 `{"configured": false}` proven *without dialling* (spies on `httpx.AsyncClient.get` + `collectors.fetch_snapshot`).
- Verified `/api/disks/history` live (`disk.temp_c@dev`, unknown names → empty series) + `test_store.py`'s 300 s rollup/prune buckets; `/api/metrics?source=disks` answers the `disks` family.
- Browser against live data: 18 drive cards (never a partition/mount), collapsed per-host `Charts` with `aria-label`, 3 `.stale` grids + one `.banner.info` why-line and **0 `.fail` classes / green `disks` dot** → criteria 3 and 4 hold; new shots `data/logs/09-verify-disks-live-dimmed.png`, `09-verify-overview-live.png` (13 total for the phase).
- Real-hardware criterion: read 4 driveripper drives through `ops`' `sudo -n smartctl -i -A -H` and ran the live output through `host_agent.parse_smart_output` → `sources/disks.parse_disks_snapshot`: `sda` 35 °C / 30 423 h, `nvme1n1` 58 % wear, `nvme5n1` 13 uncorrectable → `alarm_disks: 1`, temps graded `warn`; consistent with the 16 committed fixtures.
- `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov` → **769 passed, 11 skipped, coverage 98.33%** (floor 94); `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean; hygiene + dependency advisories clean. `uv run pytest -m live --live` → 10 passed, 1 skipped (agents still predate the SMART pass). Dev server healthy: `/api/health` `ok`, 9/9 sources.
- Docs fixed: AGENTS.md test count 698→769 and coverage 98.2→98.33 %; `disks` added to the event dashboard-name list and to the "every feed renders `Dash.eventText`" sentence; README `POLL_MS_*` row now states `/disks`' deliberate fixed 30 s; new Known gap 4 records the dev-DB `database is locked` writes (reported/recovered per invariant 4, ~590 MB DB / ~70 MB WAL, not reproduced by a controlled 25 s external read txn).
- Deviations: **no live-infra change** — granting the agent user the `disk` group and redeploying `host_agent.py` on driveripper/deskwork stay the documented paired operator action (redeploying without the grant yields 16 permission-denied lines; a sudo-spawning agent would break the confirmed "direct read, no sudo" assumption). Store-lock issue left to its own phase rather than patched speculatively.
- Next pending phase: `10_dev_direct_dial`.
This commit is contained in:
2026-10-18 00:45:01 -04:00
parent 19cfedfddf
commit fefbeb8227
81 changed files with 5525 additions and 134 deletions
+74
View File
@@ -252,3 +252,77 @@ def test_rollup_does_not_average_a_partial_minute(db) -> None:
def test_hour_constant_is_used_in_retention(db) -> None:
assert db.raw_keep_seconds == pytest.approx(48 * HOUR)
# ---- a 300-second source (phase 09: `disks` samples SMART every 5 minutes) -- #
#
# The rollup SQL groups by minute and hour buckets, so a source slower than a minute
# puts one raw row in every fifth bucket and nothing in the other four. Two things had to
# be true and are asserted rather than assumed: an empty bucket produces *no* rollup row
# (inventing a value for a minute nothing was sampled is rule 2), and the hour bucket
# still averages the samples it did get.
SMART_STEP = 300.0
def test_a_300s_source_rolls_into_minutes_without_inventing_the_empty_ones(db) -> None:
now = time.time()
# five SMART polls at exactly the phase's cadence (-30, -25, -20, -15, -10 minutes),
# so every bucket is fully in the past (+60 s slack) and maintenance may roll all five
start = int(now // MINUTE) * MINUTE - 30 * MINUTE
add_series(db, "disks", "driveripper", "disk.temp_c@nvme0n1",
[38, 39, 38, 40, 37], start=start, step=SMART_STEP)
counts = db.maintain(now=now)
minutes = db.query("disks", "driveripper", "disk.temp_c@nvme0n1", res=RES_MIN, start=0)
assert counts["to_minute"] == 5
assert [value for _ts, value in minutes] == [38.0, 39.0, 38.0, 40.0, 37.0], (
"a bucket holding one sample rolls up that sample, unchanged")
assert [int(ts) for ts, _value in minutes] == [
int(start + index * SMART_STEP // MINUTE * MINUTE) for index in range(5)], (
"every fifth minute bucket has a row; the empty ones must have none")
assert db.count(res=RES_MIN) == 5, "empty buckets became fabricated rows"
def test_a_300s_source_averages_into_the_hour_bucket_it_lands_in(db) -> None:
now = time.time()
# aligned to the epoch, and old enough that the hour bucket is complete: 12 samples
# five minutes apart inside one hour average to their mean, not to a sample count
hour = int(now // HOUR) * HOUR - 3 * HOUR
add_series(db, "disks", "driveripper", "disk.power_on_hours@nvme0n1",
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 124],
start=hour, step=SMART_STEP)
db.maintain(now=now)
db.maintain(now=now) # the 1h pass reads 1m rows, so the first pass only makes them
hours = db.query("disks", "driveripper", "disk.power_on_hours@nvme0n1", res=RES_HOUR,
start=0)
assert len(hours) == 1
assert hours[0][0] == hour
assert hours[0][1] == pytest.approx((11 * 100 + 124) / 12)
def test_a_300s_series_is_queryable_at_every_resolution(db) -> None:
"""A 5-minute series over a 1h window is 12 points; `pick_res` must find them raw."""
now = time.time()
add_series(db, "disks", "driveripper", "disk.reallocated@nvme0n1",
[8] * 12, start=now - 3600 + 60, step=SMART_STEP)
assert db.pick_res("disks", "driveripper", "disk.reallocated@nvme0n1", 3600, 600) == RES_RAW
points = db.history("disks", "driveripper", "disk.reallocated@nvme0n1", span=3600)
assert len(points) == 12, "the chart is empty while raw history exists"
db.maintain(now=now)
db.maintain(now=now)
# with rollups present, a long window reads the rolled-up series, not 288 raw rows
assert db.pick_res("disks", "driveripper", "disk.reallocated@nvme0n1",
24 * 3600, 600) in (RES_MIN, RES_RAW)
def test_a_300s_source_is_pruned_by_the_existing_retention(db) -> None:
"""288 rows a day per series is fine for 48h and not fine forever — rule 9."""
now = time.time()
expired = now - db.raw_keep_seconds - 6 * SMART_STEP
add_series(db, "disks", "h", "disk.temp_c@sda", [30] * 4, start=expired, step=SMART_STEP)
add_series(db, "disks", "h", "disk.temp_c@sdb", [30], start=now - SMART_STEP,
step=SMART_STEP)
counts = db.maintain(now=now)
assert counts["pruned"] >= 4
assert db.query("disks", "h", "disk.temp_c@sda", start=0) == []
assert len(db.query("disks", "h", "disk.temp_c@sdb", start=0)) == 1