services/dashboards
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

phase: 09_disks_smartctl

Browse Source 
**Phase 09 — final verification pass (all 4 task files already in `complete/`; nothing re-implemented)**

- Verified `GET /api/disks/snapshot` live: 3 hosts / 18 whole-`/sys/block` rows, `smartctl_available` three-way, per-row `severity`, API-summed `fleet`; 404 `{"configured": false}` proven *without dialling* (spies on `httpx.AsyncClient.get` + `collectors.fetch_snapshot`).
- Verified `/api/disks/history` live (`disk.temp_c@dev`, unknown names → empty series) + `test_store.py`'s 300 s rollup/prune buckets; `/api/metrics?source=disks` answers the `disks` family.
- Browser against live data: 18 drive cards (never a partition/mount), collapsed per-host `Charts` with `aria-label`, 3 `.stale` grids + one `.banner.info` why-line and **0 `.fail` classes / green `disks` dot** → criteria 3 and 4 hold; new shots `data/logs/09-verify-disks-live-dimmed.png`, `09-verify-overview-live.png` (13 total for the phase).
- Real-hardware criterion: read 4 driveripper drives through `ops`' `sudo -n smartctl -i -A -H` and ran the live output through `host_agent.parse_smart_output` → `sources/disks.parse_disks_snapshot`: `sda` 35 °C / 30 423 h, `nvme1n1` 58 % wear, `nvme5n1` 13 uncorrectable → `alarm_disks: 1`, temps graded `warn`; consistent with the 16 committed fixtures.
- `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov` → **769 passed, 11 skipped, coverage 98.33%** (floor 94); `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean; hygiene + dependency advisories clean. `uv run pytest -m live --live` → 10 passed, 1 skipped (agents still predate the SMART pass). Dev server healthy: `/api/health` `ok`, 9/9 sources.
- Docs fixed: AGENTS.md test count 698→769 and coverage 98.2→98.33 %; `disks` added to the event dashboard-name list and to the "every feed renders `Dash.eventText`" sentence; README `POLL_MS_*` row now states `/disks`' deliberate fixed 30 s; new Known gap 4 records the dev-DB `database is locked` writes (reported/recovered per invariant 4, ~590 MB DB / ~70 MB WAL, not reproduced by a controlled 25 s external read txn).
- Deviations: **no live-infra change** — granting the agent user the `disk` group and redeploying `host_agent.py` on driveripper/deskwork stay the documented paired operator action (redeploying without the grant yields 16 permission-denied lines; a sudo-spawning agent would break the confirmed "direct read, no sudo" assumption). Store-lock issue left to its own phase rather than patched speculatively.
- Next pending phase: `10_dev_direct_dial`.
This commit is contained in:
ducoterra
2026-10-18 00:45:01 -04:00
parent 19cfedfddf
commit fefbeb8227
81 changed files with 5525 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/conftest.py
+106
View File
@@ -107,6 +107,112 @@ def host_scrape() -> Any:
return scrape
# ---- phase 09: physical disks (the /disks source and page) ----------------- #
def _load_host_agent() -> Any:
"""The stdlib agent module, loaded by path: it is not an installed package (and must
not become one — it ships as a single file to every host). Same loader as
tests/test_agent.py, kept private here because only the disk fixtures need it."""
import importlib.util # noqa: PLC0415 - only the disk fixtures load the agent
path = FIXTURES.parent.parent / "agent" / "host_agent.py"
spec = importlib.util.spec_from_file_location("host_agent", path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules.setdefault("host_agent", module)
spec.loader.exec_module(module)
return module
#: The two `smartctl` shapes the fleet has: NVMe (health-log fields) and ATA (the vendor
#: attribute table). `tests/fixtures/smartctl_reference_*.txt` — labels verbatim from the
#: deployed smartmontools 7.5, values synthetic (see the header of each file).
SMART_SHAPES = ("nvme", "ata_hdd")
#: The agent's own words for the way SMART goes missing that the page must name: the
#: binary is not installed. Mirrors `host_agent.Collector._smart_pass()`; the agent test
#: `test_missing_smartctl_binary_is_one_error_for_the_whole_pass` pins that the line it
#: really writes still contains the word `sources.disks.MISSING_BINARY_WORD` looks for, so
#: a rewording fails a test instead of turning "install smartmontools" into "unknown".
SMART_MISSING_BINARY = ("smart: /usr/sbin/smartctl not found — install smartmontools and "
"give this user device read access (disk group) for SMART data")
@pytest.fixture
def smart_shape() -> Any:
"""`smart_shape('nvme')` / `smart_shape('ata_hdd')` -> the capture, parsed by the agent.
Fixture-driven on purpose (AGENTS.md rule 7): the `smart` blocks a `/disks` test hands
to the source are the recorded `smartctl` text run through the agent's own parser, so a
field rename on either side of the wire fails a test instead of printing a dash.
"""
agent = _load_host_agent()
def shape(kind: str = "nvme") -> dict[str, Any]:
text = (FIXTURES / f"smartctl_reference_{kind}.txt").read_text()
return agent.parse_smart_output(text)
return shape
@pytest.fixture
def disk_snapshot(agent_snapshots: dict[str, dict[str, Any]], smart_shape: Any) -> Any:
"""`build(host="driveripper", mode="healthy")` — a real capture with SMART attached.
The fleet captures predate the agent's SMART pass, so there is no single capture with
both halves. This assembles the two that exist: the physical disk rows come from the
host capture verbatim, the `smart` blocks from the recorded smartmontools shapes, and
the failure modes are the ones `host_agent.smart()` actually produces.
Modes:
* `healthy` — every disk reads, alternating the NVMe and ATA shapes
* `no_binary` — every `smart` is null plus the agent's single "not found" line
* `denied` — the last disk is null with its own permission line (SMART is installed)
* `silent` — every `smart` is null with no error line (`--no-smart`, or an agent that
predates the pass: the dashboard cannot tell why, and must not guess)
* `failed` — one drive answering `FAILED`, the rest healthy
"""
def build(host: str = "driveripper", *, mode: str = "healthy",
mutate: Any = None) -> dict[str, Any]:
import copy # noqa: PLC0415 - every build starts from the untouched capture
snapshot = copy.deepcopy(agent_snapshots[host])
devices = (snapshot.get("disk_io") or {}).get("devices") or []
errors: list[str] = list(snapshot.get("errors") or [])
for index, device in enumerate(devices):
if mode == "no_binary":
device["smart"] = None
continue
if mode == "silent":
device["smart"] = None
continue
if mode == "denied" and index == len(devices) - 1:
device["smart"] = None
continue
shape = smart_shape(SMART_SHAPES[index % len(SMART_SHAPES)])
block = dict(shape)
# the identity in the capture is the disk the row is about; the shape files
# describe one drive each, so the row keeps its own model and the counters
# come from the shape
block["model"] = block.get("model") or device.get("model")
if mode == "failed" and index == 0:
block["overall_health"] = "FAILED"
device["smart"] = block
if mode == "no_binary":
errors.append(SMART_MISSING_BINARY)
if mode == "denied" and devices:
name = str(devices[-1].get("name"))
errors.append(f"smart: {name}: Smartctl open device: /dev/{name} failed: "
"Permission denied")
snapshot["errors"] = errors
if mutate is not None:
mutate(snapshot)
return snapshot
return build
@pytest.fixture
def metrics_text() -> Any:
"""metrics_text('gufo') / metrics_text('vllm') -> the captured exposition verbatim."""
tests/fixtures/smartctl_driveripper_nvme0n1.txt
Vendored
+52
View File
@@ -0,0 +1,52 @@
# Verbatim smartctl -i -A -H /dev/nvme0n1 from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: Samsung SSD 980 PRO 2TB
Serial Number: SCRUBBEDSERIAL
Firmware Version: 5B2QGXA7
PCI Vendor/Subsystem ID: 0x144d
IEEE OUI Identifier: 0x002538
Total NVM Capacity: 2,000,398,934,016 [2.00 TB]
Unallocated NVM Capacity: 0
Controller ID: 6
NVMe Version: 1.3
Number of Namespaces: 1
Namespace 1 Size/Capacity: 2,000,398,934,016 [2.00 TB]
Namespace 1 Utilization: 1,616,151,379,968 [1.61 TB]
Namespace 1 Formatted LBA Size: 512
Namespace 1 IEEE EUI-64: SCRUBBEDEUI64
Local Time is: Sat Oct 17 23:55:46 2026 EDT
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0x1)
Critical Warning: 0x00
Temperature: 59 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 14%
Data Units Read: 281,647,984 [144 TB]
Data Units Written: 323,868,857 [165 TB]
Host Read Commands: 1,597,979,105
Host Write Commands: 9,090,416,691
Controller Busy Time: 146,032
Power Cycles: 2,339
Power On Hours: 6,301
Unsafe Shutdowns: 336
Media and Data Integrity Errors: 0
Error Information Log Entries: 0
Warning Comp. Temperature Time: 24
Critical Comp. Temperature Time: 0
Temperature Sensor 1: 59 Celsius
Temperature Sensor 2: 73 Celsius
Thermal Temp. 1 Transition Count: 87
Thermal Temp. 2 Transition Count: 54
Thermal Temp. 1 Total Time: 3708
Thermal Temp. 2 Total Time: 960
tests/fixtures/smartctl_driveripper_nvme1n1.txt
Vendored
+48
View File
@@ -0,0 +1,48 @@
# Verbatim smartctl -i -A -H /dev/nvme1n1 from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: Samsung SSD 970 EVO Plus 250GB
Serial Number: SCRUBBEDSERIAL
Firmware Version: 2B2QEXM7
PCI Vendor/Subsystem ID: 0x144d
IEEE OUI Identifier: 0x002538
Total NVM Capacity: 250,059,350,016 [250 GB]
Unallocated NVM Capacity: 0
Controller ID: 4
NVMe Version: 1.3
Number of Namespaces: 1
Namespace 1 Size/Capacity: 250,059,350,016 [250 GB]
Namespace 1 Utilization: 30,404,874,240 [30.4 GB]
Namespace 1 Formatted LBA Size: 512
Namespace 1 IEEE EUI-64: SCRUBBEDEUI64
Local Time is: Sat Oct 17 23:55:47 2026 EDT
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0x1)
Critical Warning: 0x00
Temperature: 53 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 58%
Data Units Read: 43,599,098 [22.3 TB]
Data Units Written: 92,033,256 [47.1 TB]
Host Read Commands: 500,908,623
Host Write Commands: 2,587,884,718
Controller Busy Time: 8,424
Power Cycles: 319
Power On Hours: 13,161
Unsafe Shutdowns: 177
Media and Data Integrity Errors: 0
Error Information Log Entries: 285
Warning Comp. Temperature Time: 0
Critical Comp. Temperature Time: 0
Temperature Sensor 1: 53 Celsius
Temperature Sensor 2: 55 Celsius
tests/fixtures/smartctl_driveripper_nvme2n1.txt
Vendored
+45
View File
@@ -0,0 +1,45 @@
# Verbatim smartctl -i -A -H /dev/nvme2n1 from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: WDS100T1X0E-00AFY0
Serial Number: SCRUBBEDSERIAL
Firmware Version: 613200WD
PCI Vendor/Subsystem ID: 0x15b7
IEEE OUI Identifier: 0x001b44
Total NVM Capacity: 1,000,204,886,016 [1.00 TB]
Unallocated NVM Capacity: 0
Controller ID: 8224
NVMe Version: 1.4
Number of Namespaces: 1
Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB]
Namespace 1 Formatted LBA Size: 512
Namespace 1 IEEE EUI-64: SCRUBBEDEUI64
Local Time is: Sat Oct 17 23:55:47 2026 EDT
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff)
Critical Warning: 0x00
Temperature: 49 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 16%
Data Units Read: 99,711,973 [51.0 TB]
Data Units Written: 493,282,793 [252 TB]
Host Read Commands: 943,634,622
Host Write Commands: 8,794,186,017
Controller Busy Time: 10,776
Power Cycles: 180
Power On Hours: 37,984
Unsafe Shutdowns: 90
Media and Data Integrity Errors: 0
Error Information Log Entries: 6
Warning Comp. Temperature Time: 0
Critical Comp. Temperature Time: 0
tests/fixtures/smartctl_driveripper_nvme3n1.txt
Vendored
+45
View File
@@ -0,0 +1,45 @@
# Verbatim smartctl -i -A -H /dev/nvme3n1 from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: WDBRPG0010BNC-WRSN
Serial Number: SCRUBBEDSERIAL
Firmware Version: 102000WD
PCI Vendor/Subsystem ID: 0x15b7
IEEE OUI Identifier: 0x001b44
Total NVM Capacity: 1,000,204,886,016 [1.00 TB]
Unallocated NVM Capacity: 0
Controller ID: 8215
NVMe Version: 1.3
Number of Namespaces: 1
Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB]
Namespace 1 Formatted LBA Size: 512
Namespace 1 IEEE EUI-64: SCRUBBEDEUI64
Local Time is: Sat Oct 17 23:55:48 2026 EDT
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff)
Critical Warning: 0x00
Temperature: 49 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 0%
Data Units Read: 140,935,676 [72.1 TB]
Data Units Written: 737,387,949 [377 TB]
Host Read Commands: 3,673,573,884
Host Write Commands: 15,230,962,004
Controller Busy Time: 30,285
Power Cycles: 296
Power On Hours: 52,249
Unsafe Shutdowns: 133
Media and Data Integrity Errors: 0
Error Information Log Entries: 2,038
Warning Comp. Temperature Time: 0
Critical Comp. Temperature Time: 0
tests/fixtures/smartctl_driveripper_nvme4n1.txt
Vendored
+45
View File
@@ -0,0 +1,45 @@
# Verbatim smartctl -i -A -H /dev/nvme4n1 from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: WDBRPG0010BNC-WRSN
Serial Number: SCRUBBEDSERIAL
Firmware Version: 102000WD
PCI Vendor/Subsystem ID: 0x15b7
IEEE OUI Identifier: 0x001b44
Total NVM Capacity: 1,000,204,886,016 [1.00 TB]
Unallocated NVM Capacity: 0
Controller ID: 8215
NVMe Version: 1.3
Number of Namespaces: 1
Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB]
Namespace 1 Formatted LBA Size: 512
Namespace 1 IEEE EUI-64: SCRUBBEDEUI64
Local Time is: Sat Oct 17 23:55:48 2026 EDT
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff)
Critical Warning: 0x00
Temperature: 49 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 0%
Data Units Read: 131,673,036 [67.4 TB]
Data Units Written: 737,398,022 [377 TB]
Host Read Commands: 2,970,582,194
Host Write Commands: 15,268,475,455
Controller Busy Time: 29,500
Power Cycles: 288
Power On Hours: 52,397
Unsafe Shutdowns: 134
Media and Data Integrity Errors: 0
Error Information Log Entries: 2,050
Warning Comp. Temperature Time: 0
Critical Comp. Temperature Time: 0
tests/fixtures/smartctl_driveripper_nvme5n1.txt
Vendored
+48
View File
@@ -0,0 +1,48 @@
# Verbatim smartctl -i -A -H /dev/nvme5n1 from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: Samsung SSD 980 PRO 1TB
Serial Number: SCRUBBEDSERIAL
Firmware Version: 2B2QGXA7
PCI Vendor/Subsystem ID: 0x144d
IEEE OUI Identifier: 0x002538
Total NVM Capacity: 1,000,204,886,016 [1.00 TB]
Unallocated NVM Capacity: 0
Controller ID: 6
NVMe Version: 1.3
Number of Namespaces: 1
Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB]
Namespace 1 Utilization: 699,798,638,592 [699 GB]
Namespace 1 Formatted LBA Size: 512
Namespace 1 IEEE EUI-64: SCRUBBEDEUI64
Local Time is: Sat Oct 17 23:55:49 2026 EDT
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0x1)
Critical Warning: 0x00
Temperature: 49 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 23%
Data Units Read: 134,905,771 [69.0 TB]
Data Units Written: 340,234,345 [174 TB]
Host Read Commands: 1,296,968,853
Host Write Commands: 6,485,156,779
Controller Busy Time: 88,424
Power Cycles: 716
Power On Hours: 26,278
Unsafe Shutdowns: 328
Media and Data Integrity Errors: 13
Error Information Log Entries: 13
Warning Comp. Temperature Time: 0
Critical Comp. Temperature Time: 0
Temperature Sensor 1: 49 Celsius
Temperature Sensor 2: 54 Celsius
tests/fixtures/smartctl_driveripper_nvme6n1.txt
Vendored
+45
View File
@@ -0,0 +1,45 @@
# Verbatim smartctl -i -A -H /dev/nvme6n1 from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: WDBRPG0010BNC-WRSN
Serial Number: SCRUBBEDSERIAL
Firmware Version: 102000WD
PCI Vendor/Subsystem ID: 0x15b7
IEEE OUI Identifier: 0x001b44
Total NVM Capacity: 1,000,204,886,016 [1.00 TB]
Unallocated NVM Capacity: 0
Controller ID: 8215
NVMe Version: 1.3
Number of Namespaces: 1
Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB]
Namespace 1 Formatted LBA Size: 512
Namespace 1 IEEE EUI-64: SCRUBBEDEUI64
Local Time is: Sat Oct 17 23:55:49 2026 EDT
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff)
Critical Warning: 0x00
Temperature: 50 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 0%
Data Units Read: 132,700,971 [67.9 TB]
Data Units Written: 721,481,575 [369 TB]
Host Read Commands: 3,035,612,517
Host Write Commands: 14,384,001,247
Controller Busy Time: 29,604
Power Cycles: 263
Power On Hours: 52,894
Unsafe Shutdowns: 126
Media and Data Integrity Errors: 0
Error Information Log Entries: 1,957
Warning Comp. Temperature Time: 0
Critical Comp. Temperature Time: 0
tests/fixtures/smartctl_driveripper_nvme7n1.txt
Vendored
+48
View File
@@ -0,0 +1,48 @@
# Verbatim smartctl -i -A -H /dev/nvme7n1 from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: Samsung SSD 970 EVO 1TB
Serial Number: SCRUBBEDSERIAL
Firmware Version: 2B2QEXE7
PCI Vendor/Subsystem ID: 0x144d
IEEE OUI Identifier: 0x002538
Total NVM Capacity: 1,000,204,886,016 [1.00 TB]
Unallocated NVM Capacity: 0
Controller ID: 4
NVMe Version: 1.3
Number of Namespaces: 1
Namespace 1 Size/Capacity: 1,000,204,886,016 [1.00 TB]
Namespace 1 Utilization: 694,159,712,256 [694 GB]
Namespace 1 Formatted LBA Size: 512
Namespace 1 IEEE EUI-64: SCRUBBEDEUI64
Local Time is: Sat Oct 17 23:55:50 2026 EDT
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0x1)
Critical Warning: 0x00
Temperature: 60 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 11%
Data Units Read: 131,460,130 [67.3 TB]
Data Units Written: 390,323,927 [199 TB]
Host Read Commands: 845,847,299
Host Write Commands: 6,716,124,714
Controller Busy Time: 50,013
Power Cycles: 1,012
Power On Hours: 30,083
Unsafe Shutdowns: 492
Media and Data Integrity Errors: 0
Error Information Log Entries: 5,817
Warning Comp. Temperature Time: 0
Critical Comp. Temperature Time: 0
Temperature Sensor 1: 60 Celsius
Temperature Sensor 2: 77 Celsius
tests/fixtures/smartctl_driveripper_sda.txt
Vendored
+53
View File
@@ -0,0 +1,53 @@
# Verbatim smartctl -i -A -H /dev/sda from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Family: Seagate IronWolf
Device Model: ST8000VN004-2M2101
Serial Number: SCRUBBEDSERIAL
LU WWN Device Id: SCRUBBEDWWN
Firmware Version: SC60
User Capacity: 8,001,563,222,016 bytes [8.00 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 7200 rpm
Form Factor: 3.5 inches
Device is: In smartctl database 7.5/6204
ATA Version is: ACS-4 (minor revision not indicated)
SATA Version is: SATA 3.3, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Oct 17 23:55:50 2026 EDT
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART Attributes Data Structure revision number: 10
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x000f 083 064 044 Pre-fail Always - 0/208845750
3 Spin_Up_Time 0x0003 082 081 000 Pre-fail Always - 0
4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 127
5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0
7 Seek_Error_Rate 0x000f 086 060 045 Pre-fail Always - 0/368951419
9 Power_On_Hours 0x0032 066 066 000 Old_age Always - 30423
10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0
12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 127
18 Head_Health 0x000b 100 100 050 Pre-fail Always - 0
187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0
188 Command_Timeout 0x0032 100 100 000 Old_age Always - 0 0 0
190 Airflow_Temperature_Cel 0x0022 065 049 040 Old_age Always - 35 (Min/Max 34/44)
192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 37
193 Load_Cycle_Count 0x0032 009 009 000 Old_age Always - 183890
194 Temperature_Celsius 0x0022 035 046 000 Old_age Always - 35 (0 13 0 0 0)
195 Hardware_ECC_Recovered 0x001a 083 064 000 Old_age Always - 0/208845750
197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0
198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0
199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 0
240 Head_Flying_Hours 0x0000 100 253 000 Old_age Offline - 4365h+15m+41.227s
241 Total_LBAs_Written 0x0000 100 253 000 Old_age Offline - 88499415587
242 Total_LBAs_Read 0x0000 100 253 000 Old_age Offline - 485213567179
tests/fixtures/smartctl_driveripper_sdb.txt
Vendored
+51
View File
@@ -0,0 +1,51 @@
# Verbatim smartctl -i -A -H /dev/sdb from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Family: Seagate IronWolf
Device Model: ST2000VN004-2E4164
Serial Number: SCRUBBEDSERIAL
LU WWN Device Id: SCRUBBEDWWN
Firmware Version: SC60
User Capacity: 2,000,398,934,016 bytes [2.00 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 5900 rpm
Form Factor: 3.5 inches
Device is: In smartctl database 7.5/6204
ATA Version is: ACS-2, ACS-3 T13/2161-D revision 3b
SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Oct 17 23:55:51 2026 EDT
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART Attributes Data Structure revision number: 10
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x000f 114 099 006 Pre-fail Always - 0/69728504
3 Spin_Up_Time 0x0003 095 095 000 Pre-fail Always - 0
4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 255
5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0
7 Seek_Error_Rate 0x000f 086 060 030 Pre-fail Always - 1/440523587
9 Power_On_Hours 0x0032 044 044 000 Old_age Always - 49075
10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0
12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 256
184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0
187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0
188 Command_Timeout 0x0032 100 100 000 Old_age Always - 1 1 1
189 High_Fly_Writes 0x003a 001 001 000 Old_age Always - 292
190 Airflow_Temperature_Cel 0x0022 069 060 045 Old_age Always - 31 (Min/Max 30/39)
191 G-Sense_Error_Rate 0x0032 100 100 000 Old_age Always - 0
192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 147
193 Load_Cycle_Count 0x0032 100 100 000 Old_age Always - 360
194 Temperature_Celsius 0x0022 031 040 000 Old_age Always - 31 (0 8 0 0 0)
197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0
198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0
199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 1
tests/fixtures/smartctl_driveripper_sdc.txt
Vendored
+56
View File
@@ -0,0 +1,56 @@
# Verbatim smartctl -i -A -H /dev/sdc from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Family: Seagate IronWolf
Device Model: ST6000VN001-2BB186
Serial Number: SCRUBBEDSERIAL
LU WWN Device Id: SCRUBBEDWWN
Firmware Version: SC60
User Capacity: 6,001,175,126,016 bytes [6.00 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 5425 rpm
Form Factor: 3.5 inches
Device is: In smartctl database 7.5/6204
ATA Version is: ACS-3 T13/2161-D revision 5
SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Oct 17 23:55:52 2026 EDT
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART Attributes Data Structure revision number: 10
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x000f 081 064 006 Pre-fail Always - 0/125715512
3 Spin_Up_Time 0x0003 091 091 000 Pre-fail Always - 0
4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 177
5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0
7 Seek_Error_Rate 0x000f 086 060 045 Pre-fail Always - 0/367065832
9 Power_On_Hours 0x0032 061 061 000 Old_age Always - 35014
10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0
12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 177
183 Runtime_Bad_Block 0x0032 099 099 000 Old_age Always - 1
184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0
187 Reported_Uncorrect 0x0032 095 095 000 Old_age Always - 5
188 Command_Timeout 0x0032 100 099 000 Old_age Always - 1 1 1
189 High_Fly_Writes 0x003a 100 100 000 Old_age Always - 0
190 Airflow_Temperature_Cel 0x0022 064 056 040 Old_age Always - 36 (Min/Max 34/43)
191 G-Sense_Error_Rate 0x0032 100 100 000 Old_age Always - 0
192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 1281
193 Load_Cycle_Count 0x0032 097 097 000 Old_age Always - 7402
194 Temperature_Celsius 0x0022 036 044 000 Old_age Always - 36 (0 15 0 0 0)
195 Hardware_ECC_Recovered 0x001a 081 064 000 Old_age Always - 0/125715512
197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0
198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0
199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 1
240 Head_Flying_Hours 0x0000 100 253 000 Old_age Offline - 33995h+31m+30.750s
241 Total_LBAs_Written 0x0000 100 253 000 Old_age Offline - 89359900052
242 Total_LBAs_Read 0x0000 100 253 000 Old_age Offline - 466542530444
tests/fixtures/smartctl_driveripper_sdd.txt
Vendored
+51
View File
@@ -0,0 +1,51 @@
# Verbatim smartctl -i -A -H /dev/sdd from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Family: Seagate IronWolf
Device Model: ST2000VN004-2E4164
Serial Number: SCRUBBEDSERIAL
LU WWN Device Id: SCRUBBEDWWN
Firmware Version: SC60
User Capacity: 2,000,398,934,016 bytes [2.00 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 5900 rpm
Form Factor: 3.5 inches
Device is: In smartctl database 7.5/6204
ATA Version is: ACS-2, ACS-3 T13/2161-D revision 3b
SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Oct 17 23:55:52 2026 EDT
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART Attributes Data Structure revision number: 10
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x000f 120 099 006 Pre-fail Always - 0/242964512
3 Spin_Up_Time 0x0003 095 095 000 Pre-fail Always - 0
4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 262
5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0
7 Seek_Error_Rate 0x000f 066 056 030 Pre-fail Always - 146/471494654
9 Power_On_Hours 0x0032 044 044 000 Old_age Always - 49216
10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0
12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 262
184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0
187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0
188 Command_Timeout 0x0032 100 100 000 Old_age Always - 1 1 1
189 High_Fly_Writes 0x003a 001 001 000 Old_age Always - 337
190 Airflow_Temperature_Cel 0x0022 069 060 045 Old_age Always - 31 (Min/Max 30/40)
191 G-Sense_Error_Rate 0x0032 100 100 000 Old_age Always - 0
192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 152
193 Load_Cycle_Count 0x0032 100 100 000 Old_age Always - 367
194 Temperature_Celsius 0x0022 031 040 000 Old_age Always - 31 (0 10 0 0 0)
197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0
198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0
199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 0
tests/fixtures/smartctl_driveripper_sde.txt
Vendored
+48
View File
@@ -0,0 +1,48 @@
# Verbatim smartctl -i -A -H /dev/sde from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Device Model: WDC WD180EDGZ-11B2DA0
Serial Number: SCRUBBEDSERIAL
LU WWN Device Id: SCRUBBEDWWN
Firmware Version: 85.00A85
User Capacity: 18,000,207,937,536 bytes [18.0 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 7200 rpm
Form Factor: 3.5 inches
Device is: Not in smartctl database 7.5/6204
ATA Version is: ACS-4 published, ANSI INCITS 529-2018
SATA Version is: SATA 3.3, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Oct 17 23:55:53 2026 EDT
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART Attributes Data Structure revision number: 16
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x000b 100 100 001 Pre-fail Always - 0
2 Throughput_Performance 0x0004 133 133 054 Old_age Offline - 108
3 Spin_Up_Time 0x0007 082 082 001 Pre-fail Always - 360 (Average 360)
4 Start_Stop_Count 0x0012 100 100 000 Old_age Always - 253
5 Reallocated_Sector_Ct 0x0033 100 100 001 Pre-fail Always - 0
7 Seek_Error_Rate 0x000a 100 100 001 Old_age Always - 0
8 Seek_Time_Performance 0x0004 140 140 020 Old_age Offline - 15
9 Power_On_Hours 0x0012 095 095 000 Old_age Always - 40014
10 Spin_Retry_Count 0x0012 100 100 001 Old_age Always - 0
12 Power_Cycle_Count 0x0032 097 097 000 Old_age Always - 253
22 Helium_Level 0x0023 100 100 025 Pre-fail Always - 100
192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 2210
193 Load_Cycle_Count 0x0012 100 100 000 Old_age Always - 2210
194 Temperature_Celsius 0x0002 047 047 000 Old_age Always - 35 (Min/Max 13/56)
196 Reallocated_Event_Count 0x0032 100 100 000 Old_age Always - 0
197 Current_Pending_Sector 0x0022 100 100 000 Old_age Always - 0
198 Offline_Uncorrectable 0x0008 100 100 000 Old_age Offline - 0
199 UDMA_CRC_Error_Count 0x000a 100 100 000 Old_age Always - 0
tests/fixtures/smartctl_driveripper_sdf.txt
Vendored
+56
View File
@@ -0,0 +1,56 @@
# Verbatim smartctl -i -A -H /dev/sdf from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Family: Seagate IronWolf
Device Model: ST6000VN001-2BB186
Serial Number: SCRUBBEDSERIAL
LU WWN Device Id: SCRUBBEDWWN
Firmware Version: SC60
User Capacity: 6,001,175,126,016 bytes [6.00 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 5425 rpm
Form Factor: 3.5 inches
Device is: In smartctl database 7.5/6204
ATA Version is: ACS-3 T13/2161-D revision 5
SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Oct 17 23:55:53 2026 EDT
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART Attributes Data Structure revision number: 10
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x000f 071 064 006 Pre-fail Always - 0/12209952
3 Spin_Up_Time 0x0003 091 091 000 Pre-fail Always - 0
4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 117
5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0
7 Seek_Error_Rate 0x000f 085 060 045 Pre-fail Always - 0/283506067
9 Power_On_Hours 0x0032 066 066 000 Old_age Always - 30228
10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0
12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 117
183 Runtime_Bad_Block 0x0032 100 100 000 Old_age Always - 0
184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0
187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0
188 Command_Timeout 0x0032 100 100 000 Old_age Always - 0 0 0
189 High_Fly_Writes 0x003a 100 100 000 Old_age Always - 0
190 Airflow_Temperature_Cel 0x0022 064 056 040 Old_age Always - 36 (Min/Max 34/43)
191 G-Sense_Error_Rate 0x0032 100 100 000 Old_age Always - 0
192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 1094
193 Load_Cycle_Count 0x0032 097 097 000 Old_age Always - 7121
194 Temperature_Celsius 0x0022 036 044 000 Old_age Always - 36 (0 15 0 0 0)
195 Hardware_ECC_Recovered 0x001a 071 064 000 Old_age Always - 0/12209952
197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0
198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0
199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 0
240 Head_Flying_Hours 0x0000 100 253 000 Old_age Offline - 29304h+52m+38.336s
241 Total_LBAs_Written 0x0000 100 253 000 Old_age Offline - 61796587979
242 Total_LBAs_Read 0x0000 100 253 000 Old_age Offline - 430008992227
tests/fixtures/smartctl_driveripper_sdg.txt
Vendored
+48
View File
@@ -0,0 +1,48 @@
# Verbatim smartctl -i -A -H /dev/sdg from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Family: Western Digital Blue (CMR)
Device Model: WDC WD40EZRZ-00GXCB0
Serial Number: SCRUBBEDSERIAL
LU WWN Device Id: SCRUBBEDWWN
Firmware Version: 80.00A80
User Capacity: 4,000,787,030,016 bytes [4.00 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 5400 rpm
Form Factor: 3.5 inches
Device is: In smartctl database 7.5/6204
ATA Version is: ACS-3 T13/2161-D revision 5
SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Oct 17 23:55:54 2026 EDT
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART Attributes Data Structure revision number: 16
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x002f 200 200 051 Pre-fail Always - 0
3 Spin_Up_Time 0x0027 166 164 021 Pre-fail Always - 6691
4 Start_Stop_Count 0x0032 095 095 000 Old_age Always - 5256
5 Reallocated_Sector_Ct 0x0033 200 200 140 Pre-fail Always - 0
7 Seek_Error_Rate 0x002e 100 253 000 Old_age Always - 0
9 Power_On_Hours 0x0032 088 088 000 Old_age Always - 9195
10 Spin_Retry_Count 0x0032 100 100 000 Old_age Always - 0
11 Calibration_Retry_Count 0x0032 100 100 000 Old_age Always - 0
12 Power_Cycle_Count 0x0032 095 095 000 Old_age Always - 5097
192 Power-Off_Retract_Count 0x0032 200 200 000 Old_age Always - 24
193 Load_Cycle_Count 0x0032 145 145 000 Old_age Always - 166790
194 Temperature_Celsius 0x0022 119 099 000 Old_age Always - 31
196 Reallocated_Event_Count 0x0032 200 200 000 Old_age Always - 0
197 Current_Pending_Sector 0x0032 200 200 000 Old_age Always - 0
198 Offline_Uncorrectable 0x0030 200 200 000 Old_age Offline - 0
199 UDMA_CRC_Error_Count 0x0032 200 200 000 Old_age Always - 0
200 Multi_Zone_Error_Rate 0x0008 200 200 000 Old_age Offline - 4
tests/fixtures/smartctl_driveripper_sdh.txt
Vendored
+57
View File
@@ -0,0 +1,57 @@
# Verbatim smartctl -i -A -H /dev/sdh from driveripper, recorded by
# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,
# every other byte is what the drive answered. Volatile rows (the
# Temperature raws, Local Time) move; the parse tests read shape.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Family: Seagate BarraCuda Pro
Device Model: ST10000DM0004-1ZC101
Serial Number: SCRUBBEDSERIAL
LU WWN Device Id: SCRUBBEDWWN
Firmware Version: DN01
User Capacity: 10,000,831,348,736 bytes [10.0 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 7200 rpm
Form Factor: 3.5 inches
Device is: In smartctl database 7.5/6204
ATA Version is: ACS-3 T13/2161-D revision 5
SATA Version is: SATA 3.1, 6.0 Gb/s (current: 6.0 Gb/s)
Local Time is: Sat Oct 17 23:55:54 2026 EDT
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
See vendor-specific Attribute list for marginal Attributes.
SMART Attributes Data Structure revision number: 10
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x000f 078 064 044 Pre-fail Always - 65117974
3 Spin_Up_Time 0x0003 089 089 000 Pre-fail Always - 0
4 Start_Stop_Count 0x0032 100 100 020 Old_age Always - 246
5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail Always - 0
7 Seek_Error_Rate 0x000f 093 060 045 Pre-fail Always - 1869403165
9 Power_On_Hours 0x0032 050 050 000 Old_age Always - 43894h+52m+24.646s
10 Spin_Retry_Count 0x0013 100 100 097 Pre-fail Always - 0
12 Power_Cycle_Count 0x0032 100 100 020 Old_age Always - 246
184 End-to-End_Error 0x0032 100 100 099 Old_age Always - 0
187 Reported_Uncorrect 0x0032 100 100 000 Old_age Always - 0
188 Command_Timeout 0x0032 100 099 000 Old_age Always - 1 1 2
189 High_Fly_Writes 0x003a 001 001 000 Old_age Always - 270
190 Airflow_Temperature_Cel 0x0022 067 040 040 Old_age Always In_the_past 33 (Min/Max 31/41)
191 G-Sense_Error_Rate 0x0032 095 095 000 Old_age Always - 11928
192 Power-Off_Retract_Count 0x0032 100 100 000 Old_age Always - 81
193 Load_Cycle_Count 0x0032 035 035 000 Old_age Always - 131758
194 Temperature_Celsius 0x0022 033 060 000 Old_age Always - 33 (0 13 0 0 0)
195 Hardware_ECC_Recovered 0x001a 006 001 000 Old_age Always - 65117974
197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always - 0
198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 0
199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age Always - 0
200 Pressure_Limit 0x0023 100 100 001 Pre-fail Always - 0
240 Head_Flying_Hours 0x0000 100 253 000 Old_age Offline - 6541h+31m+23.388s
241 Total_LBAs_Written 0x0000 100 253 000 Old_age Offline - 222471819122
242 Total_LBAs_Read 0x0000 100 253 000 Old_age Offline - 628009900310
tests/fixtures/smartctl_reference_ata_hdd.txt
Vendored
+42
View File
@@ -0,0 +1,42 @@
# REFERENCE capture, not a field capture — see smartctl_reference_nvme.txt for the
# provenance rules (labels verbatim from smartmontools 7.5, values synthetic). This file
# is the ATA shape: identity from IDENTIFY, health from the self-assessment line, and
# the numbers from the vendor SMART attribute table (raw value is the last column).
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Device Model: EXAMPLE-ST8000VN004
Serial Number: SCRUBBEDSERIAL2
LU WWN Device Id: SCRUBBEDWWN1
Firmware Version: SC60
User Capacity: 8,001,563,222,016 bytes [8.00 TB]
Sector Sizes: 512 bytes logical, 4096 bytes physical
Rotation Rate: 7200 rpm
Form Factor: 3.5 inches
Device is: Not in smartctl database [7.5-20250430]
Local Time is: (time of capture removed by scripts/capture-fixtures.sh)
SMART support is: Available - device has SMART capability.
SMART support is: Enabled
=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART Attributes Data Structure revision number: 10
Vendor Specific SMART Attributes with Thresholds:
ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
1 Raw_Read_Error_Rate 0x000f 079 066 006 Pre-fail ALWAYS - 219967840
5 Reallocated_Sector_Ct 0x0033 100 100 010 Pre-fail ALWAYS - 8
7 Seek_Error_Rate 0x000f 084 060 045 Pre-fail ALWAYS - 238386372
9 Power_On_Hours 0x0032 076 076 000 Old_age ALWAYS - 17,520
187 Reported_Uncorrect 0x0032 100 100 000 Old_age ALWAYS - 0
188 Command_Timeout 0x0012 100 099 000 Old_age ALWAYS - 6
193 Load_Cycle_Count 0x0032 100 100 000 Old_age ALWAYS - 4242
194 Temperature_Celsius 0x0022 031 055 000 Old_age ALWAYS - 31
195 Hardware_ECC_Recovered 0x001a 196 195 000 Old_age ALWAYS - 219967840
197 Current_Pending_Sector 0x0012 100 100 000 Old_age ALWAYS - 3
198 Offline_Uncorrectable 0x0010 100 100 000 Old_age Offline - 2
199 UDMA_CRC_Error_Count 0x003e 200 200 000 Old_age ALWAYS - 0
240 Head_Flying_Hours 0x0025 070 100 000 Pre-fail ALWAYS - 14624
241 Total_LBAs_Written 0x0032 100 253 000 Old_age ALWAYS - 117210553521
242 Total_LBAs_Read 0x0032 100 253 000 Old_age ALWAYS - 96652971088
tests/fixtures/smartctl_reference_nvme.txt
Vendored
+55
View File
@@ -0,0 +1,55 @@
# REFERENCE capture, not a field capture. Every LABEL here is the verbatim format string
# of the deployed smartmontools 7.5 binary (verified against its own strings and drive
# database on 2026-10-18); the values are synthetic, because no host reachable at
# reference-writing time could open a block device (fleet agents need the `disk` group —
# see README). Replace these files with hardware truth:
# SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh
# which records smartctl_<host>_<dev>.txt per physical disk with serials scrubbed.
# The parse tests treat this file as the NVMe shape: Temperature/Percentage Used/
# Power On Hours/Media and Data Integrity Errors come from the health log, and the ATA
# sector attributes stay absent — n/a on the page, never 0.
smartctl 7.5 2025-04-30 r5714 [x86_64-linux-7.2.11-100.fc43.x86_64] (local build)
Copyright (C) 2002-25, Bruce Allen, Christian Franke, www.smartmontools.org
=== START OF INFORMATION SECTION ===
Model Number: EXAMPLE NVMe SSD 2TB
Serial Number: SCRUBBEDSERIAL1
Firmware Version: FITMVT04
IEEE OUI Identifier: 0x000000
Total NVM Capacity: 2,000,398,934,016 (2.00 TB)
Unallocated NVM Capacity: 0
Controller ID: 0
NVMe Version: 1.4
Number of Namespaces: 1
Namespace 1 Size: 2,000,398,934,016 (2.00 TB)
Namespace 1 Capacity: 2,000,398,934,016 (2.00 TB)
Namespace 1 Utilization: 1,000,000,000,000 (1.00 TB)
Namespace 1 Formatted LBA Size: 512
Local Time is: (time of capture removed by scripts/capture-fixtures.sh)
=== START OF SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
SMART/Health Information (NVMe Log 0x02, NSID 0xffffffff)
Critical Warning: 0x00
Temperature: 38 Celsius
Available Spare: 100%
Available Spare Threshold: 10%
Percentage Used: 1%
Data Units Read: 120,000,000 [61.4 TB]
Data Units Written: 90,000,000 [46.0 TB]
Host Read Commands: 1,000,000,000
Host Write Commands: 800,000,000
Controller Busy Time: 100
Power Cycles: 57
Power On Hours: 8,760
Unsafe Shutdowns: 12
Media and Data Integrity Errors: 0
Error Information Log Entries: 0
Warning Comp. Temperature Time: 0
Critical Comp. Temperature Time: 0
Temperature Sensor 1: 42 Celsius
Temperature Sensor 2: 45 Celsius
Error Information (NVMe Log 0x01, 16 of 64 entries)
No Errors Logged
tests/test_agent.py
+432 -1
View File
@@ -117,6 +117,418 @@ def test_cpu_modes_total_excludes_guest_ticks(tmp_path: Path) -> None:
assert sample["total"] == pytest.approx(100 + 10 + 20 + 500 + 5 + 3 + 2 + 1)
# ---- SMART (smartctl) ------------------------------------------------------ #
FIXTURES = Path(__file__).resolve().parent / "fixtures"
NVME_CAPTURE = (FIXTURES / "smartctl_reference_nvme.txt").read_text()
ATA_HDD_CAPTURE = (FIXTURES / "smartctl_reference_ata_hdd.txt").read_text()
def read_fixture(text: str) -> str:
"""Strip the `#` provenance header the capture script writes above a fixture."""
return "\n".join(line for line in text.splitlines() if not line.startswith("#"))
@pytest.mark.parametrize("path", sorted(FIXTURES.glob("smartctl_*.txt")))
def test_every_smartctl_capture_parses_without_inventing_numbers(path: Path) -> None:
"""A capture the parser cannot read must fail here, not on a `/disks` card. A serial
in a fixture is a serial in the browser, so every captured serial is scrubbed."""
parsed = host_agent.parse_smart_output(read_fixture(path.read_text()))
assert parsed["model"], f"{path.name}: no model identity parsed"
# a recorded FAILED drive is a real drive failing, not a test to loosen: replace the
# drive, or capture its truth into a purpose-built fixture and say why here.
assert parsed["overall_health"] == "PASSED", f"{path.name}: health is {parsed['overall_health']!r}"
assert parsed["power_on_hours"] is not None, f"{path.name}: no power-on hours parsed"
assert parsed["serial"] is None or parsed["serial"].startswith("SCRUBBEDSERIAL"), path.name
for key in host_agent.SMART_FIELDS:
assert key in parsed, f"{path.name}: {key} missing from the parsed shape"
def test_smart_output_parser_reads_the_nvme_health_log() -> None:
parsed = host_agent.parse_smart_output(read_fixture(NVME_CAPTURE))
assert parsed["model"] == "EXAMPLE NVMe SSD 2TB"
assert parsed["serial"] == "SCRUBBEDSERIAL1"
assert parsed["temp_c"] == 38.0
assert parsed["power_on_hours"] == 8760.0, "the capture writes 8,760 — commas are notation"
assert parsed["percentage_used"] == 1.0
# NVMe's media-error counter IS this drive family's offline-uncorrectable fact
assert parsed["offline_uncorrectable"] == 0.0
# and what an NVMe does not publish stays None — reallocated/pending are ATA attributes
assert parsed["reallocated_sector_ct"] is None
assert parsed["current_pending_sector_ct"] is None
def test_smart_output_parser_reads_the_ata_attribute_table() -> None:
parsed = host_agent.parse_smart_output(read_fixture(ATA_HDD_CAPTURE))
assert parsed["model"] == "EXAMPLE-ST8000VN004"
assert parsed["overall_health"] == "PASSED"
assert parsed["temp_c"] == 31.0, "raw value (last column), not the normalised VALUE column"
assert parsed["power_on_hours"] == 17520.0
assert parsed["reallocated_sector_ct"] == 8.0
assert parsed["current_pending_sector_ct"] == 3.0
assert parsed["offline_uncorrectable"] == 2.0
assert parsed["percentage_used"] is None, "this HDD publishes no wear percentage"
def test_smart_output_parser_keeps_the_drives_own_verdict_words() -> None:
assert host_agent.parse_smart_output(
"SMART overall-health self-assessment test result: FAILED!\n")["overall_health"] == "FAILED"
assert host_agent.parse_smart_output(
"SMART overall-health self-assessment test result: UNKNOWN!\n")["overall_health"] == "UNKNOWN"
def test_smart_failure_note_quotes_the_error_not_the_version_banner() -> None:
"""Caught on a real box: smartctl prints its banner first and the failure later, so
'first line of output' reported the copyright instead of Permission denied."""
banner = ("smartctl 7.5 2025-04-30 r5714 [x86_64-linux] (local build)\n"
"Copyright (C) 2002-25, Bruce Allen, Christian Franke\n\n")
denied = "Smartctl open device: /dev/nvme0 failed: Permission denied\n"
assert "Permission denied" in host_agent.smart_failure_line(2, banner + denied, "")
assert host_agent.smart_failure_line(7, "", "") == "exit status 7"
assert host_agent.smart_failure_line(4, "some odd note\n", "") == "some odd note"
# stderr counts too — smartctl puts some failures there and nowhere else
assert "scsi hangup" in host_agent.smart_failure_line(2, banner, "scsi hangup")
def test_smart_output_parser_returns_all_none_for_output_it_cannot_read() -> None:
junk = ("smartctl 7.5 2025-04-30 r5714 (local build)\n"
"Smartctl open device: /dev/sdb failed: Permission denied\n")
parsed = host_agent.parse_smart_output(junk)
assert set(parsed) == set(host_agent.SMART_FIELDS)
assert all(value is None for value in parsed.values()), "a denied read is n/a, never 0"
for empty in (None, ""):
assert all(value is None for value in host_agent.parse_smart_output(empty).values())
def test_smart_output_parser_refuses_to_decode_vendor_hex_packed_raws() -> None:
"""Some vendors pack temperature into a hex raw whose byte layout is drive-specific.
Decoding it from memory would be an invented number, so it stays None."""
parsed = host_agent.parse_smart_output(
"194 Temperature_Celsius 0x0022 073 059 000 Old_age ALWAYS - 0x001f003c3430\n")
assert parsed["temp_c"] is None
def test_smart_attribute_rows_are_read_from_the_raw_column_not_the_line_end() -> None:
"""Every shape the deployed binary writes into RAW_VALUE, captured off real drives.
Reading the last token of the row — the obvious way to read a column table — loses the
temperature of every spinning disk on driveripper, because a Seagate ends its row with
`35 (Min/Max 34/44)` and the token there is `44)`. A Western Digital packs its hours as
`43894h+43m+32.519s`, which is hours first, minutes and seconds after: the drive's own
number is the leading group, and the value the page prints is that number."""
parsed = host_agent.parse_smart_output(
"190 Airflow_Temperature_Cel 0x0022 065 049 040 Old_age Always"
" In_the_past 35 (Min/Max 34/44)\n"
"194 Temperature_Celsius 0x0022 035 046 000 Old_age Always"
" - 35 (0 13 0 0 0)\n"
" 9 Power_On_Hours 0x0032 055 055 000 Old_age Always"
" - 43894h+43m+32.519s\n"
"197 Current_Pending_Sector 0x0012 100 100 000 Old_age Always"
" - 2 (of 4 failed when written)\n")
assert parsed["temp_c"] == 35.0
assert parsed["power_on_hours"] == 43894.0
assert parsed["current_pending_sector_ct"] == 2.0
def test_the_real_drives_answer_every_number_the_disks_page_prints() -> None:
"""`smartctl_driveripper_*.txt` is hardware truth, 16 drives, both shapes.
The values move on every re-capture, so this pins that each number `/disks` prints is
there and plausible rather than a particular figure — the check that a fleet which looks
healthy in a synthetic fixture is also healthy when read from a real device. A drive that
stops answering health fails here, and the honest response is to replace the drive or
re-capture, never to loosen the assertion."""
real = sorted(FIXTURES.glob("smartctl_driveripper_*.txt"))
assert real, "no real captures on disk: SMART_HOSTS=<alias> ./scripts/capture-fixtures.sh"
for path in real:
parsed = host_agent.parse_smart_output(read_fixture(path.read_text()))
assert parsed["model"], f"{path.name}: no identity"
hours = parsed["power_on_hours"]
assert hours is not None and hours > 0 and hours == int(hours), (
f"{path.name}: power-on hours read as {hours!r}")
temp = parsed["temp_c"]
assert temp is not None and 10 <= temp <= 85, (
f"{path.name}: temperature read as {temp!r}")
# The two shapes disagree the way the page renders them: no ATA drive on this fleet
# publishes a wear percentage, every NVMe does, and neither answer is a 0.
if "_sd" in path.name:
assert parsed["percentage_used"] is None, f"{path.name}: an HDD has no wear percent"
assert parsed["reallocated_sector_ct"] is not None, f"{path.name}: no ATA counters"
else:
assert parsed["percentage_used"] is not None, f"{path.name}: an NVMe must have wear"
assert parsed["offline_uncorrectable"] is not None, (
f"{path.name}: the media-error log is an NVMe's uncorrectable fact")
def make_fake_smartctl(tmp_path: Path, outputs: dict[str, str],
exit_codes: dict[str, int] | None = None,
slow: tuple[str, ...] = ()) -> str:
"""A stdlib fake `smartctl`: per-device stdout/exit code, plus an audit log of which
devices were asked — so 'virtual disks are skipped' is an assertion, not a hope."""
data = tmp_path / "smart-out"
data.mkdir(exist_ok=True)
for device, text in outputs.items():
(data / device).write_text(text)
for device, code in (exit_codes or {}).items():
(data / f"{device}.rc").write_text(str(code))
script = tmp_path / "bin" / "smartctl"
script.parent.mkdir(exist_ok=True)
script.write_text(f"""#!{sys.executable}
import os, sys, time
device = sys.argv[-1].rsplit("/", 1)[-1]
data = os.environ["FAKE_SMARTCTL_DIR"]
with open(os.path.join(data, "_called"), "a") as handle:
handle.write(device + "\\n")
if device in {list(slow)!r}:
time.sleep(30)
rc_path = os.path.join(data, device + ".rc")
code = int(open(rc_path).read()) if os.path.exists(rc_path) else 0
out_path = os.path.join(data, device)
text = open(out_path).read() if os.path.exists(out_path) else ""
if code and not text:
# real smartctl shape: banner first, failure later — the error note must quote
# the failure, never the banner
text = (f"smartctl 7.5 2025-04-30 r5714 [x86_64-linux] (local build)\\n"
f"Copyright (C) 2002-25, Bruce Allen, Christian Franke\\n\\n"
f"Smartctl open device: /dev/{{device}} failed: Permission denied\\n")
sys.stdout.write(text)
sys.exit(code)
""")
script.chmod(0o755)
return str(script)
def write_fake_disk(root: Path, name: str, model: str | None = None) -> None:
write_file(root, f"sys/block/{name}/stat", "1 2 3 4 5 6 7 8 9 10 11")
write_file(root, f"sys/block/{name}/queue/rotational", "0\n")
write_file(root, f"sys/block/{name}/size", "2048\n")
if model:
write_file(root, f"sys/block/{name}/device/model", f"{model}\n")
def fake_calls(tmp_path: Path) -> list[str]:
called = tmp_path / "smart-out" / "_called"
return called.read_text().split() if called.exists() else []
def test_smart_pass_attaches_parsed_health_per_disk(tmp_path, monkeypatch) -> None:
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1", model="SYSFS nvme0n1")
write_fake_disk(root, "sdb", model="SYSFS sdb")
write_fake_disk(root, "loop0") # virtual: must never be asked
write_fake_disk(root, "dm-0")
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(str(root),
smartctl=make_fake_smartctl(tmp_path, {
"nvme0n1": read_fixture(NVME_CAPTURE),
"sdb": read_fixture(ATA_HDD_CAPTURE),
}))
snap = collector.snapshot()
assert snap["errors"] == [], snap["errors"]
by_name = {device["name"]: device for device in snap["disk_io"]["devices"]}
assert by_name["nvme0n1"]["smart"]["model"] == "EXAMPLE NVMe SSD 2TB"
assert by_name["nvme0n1"]["smart"]["power_on_hours"] == 8760.0
assert by_name["sdb"]["smart"]["reallocated_sector_ct"] == 8.0
called = fake_calls(tmp_path)
assert "loop0" not in called and "dm-0" not in called, "virtual entries are not physical disks"
assert sorted(called) == ["nvme0n1", "sdb"]
def test_smart_identity_falls_back_to_the_agents_own_disk_model(tmp_path, monkeypatch) -> None:
"""Some bridges refuse the identity block while still answering the health log."""
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1", model="SYSFS DISK MODEL")
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(
str(root),
smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": "Power On Hours: 42\n"}))
device = collector.snapshot()["disk_io"]["devices"][0]
assert device["smart"]["model"] == "SYSFS DISK MODEL"
assert device["smart"]["power_on_hours"] == 42.0
assert device["smart"]["overall_health"] is None
def test_missing_smartctl_binary_is_one_error_for_the_whole_pass(tmp_path, monkeypatch) -> None:
root = tmp_path / "root"
for device in ("nvme0n1", "sdb"):
write_fake_disk(root, device)
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl")
snap = collector.snapshot()
smart_errors = [e for e in snap["errors"] if e.startswith("smart:")]
assert len(smart_errors) == 1, "one missing binary is one fact, not one per disk"
assert "smartmontools" in smart_errors[0]
# cross-module contract: `sources/disks.py` reads this word to answer
# "is smartctl installed?" (three-way: yes / no / the agent does not say). Reword the
# line here and that test fails, instead of the /disks page losing its explanation.
assert "not found" in smart_errors[0]
assert all(device["smart"] is None for device in snap["disk_io"]["devices"])
# the rest of the snapshot is intact — SMART is an add-on, not a dependency
assert snap["disk_io"]["device_names"] == ["nvme0n1", "sdb"]
def test_one_denied_device_nulls_only_that_disk(tmp_path, monkeypatch) -> None:
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1")
write_fake_disk(root, "sdb")
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(
str(root),
smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": read_fixture(NVME_CAPTURE)},
exit_codes={"sdb": 2}))
snap = collector.snapshot()
by_name = {device["name"]: device for device in snap["disk_io"]["devices"]}
assert by_name["nvme0n1"]["smart"] is not None
assert by_name["sdb"]["smart"] is None
smart_errors = [e for e in snap["errors"] if e.startswith("smart:")]
assert len(smart_errors) == 1 and "sdb" in smart_errors[0] and "Permission denied" in smart_errors[0]
def test_a_wedged_device_costs_its_timeout_not_the_snapshot(tmp_path, monkeypatch) -> None:
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1")
write_fake_disk(root, "sdb")
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(
str(root),
smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": read_fixture(NVME_CAPTURE)},
slow=("sdb",)))
collector.smart_timeout_seconds = 0.5
snap = collector.snapshot()
by_name = {device["name"]: device for device in snap["disk_io"]["devices"]}
assert by_name["nvme0n1"]["smart"] is not None
assert by_name["sdb"]["smart"] is None
assert any("timed out" in e for e in snap["errors"])
def test_the_pass_budget_skips_the_rest_with_one_error(tmp_path, monkeypatch) -> None:
root = tmp_path / "root"
for device in ("nvme0n1", "sdb", "sdc"):
write_fake_disk(root, device)
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(
str(root), smartctl=make_fake_smartctl(tmp_path, {}))
collector.smart_budget_seconds = 0.0
snap = collector.snapshot()
assert all(device["smart"] is None for device in snap["disk_io"]["devices"])
assert any("budget" in e and "3 disk(s) skipped" in e for e in snap["errors"])
def test_unexecutable_smartctl_degrades_like_a_missing_one(tmp_path, monkeypatch) -> None:
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1")
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
blocked = tmp_path / "not-executable"
blocked.write_text("not a program\n")
collector = host_agent.Collector(str(root), smartctl=str(blocked))
snap = collector.snapshot()
assert all(device["smart"] is None for device in snap["disk_io"]["devices"])
assert any(e.startswith("smart:") for e in snap["errors"])
def wait_for_calls(tmp_path, expected: int, *, timeout: float = 5.0) -> list[str]:
"""Wait for the background refresher to have run `expected` passes."""
deadline = time.time() + timeout
while time.time() < deadline:
calls = fake_calls(tmp_path)
if len(calls) >= expected:
return calls
time.sleep(0.02)
return fake_calls(tmp_path)
def test_smart_results_are_cached_and_an_expired_cache_refreshes_off_the_request_path(
tmp_path, monkeypatch) -> None:
"""The system page polls every few seconds; smartctl must not re-run every poll.
And when the cache *does* expire, the refresh runs on a background thread while the
snapshot answers from the previous result: `SMARTCTL_BUDGET_SECONDS` (20) is five
times the dashboard's `HOST_TIMEOUT_SECONDS` (4), so an inline refresh would turn the
whole `system` source red every four minutes because of an add-on it does not use.
"""
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1")
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(
str(root), smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": read_fixture(NVME_CAPTURE)}))
first = collector.snapshot()
second = collector.snapshot()
assert first["disk_io"]["devices"][0]["smart"]["temp_c"] == 38.0
assert second["disk_io"]["devices"][0]["smart"] == first["disk_io"]["devices"][0]["smart"]
assert fake_calls(tmp_path) == ["nvme0n1"], "one pass, then the cache"
collector.smart_cache_seconds = 0.0 # an expired cache re-runs the pass…
served = collector.snapshot()
assert served["disk_io"]["devices"][0]["smart"] == first["disk_io"]["devices"][0]["smart"]
assert wait_for_calls(tmp_path, 2) == ["nvme0n1", "nvme0n1"], "…in the background"
def test_an_expired_cache_never_stalls_the_snapshot_on_a_wedged_disk(tmp_path,
monkeypatch) -> None:
"""The regression that matters: a disk that takes 30 s must cost a poll nothing.
Asserted on wall-clock because the whole point is the request's latency — the agent
answers with the last SMART it had while the refresher sits in `smartctl`'s timeout.
"""
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1")
write_fake_disk(root, "sdb")
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(
str(root),
smartctl=make_fake_smartctl(tmp_path, {"nvme0n1": read_fixture(NVME_CAPTURE),
"sdb": "Power On Hours: 1\n"},
slow=("sdb",)))
collector.smart_timeout_seconds = 1.0 # the slow device costs ~1 s per pass
first = collector.snapshot()
assert any("timed out" in line for line in first["errors"])
temp = first["disk_io"]["devices"][0]["smart"]["temp_c"]
collector.smart_cache_seconds = 0.0
started = time.perf_counter()
served = collector.snapshot()
elapsed = time.perf_counter() - started
assert served["disk_io"]["devices"][0]["smart"]["temp_c"] == temp
assert elapsed < 0.5, f"a wedged disk cost the snapshot {elapsed:.2f}s"
# the refresher is single-flight: a second poll while it runs gets the cache too
started = time.perf_counter()
collector.snapshot()
assert time.perf_counter() - started < 0.5
wait_for_calls(tmp_path, 4) # let the background pass finish before teardown
def test_no_smart_flag_stays_silent_and_null(tmp_path, monkeypatch) -> None:
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1")
monkeypatch.setenv("FAKE_SMARTCTL_DIR", str(tmp_path / "smart-out"))
collector = host_agent.Collector(str(root),
smartctl=make_fake_smartctl(tmp_path, {}), smart=False)
snap = collector.snapshot()
assert snap["errors"] == []
assert snap["disk_io"]["devices"][0]["smart"] is None
assert fake_calls(tmp_path) == []
def test_an_unexpected_smart_crash_degrades_the_addon_not_the_snapshot(tmp_path, monkeypatch) -> None:
collector = host_agent.Collector("/", smart=False)
monkeypatch.setattr(collector, "smart_enabled", True)
monkeypatch.setattr(collector, "smart",
lambda names: (_ for _ in ()).throw(RuntimeError("boom")))
snap = collector.snapshot()
assert any(e.startswith("smart:") and "boom" in e for e in snap["errors"])
assert snap["disk_io"]["devices"], "the disk rows themselves come from /sys, not smartctl"
def test_smartctl_captures_carry_no_unscrubbed_serials() -> None:
"""The capture script scrubs; this fails the day it stops doing its job. The NVMe
namespace EUI-64 is on the list because it embeds the serial."""
for path in sorted(FIXTURES.glob("smartctl_*.txt")):
for line in path.read_text().splitlines():
lowered = line.lower()
if "serial number" in lowered or "wwn" in lowered or "eui" in lowered:
assert "SCRUBBED" in line, f"{path.name}: unscrubbed identifier: {line!r}"
# ---- the real collector --------------------------------------------------- #
@linux_only
@@ -133,7 +545,10 @@ def test_snapshot_against_the_live_system_is_complete() -> None:
assert len(snap["cpu"]["cores"]) >= 1
assert snap["memory"]["total_bytes"] > 0
assert snap["filesystems"], "the box it runs on has at least one mounted filesystem"
assert snap["errors"] == [], f"degraded sections must be empty on a healthy box: {snap['errors']}"
# SMART notes are allowed here: the box running pytest may have no smartctl or no
# device read access, and both are documented degradations, never snapshot failures.
hard_errors = [e for e in snap["errors"] if not e.startswith("smart")]
assert hard_errors == [], f"degraded sections must be empty on a healthy box: {hard_errors}"
def test_captured_fixture_has_the_shape_the_dashboard_depends_on(agent_snapshot) -> None:
@@ -215,6 +630,22 @@ def _get(url: str, headers: dict[str, str] | None = None) -> tuple[int, dict]:
return error.code, {"raw": body.decode(errors="replace")}
def test_health_stays_up_while_smart_is_degraded(agent_server, tmp_path) -> None:
"""Phase-09 criterion: SMART failures are add-on notes, never an agent outage."""
root = tmp_path / "root"
write_fake_disk(root, "nvme0n1")
collector = host_agent.Collector(str(root), smartctl="/nonexistent/smartctl")
base, _port = agent_server()
status, health = _get(base + "/health")
assert status == 200 and health["ok"] is True
handler = host_agent.Handler
handler.collector = collector
status, snap = _get(base + "/snapshot")
assert status == 200
assert snap["disk_io"]["devices"][0]["smart"] is None
assert any(e.startswith("smart:") for e in snap["errors"])
def test_health_info_schema_snapshot_all_answer(agent_server) -> None:
base, _port = agent_server()
for path in ("/health", "/info", "/schema", "/snapshot"):
tests/test_alerts.py
+92
View File
@@ -166,6 +166,98 @@ def test_a_dashboard_the_builder_does_not_know_about_still_gets_a_verdict_and_a_
assert "every 60s" in out[0]["message"]
def test_the_disk_job_is_stale_only_after_three_of_its_own_five_minute_polls() -> None:
"""SMART is a 300 s signal: judging it by the 10 s system interval would amber `/`
every five minutes for a source that is behaving exactly as configured."""
disks = Settings(disks_interval=300.0)
fresh = state(key="disks:driveripper", dashboard="disks", label="driveripper", age=600.0)
assert alerts.build(disks, [fresh]) == [], "600s is two polls, not an outage"
stale = {**fresh, "age_seconds": 1_000.0}
out = alerts.build(disks, [stale])
assert out[0]["severity"] == "warn"
assert "every 300s" in out[0]["message"], "the message must print the disks interval"
def test_a_disk_job_that_failed_because_its_agent_is_down_is_one_line_not_two() -> None:
"""`disks` reads the same agent snapshot `system` does.
A dead host would otherwise print the same sentence twice (`alerts` rule: one dead
source, one line), with only the internal id telling the two apart — which is a lie of
the same shape as a duplicated host name.
"""
out = alerts.build(SETTINGS, [
state(key="disks:driveripper", dashboard="disks", label="driveripper",
ok=False, error="UpstreamError: driveripper: http://h:9121/snapshot "
"unreachable (ConnectError)"),
state(ok=False, error="UpstreamError: driveripper: http://h:9121/snapshot "
"unreachable (ConnectError)"),
], now=2_000.0)
assert [item["source"] for item in out] == ["driveripper"]
assert len(out) == 1, "the echo of the same outage must not read as two problems"
assert out[0]["id"].startswith("driveripper:host:")
def test_a_disk_job_that_fails_while_its_host_answers_is_the_only_voice_it_has() -> None:
"""SMART wedged inside the agent is a `disks`-only fault, and must still be said."""
out = alerts.build(SETTINGS, [
state(key="disks:driveripper", dashboard="disks", label="driveripper",
ok=False, error="UpstreamError: driveripper: timed out"),
state(),
], now=2_000.0)
assert len(out) == 1
assert out[0]["id"].startswith("driveripper:disks:")
assert "timed out" in out[0]["message"]
def test_a_disk_only_alert_links_to_the_drive_page() -> None:
"""The overview's row is a link, so it has to point at the page that explains it.
A wedged SMART pass has no other voice (the host itself is answering), and `/system`
cannot tell the operator which drive stopped reporting — `/disks` can. Before that route
existed this link pointed at `/` on purpose; now it must not point anywhere else.
"""
out = alerts.build(SETTINGS, [
state(key="disks:driveripper", dashboard="disks", label="driveripper",
ok=False, error="UpstreamError: driveripper: timed out"),
state(),
], now=2_000.0)
assert len(out) == 1
assert out[0]["href"] == "/disks"
def test_an_unconfigured_disk_source_says_nothing_even_when_its_job_says_down() -> None:
"""No `HOST_TARGETS` means no `disks` job exists; a reported-but-unconfigured one must
still be silence (invariant 3).
The builder is fed job states rather than settings, so a hub that ever hands over a
`disks` record with `configured: false` — a config that arrives between two polls, a
future target list of its own — must not amber `/` over a page nobody installed.
"""
out = alerts.build(Settings(disks_interval=300.0), [
state(key="disks:driveripper", dashboard="disks", label="driveripper",
ok=False, configured=False, error="no hosts configured"),
], now=2_000.0)
assert out == []
def test_a_host_threshold_reads_the_system_job_not_the_disk_job(
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
"""Two jobs share a host label, so the lookup has to be filtered by dashboard.
`disks` is registered after `system`, but the builder must not depend on that: if an
unfiltered lookup found the failing disk job first, the host would be called
unreachable and its real threshold breach would vanish.
"""
payload = host_scrape(agent_snapshot, cpu_busy=95.0) # cpu.usage_pct -> fail
out = alerts.build(SETTINGS, [
state(key="disks:driveripper", dashboard="disks", label="driveripper",
ok=False, error="timed out"),
state(),
], [payload], [], None, now=2_000.0)
ids = [item["id"] for item in out]
assert "driveripper:cpu.usage_pct" in ids, "the breach is real; the disk job is not it"
def test_a_failing_source_says_when_the_poller_first_noticed() -> None:
events = [{"ts": 1_500.0, "source": "system", "host": "driveripper", "kind": "unreachable",
"severity": "error", "message": "ConnectError", "data": None},
tests/test_api.py
+672 -30
View File
@@ -14,6 +14,7 @@ from contextlib import contextmanager
from pathlib import Path
from typing import Any
import httpx
import pytest
from fastapi import HTTPException
from fastapi.testclient import TestClient
@@ -21,9 +22,11 @@ from fastapi.testclient import TestClient
from dashboards import alerts
from dashboards.api import RANGE_SECONDS, parse_range
from dashboards.app import create_app
from dashboards.collectors import UpstreamError
from dashboards.config import Settings, Target
from dashboards.severity import classify
from dashboards.sources import nut
from dashboards.sources.disks import SMART_SERIES, DisksCollector, parse_disks_snapshot
from dashboards.sources.llm import (
HISTORY_METRICS,
parse_gufo_metrics,
@@ -44,6 +47,11 @@ from slots_contract import (
# refused immediately, and never dials out of the test environment
DEAD_URL = "http://127.0.0.1:1"
#: Every dashboard route, in nav order. The markup pins below (no CDN, no `.sev-*`, no
#: narration, no orphan `id`) run across this list rather than a hand-written four, so a
#: fifth page cannot join the masthead and skip the checks the others had to pass.
PAGES = ("/", "/system", "/disks", "/llm", "/ups")
@contextmanager
def build_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, simulate: bool = False,
@@ -137,22 +145,36 @@ def data_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
def healthy_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
agent_snapshot: dict[str, Any],
host_scrape: Any) -> Iterator[TestClient]:
"""The all-clear app: one host answering, nothing past a threshold, no UPS configured.
"""The all-clear app: one host answering on both of its jobs, nothing past a threshold.
The job's collector is swapped for one that hands back the injected payload, so the
health record says `ok` truthfully instead of being overridden behind the poller.
health record says `ok` truthfully instead of being overridden behind the poller — and
that has to happen for the `disks` job too, which polls the same dead port: a host is
either answering or it is not, and a fixture that claims all-clear cannot leave one of
its two readers failing. Each reader gets **its own** projection of the same capture
(`parse_disks_snapshot` for the drive page), because a `disks` job holding a `system`
payload would make `/api/summary`'s drive block answer from the wrong shape and read as
a passing test that proved nothing.
"""
with build_client(tmp_path, monkeypatch,
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
from dashboards.poller import Result # noqa: PLC0415 - only this fixture needs it
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
disks_payload = parse_disks_snapshot("driveripper", agent_snapshot,
DisksCollector()).payload
async def collect() -> Result:
return Result(payload=payload, samples=[])
job = test_client.app.state.ctx.hub.jobs["host:driveripper"] # type: ignore[attr-defined]
async def collect_disks() -> Result:
return Result(payload=disks_payload, samples=[])
hub = test_client.app.state.ctx.hub # type: ignore[attr-defined]
job = hub.jobs["host:driveripper"]
disks_job = hub.jobs["disks:driveripper"]
job.fn = collect
disks_job.fn = collect_disks
# let the poll already in flight against the dead port land first (it fails in
# microseconds), then take the health record over: the alerts must describe a host
# that is answering, not the half-second of startup where it was not
@@ -160,11 +182,14 @@ def healthy_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
while time.time() < deadline and job.state.attempts == 0:
time.sleep(0.02)
time.sleep(0.05)
job.state.payload = payload
job.state.ok = True
job.state.error = None
job.state.consecutive_failures = 0
job.state.last_ok = time.time()
for reader in hub.jobs.values():
if reader.dashboard not in ("system", "disks"):
continue
reader.state.payload = disks_payload if reader is disks_job else payload
reader.state.ok = True
reader.state.error = None
reader.state.consecutive_failures = 0
reader.state.last_ok = time.time()
yield test_client
@@ -204,7 +229,8 @@ def test_summary_shape_did_not_shrink(data_client) -> None:
assert fleet_keys <= set(body["system"]["fleet"]), fleet_keys - set(body["system"]["fleet"])
assert host_keys <= set(body["system"]["hosts"][0]), host_keys - set(body["system"]["hosts"][0])
assert ups_keys <= set(body["ups"]["ups"][0]), ups_keys - set(body["ups"]["ups"][0])
assert set(body) == {"ts", "system", "llm", "ups", "recent_events", "alerts"}
# `disks` joined in phase 09 and changed nothing else: the set grows, never shrinks
assert set(body) == {"ts", "system", "disks", "llm", "ups", "recent_events", "alerts"}
# ...and the new keys are there beside the old ones, not instead of them
assert "severity" in body["system"]["fleet"]
assert "severity" in body["system"]["hosts"][0]
@@ -215,6 +241,7 @@ def test_summary_shape_did_not_shrink(data_client) -> None:
# read by something that is not the overview page: the live suite, an external probe, a
# notifier later. Deleting a card must never delete a key.
SUMMARY_CONTRACT = ("system.enabled", "system.fleet", "system.hosts",
"disks.enabled", "disks.fleet", "disks.hosts",
"llm.engines", "ups.ups", "recent_events", "alerts")
@@ -245,6 +272,11 @@ def test_summary_serves_every_path_another_consumer_reads(request, fixture: str)
assert (dig(body, "system.fleet") is not None) is populated
names = {host["name"] for host in dig(body, "system.hosts")}
assert names == ({"driveripper"} if populated else set())
# the drive-health block is configured exactly when the agents it reads are, and lists the
# same hosts: a `disks` job exists for every `HOST_TARGETS` entry and for no other reason
assert dig(body, "disks.enabled") is populated
assert (dig(body, "disks.fleet") is not None) is populated
assert {host["name"] for host in dig(body, "disks.hosts")} == names
batteries = {item["name"] for item in dig(body, "ups.ups")}
assert batteries == ({"apc", "cyberpower"} if populated else set())
assert dig(body, "llm.engines") == [], "neither fixture configures an engine"
@@ -342,6 +374,9 @@ def test_config_reports_every_source_as_off(client) -> None:
body = client.get("/api/config").json()
assert body["system"]["enabled"] is False
assert body["system"]["targets"] == []
assert body["disks"] == {"enabled": False, "interval_seconds": 300.0,
"timeout_seconds": 30.0}, (
"the drive-health page is configured exactly when the host agents are")
assert body["llm"]["gufo"]["enabled"] is False
assert body["llm"]["vllm"]["enabled"] is False
assert body["ups"]["enabled"] is False
@@ -357,6 +392,31 @@ def test_health_with_nothing_configured_is_not_a_failure(client) -> None:
assert body["sources"] == []
def test_the_disk_source_has_its_own_health_record_per_host(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""`disks` is a source of its own, not a field of `system` (phase 09).
Its own record is what gives the page its own dot and its own staleness, and it is
also what makes `alerts.py` need a rule: two jobs read the same agent, so one dead
host must still be one line on the overview (see `test_summary_raises_one_alert...`).
"""
with build_client(tmp_path, monkeypatch,
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
sources = {row["key"]: row for row in test_client.get("/api/health").json()["sources"]}
assert "disks:driveripper" in sources, "no separate record, no separate dot"
assert sources["disks:driveripper"]["dashboard"] == "disks"
assert sources["disks:driveripper"]["label"] == "driveripper"
assert sources["disks:driveripper"]["configured"] is True
assert sources["host:driveripper"]["dashboard"] == "system", (
"the system job is still its own source: one slow SMART poll must not "
"decide whether the host looks alive")
def test_no_targets_means_no_disk_job_that_could_look_broken(client) -> None:
"""No host agents, no `disks` task at all — so nothing red for an unconfigured page."""
assert client.get("/api/health").json()["sources"] == []
@pytest.mark.parametrize("path", ["/api/system/snapshot", "/api/llm/snapshot",
"/api/ups/snapshot", "/api/ups/commands"])
def test_unconfigured_source_is_404_configured_false(client, path) -> None:
@@ -380,6 +440,30 @@ def test_unknown_route_is_json_not_html(client) -> None:
assert response.headers["content-type"].startswith("application/json")
assert "no such route" in response.json()["detail"]
def test_an_upstream_failure_escaping_a_route_keeps_its_own_status(tmp_path: Path) -> None:
"""`collectors.UpstreamError` is the one error type a route may raise to say "upstream".
The handler turns it into `{ok: false, detail}` with the collector's status (502, or 504
for a timeout) instead of a 500 traceback, and the message is the collector's already
redacted sentence. Nothing dials here: the error is raised by the route under test.
"""
db = str(tmp_path / "upstream-error.sqlite3")
store = SampleStore(db)
app = create_app(Settings(db_path=db), store)
@app.get("/api/_boom")
async def boom() -> None:
raise UpstreamError("driveripper: http://127.0.0.1:9121/snapshot timed out", status=504)
with TestClient(app) as test_client:
response = test_client.get("/api/_boom")
assert response.status_code == 504, "a timeout stays a timeout, not a 500"
assert response.json() == {"ok": False,
"detail": "driveripper: http://127.0.0.1:9121/snapshot "
"timed out"}
assert "no-store" in response.headers["cache-control"]
store.close()
# ---- tokens --------------------------------------------------------------- #
@@ -433,12 +517,20 @@ def test_a_dead_host_never_writes_its_token_into_the_feed(tmp_path: Path) -> Non
time.sleep(0.05)
else:
pytest.fail("the dead host never recorded an error")
for path in ("/api/health", "/api/events", "/api/system/snapshot", "/api/summary"):
for path in ("/api/health", "/api/events", "/api/system/snapshot",
"/api/disks/snapshot", "/api/summary"):
body = client.get(path).text
assert "SEKRIT-AGENT-TOKEN" not in body, f"{path} leaked the agent token"
# and the outage is still legible — redaction is not the same as silence
assert "driveripper" in client.get("/api/health").text
assert "unreachable" in client.get("/api/events").text
# the `disks` job dials the same token-bearing URL on its own cadence, so its
# sentence is a second copy of the same risk: `fetch_snapshot` is the one place
# either dashboard builds a failure message, and it uses the redacted URL
disks_state = next(row for row in client.get("/api/health").json()["sources"]
if row["key"] == "disks:driveripper")
assert disks_state["error"] and "SEKRIT-AGENT-TOKEN" not in disks_state["error"]
assert "127.0.0.1:1/snapshot" in disks_state["error"], "the failure still names where"
store.close()
@@ -478,7 +570,8 @@ def test_target_problems_are_surfaced_not_swallowed(client) -> None:
# ---- pages ---------------------------------------------------------------- #
@pytest.mark.parametrize("path,title", [
("/", "overview"), ("/system", "system"), ("/llm", "llm"), ("/ups", "ups"),
("/", "overview"), ("/system", "system"), ("/disks", "disks"), ("/llm", "llm"),
("/ups", "ups"),
])
def test_pages_serve_and_declare_themselves(client, path, title) -> None:
response = client.get(path)
@@ -489,16 +582,14 @@ def test_pages_serve_and_declare_themselves(client, path, title) -> None:
def test_no_page_loads_anything_from_the_internet(client) -> None:
"""No CDN, no webfonts: this dashboard has to work with no route to the internet."""
for path in ("/", "/system", "/llm", "/ups"):
for path in PAGES:
html = client.get(path).text
for fragment in ("//cdn.", "https://fonts.", "//unpkg.com", "//cdnjs"):
assert fragment not in html, f"{path} references {fragment}"
def test_static_assets_are_served(client) -> None:
for asset in ("/static/dash.css", "/static/app.js", "/static/charts.js",
"/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js",
"/static/page-index.js"):
for asset in ASSETS:
assert client.get(asset).status_code == 200, asset
@@ -511,14 +602,26 @@ def test_static_assets_are_served(client) -> None:
# comparing the number to a threshold in JavaScript.
ASSETS = ("/static/dash.css", "/static/app.js", "/static/charts.js",
"/static/page-index.js", "/static/page-system.js", "/static/page-llm.js",
"/static/page-ups.js")
"/static/page-index.js", "/static/page-disks.js", "/static/page-system.js",
"/static/page-llm.js", "/static/page-ups.js")
def _css(client: TestClient) -> str:
return client.get("/static/dash.css").text
def _css_section(css: str, name: str) -> str:
"""One `/* ---- name … */` section of the stylesheet, up to the **next** section header.
Slicing to a named neighbour worked while the sheet had five sections. Every page has since
added a component section, and a slice that runs across someone else's rules accuses the
fleet block of colouring numbers it never coloured — so the boundary is structural: the end
of *this* section, whoever comes next.
"""
start = css.index(f"/* ---- {name}")
return css[start:css.index("/* ----", start + 10)]
def _hero_css(css: str) -> str:
"""Just the `hero numbers` section of the stylesheet."""
return css.split("/* ---- hero numbers", 1)[1].split("/* ---- bars", 1)[0]
@@ -599,7 +702,7 @@ def _fleet_field_metrics(js: str) -> list[str]:
def test_no_parallel_severity_vocabulary_exists_anywhere(client) -> None:
"""One word per colour: `ok|warn|fail`, as in severity.py and the existing components."""
for asset in (*ASSETS, "/", "/system", "/llm", "/ups"):
for asset in (*ASSETS, *PAGES):
body = _code(client.get(asset).text)
assert ".sev-" not in body, f"{asset} invented a .sev-* class"
assert not re.search(r"classList\.(?:add|toggle)\(\s*[\"'](?:danger|good|warning)[\"']",
@@ -639,7 +742,7 @@ def test_severity_for_is_kept_as_the_documented_fallback(client) -> None:
js = client.get("/static/app.js").text
assert "FALLBACK ONLY" in js
assert "severityFor" in js, "the documented fallback itself was deleted"
for page in ("page-index.js", "page-system.js", "page-llm.js", "page-ups.js"):
for page in PAGE_SCRIPTS.values():
assert "severityFor" not in client.get(f"/static/{page}").text, \
f"{page} colours something from a threshold of its own"
@@ -924,7 +1027,7 @@ def test_every_system_chart_names_itself_for_assistive_tech(client) -> None:
# ---- event feeds: a bounded, paginated window ----------------------------- #
#
# The three `ul.events` feeds are one component (`Dash.eventsPager`) because a pager that
# Every `ul.events` feed is one component (`Dash.eventsPager`) because a pager that
# works on only one page is the bug this shape prevents: the `event` table grows for as long
# as the fleet runs, and an unbounded feed turns `/system` into a scroll to nowhere
# (`TODO.md` L6 — "All the events tables need to be scrollable and paginated"). There is no
@@ -935,6 +1038,7 @@ def test_every_system_chart_names_itself_for_assistive_tech(client) -> None:
# page -> (its script, the `source=` the poller stores that page's events under)
EVENT_FEEDS = {
"/system": ("page-system.js", "system"),
"/disks": ("page-disks.js", "disks"),
"/llm": ("page-llm.js", "llm"),
"/ups": ("page-ups.js", "ups"),
}
@@ -953,11 +1057,12 @@ def _feed_fetch_url(js: str) -> str:
return found.group(1)
def test_the_event_feeds_are_one_shared_pager_not_three(client) -> None:
"""`Dash.eventsPager()` renders all three feeds, and no page grows its own pager.
def test_the_event_feeds_are_one_shared_pager_not_one_per_page(client) -> None:
"""`Dash.eventsPager()` renders every feed, and no page grows its own pager.
Three copies of a pager is three chances for one page to keep the old behaviour — and
`/system` is the one whose feed was already too tall when this was written down.
One copy per page is one chance per page to keep the old behaviour — and `/system` is the
one whose feed was already too tall when this was written down. `/disks` joined as a
fourth call site of the same helper, not as a fifth implementation of it.
"""
app_js = _code(client.get("/static/app.js").text)
assert app_js.count("function eventsPager(") == 1
@@ -1913,7 +2018,7 @@ def test_llm_fleet_block_colours_no_number(client) -> None:
"a dead engine no longer dims the tag or darkens the block's dot"
css = client.get("/static/dash.css").text
section = css[css.index("/* ---- fleet totals"):css.index("/* ---- hero numbers")]
section = _css_section(css, "fleet totals")
for token in ("var(--ok)", "var(--warn)", "var(--fail)"):
assert token not in section, f"the fleet block coloured a number with {token}"
assert ".fleet-head" in section and ".fleet-note" in section, \
@@ -2004,7 +2109,7 @@ def test_llm_slot_grid_style_is_a_marker_and_not_a_verdict(client) -> None:
the marker survives monochrome, which is the battery glyph's rule applied here.
"""
css = client.get("/static/dash.css").text
section = css[css.index("/* ---- engine slot grid"):css.index("/* ---- fleet totals")]
section = _css_section(css, "engine slot grid")
for selector in (".slots {", ".slot {", ".slot.busy", ".slot-top", ".slot-id",
".slot-model", ".slot dt", ".slot dd", ".slots-empty", ".slots-note",
".section-label"):
@@ -2292,10 +2397,11 @@ def test_overview_dims_the_fleet_row_of_a_host_that_stopped_answering(client) ->
# Which script owns which page, for the orphan check below. `charts.js` and `app.js` are
# loaded by all four pages, so an id either page code or shared chrome may read.
# loaded by every page, so an id either page code or shared chrome may read.
PAGE_SCRIPTS = {
"/": "page-index.js",
"/system": "page-system.js",
"/disks": "page-disks.js",
"/llm": "page-llm.js",
"/ups": "page-ups.js",
}
@@ -2366,7 +2472,7 @@ def test_no_page_narrates_its_own_pipeline(client, fragment: str) -> None:
the same sentence migrating into another page's card is the same slop, and a pin that
only guards its original file would let that through.
"""
for path in ("/", "/system", "/llm", "/ups", *ASSETS):
for path in (*PAGES, *ASSETS):
body = client.get(path).text
assert fragment not in body, f"{path} narrates the app again: {fragment!r}"
@@ -2378,7 +2484,7 @@ def test_pages_leave_no_empty_narration_container(client) -> None:
renders a gap where the sentence used to be. Elements the scripts fill (`#sub`,
`#alerts-count`) carry an `id` and are not in this category.
"""
for path in ("/", "/system", "/llm", "/ups"):
for path in PAGES:
html = _code(client.get(path).text)
for cls in ("sub", "faint", "mono"):
assert f'<span class="{cls}"></span>' not in html, \
@@ -2393,12 +2499,16 @@ def test_the_copy_the_operator_acts_on_survives_the_audit(client) -> None:
is this number a measurement, is this battery real, is this host's data last-good —
and each one is the reason the audit deletes sentences and not labels.
"""
pages = {path: client.get(path).text for path in ("/", "/system", "/llm", "/ups")}
pages = {path: client.get(path).text for path in PAGES}
scripts = {name: client.get(f"/static/{name}").text for name in PAGE_SCRIPTS.values()}
# an empty panel says why, in terms the operator can act on
assert "No hosts configured" in pages["/"]
assert "HOST_TARGETS" in pages["/system"]
# …including the fourth page's: a `n/a` drive card is only actionable if the page names
# the two things that make SMART readable (the binary and the device-read grant).
assert "smartmontools" in pages["/disks"] + scripts["page-disks.js"]
assert "disk group" in scripts["page-disks.js"]
assert "No engines configured" in pages["/llm"]
assert "No UPS reporting" in pages["/ups"]
assert "instcmds" in pages["/ups"], "the UPS setup note stopped naming the grant that matters"
@@ -2612,6 +2722,538 @@ def test_metrics_endpoint_lists_what_actually_has_history(client) -> None:
assert body["metrics"] == []
# ---- the `/disks` API (phase 09 task 03) ---------------------------------- #
#
# The drive-health surface answers like every other dashboard's: GETs only, the same
# stale/stub semantics for a host that is not answering, the same history mechanism for the
# `disk.*@dev` series, and the same 404-`configured: false` sentence when no agent is
# configured at all — the last one *without dialling*, because "not deployed here" must not
# cost a connect timeout or a red dot (invariant 3).
@pytest.fixture
def disks_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
disk_snapshot: Any) -> Iterator[TestClient]:
"""Two hosts configured; one of them answering SMART, through the production parser.
The payload and the stored samples are what `parse_disks_snapshot` makes of the
fixture-shaped capture (the agent's whole-disk rows plus the recorded `smartctl` shapes),
so every path a `/disks` card will read is a path the source really emits. The other host
keeps its `unreachable` stub, which is the shape a dead agent must keep (invariant 4).
Both targets point at a dead port, so nothing dials.
"""
db = str(tmp_path / "disks.sqlite3")
store = SampleStore(db)
app = create_app(Settings(db_path=db, host_targets=(Target("driveripper", DEAD_URL),
Target("deskwork", DEAD_URL))), store)
with TestClient(app) as test_client:
now = time.time()
snapshot = disk_snapshot("driveripper")
snapshot["ts"] = now # "the agent answered just now", so charts are in range
result = parse_disks_snapshot("driveripper", snapshot, DisksCollector(), now=now)
store.add_many(result.samples)
app.state.ctx.hub.jobs["disks:driveripper"].state.payload = result.payload
yield test_client
store.close()
#: (`smart` field, severity metric) for the two drive numbers the table rates — the same
#: pair `sources/disks.py:SEVERITY_FIELDS` declares, restated here so a rename that only
#: touches one side fails rather than colouring nothing.
RATED_CELLS: tuple[tuple[str, str], ...] = (
("temp_c", "disk.temp_c"),
("percentage_used", "disk.percentage_used"),
)
def disk_rows(body: dict[str, Any], host: str = "driveripper") -> list[dict[str, Any]]:
"""The disk rows of one host out of a `/api/disks/snapshot` body."""
return next(row for row in body["hosts"] if row["name"] == host)["disks"]
def flat_routes(app: Any) -> list[Any]:
"""Every route the app serves, with `include_router()` results flattened to their routes.
The FastAPI this repo pins keeps an included router as an `_IncludedRouter` wrapper with
no `path` of its own, so walking `app.routes` alone sees five anonymous objects instead of
the API surface — and the verb assertion below is only as good as this walk.
"""
out: list[Any] = []
for route in app.routes:
inner = getattr(route, "original_router", None)
out.extend(inner.routes if inner is not None else [route])
return out
def series_source(body: dict[str, Any]) -> set[str]:
"""Which sample family a history answer came from: `disks`, never the 10-second `host`."""
return {series["source"] for series in body["series"]}
def test_disks_snapshot_answers_one_entry_per_host_with_its_physical_disks(
disks_client) -> None:
"""`GET /api/disks/snapshot` is per host: rows + `smart` + `smartctl_available`."""
body = disks_client.get("/api/disks/snapshot")
assert response_ok(body)
payload = body.json()
assert [host["name"] for host in payload["hosts"]] == ["deskwork", "driveripper"], (
"every configured host is a row, in name order, answering or not")
rows = disk_rows(payload)
assert rows, "the capture has 16 whole-disk devices, so the page has 16 cards"
assert all("/" not in row["device"] for row in rows), "a mount leaked into the disk list"
first = rows[0]
assert first["path"] == f"/dev/{first['device']}"
assert set(first) >= {"device", "path", "model", "size_bytes", "rotational",
"health", "smart", "severity"}
assert first["smart"]["power_on_hours"] > 0, "the drive's own counters travel verbatim"
live = next(host for host in payload["hosts"] if host["name"] == "driveripper")
assert live["smartctl_available"] is True
assert live["smart_errors"] == []
assert live["summary"]["disks"] == len(rows)
assert "unreachable" not in live, "answering is the absence of the flag, as on /system"
assert [state["key"] for state in payload["states"]] == ["disks:driveripper",
"disks:deskwork"], (
"the page's dots come from the disks jobs, never the system ones")
def test_disks_snapshot_carries_the_verdicts_the_page_will_paint(disks_client) -> None:
"""Colour is in the payload: each row's `severity` is what `severity.classify` says.
This is the pin that keeps the threshold out of the page — `page-disks.js` reads these
and never compares a drive temperature to a bound.
"""
rows = disk_rows(disks_client.get("/api/disks/snapshot").json())
assert "ok" in {row["severity"]["temp_c"] for row in rows}, "the capture is healthy"
for row in rows:
smart = row["smart"] or {}
assert set(row["severity"]) == {field for field, _metric in RATED_CELLS} | {"health"}
for field, metric in RATED_CELLS:
assert row["severity"][field] == classify(metric, smart.get(field)), (
f"{row['device']}.{field} is coloured by something other than the one table")
assert not {key for row in rows for key in row["severity"]} & {
"reallocated_sector_ct", "pending", "uncorrectable", "power_on_hours"}, (
"a counter is an alarm chip on presence, never a graded cell")
def test_a_configured_host_with_no_payload_yet_is_a_row_of_nas(disks_client) -> None:
"""The stub shape `/api/system/snapshot` hands out, so `n/a` is never a missing row.
A host in `HOST_TARGETS` that has not answered is still in the fleet (invariants 3 and
4) with its counts `None` rather than `0`: "unknown" and "no drives" are different
sentences (rule 2), and the fleet header counts it without summing it.
"""
payload = disks_client.get("/api/disks/snapshot").json()
stub = next(host for host in payload["hosts"] if host["name"] == "deskwork")
assert stub["unreachable"] is True
assert stub["disks"] == [], "nothing to draw, but the host is still listed"
assert stub["smartctl_available"] is None, "an agent that never answered says nothing"
assert stub["smart_errors"] == []
assert set(stub["summary"]) == {"disks", "with_smart", "health_failed", "alarm_disks"}
assert all(value is None for value in stub["summary"].values()), "no invented zeros"
fleet = payload["fleet"]
assert (fleet["hosts_total"], fleet["hosts_live"]) == (2, 1)
assert fleet["disks"] == len(disk_rows(payload))
def test_the_disks_fleet_header_is_summed_by_the_api_not_the_page(disks_client) -> None:
"""`fleet` is the header's only arithmetic source, and it carries no verdict."""
payload = disks_client.get("/api/disks/snapshot").json()
live = next(host for host in payload["hosts"] if host["name"] == "driveripper")
fleet = payload["fleet"]
assert fleet["disks"] == live["summary"]["disks"]
assert fleet["alarm_disks"] == live["summary"]["alarm_disks"] > 0, (
"the recorded ATA shape carries reallocated sectors, so the fleet has chips to draw")
assert "severity" not in fleet, "a count of drives is not a graded number"
def test_the_disks_surface_is_gets_only_and_the_write_stays_the_ups_one(
disks_client) -> None:
"""Invariant 1: the app has exactly one POST, and a fourth dashboard does not add one."""
routes = flat_routes(disks_client.app)
disks = [route for route in routes
if str(getattr(route, "path", "")).startswith("/api/disks")]
assert {route.path for route in disks} == {"/api/disks/snapshot", "/api/disks/history"}
verbs = {verb for route in disks for verb in (getattr(route, "methods", None) or set())}
assert verbs == {"GET"}, "a write appeared on the drive-health surface"
for verb in ("POST", "PUT", "PATCH", "DELETE"):
response = disks_client.request(verb, "/api/disks/snapshot", json={})
assert response.status_code == 405, f"{verb} /api/disks/snapshot was accepted"
writes = sorted(str(route.path) for route in routes
if "POST" in (getattr(route, "methods", None) or set()))
assert writes == ["/api/ups/command"], "one write endpoint, four gates, unchanged"
def test_config_prints_the_disks_cadence_the_page_explains_itself_with(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""A card four minutes old is behaving, not broken — and the interval is the proof.
`/disks` has no upstream of its own, so `enabled` follows `HOST_TARGETS` and the cadence
is its own. Both numbers come from config rather than a constant in the page, so changing
`DISKS_INTERVAL_SECONDS` moves the sentence and the poller together.
"""
with build_client(tmp_path, monkeypatch, host_targets=(Target("driveripper", DEAD_URL),),
disks_interval=120.0, disks_timeout=45.0) as test_client:
disks = test_client.get("/api/config").json()["disks"]
assert disks["enabled"] is True
assert disks["interval_seconds"] == 120.0
assert disks["timeout_seconds"] == 45.0
def test_disks_snapshot_404s_without_dialling_a_host_that_was_never_configured(
client, monkeypatch: pytest.MonkeyPatch) -> None:
"""The gate runs before anything that could reach the network (invariant 3).
Both spies raise, so a route that constructed a client or fetched a snapshot before
checking `HOST_TARGETS` fails here instead of costing a timeout and a red dot. The hub
assertion is the structural half: with no targets there is no `disks` job to dial with.
"""
calls: list[str] = []
def spy_get(self: Any, url: Any, **kwargs: Any) -> Any:
calls.append(f"GET {url}")
raise AssertionError(f"the unconfigured disks route dialled {url}")
def spy_fetch(*args: Any, **kwargs: Any) -> Any:
calls.append("fetch_snapshot")
raise AssertionError("the unconfigured disks route fetched an agent snapshot")
monkeypatch.setattr(httpx.AsyncClient, "get", spy_get)
monkeypatch.setattr("dashboards.collectors.fetch_snapshot", spy_fetch)
response = client.get("/api/disks/snapshot")
assert response.status_code == 404
body = response.json()
assert body["configured"] is False
assert "HOST_TARGETS" in body["detail"], "the sentence names the fix"
assert calls == [], "an unconfigured page must not touch the network"
assert client.app.state.ctx.hub.jobs == {}, (
"no targets means no disks job, so nothing can age out or go red")
def test_disks_history_serves_the_dev_series_the_page_charts(disks_client) -> None:
"""One request, every series a host's cards could draw, points from the stored samples."""
rows = disk_rows(disks_client.get("/api/disks/snapshot").json())
metrics = [f"{base}@{row['device']}" for row in rows for _field, base in SMART_SERIES]
body = disks_client.get("/api/disks/history", params={
"hosts": "driveripper", "metrics": ",".join(metrics), "range": "1h"}).json()
assert body["hosts"] == ["driveripper"], "one host, one panel"
assert body["range_seconds"] == 3_600.0
assert [series["metric"] for series in body["series"]] == metrics
assert series_source(body) == {"disks"}, "the disks family, never the 10-second host one"
points = {(series["metric"]): series["points"] for series in body["series"]}
assert points["disk.temp_c@nvme0n1"][0]["value"] == pytest.approx(38.0)
assert points["disk.power_on_hours@nvme0n1"][0]["value"] == pytest.approx(8_760.0)
def test_disks_history_answers_empty_for_a_series_a_drive_does_not_publish(
disks_client) -> None:
"""Unknown names are an empty series, not a 404 — the /system and /llm contract.
The recorded ATA shape reports no `percentage_used`, so that drive's wear plot has
nothing to draw, and a device name the agent never listed behaves the same way. The card
asks for every series it could draw and reads `n/a` where there is nothing, instead of
the request failing.
"""
rows = disk_rows(disks_client.get("/api/disks/snapshot").json())
unworn = next(row["device"] for row in rows
if (row["smart"] or {}).get("percentage_used") is None)
body = disks_client.get("/api/disks/history", params={
"hosts": "driveripper",
"metrics": f"disk.percentage_used@{unworn},disk.temp_c@nosuchdisk,"
"disk.temp_c@nvme0n1",
"range": "1h"}).json()
by_metric = {series["metric"]: series["points"] for series in body["series"]}
assert by_metric[f"disk.percentage_used@{unworn}"] == [], "no measurement, no line"
assert by_metric["disk.temp_c@nosuchdisk"] == []
assert by_metric["disk.temp_c@nvme0n1"], "the series that does exist still answers"
def test_disks_history_charts_every_configured_host_in_one_request(disks_client) -> None:
"""With no `hosts` the answer covers every configured agent, dead ones included (empty)."""
body = disks_client.get("/api/disks/history",
params={"metrics": "disk.temp_c@nvme0n1,disk.temp_c@sda",
"range": "24h"}).json()
assert body["hosts"] == ["driveripper", "deskwork"]
assert body["range_seconds"] == 86_400.0
assert {(series["host"], series["metric"]) for series in body["series"]} == {
("driveripper", "disk.temp_c@nvme0n1"), ("driveripper", "disk.temp_c@sda"),
("deskwork", "disk.temp_c@nvme0n1"), ("deskwork", "disk.temp_c@sda")}
assert [series for series in body["series"] if series["host"] == "deskwork"], (
"a host with no history is an empty series, never a missing one")
def test_the_metrics_endpoint_answers_for_the_disks_family(disks_client) -> None:
""""Why is this chart empty" has an answer per page: /api/metrics?source=disks."""
body = disks_client.get("/api/metrics", params={"source": "disks"}).json()
assert "disk.temp_c@nvme0n1" in body["metrics"]
assert not [name for name in body["metrics"] if not name.startswith("disk.")], (
"the disks family holds only disk series")
assert not [name for name in disks_client.get(
"/api/metrics", params={"source": "host"}).json()["metrics"]
if name.startswith("disk.temp_c")], "SMART never mixes into the system family"
# ---- the `/disks` page (phase 09 task 04) ---------------------------------- #
#
# The fourth dashboard, asserted the way the other three are: on the served bytes, because a
# JS test runner would be a build step (a locked anchor). What is pinned is not that the page
# exists but that it keeps the four promises the phase wrote down — one card per **physical**
# drive and never one per partition, alarm chips that exist only while a counter is non-zero,
# a host without `smartctl` dimmed rather than reddened, and charts that belong to one host,
# stay collapsed until opened, and cost one history request per host per refresh.
def _disks_js(client: TestClient) -> str:
return client.get("/static/page-disks.js").text
def test_the_disks_tab_is_in_the_masthead_of_every_page(client) -> None:
"""Four tabs became five, on every page, with exactly one marking itself current.
The masthead is the app's only "which dashboard do I need" surface since phase 02 deleted
the overview's table, so a tab missing from one page is a page you cannot get to, and a
tab marked current on the wrong page is a lie about where you are.
"""
for path in PAGES:
html = client.get(path).text
assert '<a href="/disks"' in html, f"{path}'s nav has no Disks tab"
assert html.count('aria-current="page"') == 1, f"{path} marks more than one tab current"
assert f'<a href="{path}" aria-current="page">' in html, f"{path} does not mark itself"
assert (html.index('href="/system"') < html.index('href="/disks"')
< html.index('href="/llm"')), "tab order is machine, drive, engine, power"
def test_the_header_status_dot_picks_up_the_disks_job(client) -> None:
"""`disks` gets its own dot instead of hiding inside `hosts`.
The same agent can answer the 10-second utilisation poll and have a wedged SMART pass;
one dot for both would have to be right about neither.
"""
app_js = client.get("/static/app.js").text
assert "{ key: 'disks', label: 'disks', href: '/disks' }" in app_js
assert "s.dashboard === job.key" in app_js, "the dot stopped filtering by dashboard"
def test_disks_page_draws_one_card_per_physical_drive_from_the_api(client) -> None:
"""The card is a drive, keyed by the agent's device name, patched in place.
A mount or a partition would need a filesystem field, and `/system` already answers "is
the filesystem full"; if one leaks back in here, the page stops being about the drive.
"""
html = client.get("/disks").text
js = _disks_js(client)
assert 'id="hosts"' in html and 'id="hosts-empty"' in html
assert "fetchJSON('/api/disks/snapshot')" in js
assert "row.path" in js and "/dev/" in js, "the card stopped naming the device it is about"
# identity + verdict lead the card: no hero row, no big single number
assert 'class="heroes"' not in html, "the disk card grew a hero row"
assert "Dash.hero(" not in js
for filesystem in ("mountpoint", "fstype", "free_bytes", "worst_mount"):
assert filesystem not in js, f"{filesystem} leaked into the drive page"
# built once per device and patched, so a 16-drive host does not rebuild 16 panels a poll
assert "seen.get(row.device)" in js and "renderDiskCard(panel, row)" in js
assert "pruneHosts(" in js, "a drive that left the host keeps its card forever"
def test_disk_cards_print_the_drive_numbers_and_colour_only_rated_ones(client) -> None:
"""Temp, power-on hours and wear, coloured by the API's verdicts and nothing else.
The odometer is the deliberate exception: `severity.THRESHOLDS` has no `disk.power_on_hours`
row, so the cell is written with a `null` verdict and cannot go green at 100 h or red at
60 000 h. A drive that publishes no wear (the recorded HDD shape) prints `n/a` because
`setTxt` is the only writer.
"""
js = _disks_js(client)
for field in ("smart.temp_c", "smart.power_on_hours", "smart.percentage_used",
"row.health"):
assert field in js, f"{field} is no longer on the card"
for verdict in ("severity.temp_c", "severity.percentage_used", "severity.health"):
assert verdict in js, f"the card stopped rendering {verdict}"
assert "severityFor" not in js, "the drive page wrote a threshold of its own"
body = js[js.index("function renderDiskCard("):js.index("/** Text that prints nothing")]
assert "severity.temp_c" in body and "severity.percentage_used" in body
assert re.search(r"setMetric\(card, 'hours',[^;]*null\)", body), (
"the odometer is being graded by something severity.py does not have")
# the drive's own word, and `unknown` for a drive that gave none — never a green pass
assert "'FAILED'" in body and "'unknown'" in body
def test_disk_alarm_chips_exist_only_while_a_counter_is_non_zero(client) -> None:
"""Zero and unknown both draw no chip; only a non-zero counter draws a red one.
The `/llm` preemption rule: a permanent "0 pending sectors" row trains the eye to ignore
it, and grading a counter green would call a quiet drive an achievement. `severity.py`
rates none of these, so the chip's presence is the entire signal here.
"""
js = _disks_js(client)
for counter in ("reallocated_sector_ct", "current_pending_sector_ct",
"offline_uncorrectable"):
assert counter in js, f"{counter} is no longer an alarm"
assert "value <= 0) continue" in js, "a zero now draws a chip"
assert "alarms.hidden = chips.length === 0" in js, "the empty chip row stays on screen"
assert 'class="chip fail"' in js
css = client.get("/static/dash.css").text
assert ".alarms {" in css and ".chip.fail" in css
def test_a_host_without_smartctl_is_dimmed_with_a_one_line_why_and_never_red(client) -> None:
"""Unconfigured is not broken (invariant 3), and the three answers stay three.
`smartctl_available` is `false` (install smartmontools — the agent said so, so the page
may say it), `null` (nobody told us either way, so name the fix without claiming to know
which half is missing), or the agent is not answering at all, which is a different
sentence again. None of the three reddens anything.
"""
js = _disks_js(client)
html = client.get("/disks").text
assert "smartctl not available on" in js
assert "disk group" in js, "the why stopped naming the grant that fixes it"
assert "smartctl_available === false" in js and "smartctl_available === true" in js
assert "host.unreachable" in js, "a dead agent must not be reported as a missing binary"
assert "smart_errors" in js, "the agent's own sentence is no longer printed"
assert "classList.toggle('stale'" in js
assert 'class="banner info why" hidden' in js, "the why-line has no home on the card"
assert "banner fail" not in js and html.count("banner") == 0, (
"a missing binary reddens the page")
def test_disks_charts_are_collapsed_lazy_and_cost_one_history_request_per_host(client) -> None:
"""`/system`'s chart discipline, reused: collapsed, painted only while open, one XHR.
The request still names every series the host could draw (temperature and wear per
drive), because `/api/disks/history` answers an unknown name as an empty series and a
panel should never have to discover series it did not ask for.
"""
js = _disks_js(client)
assert '<details class="charts-box"' in js
guard = js[js.index("function drawHostCharts("):js.index("async function renderHostCharts(")]
assert "if (!chartsOpen(card)) return;" in guard, (
"a collapsed stack is being built and drawn every poll")
history = js[js.index("async function renderHostCharts("):js.index("/**\n * Drop cards")]
assert history.count("fetchJSON(") == 1, "more than one history request for one host"
assert "/api/disks/history" in history
assert "streams.temp.concat(streams.wear)" in history, (
"the request no longer names every series the panel could draw")
prune = js[js.index("function pruneHosts("):js.index("// ---- fleet header")]
for entry in ("state.hosts.delete", "state.disks.delete", "state.charts.delete",
"state.history.delete", "state.payloads.delete"):
assert entry in prune, f"a pruned host keeps its {entry}"
def test_every_disks_chart_names_itself_for_assistive_tech(client) -> None:
"""An unlabelled plot is silence, and the legend is a sibling that labels nothing."""
js = _code(_disks_js(client))
labels = re.findall(r"ariaLabel: `([^`]+)`", js)
assert len(labels) == 2, "every drive chart must name itself (temperature, wear)"
assert all(label.startswith("${host.name}") for label in labels), (
"a chart that does not say which host it plots belongs on no page")
def test_the_disks_fleet_header_reads_the_api_sums_and_invents_none(client) -> None:
"""The header's arithmetic is the API's (`fleet_disk_summary`), never the page's.
Hidden below two hosts, because with one box in `HOST_TARGETS` these sums only duplicate
that host's own cards — `/llm`'s fleet-block rule. And no cell is coloured: a count of
drives is not a graded number (`severity.py` has no row for one).
"""
html = client.get("/disks").text
js = _disks_js(client)
for element in ("fleet", "fleet-dot", "fleet-count", "fleet-disks", "fleet-smart",
"fleet-failed", "fleet-alarms"):
assert f'id="{element}"' in html, f"the header lost #{element}"
fleet = js[js.index("function renderFleet("):js.index("// ---- snapshot")]
for path in ("fleet.hosts_live", "fleet.disks", "fleet.with_smart",
"fleet.health_failed", "fleet.alarm_disks"):
assert path in fleet, f"the header no longer reads {path}"
assert "reduce(" not in fleet and "+=" not in fleet, "the page is summing drives itself"
assert "severity" not in fleet, "a drive count grew a verdict"
assert "block.hidden = true" in fleet, "one host's sums duplicate its own cards"
def test_the_disk_panel_style_is_existing_vocabulary_with_no_new_palette(client) -> None:
"""`.disk` is the `.slot` shape, and the only colours it reaches for are the verdicts."""
css = _css(client)
block = _css_section(css, "drive panels")
for selector in (".disks {", ".disk {", ".disk-head", ".disk-dev", ".disk-model",
".disk .metrics", ".disk .alarms", ".disk .metric.warn .value",
".disk .metric.fail .value"):
assert selector in block, f"dash.css lost {selector}"
for token in ("var(--warn)", "var(--fail)", "var(--border)", "var(--bg-sunken)",
"var(--text)"):
assert token in block, f"the drive panel no longer uses {token}"
assert "--disk" not in css, "a drive-only colour token appeared"
assert ".stale" in css, "the dim that marks an unreadable host is gone"
def test_the_overview_prints_the_fourth_source_cadence(client) -> None:
"""`/` names every source's interval, so a drive card four minutes old reads as behaving."""
js = client.get("/static/page-index.js").text
assert "config.disks.interval_seconds" in js
@pytest.mark.parametrize("fixture", ["client", "data_client"])
def test_summary_carries_the_drive_block_for_programmatic_consumers(
request, fixture: str) -> None:
"""`disks` is additive: the same configured-ness, the same hosts, no key retired.
`client` has no targets, so it proves the block answers as `False`/`None`/`[]` instead of
disappearing; `data_client` proves it lists the host the fleet has.
"""
test_client = request.getfixturevalue(fixture)
body = test_client.get("/api/summary").json()
populated = fixture == "data_client"
assert body["disks"]["enabled"] is populated
assert (body["disks"]["fleet"] is not None) is populated
assert body["disks"]["hosts"] == ([] if not populated else [
{"name": "driveripper", "disks": None, "with_smart": None, "health_failed": None,
"alarm_disks": None, "smartctl_available": None, "failed_devices": [],
"unreachable": True}]), "a configured host with no payload is a row of nones"
assert [row["name"] for row in body["disks"]["hosts"]] == [
row["name"] for row in body["system"]["hosts"]]
def test_summary_serves_the_same_counts_the_disks_page_prints(disks_client) -> None:
"""One projection, two consumers: the summary's numbers *are* the snapshot's."""
body = disks_client.get("/api/summary").json()
snapshot = disks_client.get("/api/disks/snapshot").json()
disks = body["disks"]
assert disks["fleet"] == snapshot["fleet"], "the header would disagree with the page"
rows = {row["name"]: row for row in disks["hosts"]}
live = next(host for host in snapshot["hosts"] if host["name"] == "driveripper")
assert (rows["driveripper"]["disks"], rows["driveripper"]["with_smart"],
rows["driveripper"]["health_failed"], rows["driveripper"]["alarm_disks"]) == (
live["summary"]["disks"], live["summary"]["with_smart"],
live["summary"]["health_failed"], live["summary"]["alarm_disks"])
assert rows["driveripper"]["smartctl_available"] is True
assert rows["driveripper"]["unreachable"] is False
assert rows["deskwork"]["unreachable"] is True
assert rows["deskwork"]["disks"] is None, "unknown, never 0 (rule 2)"
assert sorted(state["key"] for state in disks["states"]) == ["disks:deskwork",
"disks:driveripper"]
def test_summary_names_the_drive_the_health_verdict_called_failed(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, disk_snapshot: Any) -> None:
"""Which drive the hardware itself called `FAILED` is part of the fleet answer.
A notifier reading `/api/summary` should not have to fetch a second endpoint to name the
device, and the count and the list have to agree or one of them is wrong.
"""
db = str(tmp_path / "summary-failed.sqlite3")
store = SampleStore(db)
app = create_app(Settings(db_path=db,
host_targets=(Target("driveripper", DEAD_URL),)), store)
with TestClient(app) as test_client:
now = time.time()
snapshot = disk_snapshot("driveripper", mode="failed")
snapshot["ts"] = now
result = parse_disks_snapshot("driveripper", snapshot, DisksCollector(), now=now)
app.state.ctx.hub.jobs["disks:driveripper"].state.payload = result.payload
row = test_client.get("/api/summary").json()["disks"]["hosts"][0]
assert len(row["failed_devices"]) == 1
assert row["failed_devices"][0] == result.payload["disks"][0]["device"]
assert row["health_failed"] == result.payload["summary"]["health_failed"] == 1
store.close()
# ---- events --------------------------------------------------------------- #
def test_events_are_returned_in_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
tests/test_collectors.py
+145
View File
@@ -19,12 +19,15 @@ import pytest
from dashboards.collectors import (
NutMonitor,
UpstreamError,
fetch_disk_snapshot,
fetch_engine,
fetch_host_snapshot,
fetch_snapshot,
)
from dashboards.config import Target
from dashboards.rates import RateEstimator
from dashboards.sources import nut
from dashboards.sources.disks import DisksCollector
from dashboards.sources.host import HostCollector
FIXTURES = Path(__file__).parent / "fixtures"
@@ -203,6 +206,148 @@ async def test_a_degraded_agent_still_renders_and_says_what_broke() -> None:
assert result.payload["memory"]["total_bytes"] > 0
async def test_smart_addon_errors_do_not_degrade_the_system_source() -> None:
"""A host without smartmontools (or without device read access) is unconfigured, not
broken (invariant 3): its `smart:` lines must never hold the system page amber. They
stay in `agent.errors`, where the /disks page explains them in its own panel.
"""
body = agent_body()
body["errors"] = ["smart: sdb: Smartctl open device: /dev/sdb failed: Permission denied"]
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=body)
async with client_for(handler) as client:
result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
HostCollector(RateEstimator()))
assert "degraded" not in result.payload
assert any("smart:" in str(e) for e in result.payload["agent"]["errors"])
async def test_real_degradations_still_show_when_smart_noise_is_alongside() -> None:
body = agent_body()
body["errors"] = ["smart: smartctl not found", "temperatures: no hwmon sensors"]
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=body)
async with client_for(handler) as client:
result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
HostCollector(RateEstimator()))
assert result.payload["degraded"] == ["temperatures: no hwmon sensors"]
# --------------------------------------------------------------------------- #
# physical disks (the /disks source's fetch half)
# --------------------------------------------------------------------------- #
async def test_the_disk_fetch_reads_the_same_agent_snapshot_as_the_system_one(
disk_snapshot: Any) -> None:
body = disk_snapshot(mode="healthy")
seen: list[str] = []
def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/snapshot", "disks has no endpoint of its own"
seen.append(request.url.path)
return httpx.Response(200, json=body)
target = Target("driveripper", "http://agent.test")
async with client_for(handler) as client:
result = await fetch_disk_snapshot(client, target, DisksCollector(), timeout=30)
assert seen == ["/snapshot"]
assert result.payload["name"] == "driveripper"
assert result.payload["disks"], "the physical rows come with the snapshot"
assert any(sample.metric.startswith("disk.temp_c@") for sample in result.samples)
async def test_the_disk_fetch_reports_the_agent_failure_like_the_system_one(
monkeypatch: pytest.MonkeyPatch) -> None:
"""One dead host must say the same sentence on both pages (shared `fetch_snapshot`)."""
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(500, text="boom")
target = Target("gw", "http://agent.test")
async with client_for(handler) as client:
with pytest.raises(UpstreamError) as system_error:
await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
with pytest.raises(UpstreamError) as disk_error:
await fetch_disk_snapshot(client, target, DisksCollector())
assert disk_error.value.message == system_error.value.message
async def test_an_agent_that_is_not_ok_fails_the_disk_poll_too() -> None:
"""A degraded agent that says `ok: false` has no trustworthy disk rows to hand over."""
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"})
async with client_for(handler) as client:
with pytest.raises(UpstreamError, match="procfs unreadable"):
await fetch_disk_snapshot(client, Target("gw", "http://agent.test"),
DisksCollector())
class TimeoutRecordingClient(httpx.AsyncClient):
"""An `AsyncClient` that remembers the `timeout=` each request was handed.
A `MockTransport` replaces httpcore's connection pool, so it does not enforce read
timeouts and a slow-handler test would prove nothing. The argument is what matters
anyway: `timeout=None` means *no timeout at all* to httpx, which would hang a poll
loop forever, and the client default means "the system poll's 4 s".
"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.timeouts: list[Any] = []
async def get(self, url: Any, **kwargs: Any) -> httpx.Response: # type: ignore[override]
self.timeouts.append(kwargs.get("timeout", "<unset>"))
return await super().get(url, **kwargs)
async def test_the_slow_disk_poll_carries_its_own_timeout_and_never_disables_one() -> None:
"""`HOST_TIMEOUT_SECONDS` (4 s) stays `/system`'s budget; `disks` brings its own 30 s.
The SMART pass runs inside the agent's snapshot handler, so the two dashboards cannot
share one budget: too short and `/disks` reads red on every SMART poll, too long and
`/system` stops noticing a dead host. And the default must stay httpx's sentinel —
passing a literal `None` silently turns timeout protection off altogether.
"""
body = {"ok": True, "ts": 1.0, "disk_io": {"devices": []}, "errors": []}
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=body)
client = TimeoutRecordingClient(transport=httpx.MockTransport(handler),
timeout=4.0, base_url="http://agent.test")
target = Target("gw", "http://agent.test")
async with client:
await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
assert client.timeouts == [httpx.USE_CLIENT_DEFAULT]
await fetch_disk_snapshot(client, target, DisksCollector(), timeout=30)
assert client.timeouts[-1] == 30, "the disk poll waits for the SMART pass"
await fetch_snapshot(client, target)
assert client.timeouts[-1] is httpx.USE_CLIENT_DEFAULT
assert all(timeout is not None for timeout in client.timeouts), (
"`timeout=None` disables httpx's timeouts entirely — translate it, never pass it")
async def test_smart_addon_errors_reach_the_disk_payload_verbatim(
disk_snapshot: Any) -> None:
"""`collectors` keeps the `smart:` lines out of `/system`'s `degraded`; `/disks` is
where they earn their keep — they are the page's reason for an empty card."""
body = disk_snapshot(mode="denied")
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, json=body)
async with client_for(handler) as client:
disks = await fetch_disk_snapshot(client, Target("driveripper", "http://agent.test"),
DisksCollector())
system = await fetch_host_snapshot(client, Target("driveripper", "http://agent.test"),
HostCollector(RateEstimator()))
assert disks.payload["smart_errors"], "the /disks page gets the agent's own sentence"
assert "degraded" not in system.payload, "and /system stays unbothered by it"
# --------------------------------------------------------------------------- #
# LLM engines
# --------------------------------------------------------------------------- #
tests/test_config.py
+22
View File
@@ -86,6 +86,28 @@ def test_settings_default_to_no_sources_configured() -> None:
assert settings.any_source_enabled is False
def test_the_disk_source_defaults_to_a_five_minute_cadence() -> None:
"""SMART is not a per-second signal (phase 09), and the fetch waits for the agent's
own 20 s smartctl budget. Both are env-only, like every other interval."""
settings = Settings()
assert settings.disks_interval == 300.0
assert settings.disks_timeout == 30.0
def test_the_disk_cadence_is_an_env_var_like_every_other_one(monkeypatch: pytest.MonkeyPatch) -> None:
"""`DISKS_INTERVAL_SECONDS=60` is how an operator watches a failing drive closely."""
monkeypatch.setenv("DISKS_INTERVAL_SECONDS", "60")
monkeypatch.setenv("DISKS_TIMEOUT_SECONDS", "45")
reloaded = importlib.reload(config)
try:
assert reloaded.SETTINGS.disks_interval == 60.0
assert reloaded.SETTINGS.disks_timeout == 45.0
finally:
monkeypatch.delenv("DISKS_INTERVAL_SECONDS")
monkeypatch.delenv("DISKS_TIMEOUT_SECONDS")
importlib.reload(config)
def test_actions_need_the_flag_and_are_dead_under_read_only() -> None:
assert Settings(nut_allow_actions=True, read_only=False).actions_allowed is True
assert Settings(nut_allow_actions=True, read_only=True).actions_allowed is False
tests/test_disks_source.py
+474
View File
@@ -0,0 +1,474 @@
"""The `disks` source: physical drives in, `disk.*@dev` samples out, nothing invented.
Phase 09 task 02. The behaviours this pins, in the order it hurts to break them:
* **One row per physical disk, never per partition.** The rows come from the agent's
whole-`/sys/block` list; the filesystems in the same snapshot must not grow a card.
* **A value the drive does not publish stays absent.** Not 0, not a stored sample — a
stored 0 and "this drive reports nothing" are indistinguishable on a 90-day chart, and
one of them is a lie (`AGENTS.md` rule 2).
* **Nothing is a rate.** Power-on hours and the sector counters are stored as reported;
a second poll produces the same number, not a delta.
* **Why a card is empty is a fact, not a mystery.** `smartctl_available` is three-way
(yes / no, install smartmontools / the agent does not say) because those are three
different sentences for the operator, and `smart_errors` carries the agent's own line.
* **The colour comes from the shared table.** A row carries a `severity` dict for the two
rated numbers (`temp_c`, `percentage_used`) and the drive's own `health` word, so the page
paints a class instead of comparing a number to a bound — and the sector counters get no
verdict at all, because a non-zero one is a chip and zero is not an achievement.
* **Events go under the dashboard name** `disks`, once per verdict change — the phase-03
lesson: events filed under the sample source leave the page's feed empty forever.
* **The job is its own, at a 5-minute cadence**, and no host targets means no job at all,
so an unconfigured deployment never dials (invariant 3).
"""
from __future__ import annotations
from typing import Any
import httpx
import pytest
from dashboards.collectors import disks_jobs
from dashboards.config import Settings, Target
from dashboards.severity import THRESHOLDS
from dashboards.sources.disks import (
DASHBOARD,
SEVERITY_FIELDS,
SMART_SERIES,
SOURCE,
UNGRADED_FIELDS,
DisksCollector,
disk_summary,
fleet_disk_summary,
normalize_health,
parse_disks_snapshot,
row_severity,
smartctl_available,
)
class NeverDialClient(httpx.AsyncClient): # pragma: no cover - reaching it fails a test
"""A client that fails the test the moment anything tries to dial it."""
async def get(self, *args: Any, **kwargs: Any) -> httpx.Response:
raise AssertionError("a job factory must not dial; only the poll may")
def parse(snapshot: dict[str, Any], *, name: str = "driveripper",
collector: DisksCollector | None = None,
now: float = 1_700_000_000.0):
"""One poll of the source, with the defaults the tests do not care about filled in."""
return parse_disks_snapshot(name, snapshot, collector or DisksCollector(), now=now)
def metric_names(result: Any) -> set[str]:
return {sample.metric for sample in result.samples}
def values(result: Any) -> dict[str, float]:
return {sample.metric: sample.value for sample in result.samples}
# ---- the rows: one per physical disk -------------------------------------- #
def test_every_physical_disk_the_agent_lists_becomes_a_row(disk_snapshot: Any) -> None:
snapshot = disk_snapshot()
result = parse(snapshot)
rows = result.payload["disks"]
assert [row["device"] for row in rows] == [
device["name"] for device in snapshot["disk_io"]["devices"]
], "the card set is the agent's whole-disk list, in its order"
agent_rows = {device["name"]: device for device in snapshot["disk_io"]["devices"]}
for row in rows:
source_row = agent_rows[row["device"]]
assert row["path"] == f"/dev/{row['device']}"
assert row["size_bytes"] == source_row["size_bytes"], "capacity is the agent's, not ours"
assert row["rotational"] == source_row["rotational"], "SSD-vs-spinning is a fact"
assert row["smart"] is not None
def test_filesystems_never_grow_a_disk_card(disk_snapshot: Any) -> None:
""""Not per-partition" is the phase's one hard UI constraint, so it is asserted, not hoped.
The snapshot carries a dozen mounts (`/`, `/srv`, `/boot`); a card per mount would
double-count the same platter and bury the drive that is actually failing.
"""
snapshot = disk_snapshot(mutate=lambda snap: snap["filesystems"].append(
{"device": "/dev/mapper/srv", "mountpoint": "/srv", "fstype": "xfs",
"used_pct": 91.0}))
rows = parse(snapshot).payload["disks"]
assert all("/" not in row["device"] for row in rows), "a mountpoint leaked into the cards"
assert {row["device"] for row in rows}.isdisjoint(
{fs["mountpoint"] for fs in snapshot["filesystems"]})
def test_the_smart_block_is_carried_verbatim_and_the_verdict_is_normalised(
disk_snapshot: Any) -> None:
snapshot = disk_snapshot()
rows = parse(snapshot).payload["disks"]
ata = next(row for row in rows if row["smart"] and row["smart"]["reallocated_sector_ct"])
assert ata["smart"]["reallocated_sector_ct"] == 8.0
assert ata["smart"]["current_pending_sector_ct"] == 3.0
assert ata["smart"]["offline_uncorrectable"] == 2.0
assert ata["smart"]["power_on_hours"] == 17_520.0
assert ata["smart"]["temp_c"] == 31.0
# `PASSED` is smartctl's word; the app's colour vocabulary is `ok`
assert ata["smart"]["overall_health"] == "PASSED", "the drive's own word stays verbatim"
assert ata["health"] == "ok"
@pytest.mark.parametrize("word,expected", [
("PASSED", "ok"), ("OK", "ok"), (" ok ", "ok"), ("passed", "ok"),
("FAILED", "failed"), ("failed!", "failed"),
("UNKNOWN", None), ("", None), (None, None), (7, None),
])
def test_only_the_drives_own_verdict_words_count(word: Any, expected: str | None) -> None:
"""`UNKNOWN` is not a pass: no measurement, no colour (`severity.py`'s first rule)."""
assert normalize_health(word) == expected
def test_a_host_that_reports_no_disks_answers_with_an_empty_list_not_an_error() -> None:
"""gateway is a VM on virtio with nothing behind it; `/disks` renders "no disks"."""
result = parse({"ts": 1.0, "disk_io": {"devices": []}, "errors": []}, name="gateway")
assert result.payload["disks"] == []
assert result.payload["summary"]["disks"] == 0
assert result.samples == []
assert result.payload["smartctl_available"] is None
def test_a_malformed_device_row_is_skipped_rather_than_crashing(disk_snapshot: Any) -> None:
"""`/disks` must survive an agent that answered something odd.
A device with no name would render a card called `/dev/`, which is a worse lie than a
missing card; a non-object row is what a truncated JSON body looks like.
"""
clean = parse(disk_snapshot(mode="healthy")).payload["disks"]
def mangle(snapshot: dict[str, Any]) -> None:
snapshot["disk_io"]["devices"].append("not-a-dict")
snapshot["disk_io"]["devices"].append({"name": " ", "smart": {"temp_c": 44.0}})
mangled = parse(disk_snapshot(mode="healthy", mutate=mangle)).payload["disks"]
assert mangled == clean, "only the two junk rows were dropped, in place and unchanged"
assert all(row["device"].strip() for row in mangled)
# ---- samples: the history namespace, counters as reported ----------------- #
def test_the_series_are_the_pinned_disk_names_per_device(disk_snapshot: Any) -> None:
snapshot = disk_snapshot(mode="healthy")
result = parse(snapshot)
devices = [device["name"] for device in snapshot["disk_io"]["devices"]]
# every series this source may write is `<base>@<device>` from the pinned table
allowed = {f"{base}@{device}" for device in devices for _field, base in SMART_SERIES}
assert metric_names(result) <= allowed
assert all(name.startswith("disk.") and "@" in name for name in metric_names(result)), \
"an unqualified disk name would collide with /system's per-mount series"
# all six bases show up: the ATA shape supplies the sector counters, the NVMe shape
# the wear percentage, so between them the whole table is exercised
assert {name.split("@")[0] for name in metric_names(result)} == {
base for _field, base in SMART_SERIES}
# and the family is the disks one, so /api/metrics?source=disks answers for this page
assert {sample.source for sample in result.samples} == {SOURCE} == {"disks"}
# ---- why a card is `n/a` --------------------------------------------------- #
def test_a_denied_device_nulls_only_that_disk(disk_snapshot: Any) -> None:
snapshot = disk_snapshot(mode="denied")
result = parse(snapshot)
denied = snapshot["disk_io"]["devices"][-1]["name"]
by_device = {row["device"]: row for row in result.payload["disks"]}
assert by_device[denied]["smart"] is None
assert by_device[denied]["health"] is None
assert not [name for name in metric_names(result) if name.endswith(f"@{denied}")]
assert any(row["smart"] for row in result.payload["disks"]), "its neighbours are fine"
assert result.payload["smart_errors"] == [
line for line in snapshot["errors"] if line.startswith("smart:")]
def test_a_missing_binary_is_answered_as_not_installed(disk_snapshot: Any) -> None:
""""smartctl not available" is the sentence the page needs, and it comes from here."""
result = parse(disk_snapshot(mode="no_binary"))
assert result.payload["smartctl_available"] is False
assert all(row["smart"] is None for row in result.payload["disks"])
assert len(result.payload["smart_errors"]) == 1, "one missing binary is one fact"
assert result.samples == [], "nothing was read, so nothing is stored"
def test_a_silent_opt_out_is_unknown_rather_than_not_installed(disk_snapshot: Any) -> None:
"""`--no-smart` and an agent predating the pass say nothing; guessing is worse.
The page still has an honest state: `None` renders `n/a` with no colour, and the cards
are dimmed rather than red (invariant 3).
"""
result = parse(disk_snapshot(mode="silent"))
assert result.payload["smartctl_available"] is None
assert result.payload["smart_errors"] == []
def test_smartctl_available_is_true_the_moment_any_disk_answers() -> None:
rows = [{"smart": {"temp_c": 31.0}}, {"smart": None}]
assert smartctl_available(rows, ["smart: sdb: Permission denied"]) is True
def test_smart_errors_ignore_the_lines_that_are_not_about_smart() -> None:
"""`errors` also carries /proc failures, and those belong to `/system`, not here."""
result = parse({"ts": 1.0, "disk_io": {"devices": []},
"errors": ["cpu: OSError: /proc/stat", "smart: sdb: denied"]})
assert result.payload["smart_errors"] == ["smart: sdb: denied"]
# ---- the host summary ------------------------------------------------------ #
def test_summary_counts_reads_alarm_drives_and_failures_apart(disk_snapshot: Any) -> None:
result = parse(disk_snapshot(mode="healthy"))
summary = result.payload["summary"]
devices = result.payload["disks"]
assert summary["disks"] == len(devices) == summary["with_smart"]
assert summary["health_failed"] == 0
# every ATA shape carries reallocated/pending/uncorrectable sectors; the NVMe shapes
# carry zeros, so the alarm count is exactly the number of ATA-shaped disks
ata_disks = [row for row in devices if row["smart"]["reallocated_sector_ct"]]
assert summary["alarm_disks"] == len(ata_disks) > 0
def test_a_drive_answering_failed_is_counted_and_the_neighbours_are_not(
disk_snapshot: Any) -> None:
result = parse(disk_snapshot(mode="failed"))
assert result.payload["summary"]["health_failed"] == 1
assert result.payload["disks"][0]["health"] == "failed"
def test_an_unreadable_drive_is_not_counted_as_failed(disk_snapshot: Any) -> None:
""""Cannot read it" and "it is failing" are different alarms (rule 2)."""
result = parse(disk_snapshot(mode="no_binary"))
summary = result.payload["summary"]
assert summary["disks"] > 0
assert summary["with_smart"] == summary["health_failed"] == summary["alarm_disks"] == 0
def test_disk_summary_ignores_a_zero_counter(disk_snapshot: Any) -> None:
rows = parse(disk_snapshot(mode="healthy")).payload["disks"]
clean = [{**row, "smart": {**(row["smart"] or {}),
"reallocated_sector_ct": 0.0,
"current_pending_sector_ct": 0.0,
"offline_uncorrectable": 0.0}} for row in rows]
assert disk_summary(clean)["alarm_disks"] == 0
# ---- events: one per verdict change, filed under the dashboard ------------- #
def test_a_drive_crossing_into_failed_writes_one_error_event(disk_snapshot: Any) -> None:
collector = DisksCollector()
snapshot = disk_snapshot(mode="healthy")
device = snapshot["disk_io"]["devices"][0]["name"]
assert parse(snapshot, collector=collector).events == [], "first sighting is not news"
assert parse(snapshot, collector=collector).events == [], "an unchanged verdict is quiet"
degraded = disk_snapshot(mode="failed")
events = parse(degraded, collector=collector).events
assert len(events) == 1
event = events[0]
assert event.source == DASHBOARD == "disks", (
"a feed filtered by dashboard must find these — phase 03's empty-feed bug")
assert event.host == "driveripper" and event.kind == "smart-health"
assert event.severity == "error"
assert device in event.message and "failed" in event.message
assert event.data == {"device": device, "from": "ok", "to": "failed"}
assert parse(degraded, collector=collector).events == []
def test_recovering_from_failed_is_a_quiet_info_event(disk_snapshot: Any) -> None:
collector = DisksCollector()
parse(disk_snapshot(mode="failed"), collector=collector)
events = parse(disk_snapshot(mode="healthy"), collector=collector).events
assert [event.severity for event in events] == ["info"]
def test_a_disk_that_left_and_came_back_is_a_first_sighting_again(disk_snapshot: Any) -> None:
"""State is replaced per poll, so a removed drive cannot compare against its old self."""
collector = DisksCollector()
snapshot = disk_snapshot(mode="healthy")
device = snapshot["disk_io"]["devices"][0]["name"]
parse(snapshot, collector=collector)
empty = {"ts": snapshot["ts"], "errors": [], "disk_io": {"devices": []}}
parse(empty, collector=collector)
assert collector.previous_health("driveripper") == {}
assert parse(disk_snapshot(mode="failed"), collector=collector).events == []
assert collector.previous_health("driveripper")[device] == "failed"
def test_two_hosts_do_not_share_verdict_state(disk_snapshot: Any) -> None:
collector = DisksCollector()
parse(disk_snapshot("driveripper"), name="driveripper", collector=collector)
parse(disk_snapshot("deskwork"), name="deskwork", collector=collector)
assert set(collector.hosts) == {"driveripper", "deskwork"}
assert len(collector.hosts["deskwork"]) == 1, "a host compares only against its own disks"
# ---- the job: its own, slow, and absent rather than dialling --------------- #
def test_one_disks_job_per_host_agent_at_the_slow_cadence() -> None:
targets = [Target("driveripper", "http://a:9121"), Target("deskwork", "http://b:9122")]
# the client is never used by the factory, only captured by the job closures
jobs = disks_jobs(targets, NeverDialClient(), DisksCollector(), 300.0, timeout=30.0)
assert [job.key for job in jobs] == ["disks:driveripper", "disks:deskwork"]
assert [job.dashboard for job in jobs] == ["disks", "disks"]
assert [job.label for job in jobs] == ["driveripper", "deskwork"], (
"the label is the host: it is what the event feed and the alert print")
assert [job.interval for job in jobs] == [300.0, 300.0], "SMART is not a 10 s signal"
def test_the_default_cadence_is_five_minutes_with_a_slow_fetch_timeout() -> None:
"""The defaults are the phase's decision (300 s) and the agent's SMART budget (30 s)."""
settings = Settings()
assert settings.disks_interval == 300.0
assert settings.disks_timeout == 30.0, "a wedged smartctl pass costs the agent 20 s"
def test_no_host_targets_means_no_disks_job_and_nothing_dialled() -> None:
"""`disks` has no upstream of its own, so unconfigured means *no task at all*.
The 404 `{"configured": false}` half of invariant 3 is task 03's route; this is the
half that matters first: with no targets the app must not open a socket, and
`/api/health` must not carry a `disks` source that is red because nobody installed one.
"""
assert disks_jobs([], NeverDialClient(), DisksCollector(), 300.0) == []
def test_a_host_that_has_never_polled_has_no_previous_verdicts() -> None:
assert DisksCollector().previous_health("never-seen") == {}
# ---- the verdicts a card paints (task 03's severity rows) ------------------ #
def hot_drive(disk_snapshot: Any, **fields: Any) -> dict[str, Any]:
"""The healthy capture with one drive's rated numbers moved onto a chosen bound.
Driven through the fixture's `mutate` so the numbers under test are the ones the agent's
own parser would have produced for a drive running hot and half-worn out.
"""
def tweak(snapshot: dict[str, Any]) -> None:
(snapshot["disk_io"]["devices"][0].get("smart") or {}).update(fields)
return disk_snapshot(mode="healthy", mutate=tweak)
def test_a_row_carries_a_verdict_for_each_rated_number_and_for_the_drives_word(
disk_snapshot: Any) -> None:
""""72 °C, 85 % wear, PASSED" arrives as `fail`/`warn`/`ok`, never as a bound in JS."""
rows = parse(hot_drive(disk_snapshot, temp_c=72.0, percentage_used=85.0)).payload["disks"]
assert rows[0]["severity"] == {"temp_c": "fail", "percentage_used": "warn",
"health": "ok"}
def test_a_drive_answering_failed_is_the_row_that_reads_red(disk_snapshot: Any) -> None:
"""`health` is the drive's own verdict, so its colour is a translation, not a threshold."""
rows = parse(disk_snapshot(mode="failed")).payload["disks"]
assert rows[0]["health"] == "failed" and rows[0]["severity"]["health"] == "fail"
assert rows[1]["severity"]["health"] == "ok", "its neighbours keep their own verdict"
def test_an_unreadable_drive_gets_no_verdict_at_all(disk_snapshot: Any) -> None:
"""No `smart` block, no colour: the keys are present and every one is `None`.
The card must be able to read `severity.temp_c` without branching on a missing key, and
must not render green for a drive the agent could not open (`AGENTS.md` rule 2).
"""
rows = parse(disk_snapshot(mode="no_binary")).payload["disks"]
assert rows, "the drive is still listed, dimmed"
for row in rows:
assert row["severity"] == {"temp_c": None, "percentage_used": None, "health": None}
def test_a_missing_number_stays_unjudged_rather_than_good(disk_snapshot: Any) -> None:
"""An ATA spinning disk publishes no `percentage_used`; that row reads `n/a`, not `ok`."""
rows = parse(disk_snapshot(mode="healthy")).payload["disks"]
spinning = [row for row in rows if row["smart"]["percentage_used"] is None]
assert spinning, "the recorded ATA shape has no wear field, which is the case here"
for row in spinning:
assert row["severity"]["percentage_used"] is None
assert row["severity"]["temp_c"] == "ok", "the number it does report is still judged"
def test_the_sector_counters_are_never_part_of_a_rows_verdict(disk_snapshot: Any) -> None:
"""The alarm fields appear on the row and in `summary`, never as an `ok|warn|fail`."""
rows = parse(disk_snapshot(mode="healthy")).payload["disks"]
alarming = next(row for row in rows if row["smart"]["reallocated_sector_ct"])
assert set(alarming["severity"]) == {field for field, _metric in SEVERITY_FIELDS} | {"health"}
assert not set(alarming["severity"]) & set(UNGRADED_FIELDS)
assert alarming["smart"]["reallocated_sector_ct"] == 8.0, "the number still travels"
def test_the_rated_rows_exist_in_the_one_threshold_table() -> None:
"""`SEVERITY_FIELDS` may only name metrics `severity.THRESHOLDS` actually carries.
A row here without a bound there would silently colour nothing; a bound for a field the
source never reads is the same drift seen from the other side.
"""
for field, metric in SEVERITY_FIELDS:
assert metric in THRESHOLDS, f"{field} is rated by a metric nobody defines"
assert UNGRADED_FIELDS and not set(UNGRADED_FIELDS) & {
field for field, _metric in SEVERITY_FIELDS}
assert {base for _field, base in SMART_SERIES} >= {
metric for _field, metric in SEVERITY_FIELDS}, (
"a rated number that is not stored cannot be charted in its own colour")
def test_row_severity_tolerates_a_row_with_nothing_in_it() -> None:
"""The API hands this a dict built from an agent that answered oddly; it must not raise."""
assert row_severity({}) == {"temp_c": None, "percentage_used": None, "health": None}
assert row_severity({"health": "no-such-word", "smart": {"temp_c": "hot"}}) == {
"temp_c": None, "percentage_used": None, "health": None}
# ---- the fleet header the API sums, not the page --------------------------- #
def test_the_fleet_counts_drives_across_the_hosts_that_answered(
disk_snapshot: Any) -> None:
"""`/disks` says how many drives the fleet has and how many it could read."""
hosts = [parse(disk_snapshot("driveripper"), name="driveripper").payload,
parse(disk_snapshot("deskwork"), name="deskwork").payload]
fleet = fleet_disk_summary(hosts)
disks = sum(host["summary"]["disks"] for host in hosts)
assert fleet["hosts_total"] == fleet["hosts_live"] == 2
assert fleet["disks"] == disks == len(hosts[0]["disks"]) + len(hosts[1]["disks"])
assert fleet["with_smart"] == disks
assert fleet["health_failed"] == 0
# the recorded ATA shape carries reallocated sectors, so the fleet's alarm count is the
# sum of the hosts' own chip counts, not a number this function invented
assert fleet["alarm_disks"] == sum(host["summary"]["alarm_disks"] for host in hosts) > 0
def test_a_host_that_never_answered_is_counted_but_never_summed(
disk_snapshot: Any) -> None:
"""A dead agent shrinks `hosts_live`, not the drive totals presented as current.
Adding a stub's zeros to the sums would halve the fleet's alarm count on the strength of
a host that said nothing — the `/llm` fleet block's rule, same failure shape.
"""
live = parse(disk_snapshot(mode="failed"), name="driveripper").payload
stub = {"name": "deskwork", "ts": None, "unreachable": True, "disks": []}
fleet = fleet_disk_summary([live, stub])
assert fleet["hosts_total"] == 2 and fleet["hosts_live"] == 1
assert fleet["disks"] == len(live["disks"]), "only the answering host is in the sum"
assert fleet["health_failed"] == 1
def test_a_payload_with_no_countable_summary_contributes_nothing() -> None:
""""Answered" is not "counted": a summary that is missing or junk adds zero, not a guess."""
assert fleet_disk_summary([{"name": "gw", "disks": []}])["disks"] == 0
weird = {"name": "gw", "summary": {"disks": True, "with_smart": "many",
"health_failed": None, "alarm_disks": 2}}
fleet = fleet_disk_summary([weird])
assert fleet["disks"] == fleet["with_smart"] == fleet["health_failed"] == 0
assert fleet["alarm_disks"] == 2
assert fleet["hosts_live"] == 1
def test_an_empty_fleet_answers_with_zeros_rather_than_none() -> None:
"""No hosts configured never reaches this route, but an empty list must not crash either."""
assert fleet_disk_summary([]) == {"hosts_total": 0, "hosts_live": 0, "disks": 0,
"with_smart": 0, "health_failed": 0, "alarm_disks": 0}
tests/test_live_fleet.py
+85
View File
@@ -32,6 +32,7 @@ from dashboards.config import Settings, load
from dashboards.rates import RateEstimator
from dashboards.severity import classify
from dashboards.sources import nut
from dashboards.sources.disks import SMART_SERIES, DisksCollector
from dashboards.sources.host import HostCollector
from slots_contract import assert_slot_shape
from ups_contract import assert_ups_hero_contract
@@ -95,6 +96,90 @@ async def test_every_host_serves_the_hero_card_contract(settings: Settings,
)
async def test_agent_disk_rows_carry_the_smart_shape(settings: Settings) -> None:
"""Phase-09: a redeployed agent answers `smart` per physical disk of `/snapshot`.
Agents are redeployed by hand, so an agent whose rows carry no `smart` key at all
SKIPS this host rather than failing it. `smart: null` is a documented state (no
smartctl binary, or the agent user lacks device read access) — what must hold is the
shape of a non-null block and that unknowns are null, never 0. Read-only: nothing
here sends `smartctl -t`; a self-test on hardware something is plugged into is the
operator's call, not a unit test.
"""
if not settings.host_targets:
pytest.skip("HOST_TARGETS is unset — start ./scripts/dev-tunnels.sh")
seen_smart = 0
async with httpx.AsyncClient(timeout=settings.host_timeout) as client:
for target in settings.host_targets:
response = await client.get(f"{target.url}/snapshot", headers=target.headers)
if response.status_code >= 400:
continue # reachability has its own live test
devices = (response.json().get("disk_io") or {}).get("devices") or []
if devices and not all("smart" in device for device in devices):
continue # agent predates the smart pass — redeploy it to test this
for device in devices:
seen_smart += 1
smart = device.get("smart")
if smart is None:
continue # degraded by design; the errors[] entry says why
# the literal set is the pin: a rename on the agent must fail here too
assert set(smart) == {"model", "serial", "overall_health", "temp_c",
"power_on_hours", "reallocated_sector_ct",
"current_pending_sector_ct",
"offline_uncorrectable", "percentage_used"}, \
f"{target.name}/{device['name']}"
if smart["power_on_hours"] is not None:
assert smart["power_on_hours"] > 0
if smart["temp_c"] is not None:
assert -20.0 < smart["temp_c"] < 125.0
assert smart["overall_health"] in (None, "PASSED", "FAILED", "UNKNOWN")
if not seen_smart:
pytest.skip("no agent redeployed with the smart pass yet (deploy/install-agent.sh)")
async def test_disks_source_charts_the_physical_drives_the_agent_reports(
settings: Settings) -> None:
"""Phase 09 task 02: the `disks` source over a real agent snapshot.
This is the production code path — the fetch the poller runs, not a hand-built dict —
so what fails here is a rename between the agent and the page, not a typo in a fixture.
The assertions hold whether or not this fleet can read SMART yet: one row per whole
disk, `disk.*@<dev>` series only for values that exist, and no invented numbers.
"""
if not settings.host_targets:
pytest.skip("HOST_TARGETS is unset — start ./scripts/dev-tunnels.sh")
collector = DisksCollector()
checked = 0
async with httpx.AsyncClient(timeout=settings.disks_timeout) as client:
for target in settings.host_targets:
try:
result = await collectors.fetch_disk_snapshot(client, target, collector,
timeout=settings.disks_timeout)
except collectors.UpstreamError:
continue # reachability has its own live test
checked += 1
payload = result.payload
assert payload["name"] == target.name
assert payload["disks"], f"{target.name}: no physical disks at all?"
for row in payload["disks"]:
assert row["path"] == f"/dev/{row['device']}"
assert "/" not in row["device"], f"partition {row['device']} in a disk list"
assert row["health"] in (None, "ok", "failed")
devices = {row["device"] for row in payload["disks"]}
bases = {base for _field, base in SMART_SERIES}
for sample in result.samples:
assert sample.source == "disks", "the wrong family would chart nowhere"
base, _, device = sample.metric.partition("@")
assert base in bases and device in devices, (
f"{target.name}: {sample.metric} is not a pinned disk series")
assert sample.value == sample.value and abs(sample.value) != float("inf"), \
"NaN must never reach the store"
if any(row["smart"] for row in payload["disks"]):
assert payload["smartctl_available"] is True
if not checked:
pytest.skip("no host agent answered — start ./scripts/dev-tunnels.sh")
@pytest.mark.parametrize("engine", ["gufo", "vllm"])
async def test_engine_still_publishes_the_series_we_parse(engine: str,
settings: Settings) -> None:
tests/test_poller.py
+34
View File
@@ -113,6 +113,40 @@ async def test_grace_allows_a_blip_without_an_event(store) -> None:
assert len(store.recent_events(limit=10)) == 1, "one event per outage, not per poll"
async def test_a_failed_disks_poll_ages_its_cards_instead_of_zeroing_them(store) -> None:
"""Invariant 4 on the phase-09 job: SMART goes quiet, the numbers stay and age.
The `/disks` page is the one a operator reads *because* something looks wrong, so a
failed poll must not turn "last read 12 minutes ago" into a page of zeros — and the
outage must be filed under the `disks` dashboard, which is where that page filters.
"""
payload = {"name": "driveripper", "smartctl_available": True,
"disks": [{"device": "nvme0n1", "health": "ok",
"smart": {"temp_c": 38.0, "power_on_hours": 8760.0}}],
"summary": {"disks": 1, "with_smart": 1, "health_failed": 0,
"alarm_disks": 0}}
async def good() -> Result:
return Result(payload=payload,
samples=[Sample("disks", "driveripper", "disk.temp_c@nvme0n1",
38.0, time.time())])
job = Job("disks:driveripper", "disks", "driveripper", 300.0, good)
assert job.interval == 300.0, "SMART is not a 10 s signal"
assert await job.run_once(store) is True
assert store.query("disks", "driveripper", "disk.temp_c@nvme0n1",
start=time.time() - 60), "the series is written under `disks`"
job.fn = failing
assert await job.run_once(store) is False
assert job.state.payload == payload, "the card keeps the last temperature it read"
assert job.state.ok is False
assert job.state.as_json()["age_seconds"] is not None
events = store.recent_events(limit=10)
assert [event.source for event in events] == ["disks"], (
"the outage must land where /disks filters its feed, not under the sample source")
async def test_a_slow_poll_never_overlaps_itself(store) -> None:
calls = 0
gate = asyncio.Event()
tests/test_severity.py
+42
View File
@@ -22,6 +22,8 @@ RATIFIED = {
"cpu.usage_pct": (75.0, 90.0),
"mem.used_pct": (80.0, 92.0),
"disk.used_pct": (80.0, 90.0),
"disk.temp_c": (50.0, 70.0),
"disk.percentage_used": (80.0, 100.0),
"cpu.temp_c": (75.0, 90.0),
"llm.kv_cache_usage_pct": (75.0, 90.0),
"ups.charge_pct": (50.0, 20.0),
@@ -134,10 +136,50 @@ def test_non_finite_values_are_treated_as_missing(metric: str, value: float) ->
assert classify(metric, value) is None
def test_a_drive_temperature_and_its_wear_are_the_only_disk_verdicts() -> None:
"""Phase 09: the two `/disks` numbers worth a colour, with the bounds the phase set.
A spinning drive at 70 °C is a fire and an SSD that has consumed its rated wear is a
replacement, so both trip red; the amber in between is the "watch it" band. An SSD with
no temperature sensor has **no value**, and therefore no verdict — the absence of a
number is not a healthy reading (`classify(metric, None)` stays `None`).
"""
assert classify("disk.temp_c", 49.9) == OK
assert classify("disk.temp_c", 50) == WARN
assert classify("disk.temp_c", 69.9) == WARN
assert classify("disk.temp_c", 70) == FAIL
assert classify("disk.temp_c", 75) == FAIL
assert classify("disk.temp_c", None) is None
assert classify("disk.percentage_used", 79.9) == OK
assert classify("disk.percentage_used", 80) == WARN
assert classify("disk.percentage_used", 100) == FAIL
assert classify("disk.percentage_used", None) is None
def test_sector_counters_and_power_on_hours_are_never_graded() -> None:
"""A non-zero sector count is an alarm, not a threshold breach (`/llm`'s chip rule).
Colouring 0 reallocated sectors green would call a drive healthy because a number is
small, and colouring 1 amber would make the chip redundant: the page shows an alarm on
**presence** and says nothing at all at zero. `power_on_hours` is an odometer with no bad
value. So none of them may ever grow a row in `THRESHOLDS`.
"""
counters = ("disk.reallocated", "disk.pending", "disk.uncorrectable",
"disk.power_on_hours", "disk.health_ok")
assert set(counters).isdisjoint(THRESHOLDS), "a counter is not a graded number"
for metric in counters:
assert classify(metric, 0) is None
assert classify(metric, 1) is None
assert classify(metric, 99_000) is None
assert classify(f"{metric}@sda", 1) is None
def test_per_device_streams_inherit_their_base_metric() -> None:
assert classify("disk.used_pct@/srv", 95) == "fail"
assert classify("disk.used_pct@/srv", 10) == "ok"
assert classify("cpu.usage_pct@core7", 80) == "warn"
assert classify("disk.temp_c@nvme0n1", 71) == "fail"
assert classify("disk.percentage_used@nvme0n1", 80) == "warn"
assert classify("mem.used_pct@any", 95) == "fail"
tests/test_store.py
+74
View File
@@ -252,3 +252,77 @@ def test_rollup_does_not_average_a_partial_minute(db) -> None:
def test_hour_constant_is_used_in_retention(db) -> None:
assert db.raw_keep_seconds == pytest.approx(48 * HOUR)
# ---- a 300-second source (phase 09: `disks` samples SMART every 5 minutes) -- #
#
# The rollup SQL groups by minute and hour buckets, so a source slower than a minute
# puts one raw row in every fifth bucket and nothing in the other four. Two things had to
# be true and are asserted rather than assumed: an empty bucket produces *no* rollup row
# (inventing a value for a minute nothing was sampled is rule 2), and the hour bucket
# still averages the samples it did get.
SMART_STEP = 300.0
def test_a_300s_source_rolls_into_minutes_without_inventing_the_empty_ones(db) -> None:
now = time.time()
# five SMART polls at exactly the phase's cadence (-30, -25, -20, -15, -10 minutes),
# so every bucket is fully in the past (+60 s slack) and maintenance may roll all five
start = int(now // MINUTE) * MINUTE - 30 * MINUTE
add_series(db, "disks", "driveripper", "disk.temp_c@nvme0n1",
[38, 39, 38, 40, 37], start=start, step=SMART_STEP)
counts = db.maintain(now=now)
minutes = db.query("disks", "driveripper", "disk.temp_c@nvme0n1", res=RES_MIN, start=0)
assert counts["to_minute"] == 5
assert [value for _ts, value in minutes] == [38.0, 39.0, 38.0, 40.0, 37.0], (
"a bucket holding one sample rolls up that sample, unchanged")
assert [int(ts) for ts, _value in minutes] == [
int(start + index * SMART_STEP // MINUTE * MINUTE) for index in range(5)], (
"every fifth minute bucket has a row; the empty ones must have none")
assert db.count(res=RES_MIN) == 5, "empty buckets became fabricated rows"
def test_a_300s_source_averages_into_the_hour_bucket_it_lands_in(db) -> None:
now = time.time()
# aligned to the epoch, and old enough that the hour bucket is complete: 12 samples
# five minutes apart inside one hour average to their mean, not to a sample count
hour = int(now // HOUR) * HOUR - 3 * HOUR
add_series(db, "disks", "driveripper", "disk.power_on_hours@nvme0n1",
[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 124],
start=hour, step=SMART_STEP)
db.maintain(now=now)
db.maintain(now=now) # the 1h pass reads 1m rows, so the first pass only makes them
hours = db.query("disks", "driveripper", "disk.power_on_hours@nvme0n1", res=RES_HOUR,
start=0)
assert len(hours) == 1
assert hours[0][0] == hour
assert hours[0][1] == pytest.approx((11 * 100 + 124) / 12)
def test_a_300s_series_is_queryable_at_every_resolution(db) -> None:
"""A 5-minute series over a 1h window is 12 points; `pick_res` must find them raw."""
now = time.time()
add_series(db, "disks", "driveripper", "disk.reallocated@nvme0n1",
[8] * 12, start=now - 3600 + 60, step=SMART_STEP)
assert db.pick_res("disks", "driveripper", "disk.reallocated@nvme0n1", 3600, 600) == RES_RAW
points = db.history("disks", "driveripper", "disk.reallocated@nvme0n1", span=3600)
assert len(points) == 12, "the chart is empty while raw history exists"
db.maintain(now=now)
db.maintain(now=now)
# with rollups present, a long window reads the rolled-up series, not 288 raw rows
assert db.pick_res("disks", "driveripper", "disk.reallocated@nvme0n1",
24 * 3600, 600) in (RES_MIN, RES_RAW)
def test_a_300s_source_is_pruned_by_the_existing_retention(db) -> None:
"""288 rows a day per series is fine for 48h and not fine forever — rule 9."""
now = time.time()
expired = now - db.raw_keep_seconds - 6 * SMART_STEP
add_series(db, "disks", "h", "disk.temp_c@sda", [30] * 4, start=expired, step=SMART_STEP)
add_series(db, "disks", "h", "disk.temp_c@sdb", [30], start=now - SMART_STEP,
step=SMART_STEP)
counts = db.maintain(now=now)
assert counts["pruned"] >= 4
assert db.query("disks", "h", "disk.temp_c@sda", start=0) == []
assert len(db.query("disks", "h", "disk.temp_c@sdb", start=0)) == 1