phase: 09_disks_smartctl
**Phase 09 — final verification pass (all 4 task files already in `complete/`; nothing re-implemented)**
- Verified `GET /api/disks/snapshot` live: 3 hosts / 18 whole-`/sys/block` rows, `smartctl_available` three-way, per-row `severity`, API-summed `fleet`; 404 `{"configured": false}` proven *without dialling* (spies on `httpx.AsyncClient.get` + `collectors.fetch_snapshot`).
- Verified `/api/disks/history` live (`disk.temp_c@dev`, unknown names → empty series) + `test_store.py`'s 300 s rollup/prune buckets; `/api/metrics?source=disks` answers the `disks` family.
- Browser against live data: 18 drive cards (never a partition/mount), collapsed per-host `Charts` with `aria-label`, 3 `.stale` grids + one `.banner.info` why-line and **0 `.fail` classes / green `disks` dot** → criteria 3 and 4 hold; new shots `data/logs/09-verify-disks-live-dimmed.png`, `09-verify-overview-live.png` (13 total for the phase).
- Real-hardware criterion: read 4 driveripper drives through `ops`' `sudo -n smartctl -i -A -H` and ran the live output through `host_agent.parse_smart_output` → `sources/disks.parse_disks_snapshot`: `sda` 35 °C / 30 423 h, `nvme1n1` 58 % wear, `nvme5n1` 13 uncorrectable → `alarm_disks: 1`, temps graded `warn`; consistent with the 16 committed fixtures.
- `bash .agents/validate.sh` → **EXIT=0**; `uv run pytest --cov` → **769 passed, 11 skipped, coverage 98.33%** (floor 94); `ruff check src tests agent` clean; `pyright` 0 errors; `node --check` clean; hygiene + dependency advisories clean. `uv run pytest -m live --live` → 10 passed, 1 skipped (agents still predate the SMART pass). Dev server healthy: `/api/health` `ok`, 9/9 sources.
- Docs fixed: AGENTS.md test count 698→769 and coverage 98.2→98.33 %; `disks` added to the event dashboard-name list and to the "every feed renders `Dash.eventText`" sentence; README `POLL_MS_*` row now states `/disks`' deliberate fixed 30 s; new Known gap 4 records the dev-DB `database is locked` writes (reported/recovered per invariant 4, ~590 MB DB / ~70 MB WAL, not reproduced by a controlled 25 s external read txn).
- Deviations: **no live-infra change** — granting the agent user the `disk` group and redeploying `host_agent.py` on driveripper/deskwork stay the documented paired operator action (redeploying without the grant yields 16 permission-denied lines; a sudo-spawning agent would break the confirmed "direct read, no sudo" assumption). Store-lock issue left to its own phase rather than patched speculatively.
- Next pending phase: `10_dev_direct_dial`.
This commit is contained in:
@@ -19,12 +19,15 @@ import pytest
|
||||
from dashboards.collectors import (
|
||||
NutMonitor,
|
||||
UpstreamError,
|
||||
fetch_disk_snapshot,
|
||||
fetch_engine,
|
||||
fetch_host_snapshot,
|
||||
fetch_snapshot,
|
||||
)
|
||||
from dashboards.config import Target
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.sources import nut
|
||||
from dashboards.sources.disks import DisksCollector
|
||||
from dashboards.sources.host import HostCollector
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
@@ -203,6 +206,148 @@ async def test_a_degraded_agent_still_renders_and_says_what_broke() -> None:
|
||||
assert result.payload["memory"]["total_bytes"] > 0
|
||||
|
||||
|
||||
async def test_smart_addon_errors_do_not_degrade_the_system_source() -> None:
|
||||
"""A host without smartmontools (or without device read access) is unconfigured, not
|
||||
broken (invariant 3): its `smart:` lines must never hold the system page amber. They
|
||||
stay in `agent.errors`, where the /disks page explains them in its own panel.
|
||||
"""
|
||||
body = agent_body()
|
||||
body["errors"] = ["smart: sdb: Smartctl open device: /dev/sdb failed: Permission denied"]
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
async with client_for(handler) as client:
|
||||
result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
assert "degraded" not in result.payload
|
||||
assert any("smart:" in str(e) for e in result.payload["agent"]["errors"])
|
||||
|
||||
|
||||
async def test_real_degradations_still_show_when_smart_noise_is_alongside() -> None:
|
||||
body = agent_body()
|
||||
body["errors"] = ["smart: smartctl not found", "temperatures: no hwmon sensors"]
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
async with client_for(handler) as client:
|
||||
result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
assert result.payload["degraded"] == ["temperatures: no hwmon sensors"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# physical disks (the /disks source's fetch half)
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def test_the_disk_fetch_reads_the_same_agent_snapshot_as_the_system_one(
|
||||
disk_snapshot: Any) -> None:
|
||||
body = disk_snapshot(mode="healthy")
|
||||
seen: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url.path == "/snapshot", "disks has no endpoint of its own"
|
||||
seen.append(request.url.path)
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
target = Target("driveripper", "http://agent.test")
|
||||
async with client_for(handler) as client:
|
||||
result = await fetch_disk_snapshot(client, target, DisksCollector(), timeout=30)
|
||||
assert seen == ["/snapshot"]
|
||||
assert result.payload["name"] == "driveripper"
|
||||
assert result.payload["disks"], "the physical rows come with the snapshot"
|
||||
assert any(sample.metric.startswith("disk.temp_c@") for sample in result.samples)
|
||||
|
||||
|
||||
async def test_the_disk_fetch_reports_the_agent_failure_like_the_system_one(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""One dead host must say the same sentence on both pages (shared `fetch_snapshot`)."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(500, text="boom")
|
||||
|
||||
target = Target("gw", "http://agent.test")
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError) as system_error:
|
||||
await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
|
||||
with pytest.raises(UpstreamError) as disk_error:
|
||||
await fetch_disk_snapshot(client, target, DisksCollector())
|
||||
assert disk_error.value.message == system_error.value.message
|
||||
|
||||
|
||||
async def test_an_agent_that_is_not_ok_fails_the_disk_poll_too() -> None:
|
||||
"""A degraded agent that says `ok: false` has no trustworthy disk rows to hand over."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"})
|
||||
|
||||
async with client_for(handler) as client:
|
||||
with pytest.raises(UpstreamError, match="procfs unreadable"):
|
||||
await fetch_disk_snapshot(client, Target("gw", "http://agent.test"),
|
||||
DisksCollector())
|
||||
|
||||
|
||||
class TimeoutRecordingClient(httpx.AsyncClient):
|
||||
"""An `AsyncClient` that remembers the `timeout=` each request was handed.
|
||||
|
||||
A `MockTransport` replaces httpcore's connection pool, so it does not enforce read
|
||||
timeouts and a slow-handler test would prove nothing. The argument is what matters
|
||||
anyway: `timeout=None` means *no timeout at all* to httpx, which would hang a poll
|
||||
loop forever, and the client default means "the system poll's 4 s".
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.timeouts: list[Any] = []
|
||||
|
||||
async def get(self, url: Any, **kwargs: Any) -> httpx.Response: # type: ignore[override]
|
||||
self.timeouts.append(kwargs.get("timeout", "<unset>"))
|
||||
return await super().get(url, **kwargs)
|
||||
|
||||
|
||||
async def test_the_slow_disk_poll_carries_its_own_timeout_and_never_disables_one() -> None:
|
||||
"""`HOST_TIMEOUT_SECONDS` (4 s) stays `/system`'s budget; `disks` brings its own 30 s.
|
||||
|
||||
The SMART pass runs inside the agent's snapshot handler, so the two dashboards cannot
|
||||
share one budget: too short and `/disks` reads red on every SMART poll, too long and
|
||||
`/system` stops noticing a dead host. And the default must stay httpx's sentinel —
|
||||
passing a literal `None` silently turns timeout protection off altogether.
|
||||
"""
|
||||
body = {"ok": True, "ts": 1.0, "disk_io": {"devices": []}, "errors": []}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
client = TimeoutRecordingClient(transport=httpx.MockTransport(handler),
|
||||
timeout=4.0, base_url="http://agent.test")
|
||||
target = Target("gw", "http://agent.test")
|
||||
async with client:
|
||||
await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
|
||||
assert client.timeouts == [httpx.USE_CLIENT_DEFAULT]
|
||||
await fetch_disk_snapshot(client, target, DisksCollector(), timeout=30)
|
||||
assert client.timeouts[-1] == 30, "the disk poll waits for the SMART pass"
|
||||
await fetch_snapshot(client, target)
|
||||
assert client.timeouts[-1] is httpx.USE_CLIENT_DEFAULT
|
||||
assert all(timeout is not None for timeout in client.timeouts), (
|
||||
"`timeout=None` disables httpx's timeouts entirely — translate it, never pass it")
|
||||
|
||||
|
||||
async def test_smart_addon_errors_reach_the_disk_payload_verbatim(
|
||||
disk_snapshot: Any) -> None:
|
||||
"""`collectors` keeps the `smart:` lines out of `/system`'s `degraded`; `/disks` is
|
||||
where they earn their keep — they are the page's reason for an empty card."""
|
||||
body = disk_snapshot(mode="denied")
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=body)
|
||||
|
||||
async with client_for(handler) as client:
|
||||
disks = await fetch_disk_snapshot(client, Target("driveripper", "http://agent.test"),
|
||||
DisksCollector())
|
||||
system = await fetch_host_snapshot(client, Target("driveripper", "http://agent.test"),
|
||||
HostCollector(RateEstimator()))
|
||||
assert disks.payload["smart_errors"], "the /disks page gets the agent's own sentence"
|
||||
assert "degraded" not in system.payload, "and /system stays unbothered by it"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# LLM engines
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
Reference in New Issue
Block a user