Add homelab dashboards: system, LLM, and UPS on one domain

Three dashboards served by one FastAPI app with a SQLite history store (raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana, no CDN, no build step. Nothing is fetched from the internet at runtime. * system: a stdlib-only agent (agent/host_agent.py) reports /proc and /sys counters; the dashboard turns them into rates over a labelled window, so every number says how it was measured. * llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house exposition parser. Measured delta/elapsed throughput beats a native per-request gauge, and each value carries its provenance. * ups: a NUT wire-protocol client plus health checks that name what they looked at. Commands are gated by NUT_ALLOW_ACTIONS, DASH_READ_ONLY and a DANGER confirmation, re-checked server-side. Missing values render as n/a rather than 0, and an unconfigured source answers 404 {"configured": false} without dialling anything, so "not deployed here" never looks like an outage. 253 tests at 94% coverage against captured real fixtures; ruff and pyright clean. LLM-Generated: true
2026-10-16 17:20:45 -04:00
commit c4bb374ccc
56 changed files with 13018 additions and 0 deletions
@@ -0,0 +1,443 @@
+"""Collectors: the fetch half of every poll, exercised with a fake transport.
+
+`httpx.MockTransport` means these run at the speed of the parser and still cover what the
+live hosts do badly: a 401 from a rotated agent token, a half-degraded agent snapshot, a
+gufo server that only answers `/v1/slots`, an engine that counts nothing yet.
+
+NUT is covered here for the *monitor* bookkeeping (payload shape, command gating, status
+transitions); the socket protocol itself is in test_nut.py.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, ClassVar
+
+import httpx
+import pytest
+
+from dashboards.collectors import (
+    NutMonitor,
+    UpstreamError,
+    fetch_engine,
+    fetch_host_snapshot,
+)
+from dashboards.config import Target
+from dashboards.rates import RateEstimator
+from dashboards.sources import nut
+from dashboards.sources.host import HostCollector
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def client_for(handler: Any) -> httpx.AsyncClient:
+    return httpx.AsyncClient(transport=httpx.MockTransport(handler),
+                             base_url="http://agent.test")
+
+
+# --------------------------------------------------------------------------- #
+# host agent
+# --------------------------------------------------------------------------- #
+def agent_body() -> dict[str, Any]:
+    return json.loads((FIXTURES / "agent_driveripper.json").read_text())
+
+
+def advance_cpu(body: dict[str, Any], *, total: float, idle: float) -> None:
+    """Move the jiffy counters the way a real /proc/stat does between polls."""
+    modes = body["cpu"]["modes"]
+    for mode, value in modes.items():
+        modes[mode] = value + (total if mode == "total" else idle if mode == "idle" else 0)
+
+
+async def test_a_real_agent_snapshot_parses() -> None:
+    body = agent_body()
+    calls: list[str] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.path == "/snapshot"
+        calls.append(request.url.path)
+        return httpx.Response(200, json=body)
+
+    target = Target("driveripper", "http://agent.test")
+    collector = HostCollector(RateEstimator())
+    async with client_for(handler) as client:
+        first = (await fetch_host_snapshot(client, target, collector)).payload
+        # CPU usage is Δjiffies/Δt: one scrape has no window, so it is n/a and not 0 %
+        assert first["cpu"]["usage_pct"] is None
+        # the counters have to actually move — total 0 delta means "cannot divide", not idle
+        same = (await fetch_host_snapshot(client, target, collector)).payload
+        assert same["cpu"]["usage_pct"] is None, "no counter movement must read as n/a"
+        advance_cpu(body, total=1000, idle=700)
+        result = await fetch_host_snapshot(client, target, collector)
+    payload = result.payload
+    assert payload["cpu"]["usage_pct"] == pytest.approx(30.0, abs=1.0)
+    assert payload["name"] == "driveripper"
+    assert payload["cpu"]["usage_pct"] is not None
+    assert payload["memory"]["total_bytes"] > 0
+    # real captures have real GPUs; the legend wants the short name, the tooltip the full one
+    assert payload["gpus"], "driveripper's capture must yield at least one GPU"
+    assert payload["gpus"][0]["model"] and "model_full" in payload["gpus"][0]
+    assert any(s.metric == "cpu.usage_pct" for s in result.samples)
+
+
+async def test_target_headers_are_sent_to_the_agent() -> None:
+    seen: dict[str, str] = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen["authorization"] = request.headers.get("authorization", "")
+        return httpx.Response(200, json=agent_body())
+
+    target = Target("gateway", "http://agent.test", headers={"Authorization": "Bearer tok"})
+    async with client_for(handler) as client:
+        await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
+    assert seen["authorization"] == "Bearer tok"
+
+
+@pytest.mark.parametrize("status,fragment", [
+    (401, "token"),
+    (403, "HTTP 403"),
+    (500, "HTTP 500"),
+])
+async def test_agent_rejections_say_which_one_it_was(status, fragment) -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(status, text="nope")
+
+    async with client_for(handler) as client:
+        with pytest.raises(UpstreamError) as excinfo:
+            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
+                                      HostCollector(RateEstimator()))
+    assert fragment in excinfo.value.message
+    assert excinfo.value.status == 502
+
+
+async def test_a_refused_connection_is_an_upstream_error_not_a_traceback() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        raise httpx.ConnectError("connection refused")
+
+    async with client_for(handler) as client:
+        with pytest.raises(UpstreamError, match="unreachable"):
+            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
+                                      HostCollector(RateEstimator()))
+
+
+async def test_a_slow_agent_is_a_gateway_timeout() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        raise httpx.ReadTimeout("too slow")
+
+    async with client_for(handler) as client:
+        with pytest.raises(UpstreamError) as excinfo:
+            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
+                                      HostCollector(RateEstimator()))
+    assert excinfo.value.status == 504
+
+
+async def test_html_instead_of_json_is_reported_as_such() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, text="<html><body>502 Bad Gateway</body></html>")
+
+    async with client_for(handler) as client:
+        with pytest.raises(UpstreamError, match="non-JSON"):
+            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
+                                      HostCollector(RateEstimator()))
+
+
+async def test_an_agent_answering_not_ok_is_not_silently_empty() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"})
+
+    async with client_for(handler) as client:
+        with pytest.raises(UpstreamError, match="procfs unreadable"):
+            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
+                                      HostCollector(RateEstimator()))
+
+
+async def test_a_degraded_agent_still_renders_and_says_what_broke() -> None:
+    body = agent_body()
+    body["errors"] = {"temperatures": "no hwmon sensors"}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, json=body)
+
+    async with client_for(handler) as client:
+        result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
+                                           HostCollector(RateEstimator()))
+    assert result.payload["degraded"] == {"temperatures": "no hwmon sensors"}
+    assert result.payload["memory"]["total_bytes"] > 0
+
+
+# --------------------------------------------------------------------------- #
+# LLM engines
+# --------------------------------------------------------------------------- #
+GUFO = (FIXTURES / "metrics_gufo.txt").read_text()
+VLLM = (FIXTURES / "metrics_vllm.txt").read_text()
+
+
+def bump(text: str, series: str, amount: float) -> str:
+    lines = []
+    for line in text.splitlines():
+        if line.startswith(series + " "):
+            head, _, value = line.rpartition(" ")
+            line = f"{head} {float(value) + amount}"
+        lines.append(line)
+    return "\n".join(lines) + "\n"
+
+
+async def test_first_gufo_poll_has_no_measured_rate_yet() -> None:
+    """One scrape cannot produce Δ/Δt. The cell is n/a, not a stale single-request gauge."""
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/metrics":
+            return httpx.Response(200, text=GUFO)
+        return httpx.Response(404, text="nope")
+
+    async with client_for(handler) as client:
+        result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(),
+                                    with_slots=True)
+    decode = result.payload["metrics"]["decode_tokens_per_second"]
+    assert decode["value"] is None
+    assert decode["source"] is None
+    assert result.payload["sessions"]["active"] == 0.0
+    assert "no /slots answer" in result.payload["sessions"]["detail"]
+
+
+async def test_second_gufo_poll_measures_throughput_over_a_window() -> None:
+    state = {"text": GUFO}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/metrics":
+            return httpx.Response(200, text=state["text"])
+        if request.url.path == "/slots":
+            return httpx.Response(200, json=[{"index": 0, "n_ctx": 8192, "predicted": 12}])
+        return httpx.Response(404)
+
+    rates = RateEstimator(span=10.0)
+    async with client_for(handler) as client:
+        await fetch_engine(client, "gufo", "http://engine.test", "", rates)
+        state["text"] = bump(GUFO, "llamacpp:tokens_predicted_total", 40.0)
+        result = await fetch_engine(client, "gufo", "http://engine.test", "", rates,
+                                    with_slots=True)
+    decode = result.payload["metrics"]["decode_tokens_per_second"]
+    assert decode["source"].startswith("computed:Δllamacpp:")
+    assert decode["derived"] is True
+    assert decode["window"] > 0
+    assert decode["value"] == pytest.approx(40.0 / decode["window"], rel=0.05)
+    assert result.payload["sessions"]["source"] == "gufo /slots"
+    assert result.payload["sessions"]["list"][0]["index"] == 0
+
+
+async def test_gufo_slots_alias_is_used_when_plain_slots_is_missing() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/metrics":
+            return httpx.Response(200, text=GUFO)
+        if request.url.path == "/v1/slots":
+            return httpx.Response(200, json={"slots": [{"index": 1}]})
+        return httpx.Response(404)
+
+    async with client_for(handler) as client:
+        result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(),
+                                    with_slots=True)
+    assert result.payload["sessions"]["list"] == [{"index": 1}]
+
+
+async def test_vllm_metrics_parse_and_carry_the_api_key() -> None:
+    seen: list[str] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.append(request.headers.get("authorization", ""))
+        return httpx.Response(200, text=VLLM)
+
+    async with client_for(handler) as client:
+        result = await fetch_engine(client, "vllm", "http://engine.test", "SECRET",
+                                    RateEstimator())
+    assert seen == ["Bearer SECRET"]
+    metrics = result.payload["metrics"]
+    assert metrics["num_requests_running"]["value"] is not None
+    # vLLM publishes no per-session list: the page is told, rather than shown an empty table
+    assert "counts, not sessions" in result.payload["sessions"]["detail"]
+    assert any(s.metric.startswith("llm.") for s in result.samples)
+
+
+@pytest.mark.parametrize("status", [404, 500, 503])
+async def test_engine_http_errors_name_the_status(status) -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(status, text="")
+
+    async with client_for(handler) as client:
+        with pytest.raises(UpstreamError, match=f"HTTP {status}"):
+            await fetch_engine(client, "vllm", "http://engine.test", "", RateEstimator())
+
+
+async def test_engine_timeout_is_504() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        raise httpx.ReadTimeout("engine busy")
+
+    async with client_for(handler) as client:
+        with pytest.raises(UpstreamError) as excinfo:
+            await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator())
+    assert excinfo.value.status == 504
+
+
+# --------------------------------------------------------------------------- #
+# NUT monitor
+# --------------------------------------------------------------------------- #
+class FakeSimulator:
+    """Stands in for NutSimulator: hands back readings we control."""
+
+    def __init__(self, readings: list[nut.UpsReading]) -> None:
+        self.readings = readings
+
+    async def read(self) -> list[nut.UpsReading]:
+        return list(self.readings)
+
+
+def reading(status: str, **vars: str) -> nut.UpsReading:
+    item = nut.UpsReading(name="cyberpower", description="CP1350PFCLCD", ts=1_700_000_000.0)
+    item.vars = {"ups.status": status, "battery.charge": "92", "ups.load": "35",
+                 "battery.runtime": "1800", **vars}
+    item.commands = ["beeper.mute", "test.panel.start", "test.battery.start.deep"]
+    return item
+
+
+class SettingsStub:
+    nut_host, nut_port, nut_timeout = "ups.test", 3493, 5.0
+    nut_user = nut_password = ""
+    nut_interval = 5.0
+    nut_enabled = True
+    actions_allowed = True
+    read_only = False
+
+
+async def test_monitor_payload_labels_the_simulator() -> None:
+    monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
+    result = await monitor.collect()
+    payload = result.payload
+    assert payload["simulated"] is True
+    assert payload["server"] is None, "a simulator must not claim a server address"
+    assert payload["actions_enabled"] is True
+    card = payload["ups"][0]
+    assert card["name"] == "cyberpower"
+    assert card["status"] == "OL"
+    assert card["numbers"]["charge_pct"] == 92.0
+    # a driver that reports no battery date, temperature, or self-test result is `unknown`,
+    # never a silent "healthy" — and never a failure either
+    assert card["health"]["overall"] == "unknown", card["health"]
+    assert card["health"]["counts"]["fail"] == 0
+    assert card["health"]["counts"]["warn"] == 0
+    # commands are offered with their risk, and the dangerous one is present but marked
+    risks = {item["command"]: item for item in card["commands"]}
+    assert risks["beeper.mute"]["enabled"] is True
+    assert risks["test.battery.start.deep"]["risk"] == "dangerous"
+
+
+async def test_a_fully_report_ups_is_called_healthy() -> None:
+    full = reading("OL", **{"battery.temperature": "27.4", "battery.mfr.date": "2024-06-01",
+                            "battery.test.result": "OK", "input.voltage": "230"})
+    full.vars["battery.charge"] = "100"
+    card = (await NutMonitor(SettingsStub(), simulator=FakeSimulator([full])).collect()).payload["ups"][0]
+    assert card["health"]["overall"] == "healthy", card["health"]
+    assert card["health"]["counts"]["unknown"] == 0
+
+
+async def test_monitor_sends_history_samples_for_every_ups_metric() -> None:
+    monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OB")]))
+    result = await monitor.collect()
+    names = {sample.metric for sample in result.samples}
+    assert "ups.charge_pct" in names
+    assert "ups.load_pct" in names
+    assert all(sample.source == "ups" for sample in result.samples)
+
+
+async def test_status_changes_become_one_event_each() -> None:
+    monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
+    assert (await monitor.collect()).events == []
+    monitor.simulator = FakeSimulator([reading("OL")])
+    assert (await monitor.collect()).events == [], "an unchanged status is not news"
+    monitor.simulator = FakeSimulator([reading("OB")])
+    events = (await monitor.collect()).events
+    assert len(events) == 1
+    assert events[0].kind == "status"
+    assert "OL → OB" in events[0].message
+    assert events[0].severity == "warn", "on battery is a warning, not a failure"
+    monitor.simulator = FakeSimulator([reading("OL FUB")])
+    events = (await monitor.collect()).events
+    assert events[0].severity == "error", "FUB (the UPS decided to power off) is a failure"
+
+
+async def test_a_self_test_result_is_reported_when_it_changes() -> None:
+    monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
+    await monitor.collect()
+    # the variable is `ups.test.result` (drivers also use battery.test.result) — see VAR_MAP
+    monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "INPROGRESS"})])
+    assert (await monitor.collect()).events == [], "first sighting is not news"
+    monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "FAIL"})])
+    events = (await monitor.collect()).events
+    assert [event.kind for event in events] == ["test"]
+    assert events[0].severity == "warn"
+    monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "OK"})])
+    assert (await monitor.collect()).events[0].severity == "info"
+
+
+async def test_gating_is_applied_where_the_commands_are_built() -> None:
+    class ReadOnly(SettingsStub):
+        actions_allowed = False
+
+    monitor = NutMonitor(ReadOnly(), simulator=FakeSimulator([reading("OL")]))
+    card = (await monitor.collect()).payload["ups"][0]
+    assert all(item["enabled"] is False for item in card["commands"])
+    assert all(item["reason"] for item in card["commands"]), "every refusal must explain itself"
+    assert set(card["advertised_commands"]) == {"beeper.mute", "test.panel.start",
+                                                "test.battery.start.deep"}, (
+        "the driver still advertises them — the refusal is ours, and must say so")
+
+
+class FakeNutClient:
+    """The real upsd branch without a socket: vars work, the optional lists do not."""
+
+    instances: ClassVar[list[FakeNutClient]] = []
+
+    def __init__(self, host: str, port: int, **kwargs: Any) -> None:
+        self.host, self.port, self.kwargs = host, port, kwargs
+        self.entered = False
+        FakeNutClient.instances.append(self)
+
+    async def __aenter__(self) -> FakeNutClient:
+        self.entered = True
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        self.entered = False
+
+    async def list_ups(self) -> list[dict[str, str]]:
+        return [{"name": "cyberpower", "description": "CP1350PFCLCD"}]
+
+    async def list_vars(self, ups: str) -> dict[str, str]:
+        return {"ups.status": "OL", "battery.charge": "100", "battery.temperature": "301.1"}
+
+    async def list_cmds(self, ups: str) -> list[str]:
+        raise nut.NutError("this driver exports no commands")
+
+    async def list_rw_vars(self, ups: str) -> list[str]:
+        raise nut.NutError("LIST RW denied")
+
+
+async def test_real_upsd_branch_survives_drivers_that_refuse_the_optional_lists(
+        monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(nut, "NutClient", FakeNutClient)
+    monitor = NutMonitor(SettingsStub())
+    result = await monitor.collect()
+    client = FakeNutClient.instances[-1]
+    assert client.entered is False, "the connection must not outlive one poll"
+    assert client.kwargs["timeout"] == 5.0
+    card = result.payload["ups"][0]
+    assert card["commands"] == [] and card["advertised_commands"] == []
+    assert card["numbers"]["charge_pct"] == 100.0
+    # a driver reporting Kelvin is normalised, not charted as 301 °C
+    assert card["numbers"]["battery_temperature"] == pytest.approx(301.1 - 273.15, abs=0.01)
+    samples = {sample.metric: sample.value for sample in result.samples}
+    assert samples["ups.battery_temperature_c"] == pytest.approx(27.95, abs=0.01)
+
+
+async def test_a_real_server_names_itself_in_the_payload(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(nut, "NutClient", FakeNutClient)
+    payload = (await NutMonitor(SettingsStub()).collect()).payload
+    assert payload["simulated"] is False
+    assert payload["server"] == "ups.test:3493"