ducoterra
c4bb374ccc
Three dashboards served by one FastAPI app with a SQLite history store
(raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana,
no CDN, no build step. Nothing is fetched from the internet at runtime.
* system: a stdlib-only agent (agent/host_agent.py) reports /proc and
/sys counters; the dashboard turns them into rates over a labelled
window, so every number says how it was measured.
* llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house
exposition parser. Measured delta/elapsed throughput beats a native
per-request gauge, and each value carries its provenance.
* ups: a NUT wire-protocol client plus health checks that name what
they looked at. Commands are gated by NUT_ALLOW_ACTIONS,
DASH_READ_ONLY and a DANGER confirmation, re-checked server-side.
Missing values render as n/a rather than 0, and an unconfigured source
answers 404 {"configured": false} without dialling anything, so "not
deployed here" never looks like an outage.
253 tests at 94% coverage against captured real fixtures; ruff and
pyright clean.
LLM-Generated: true
444 lines
19 KiB
Python
444 lines
19 KiB
Python
"""Collectors: the fetch half of every poll, exercised with a fake transport.
|
|
|
|
`httpx.MockTransport` means these run at the speed of the parser and still cover what the
|
|
live hosts do badly: a 401 from a rotated agent token, a half-degraded agent snapshot, a
|
|
gufo server that only answers `/v1/slots`, an engine that counts nothing yet.
|
|
|
|
NUT is covered here for the *monitor* bookkeeping (payload shape, command gating, status
|
|
transitions); the socket protocol itself is in test_nut.py.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any, ClassVar
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from dashboards.collectors import (
|
|
NutMonitor,
|
|
UpstreamError,
|
|
fetch_engine,
|
|
fetch_host_snapshot,
|
|
)
|
|
from dashboards.config import Target
|
|
from dashboards.rates import RateEstimator
|
|
from dashboards.sources import nut
|
|
from dashboards.sources.host import HostCollector
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures"
|
|
|
|
|
|
def client_for(handler: Any) -> httpx.AsyncClient:
|
|
return httpx.AsyncClient(transport=httpx.MockTransport(handler),
|
|
base_url="http://agent.test")
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# host agent
|
|
# --------------------------------------------------------------------------- #
|
|
def agent_body() -> dict[str, Any]:
|
|
return json.loads((FIXTURES / "agent_driveripper.json").read_text())
|
|
|
|
|
|
def advance_cpu(body: dict[str, Any], *, total: float, idle: float) -> None:
|
|
"""Move the jiffy counters the way a real /proc/stat does between polls."""
|
|
modes = body["cpu"]["modes"]
|
|
for mode, value in modes.items():
|
|
modes[mode] = value + (total if mode == "total" else idle if mode == "idle" else 0)
|
|
|
|
|
|
async def test_a_real_agent_snapshot_parses() -> None:
|
|
body = agent_body()
|
|
calls: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
assert request.url.path == "/snapshot"
|
|
calls.append(request.url.path)
|
|
return httpx.Response(200, json=body)
|
|
|
|
target = Target("driveripper", "http://agent.test")
|
|
collector = HostCollector(RateEstimator())
|
|
async with client_for(handler) as client:
|
|
first = (await fetch_host_snapshot(client, target, collector)).payload
|
|
# CPU usage is Δjiffies/Δt: one scrape has no window, so it is n/a and not 0 %
|
|
assert first["cpu"]["usage_pct"] is None
|
|
# the counters have to actually move — total 0 delta means "cannot divide", not idle
|
|
same = (await fetch_host_snapshot(client, target, collector)).payload
|
|
assert same["cpu"]["usage_pct"] is None, "no counter movement must read as n/a"
|
|
advance_cpu(body, total=1000, idle=700)
|
|
result = await fetch_host_snapshot(client, target, collector)
|
|
payload = result.payload
|
|
assert payload["cpu"]["usage_pct"] == pytest.approx(30.0, abs=1.0)
|
|
assert payload["name"] == "driveripper"
|
|
assert payload["cpu"]["usage_pct"] is not None
|
|
assert payload["memory"]["total_bytes"] > 0
|
|
# real captures have real GPUs; the legend wants the short name, the tooltip the full one
|
|
assert payload["gpus"], "driveripper's capture must yield at least one GPU"
|
|
assert payload["gpus"][0]["model"] and "model_full" in payload["gpus"][0]
|
|
assert any(s.metric == "cpu.usage_pct" for s in result.samples)
|
|
|
|
|
|
async def test_target_headers_are_sent_to_the_agent() -> None:
|
|
seen: dict[str, str] = {}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen["authorization"] = request.headers.get("authorization", "")
|
|
return httpx.Response(200, json=agent_body())
|
|
|
|
target = Target("gateway", "http://agent.test", headers={"Authorization": "Bearer tok"})
|
|
async with client_for(handler) as client:
|
|
await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
|
|
assert seen["authorization"] == "Bearer tok"
|
|
|
|
|
|
@pytest.mark.parametrize("status,fragment", [
|
|
(401, "token"),
|
|
(403, "HTTP 403"),
|
|
(500, "HTTP 500"),
|
|
])
|
|
async def test_agent_rejections_say_which_one_it_was(status, fragment) -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(status, text="nope")
|
|
|
|
async with client_for(handler) as client:
|
|
with pytest.raises(UpstreamError) as excinfo:
|
|
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
|
HostCollector(RateEstimator()))
|
|
assert fragment in excinfo.value.message
|
|
assert excinfo.value.status == 502
|
|
|
|
|
|
async def test_a_refused_connection_is_an_upstream_error_not_a_traceback() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ConnectError("connection refused")
|
|
|
|
async with client_for(handler) as client:
|
|
with pytest.raises(UpstreamError, match="unreachable"):
|
|
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
|
HostCollector(RateEstimator()))
|
|
|
|
|
|
async def test_a_slow_agent_is_a_gateway_timeout() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ReadTimeout("too slow")
|
|
|
|
async with client_for(handler) as client:
|
|
with pytest.raises(UpstreamError) as excinfo:
|
|
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
|
HostCollector(RateEstimator()))
|
|
assert excinfo.value.status == 504
|
|
|
|
|
|
async def test_html_instead_of_json_is_reported_as_such() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, text="<html><body>502 Bad Gateway</body></html>")
|
|
|
|
async with client_for(handler) as client:
|
|
with pytest.raises(UpstreamError, match="non-JSON"):
|
|
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
|
HostCollector(RateEstimator()))
|
|
|
|
|
|
async def test_an_agent_answering_not_ok_is_not_silently_empty() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"})
|
|
|
|
async with client_for(handler) as client:
|
|
with pytest.raises(UpstreamError, match="procfs unreadable"):
|
|
await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
|
HostCollector(RateEstimator()))
|
|
|
|
|
|
async def test_a_degraded_agent_still_renders_and_says_what_broke() -> None:
|
|
body = agent_body()
|
|
body["errors"] = {"temperatures": "no hwmon sensors"}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, json=body)
|
|
|
|
async with client_for(handler) as client:
|
|
result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
|
|
HostCollector(RateEstimator()))
|
|
assert result.payload["degraded"] == {"temperatures": "no hwmon sensors"}
|
|
assert result.payload["memory"]["total_bytes"] > 0
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# LLM engines
|
|
# --------------------------------------------------------------------------- #
|
|
GUFO = (FIXTURES / "metrics_gufo.txt").read_text()
|
|
VLLM = (FIXTURES / "metrics_vllm.txt").read_text()
|
|
|
|
|
|
def bump(text: str, series: str, amount: float) -> str:
|
|
lines = []
|
|
for line in text.splitlines():
|
|
if line.startswith(series + " "):
|
|
head, _, value = line.rpartition(" ")
|
|
line = f"{head} {float(value) + amount}"
|
|
lines.append(line)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
async def test_first_gufo_poll_has_no_measured_rate_yet() -> None:
|
|
"""One scrape cannot produce Δ/Δt. The cell is n/a, not a stale single-request gauge."""
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path == "/metrics":
|
|
return httpx.Response(200, text=GUFO)
|
|
return httpx.Response(404, text="nope")
|
|
|
|
async with client_for(handler) as client:
|
|
result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(),
|
|
with_slots=True)
|
|
decode = result.payload["metrics"]["decode_tokens_per_second"]
|
|
assert decode["value"] is None
|
|
assert decode["source"] is None
|
|
assert result.payload["sessions"]["active"] == 0.0
|
|
assert "no /slots answer" in result.payload["sessions"]["detail"]
|
|
|
|
|
|
async def test_second_gufo_poll_measures_throughput_over_a_window() -> None:
|
|
state = {"text": GUFO}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path == "/metrics":
|
|
return httpx.Response(200, text=state["text"])
|
|
if request.url.path == "/slots":
|
|
return httpx.Response(200, json=[{"index": 0, "n_ctx": 8192, "predicted": 12}])
|
|
return httpx.Response(404)
|
|
|
|
rates = RateEstimator(span=10.0)
|
|
async with client_for(handler) as client:
|
|
await fetch_engine(client, "gufo", "http://engine.test", "", rates)
|
|
state["text"] = bump(GUFO, "llamacpp:tokens_predicted_total", 40.0)
|
|
result = await fetch_engine(client, "gufo", "http://engine.test", "", rates,
|
|
with_slots=True)
|
|
decode = result.payload["metrics"]["decode_tokens_per_second"]
|
|
assert decode["source"].startswith("computed:Δllamacpp:")
|
|
assert decode["derived"] is True
|
|
assert decode["window"] > 0
|
|
assert decode["value"] == pytest.approx(40.0 / decode["window"], rel=0.05)
|
|
assert result.payload["sessions"]["source"] == "gufo /slots"
|
|
assert result.payload["sessions"]["list"][0]["index"] == 0
|
|
|
|
|
|
async def test_gufo_slots_alias_is_used_when_plain_slots_is_missing() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path == "/metrics":
|
|
return httpx.Response(200, text=GUFO)
|
|
if request.url.path == "/v1/slots":
|
|
return httpx.Response(200, json={"slots": [{"index": 1}]})
|
|
return httpx.Response(404)
|
|
|
|
async with client_for(handler) as client:
|
|
result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(),
|
|
with_slots=True)
|
|
assert result.payload["sessions"]["list"] == [{"index": 1}]
|
|
|
|
|
|
async def test_vllm_metrics_parse_and_carry_the_api_key() -> None:
|
|
seen: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen.append(request.headers.get("authorization", ""))
|
|
return httpx.Response(200, text=VLLM)
|
|
|
|
async with client_for(handler) as client:
|
|
result = await fetch_engine(client, "vllm", "http://engine.test", "SECRET",
|
|
RateEstimator())
|
|
assert seen == ["Bearer SECRET"]
|
|
metrics = result.payload["metrics"]
|
|
assert metrics["num_requests_running"]["value"] is not None
|
|
# vLLM publishes no per-session list: the page is told, rather than shown an empty table
|
|
assert "counts, not sessions" in result.payload["sessions"]["detail"]
|
|
assert any(s.metric.startswith("llm.") for s in result.samples)
|
|
|
|
|
|
@pytest.mark.parametrize("status", [404, 500, 503])
|
|
async def test_engine_http_errors_name_the_status(status) -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(status, text="")
|
|
|
|
async with client_for(handler) as client:
|
|
with pytest.raises(UpstreamError, match=f"HTTP {status}"):
|
|
await fetch_engine(client, "vllm", "http://engine.test", "", RateEstimator())
|
|
|
|
|
|
async def test_engine_timeout_is_504() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
raise httpx.ReadTimeout("engine busy")
|
|
|
|
async with client_for(handler) as client:
|
|
with pytest.raises(UpstreamError) as excinfo:
|
|
await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator())
|
|
assert excinfo.value.status == 504
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# NUT monitor
|
|
# --------------------------------------------------------------------------- #
|
|
class FakeSimulator:
|
|
"""Stands in for NutSimulator: hands back readings we control."""
|
|
|
|
def __init__(self, readings: list[nut.UpsReading]) -> None:
|
|
self.readings = readings
|
|
|
|
async def read(self) -> list[nut.UpsReading]:
|
|
return list(self.readings)
|
|
|
|
|
|
def reading(status: str, **vars: str) -> nut.UpsReading:
|
|
item = nut.UpsReading(name="cyberpower", description="CP1350PFCLCD", ts=1_700_000_000.0)
|
|
item.vars = {"ups.status": status, "battery.charge": "92", "ups.load": "35",
|
|
"battery.runtime": "1800", **vars}
|
|
item.commands = ["beeper.mute", "test.panel.start", "test.battery.start.deep"]
|
|
return item
|
|
|
|
|
|
class SettingsStub:
|
|
nut_host, nut_port, nut_timeout = "ups.test", 3493, 5.0
|
|
nut_user = nut_password = ""
|
|
nut_interval = 5.0
|
|
nut_enabled = True
|
|
actions_allowed = True
|
|
read_only = False
|
|
|
|
|
|
async def test_monitor_payload_labels_the_simulator() -> None:
|
|
monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
|
|
result = await monitor.collect()
|
|
payload = result.payload
|
|
assert payload["simulated"] is True
|
|
assert payload["server"] is None, "a simulator must not claim a server address"
|
|
assert payload["actions_enabled"] is True
|
|
card = payload["ups"][0]
|
|
assert card["name"] == "cyberpower"
|
|
assert card["status"] == "OL"
|
|
assert card["numbers"]["charge_pct"] == 92.0
|
|
# a driver that reports no battery date, temperature, or self-test result is `unknown`,
|
|
# never a silent "healthy" — and never a failure either
|
|
assert card["health"]["overall"] == "unknown", card["health"]
|
|
assert card["health"]["counts"]["fail"] == 0
|
|
assert card["health"]["counts"]["warn"] == 0
|
|
# commands are offered with their risk, and the dangerous one is present but marked
|
|
risks = {item["command"]: item for item in card["commands"]}
|
|
assert risks["beeper.mute"]["enabled"] is True
|
|
assert risks["test.battery.start.deep"]["risk"] == "dangerous"
|
|
|
|
|
|
async def test_a_fully_report_ups_is_called_healthy() -> None:
|
|
full = reading("OL", **{"battery.temperature": "27.4", "battery.mfr.date": "2024-06-01",
|
|
"battery.test.result": "OK", "input.voltage": "230"})
|
|
full.vars["battery.charge"] = "100"
|
|
card = (await NutMonitor(SettingsStub(), simulator=FakeSimulator([full])).collect()).payload["ups"][0]
|
|
assert card["health"]["overall"] == "healthy", card["health"]
|
|
assert card["health"]["counts"]["unknown"] == 0
|
|
|
|
|
|
async def test_monitor_sends_history_samples_for_every_ups_metric() -> None:
|
|
monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OB")]))
|
|
result = await monitor.collect()
|
|
names = {sample.metric for sample in result.samples}
|
|
assert "ups.charge_pct" in names
|
|
assert "ups.load_pct" in names
|
|
assert all(sample.source == "ups" for sample in result.samples)
|
|
|
|
|
|
async def test_status_changes_become_one_event_each() -> None:
|
|
monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
|
|
assert (await monitor.collect()).events == []
|
|
monitor.simulator = FakeSimulator([reading("OL")])
|
|
assert (await monitor.collect()).events == [], "an unchanged status is not news"
|
|
monitor.simulator = FakeSimulator([reading("OB")])
|
|
events = (await monitor.collect()).events
|
|
assert len(events) == 1
|
|
assert events[0].kind == "status"
|
|
assert "OL → OB" in events[0].message
|
|
assert events[0].severity == "warn", "on battery is a warning, not a failure"
|
|
monitor.simulator = FakeSimulator([reading("OL FUB")])
|
|
events = (await monitor.collect()).events
|
|
assert events[0].severity == "error", "FUB (the UPS decided to power off) is a failure"
|
|
|
|
|
|
async def test_a_self_test_result_is_reported_when_it_changes() -> None:
|
|
monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
|
|
await monitor.collect()
|
|
# the variable is `ups.test.result` (drivers also use battery.test.result) — see VAR_MAP
|
|
monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "INPROGRESS"})])
|
|
assert (await monitor.collect()).events == [], "first sighting is not news"
|
|
monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "FAIL"})])
|
|
events = (await monitor.collect()).events
|
|
assert [event.kind for event in events] == ["test"]
|
|
assert events[0].severity == "warn"
|
|
monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "OK"})])
|
|
assert (await monitor.collect()).events[0].severity == "info"
|
|
|
|
|
|
async def test_gating_is_applied_where_the_commands_are_built() -> None:
|
|
class ReadOnly(SettingsStub):
|
|
actions_allowed = False
|
|
|
|
monitor = NutMonitor(ReadOnly(), simulator=FakeSimulator([reading("OL")]))
|
|
card = (await monitor.collect()).payload["ups"][0]
|
|
assert all(item["enabled"] is False for item in card["commands"])
|
|
assert all(item["reason"] for item in card["commands"]), "every refusal must explain itself"
|
|
assert set(card["advertised_commands"]) == {"beeper.mute", "test.panel.start",
|
|
"test.battery.start.deep"}, (
|
|
"the driver still advertises them — the refusal is ours, and must say so")
|
|
|
|
|
|
class FakeNutClient:
|
|
"""The real upsd branch without a socket: vars work, the optional lists do not."""
|
|
|
|
instances: ClassVar[list[FakeNutClient]] = []
|
|
|
|
def __init__(self, host: str, port: int, **kwargs: Any) -> None:
|
|
self.host, self.port, self.kwargs = host, port, kwargs
|
|
self.entered = False
|
|
FakeNutClient.instances.append(self)
|
|
|
|
async def __aenter__(self) -> FakeNutClient:
|
|
self.entered = True
|
|
return self
|
|
|
|
async def __aexit__(self, *exc: Any) -> None:
|
|
self.entered = False
|
|
|
|
async def list_ups(self) -> list[dict[str, str]]:
|
|
return [{"name": "cyberpower", "description": "CP1350PFCLCD"}]
|
|
|
|
async def list_vars(self, ups: str) -> dict[str, str]:
|
|
return {"ups.status": "OL", "battery.charge": "100", "battery.temperature": "301.1"}
|
|
|
|
async def list_cmds(self, ups: str) -> list[str]:
|
|
raise nut.NutError("this driver exports no commands")
|
|
|
|
async def list_rw_vars(self, ups: str) -> list[str]:
|
|
raise nut.NutError("LIST RW denied")
|
|
|
|
|
|
async def test_real_upsd_branch_survives_drivers_that_refuse_the_optional_lists(
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(nut, "NutClient", FakeNutClient)
|
|
monitor = NutMonitor(SettingsStub())
|
|
result = await monitor.collect()
|
|
client = FakeNutClient.instances[-1]
|
|
assert client.entered is False, "the connection must not outlive one poll"
|
|
assert client.kwargs["timeout"] == 5.0
|
|
card = result.payload["ups"][0]
|
|
assert card["commands"] == [] and card["advertised_commands"] == []
|
|
assert card["numbers"]["charge_pct"] == 100.0
|
|
# a driver reporting Kelvin is normalised, not charted as 301 °C
|
|
assert card["numbers"]["battery_temperature"] == pytest.approx(301.1 - 273.15, abs=0.01)
|
|
samples = {sample.metric: sample.value for sample in result.samples}
|
|
assert samples["ups.battery_temperature_c"] == pytest.approx(27.95, abs=0.01)
|
|
|
|
|
|
async def test_a_real_server_names_itself_in_the_payload(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(nut, "NutClient", FakeNutClient)
|
|
payload = (await NutMonitor(SettingsStub()).collect()).payload
|
|
assert payload["simulated"] is False
|
|
assert payload["server"] == "ups.test:3493"
|