ducoterra
8b048599f3
**Phase 02 final verification pass — all three tasks already complete; verified criteria, fixed 3 defects found live.**
- Verified: `/` = exactly 2 `<section>`s (Warnings & errors + Fleet), alerts render from `/api/summary` verdicts with each alert's own `href`; `/system`, `/llm`, `/ups` unchanged (charts + data live).
- Fixed (`src/dashboards/alerts.py`): a dead source's message printed the name twice — `gateway: UpstreamError: gateway: http://…`; the poller's `UpstreamError:` wrapper is now stripped (other exception names kept). Tests: 2 new in `tests/test_alerts.py`.
- Fixed (`src/dashboards/api.py`): `/api/summary` dropped a configured host/engine that had never answered post-start, so the fleet said `2/2 hosts` while a third was down. Stub rows (`unreachable: true`, all `n/a`) now match `/api/*/snapshot`; `hosts_total=3 / hosts_live=2`. Test: `test_a_host_that_has_never_answered_is_still_a_row_in_the_fleet`.
- Fixed (`web/static/page-index.js`): fleet row of a non-answering host is now dimmed with the shared `.stale` class (invariant 4 on `/`). Test: `test_overview_dims_the_fleet_row_of_a_host_that_stopped_answering`.
- `bash .agents/validate.sh` exit 0: `uv run pytest --cov --cov-fail-under=94` → **439 passed, 6 skipped, 96.82%** (`alerts.py` 100%, `api.py` 100%); `ruff check src tests agent` clean; `pyright` 0 errors; `node --check web/static` clean; hygiene clean; `uv pip list --outdated` empty; `uv run pytest -m live --live` → **6 passed**.
- Live proof (tunnel kill, then restored — all 6 sources `ok` again): killing the gateway forward produced exactly **one** alert, `fail gateway /system … unreachable (ConnectError)`; screenshots `data/logs/phase02-final-{alert,system,llm,ups,recovered}.png`.
- Completion criteria: two panels ✅ · `[]` when healthy ✅ (hermetic `healthy_client` + a live gateway-only instance returned `[]`; the real fleet can never be `[]` — its UPSes permanently warn on battery age/self-test) · one alert per dead host with `/system` link ✅ · pytest/live/coverage/ruff/pyright ✅ · no change on `/system`,`/llm`,`/ups` ✅ · phase 01 verdict vocabulary intact ✅.
- Docs: README `web/` bullet now states the dimmed `n/a` row; AGENTS.md invariant 4, `alerts.py`/`api.py` bullets, UI-conventions stale-row + window-chip ranges (5m…30d vs 1h…90d), metric example `llm.decode_tps` → `llm.decode_tokens_per_second` (verified against `/api/metrics`), test count 435 → 439. Env table, Containerfile, agent/install-script claims, fleet facts re-checked and consistent.
- Notable: left the fleet's real hostnames/domain in AGENTS.md's "real environment" section (deliberate, prior-phase content) even though rule 6 discourages personal hostnames in tracked files — worth a separate decision; no live config was changed beyond stopping/starting a local dev ssh forward.
- Next pending phase: `.agents/phases/todo/03_system_per_host_panels/`.
1017 lines
48 KiB
Python
1017 lines
48 KiB
Python
"""HTTP contract: what the pages may assume, and what must never leak.
|
|
|
|
The behaviours here are the ones a screenshot cannot show: an unconfigured source answering
|
|
404 `configured: false` instead of dialling a hostname that does not exist; a bearer token
|
|
that stops at the backend; the single write endpoint refusing on every gate; no external
|
|
asset the network can fail to deliver.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import time
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
from fastapi.testclient import TestClient
|
|
|
|
from dashboards import alerts
|
|
from dashboards.api import RANGE_SECONDS, parse_range
|
|
from dashboards.app import create_app
|
|
from dashboards.config import Settings, Target
|
|
from dashboards.severity import classify
|
|
from dashboards.sources import nut
|
|
from dashboards.store import Event, Sample, SampleStore
|
|
|
|
# refused immediately, and never dials out of the test environment
|
|
DEAD_URL = "http://127.0.0.1:1"
|
|
|
|
|
|
@contextmanager
|
|
def build_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, simulate: bool = False,
|
|
**overrides: Any) -> Iterator[TestClient]:
|
|
"""A TestClient whose lifespan runs, against a private DB and no real network.
|
|
|
|
The store is injected, so `create_app()` deliberately does not close it (production
|
|
owns its own); the caller here does, or every test leaks a SQLite handle and the
|
|
`--cov` log fills with ResourceWarnings that bury the warnings worth reading.
|
|
"""
|
|
if simulate:
|
|
monkeypatch.setenv("NUT_SIMULATE", "1")
|
|
else:
|
|
monkeypatch.delenv("NUT_SIMULATE", raising=False)
|
|
db = str(tmp_path / "api.sqlite3")
|
|
settings = Settings(db_path=db, **overrides)
|
|
store = SampleStore(db)
|
|
client = TestClient(create_app(settings, store))
|
|
client.__enter__()
|
|
try:
|
|
yield client
|
|
finally:
|
|
client.__exit__(None, None, None)
|
|
store.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
|
with build_client(tmp_path, monkeypatch) as test_client:
|
|
yield test_client
|
|
|
|
|
|
@pytest.fixture
|
|
def ups_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
|
"""NUT on the labelled simulator, actions allowed — the dev configuration."""
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True) as c:
|
|
yield c
|
|
|
|
|
|
def wait_for_payload(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
response = client.get(path)
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
time.sleep(0.1)
|
|
pytest.fail(f"{path} never became available")
|
|
|
|
|
|
def failing_host_payload(scrape: Any, agent_snapshot: dict[str, Any]) -> dict[str, Any]:
|
|
"""Two scrapes of the real driveripper capture, nudged onto known sides of the table.
|
|
|
|
Driven from the fixture rather than hand-written so a schema drift breaks here first.
|
|
"""
|
|
def nudge(snap: dict[str, Any]) -> None:
|
|
snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.95 # mem.used_pct -> fail
|
|
snap["cpu_temperature_celsius"] = 91.0 # cpu.temp_c -> fail
|
|
for fs in snap["filesystems"]:
|
|
fs["used_pct"] = 93.0 # disk -> fail
|
|
|
|
return scrape(agent_snapshot, cpu_busy=10.0, mutate=nudge)
|
|
|
|
|
|
@pytest.fixture
|
|
def data_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
agent_snapshot: dict[str, Any], host_scrape: Any) -> Iterator[TestClient]:
|
|
"""Overview-shaped app: one host payload in the hub plus the simulated UPSes.
|
|
|
|
The host job points at a dead port, so a poll can never overwrite the injected payload
|
|
(a failed poll keeps the last good one) — the numbers below are ours, not the fleet's.
|
|
"""
|
|
monkeypatch.setenv("NUT_SIMULATE", "1")
|
|
db = str(tmp_path / "summary.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, nut_host="127.0.0.1",
|
|
host_targets=(Target("driveripper", DEAD_URL),)), store)
|
|
test_client = TestClient(app)
|
|
test_client.__enter__()
|
|
try:
|
|
app.state.ctx.hub.jobs["host:driveripper"].state.payload = failing_host_payload(
|
|
host_scrape, agent_snapshot)
|
|
wait_for_payload(test_client, "/api/ups/snapshot")
|
|
yield test_client
|
|
finally:
|
|
test_client.__exit__(None, None, None)
|
|
store.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def healthy_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
agent_snapshot: dict[str, Any],
|
|
host_scrape: Any) -> Iterator[TestClient]:
|
|
"""The all-clear app: one host answering, nothing past a threshold, no UPS configured.
|
|
|
|
The job's collector is swapped for one that hands back the injected payload, so the
|
|
health record says `ok` truthfully instead of being overridden behind the poller.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
from dashboards.poller import Result # noqa: PLC0415 - only this fixture needs it
|
|
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
|
|
async def collect() -> Result:
|
|
return Result(payload=payload, samples=[])
|
|
|
|
job = test_client.app.state.ctx.hub.jobs["host:driveripper"] # type: ignore[attr-defined]
|
|
job.fn = collect
|
|
# let the poll already in flight against the dead port land first (it fails in
|
|
# microseconds), then take the health record over: the alerts must describe a host
|
|
# that is answering, not the half-second of startup where it was not
|
|
deadline = time.time() + 2.0
|
|
while time.time() < deadline and job.state.attempts == 0:
|
|
time.sleep(0.02)
|
|
time.sleep(0.05)
|
|
job.state.payload = payload
|
|
job.state.ok = True
|
|
job.state.error = None
|
|
job.state.consecutive_failures = 0
|
|
job.state.last_ok = time.time()
|
|
yield test_client
|
|
|
|
|
|
# ---- severity travels with the value --------------------------------------- #
|
|
|
|
def test_summary_carries_the_verdict_for_every_column_the_page_colours(data_client) -> None:
|
|
body = data_client.get("/api/summary").json()
|
|
row = body["system"]["hosts"][0]
|
|
assert row["severity"] == {"cpu_usage_pct": "ok", "mem_used_pct": "fail",
|
|
"disk_worst_used_pct": "fail", "temp_c": "fail"}
|
|
# the fleet card is judged by the same table, so amber means the same thing at both sizes
|
|
assert body["system"]["fleet"]["severity"] == {
|
|
"cpu_usage_pct_avg": "ok", "memory_used_pct": "fail",
|
|
"disk_worst_used_pct": "fail", "gpu_busy_pct_avg": "ok",
|
|
}
|
|
for ups in body["ups"]["ups"]:
|
|
assert set(ups["severity"]) == {"charge_pct", "load_pct", "runtime_seconds"}
|
|
assert ups["severity"]["charge_pct"] == classify("ups.charge_pct", ups["charge_pct"])
|
|
assert ups["severity"]["charge_pct"] == "ok", "the simulator starts at float charge"
|
|
|
|
|
|
def test_summary_shape_did_not_shrink(data_client) -> None:
|
|
"""Additive-only anchor: /api/summary gained `severity` and lost nothing.
|
|
|
|
The overview, the live tests, and any external consumer read the flat keys; this test is
|
|
the tripwire for a well-meaning refactor that "tidies" them away.
|
|
"""
|
|
body = data_client.get("/api/summary").json()
|
|
fleet_keys = {"hosts_total", "hosts_live", "cpu_cores", "cpu_usage_pct_avg",
|
|
"memory_total_bytes", "memory_used_bytes", "memory_used_pct",
|
|
"disk_worst_used_pct", "load1_total", "net_rx_bytes_per_s",
|
|
"net_tx_bytes_per_s", "gpu_count", "gpu_busy_pct_avg", "power_estimate_w"}
|
|
host_keys = {"name", "os", "cpu_usage_pct", "load1", "cores", "mem_used_pct",
|
|
"disk_worst_used_pct", "gpu_busy_pct", "temp_c", "uptime_seconds"}
|
|
ups_keys = {"name", "overall", "status", "charge_pct", "load_pct", "runtime_seconds"}
|
|
assert fleet_keys <= set(body["system"]["fleet"]), fleet_keys - set(body["system"]["fleet"])
|
|
assert host_keys <= set(body["system"]["hosts"][0]), host_keys - set(body["system"]["hosts"][0])
|
|
assert ups_keys <= set(body["ups"]["ups"][0]), ups_keys - set(body["ups"]["ups"][0])
|
|
assert set(body) == {"ts", "system", "llm", "ups", "recent_events", "alerts"}
|
|
# ...and the new keys are there beside the old ones, not instead of them
|
|
assert "severity" in body["system"]["fleet"]
|
|
assert "severity" in body["system"]["hosts"][0]
|
|
assert "severity" in body["ups"]["ups"][0]
|
|
|
|
|
|
# `/` paints two panels; the payload is still the whole-fleet contract. Every path here is
|
|
# read by something that is not the overview page: the live suite, an external probe, a
|
|
# notifier later. Deleting a card must never delete a key.
|
|
SUMMARY_CONTRACT = ("system.enabled", "system.fleet", "system.hosts",
|
|
"llm.engines", "ups.ups", "recent_events", "alerts")
|
|
|
|
|
|
def dig(body: dict[str, Any], path: str) -> Any:
|
|
"""`body["a"]["b"]` that names the missing path when it is not there."""
|
|
node: Any = body
|
|
for part in path.split("."):
|
|
assert isinstance(node, dict) and part in node, f"/api/summary lost `{path}`"
|
|
node = node[part]
|
|
return node
|
|
|
|
|
|
@pytest.mark.parametrize("fixture", ["client", "data_client"])
|
|
def test_summary_serves_every_path_another_consumer_reads(request, fixture: str) -> None:
|
|
"""The keys survive the overview cleanup whether or not anything is configured.
|
|
|
|
Run twice on purpose: `client` has no targets at all, so it proves the keys are present
|
|
as `None`/`[]` rather than omitted when a source is unconfigured (invariant 3 is about
|
|
the payload shape as much as about colour); `data_client` proves the same paths carry
|
|
the fleet once there is one.
|
|
"""
|
|
test_client = request.getfixturevalue(fixture)
|
|
body = test_client.get("/api/summary").json()
|
|
populated = fixture == "data_client"
|
|
for path in SUMMARY_CONTRACT:
|
|
dig(body, path) # presence is the assertion; a missing path raises with its name
|
|
assert dig(body, "system.enabled") is populated
|
|
assert (dig(body, "system.fleet") is not None) is populated
|
|
names = {host["name"] for host in dig(body, "system.hosts")}
|
|
assert names == ({"driveripper"} if populated else set())
|
|
batteries = {item["name"] for item in dig(body, "ups.ups")}
|
|
assert batteries == ({"apc", "cyberpower"} if populated else set())
|
|
assert dig(body, "llm.engines") == [], "neither fixture configures an engine"
|
|
assert isinstance(dig(body, "recent_events"), list)
|
|
assert isinstance(dig(body, "alerts"), list)
|
|
# and the payload as a whole still carries every top-level section it always did
|
|
assert set(body) >= {"ts", "system", "llm", "ups", "recent_events", "alerts"}
|
|
|
|
|
|
# ---- alerts: the landing page's answer to "what is wrong" ------------------- #
|
|
|
|
def test_summary_is_empty_of_alerts_when_everything_is_healthy(healthy_client) -> None:
|
|
"""`[]`, not a green banner: an empty list is the clearest possible all-clear."""
|
|
body = healthy_client.get("/api/summary").json()
|
|
assert body["alerts"] == []
|
|
assert body["system"]["fleet"]["hosts_live"] == 1
|
|
|
|
|
|
def test_summary_raises_one_alert_for_the_host_that_stopped_answering(data_client) -> None:
|
|
"""One dead host is one line naming it, with the link to the page that explains it."""
|
|
body = data_client.get("/api/summary").json()
|
|
down = [item for item in body["alerts"] if item["source"] == "driveripper"]
|
|
assert len(down) == 1, "its stale payload must not add amber lines on top of the red one"
|
|
assert down[0]["severity"] == "fail"
|
|
assert down[0]["href"] == "/system"
|
|
assert "unreachable" in down[0]["message"]
|
|
|
|
|
|
def test_a_host_that_has_never_answered_is_still_a_row_in_the_fleet(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A dead agent is a red row in the fleet, not a fleet that quietly got smaller.
|
|
|
|
`/api/system/snapshot` already stubs a job with no payload; `/api/summary` has to agree
|
|
with it, or a target that never answered since startup leaves the table at "2/2 hosts"
|
|
while the panel above says a third box is down (`AGENTS.md` invariants 3 and 4).
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
body = test_client.get("/api/summary").json()
|
|
rows = body["system"]["hosts"]
|
|
assert [row["name"] for row in rows] == ["driveripper"]
|
|
assert rows[0]["unreachable"] is True
|
|
assert rows[0]["cpu_usage_pct"] is None, "n/a, never a 0.0 nobody measured"
|
|
assert body["system"]["fleet"]["hosts_total"] == 1
|
|
assert body["system"]["fleet"]["hosts_live"] == 0
|
|
# and the alert list stays exactly one line for it, not a row plus a stale payload
|
|
assert [item["source"] for item in body["alerts"]] == ["driveripper"]
|
|
|
|
|
|
def test_summary_alerts_are_worst_first_and_carry_the_contract(data_client) -> None:
|
|
body = data_client.get("/api/summary").json()
|
|
alerts_list = body["alerts"]
|
|
assert alerts_list, "a dead host plus simulated UPS data cannot read as all-clear"
|
|
for item in alerts_list:
|
|
assert set(item) == {"id", "severity", "source", "message", "href", "at"}
|
|
assert item["severity"] in ("warn", "fail")
|
|
ranks = [0 if item["severity"] == "fail" else 1 for item in alerts_list]
|
|
assert ranks == sorted(ranks), "fails before warns"
|
|
assert any("simulat" in item["message"].lower() and item["href"] == "/ups"
|
|
for item in alerts_list), "NUT_SIMULATE must be said out loud on the landing page"
|
|
|
|
|
|
def test_summary_reuses_the_alert_builder_verbatim(data_client) -> None:
|
|
"""The endpoint is a thin call: what `alerts.build` says over the same state is the body."""
|
|
body = data_client.get("/api/summary").json()
|
|
hub = data_client.app.state.ctx.hub # type: ignore[attr-defined]
|
|
settings = data_client.app.state.ctx.settings # type: ignore[attr-defined]
|
|
hosts = [job.state.payload for job in hub.jobs.values()
|
|
if job.dashboard == "system" and job.state.payload]
|
|
ups_payload = hub.jobs["nut"].state.payload
|
|
expected = alerts.build(settings, hub.states(), hosts, [], ups_payload, [])
|
|
assert [item["id"] for item in body["alerts"]] == [item["id"] for item in expected]
|
|
|
|
|
|
def test_snapshot_payloads_carry_their_own_severity_sections(data_client) -> None:
|
|
"""/api/system/snapshot and /api/ups/snapshot ship the same verdicts the summary reuses."""
|
|
system = data_client.get("/api/system/snapshot").json()
|
|
host = system["hosts"][0]
|
|
assert set(host["cpu"]["severity"]) == {"usage_pct", "temp_c"}
|
|
assert set(host["memory"]["severity"]) == {"used_pct"}
|
|
assert set(host["disks"]["severity"]) == {"worst_used_pct"}
|
|
assert set(host["gpus"][0]["severity"]) == {"busy_pct", "mem_used_pct"}
|
|
ups = data_client.get("/api/ups/snapshot").json()
|
|
for card in ups["ups"]:
|
|
assert set(card["severity"]) == {"charge_pct", "load_pct", "battery_runtime"}
|
|
# numbers keep their own shape; severity is a sibling, never a replacement
|
|
assert isinstance(card["numbers"], dict) and "charge_pct" in card["numbers"]
|
|
|
|
|
|
# ---- unconfigured: say so, and do not dial -------------------------------- #
|
|
|
|
def test_config_reports_every_source_as_off(client) -> None:
|
|
body = client.get("/api/config").json()
|
|
assert body["system"]["enabled"] is False
|
|
assert body["system"]["targets"] == []
|
|
assert body["llm"]["gufo"]["enabled"] is False
|
|
assert body["llm"]["vllm"]["enabled"] is False
|
|
assert body["ups"]["enabled"] is False
|
|
assert body["ups"]["actions_enabled"] is False
|
|
assert body["ranges"], "the page builds its window buttons from this list"
|
|
|
|
|
|
def test_health_with_nothing_configured_is_not_a_failure(client) -> None:
|
|
"""Nothing configured is 'empty', not 'broken' — and the page must not go red."""
|
|
body = client.get("/api/health").json()
|
|
assert body["status"] in ("ok", "empty")
|
|
assert body["ok"] is True
|
|
assert body["sources"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("path", ["/api/system/snapshot", "/api/llm/snapshot",
|
|
"/api/ups/snapshot", "/api/ups/commands"])
|
|
def test_unconfigured_source_is_404_configured_false(client, path) -> None:
|
|
response = client.get(path)
|
|
assert response.status_code == 404
|
|
assert response.json()["configured"] is False
|
|
assert "detail" in response.json()
|
|
|
|
|
|
def test_summary_still_answers_with_no_sources(client) -> None:
|
|
body = client.get("/api/summary").json()
|
|
assert body["system"]["enabled"] is False
|
|
assert body["system"]["fleet"] is None
|
|
assert body["llm"]["enabled"] is False
|
|
assert body["ups"]["enabled"] is False
|
|
|
|
|
|
def test_unknown_route_is_json_not_html(client) -> None:
|
|
response = client.get("/api/nope")
|
|
assert response.status_code == 404
|
|
assert response.headers["content-type"].startswith("application/json")
|
|
assert "no such route" in response.json()["detail"]
|
|
|
|
|
|
# ---- tokens --------------------------------------------------------------- #
|
|
|
|
def test_agent_tokens_never_reach_the_browser(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("NUT_SIMULATE", "")
|
|
targets = [Target(name="driveripper", url="http://driveripper:9120",
|
|
headers={"Authorization": "Bearer SEKRIT-TOKEN"})]
|
|
db = str(tmp_path / "tokens.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, host_targets=tuple(targets)), store)
|
|
with TestClient(app) as client:
|
|
body = client.get("/api/config")
|
|
assert "SEKRIT-TOKEN" not in body.text
|
|
target = body.json()["system"]["targets"][0]
|
|
assert target["name"] == "driveripper"
|
|
assert "SEKRIT" not in target["url"] and "@" not in target["url"]
|
|
# and a host that never answered stays in the list, marked, with the fleet card intact
|
|
snapshot = wait_for_state(client, "/api/system/snapshot")
|
|
assert snapshot["hosts"][0]["name"] == "driveripper"
|
|
assert snapshot["hosts"][0].get("unreachable") is True
|
|
assert snapshot["fleet"]["hosts_total"] == 1
|
|
assert snapshot["fleet"]["hosts_live"] == 0
|
|
assert snapshot["fleet"]["memory_total_bytes"] is None
|
|
store.close()
|
|
|
|
|
|
def wait_for_state(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
response = client.get(path)
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
time.sleep(0.1)
|
|
pytest.fail(f"{path} never answered")
|
|
|
|
|
|
def test_target_problems_are_surfaced_not_swallowed(client) -> None:
|
|
body = client.get("/api/config").json()
|
|
assert body["system"]["target_problems"] == []
|
|
|
|
|
|
# ---- pages ---------------------------------------------------------------- #
|
|
|
|
@pytest.mark.parametrize("path,title", [
|
|
("/", "overview"), ("/system", "system"), ("/llm", "llm"), ("/ups", "ups"),
|
|
])
|
|
def test_pages_serve_and_declare_themselves(client, path, title) -> None:
|
|
response = client.get(path)
|
|
assert response.status_code == 200
|
|
assert "text/html" in response.headers["content-type"]
|
|
assert title in response.text.lower()
|
|
|
|
|
|
def test_no_page_loads_anything_from_the_internet(client) -> None:
|
|
"""No CDN, no webfonts: this dashboard has to work with no route to the internet."""
|
|
for path in ("/", "/system", "/llm", "/ups"):
|
|
html = client.get(path).text
|
|
for fragment in ("//cdn.", "https://fonts.", "//unpkg.com", "//cdnjs"):
|
|
assert fragment not in html, f"{path} references {fragment}"
|
|
|
|
|
|
def test_static_assets_are_served(client) -> None:
|
|
for asset in ("/static/dash.css", "/static/app.js", "/static/charts.js",
|
|
"/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js",
|
|
"/static/page-index.js"):
|
|
assert client.get(asset).status_code == 200, asset
|
|
|
|
|
|
# ---- the severity language in the static assets --------------------------- #
|
|
#
|
|
# There is no JS test runner (a locked anchor: adding one would mean a build step).
|
|
# What *can* be asserted end-to-end, from the served bytes, is the contract the pages
|
|
# depend on: the hero style exists, its colour comes from the shared ok/warn/fail tokens,
|
|
# no parallel `.sev-*` vocabulary appeared, and `Dash.hero()` reads the verdict instead of
|
|
# comparing the number to a threshold in JavaScript.
|
|
|
|
ASSETS = ("/static/dash.css", "/static/app.js", "/static/charts.js",
|
|
"/static/page-index.js", "/static/page-system.js", "/static/page-llm.js",
|
|
"/static/page-ups.js")
|
|
|
|
|
|
def _css(client: TestClient) -> str:
|
|
return client.get("/static/dash.css").text
|
|
|
|
|
|
def _hero_css(css: str) -> str:
|
|
"""Just the `hero numbers` section of the stylesheet."""
|
|
return css.split("/* ---- hero numbers", 1)[1].split("/* ---- bars", 1)[0]
|
|
|
|
|
|
def test_css_ships_the_hero_number_style(client) -> None:
|
|
css = _css(client)
|
|
hero_block = _hero_css(css)
|
|
for rule in (".hero {", ".hero .label", ".hero .value", ".hero .unit", ".hero .sub",
|
|
".hero.ok .value", ".hero.warn .value", ".hero.fail .value",
|
|
".hero-bar", ".hero-bar.ok .bar"):
|
|
assert rule in hero_block, f"dash.css lost the {rule} rule"
|
|
# a display size with fixed-width digits, so the number does not jitter between polls
|
|
assert "2.4rem" in hero_block
|
|
assert "tabular-nums" in hero_block
|
|
|
|
|
|
def test_hero_colour_comes_from_the_shared_verdict_tokens(client) -> None:
|
|
"""`.hero` reuses `--ok/--warn/--fail`; it does not invent a palette of its own."""
|
|
hero_block = _hero_css(_css(client))
|
|
for token in ("var(--ok)", "var(--warn)", "var(--fail)", "var(--text)",
|
|
"var(--text-muted)", "var(--text-faint)"):
|
|
assert token in hero_block, f"the hero style no longer uses {token}"
|
|
|
|
|
|
def test_no_stylesheet_rule_uses_an_undeclared_token(client) -> None:
|
|
"""A typo'd `var(--x)` renders unstyled and reads as a broken panel, not an error."""
|
|
css = _css(client)
|
|
root = re.search(r":root\s*\{(.*?)\n\}", css, re.S)
|
|
assert root is not None
|
|
declared = set(re.findall(r"(--[a-z0-9-]+)\s*:", root.group(1)))
|
|
used = set(re.findall(r"var\(\s*(--[a-z0-9-]+)", css))
|
|
assert not used - declared
|
|
|
|
|
|
def _code(body: str) -> str:
|
|
"""An asset with its comments stripped — prose may *name* a class the code must not define."""
|
|
body = re.sub(r"/\*.*?\*/", " ", body, flags=re.S)
|
|
body = re.sub(r"<!--.*?-->", " ", body, flags=re.S)
|
|
return re.sub(r"^\s*//.*$", " ", body, flags=re.M)
|
|
|
|
|
|
def test_no_parallel_severity_vocabulary_exists_anywhere(client) -> None:
|
|
"""One word per colour: `ok|warn|fail`, as in severity.py and the existing components."""
|
|
for asset in (*ASSETS, "/", "/system", "/llm", "/ups"):
|
|
body = _code(client.get(asset).text)
|
|
assert ".sev-" not in body, f"{asset} invented a .sev-* class"
|
|
assert not re.search(r"classList\.(?:add|toggle)\(\s*[\"'](?:danger|good|warning)[\"']",
|
|
body), f"{asset} invented a synonym for fail"
|
|
|
|
|
|
def test_app_js_exposes_the_hero_helpers(client) -> None:
|
|
js = client.get("/static/app.js").text
|
|
assert "function hero(" in js
|
|
assert "function severityFrom(" in js
|
|
exports = js.rsplit("return {", 1)[1].split("}", 1)[0]
|
|
for name in ("hero", "severityFrom", "severityFor"):
|
|
assert re.search(rf"\b{name}\b", exports), f"Dash.{name} is not exported"
|
|
|
|
|
|
def test_hero_reads_the_verdict_instead_of_a_threshold(client) -> None:
|
|
"""`hero()` may only colour what the API rated — no thresholds re-derived in JS."""
|
|
js = client.get("/static/app.js").text
|
|
start = js.index("function hero(")
|
|
body = js[start:js.index("\n function setDot", start)]
|
|
assert "severityFrom(" in body
|
|
assert "severityFor(" not in body
|
|
# an unknown value gets n/a and no colour class at all
|
|
assert "naText" in body
|
|
assert re.search(r"raw != null", body)
|
|
|
|
|
|
def test_severity_for_is_kept_as_the_documented_fallback(client) -> None:
|
|
"""The old call sites still work until the remaining phases convert each page.
|
|
|
|
`/` was converted in phase 02: the overview colours from the `severity` verdicts
|
|
`/api/summary` already sends, so it no longer carries a threshold of its own. The
|
|
three pages still to come must keep compiling against the fallback until they are.
|
|
"""
|
|
js = client.get("/static/app.js").text
|
|
assert "FALLBACK ONLY" in js
|
|
for page in ("/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js"):
|
|
assert "severityFor" in client.get(page).text, f"{page} lost its fallback call site"
|
|
|
|
|
|
def test_overview_colours_from_the_payload_not_from_js_thresholds(client) -> None:
|
|
"""`page-index.js` reads verdicts; it does not compare a number to a bound."""
|
|
js = client.get("/static/page-index.js").text
|
|
assert "severityFrom" in js
|
|
assert "severityFor" not in js
|
|
|
|
|
|
def test_overview_is_exactly_two_panels(client) -> None:
|
|
"""The landing page answers one question: what is wrong, and what is the fleet doing.
|
|
|
|
Asserted on the served bytes because there is no JS test runner (a locked anchor):
|
|
the summary cards and the raw event feed are what made this page busy, and once a
|
|
panel is gone a browser is the only thing that would notice its return.
|
|
"""
|
|
html = client.get("/").text
|
|
assert html.count("<section") == 2, "the overview grew a third panel"
|
|
assert 'id="alerts"' in html and 'id="alerts-clear"' in html
|
|
assert 'aria-live="polite"' in html
|
|
assert 'id="fleet-table"' in html
|
|
for gone in ("sys-cpu", "sys-mem", "sys-disk", "sys-gpus", "llm-decode", "llm-kv",
|
|
"ups-list", 'id="events"'):
|
|
assert gone not in html, f"the overview still renders {gone}"
|
|
# the header's source dots are chrome owned by app.js, not the deleted card dots
|
|
assert 'id="dot-system"' not in html and 'id="nav-status"' in html
|
|
|
|
|
|
def test_overview_alert_rows_link_to_the_dashboard_that_explains_them(client) -> None:
|
|
"""Every alert carries the `href` its row turns into a link; the page invents none."""
|
|
page_js = client.get("/static/page-index.js").text
|
|
assert "alert.href" in page_js
|
|
css = client.get("/static/dash.css").text
|
|
for selector in ("ul.alerts li.ok .src", "ul.alerts li.warn .src", "ul.alerts li.fail .src",
|
|
"table.data td.warn", "table.data td.fail"):
|
|
assert selector in css, f"dash.css lost {selector}"
|
|
# the swap that puts a live problem above the fleet table
|
|
assert 'data-state="alerting"' in css
|
|
|
|
|
|
def test_overview_dims_the_fleet_row_of_a_host_that_stopped_answering(client) -> None:
|
|
"""A dead host keeps its last good numbers, so the row has to say they are stale.
|
|
|
|
`AGENTS.md` invariant 4: keep the payload and mark it stale, never zero it. The
|
|
overview's table is the one place those last good numbers still print, so it reads the
|
|
per-source verdicts `/api/summary` sends (`system.states`) and applies the shared
|
|
`.stale` class the other pages already use — no new palette, no JS threshold.
|
|
"""
|
|
page_js = client.get("/static/page-index.js").text
|
|
assert "system.states" in page_js, "the fleet table stopped reading reachability"
|
|
assert 'class="stale"' in page_js
|
|
assert ".stale" in client.get("/static/dash.css").text
|
|
|
|
|
|
# Which script owns which page, for the orphan check below. `charts.js` and `app.js` are
|
|
# loaded by all four pages, so an id either page code or shared chrome may read.
|
|
PAGE_SCRIPTS = {
|
|
"/": "page-index.js",
|
|
"/system": "page-system.js",
|
|
"/llm": "page-llm.js",
|
|
"/ups": "page-ups.js",
|
|
}
|
|
SHARED_SCRIPTS = ("app.js", "charts.js")
|
|
|
|
|
|
def test_no_page_carries_an_id_nothing_reads(client) -> None:
|
|
"""An id no script queries, no stylesheet styles and no attribute points at is dead markup.
|
|
|
|
The summary cards were retired this way — a `<span id="sys-cpu">` left behind after its
|
|
writer was deleted is the same mistake in slower motion. Asserted on served bytes: the
|
|
browser console is otherwise the only thing that would notice.
|
|
"""
|
|
scripts = {name: client.get(f"/static/{name}").text for name in
|
|
(*SHARED_SCRIPTS, *PAGE_SCRIPTS.values(), "dash.css")}
|
|
for path, own_script in PAGE_SCRIPTS.items():
|
|
html = _code(client.get(path).text)
|
|
corpus = "".join(_code(scripts[name])
|
|
for name in (*SHARED_SCRIPTS, own_script, "dash.css"))
|
|
# ids the markup uses itself: labels, targets, `<label for>`
|
|
anchors = set(re.findall(
|
|
r'(?:aria-labelledby|aria-describedby|aria-controls|for)="([^"]+)"', html))
|
|
for element_id in re.findall(r'id="([^"]+)"', html):
|
|
quoted = (f'"{element_id}"' in corpus or f"'{element_id}'" in corpus
|
|
or f"#{element_id}" in corpus or element_id in anchors)
|
|
assert quoted, f"{path} has id=\"{element_id}\" that nothing reads"
|
|
|
|
|
|
def test_api_responses_are_not_cacheable(client) -> None:
|
|
for path in ("/api/config", "/api/health", "/api/summary"):
|
|
assert "no-store" in client.get(path).headers.get("cache-control", "")
|
|
|
|
|
|
# ---- history -------------------------------------------------------------- #
|
|
|
|
def test_history_returns_seeded_samples(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
db = str(tmp_path / "hist.sqlite3")
|
|
store = SampleStore(db)
|
|
now = time.time()
|
|
for i in range(30):
|
|
store.add("host", "driveripper", "cpu.usage_pct", 10.0 + i, ts=now - 300 + i)
|
|
app = create_app(Settings(db_path=db, host_targets=(Target("driveripper", DEAD_URL),)), store)
|
|
with TestClient(app) as client:
|
|
body = client.get("/api/history", params={"source": "host", "host": "driveripper",
|
|
"metric": "cpu.usage_pct", "range": "15m"})
|
|
assert response_ok(body)
|
|
points = body.json()["points"]
|
|
assert len(points) >= 25
|
|
assert points[-1]["value"] == pytest.approx(39.0)
|
|
store.close()
|
|
|
|
|
|
def test_history_rejects_an_unknown_range(client) -> None:
|
|
response = client.get("/api/history", params={"source": "host", "host": "h",
|
|
"metric": "m", "range": "banana"})
|
|
assert response.status_code == 400
|
|
|
|
|
|
def test_history_requires_the_series_coordinates(client) -> None:
|
|
assert client.get("/api/history").status_code == 422
|
|
|
|
|
|
@pytest.mark.parametrize(("raw", "seconds"), [
|
|
*[(key, float(seconds)) for key, seconds in RANGE_SECONDS.items()],
|
|
("45s", 45.0), ("13h", 46_800.0), ("7200", 7_200.0), (" 2D ", 172_800.0),
|
|
(None, 3_600.0), ("", 3_600.0),
|
|
])
|
|
def test_parse_range_reads_every_window_the_ui_offers(raw: Any, seconds: float) -> None:
|
|
"""Every chip key (5m…90d), any `<n><s|m|h|d>`, raw seconds, and the 1h default.
|
|
|
|
`parse_range` is the only thing between a typo'd query string and a chart of the wrong
|
|
window, and a wrong window is worse than an error: it still looks like data.
|
|
"""
|
|
assert parse_range(raw) == seconds
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["banana", "1w", "h", "-4h", "one"])
|
|
def test_parse_range_refuses_a_window_it_cannot_interpret(raw: str) -> None:
|
|
"""A 400 that names the value, never a silent fallback to 1h."""
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
parse_range(raw)
|
|
assert excinfo.value.status_code == 400
|
|
assert raw in str(excinfo.value.detail)
|
|
|
|
|
|
# ---- the routes the *other* pages chart from -------------------------------- #
|
|
#
|
|
# `/` renders two panels now; `/system`, `/llm` and `/ups` still need these, and so does
|
|
# anything outside the browser. Retiring overview code must not retire them, so the shapes
|
|
# are pinned here rather than discovered by a page three phases from now.
|
|
|
|
def test_system_history_charts_every_configured_host_in_one_request(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""N hosts x M metrics, one round trip — the chart endpoint's whole reason to exist."""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),
|
|
Target("deskwork", DEAD_URL))) as test_client:
|
|
body = test_client.get("/api/system/history",
|
|
params={"metrics": "cpu.usage_pct,mem.used_pct",
|
|
"range": "1h"}).json()
|
|
assert body["hosts"] == ["driveripper", "deskwork"]
|
|
assert body["range_seconds"] == 3_600.0
|
|
assert len(body["series"]) == 4
|
|
assert {(item["host"], item["metric"]) for item in body["series"]} == {
|
|
("driveripper", "cpu.usage_pct"), ("driveripper", "mem.used_pct"),
|
|
("deskwork", "cpu.usage_pct"), ("deskwork", "mem.used_pct")}
|
|
# an explicit `hosts` list narrows it instead of adding to the default
|
|
narrowed = test_client.get("/api/system/history",
|
|
params={"metrics": "cpu.usage_pct",
|
|
"hosts": "deskwork"}).json()
|
|
assert narrowed["hosts"] == ["deskwork"]
|
|
|
|
|
|
def test_llm_snapshot_keeps_an_engine_that_never_answered_in_the_list(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A configured-but-dead engine is listed as unreachable, not dropped (invariant 4)."""
|
|
# a host target too, so the snapshot's job filter has to skip a non-LLM job
|
|
with build_client(tmp_path, monkeypatch, vllm_url=DEAD_URL,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
body = test_client.get("/api/llm/snapshot").json()
|
|
assert [engine["engine"] for engine in body["engines"]] == ["vLLM"]
|
|
assert body["engines"][0]["unreachable"] is True
|
|
assert body["fleet"]["engines_total"] == 1
|
|
assert body["fleet"]["engines_live"] == 0
|
|
assert [state["key"] for state in body["states"]] == ["llm:vllm"]
|
|
|
|
|
|
def test_llm_history_charts_both_engines_by_default(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, gufo_url=DEAD_URL,
|
|
vllm_url=DEAD_URL) as test_client:
|
|
body = test_client.get("/api/llm/history",
|
|
params={"metrics": "decode_tokens_per_second,kv_cache_usage_ratio",
|
|
"range": "15m"}).json()
|
|
assert body["engines"] == ["gufo", "vllm"]
|
|
assert body["range_seconds"] == 900.0
|
|
assert {(item["host"], item["metric"]) for item in body["series"]} == {
|
|
("gufo", "llm.decode_tokens_per_second"),
|
|
("gufo", "llm.kv_cache_usage_ratio"),
|
|
("vllm", "llm.decode_tokens_per_second"),
|
|
("vllm", "llm.kv_cache_usage_ratio")}
|
|
|
|
|
|
def test_metrics_endpoint_lists_what_actually_has_history(client) -> None:
|
|
body = client.get("/api/metrics").json()
|
|
assert body["metrics"] == []
|
|
|
|
|
|
# ---- events --------------------------------------------------------------- #
|
|
|
|
def test_events_are_returned_in_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
db = str(tmp_path / "events.sqlite3")
|
|
store = SampleStore(db)
|
|
store.log_event(Event(ts=time.time(), source="host", host="deskwork", kind="down",
|
|
severity="error", message="deskwork is not answering"))
|
|
app = create_app(Settings(db_path=db), store)
|
|
with TestClient(app) as client:
|
|
events = client.get("/api/events").json()["events"]
|
|
assert events[0]["host"] == "deskwork"
|
|
assert events[0]["severity"] == "error"
|
|
store.close()
|
|
|
|
|
|
# ---- the one write endpoint ------------------------------------------------ #
|
|
|
|
def test_command_endpoint_is_post_only(ups_client) -> None:
|
|
assert ups_client.post("/api/ups/command", json={}).status_code in (400, 403, 404)
|
|
assert ups_client.put("/api/ups/command", json={}).status_code == 405
|
|
assert ups_client.delete("/api/ups/command").status_code == 405
|
|
|
|
|
|
def test_commands_are_offered_when_actions_are_enabled(ups_client) -> None:
|
|
body = wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
assert body["simulated"] is True, "the simulator must label itself"
|
|
assert body["actions_enabled"] is True
|
|
ups = body["ups"][0]
|
|
enabled = [item for item in ups["commands"] if item["enabled"]]
|
|
assert enabled, "with NUT_ALLOW_ACTIONS=1 and a simulator, safe commands are sendable"
|
|
assert all(item["risk"] in ("safe", "caution", "dangerous") for item in ups["commands"])
|
|
|
|
|
|
def test_a_safe_command_actually_reaches_the_driver(ups_client) -> None:
|
|
body = wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.panel.start"})
|
|
assert response.status_code == 200, response.text
|
|
payload = response.json()
|
|
assert payload["ok"] is True and payload["simulated"] is True
|
|
events = ups_client.get("/api/events").json()["events"]
|
|
assert any("test.panel.start" in event["message"] or "Panel test" in event["message"]
|
|
for event in events), "every sent command must leave an audit trail"
|
|
|
|
|
|
def test_a_dangerous_command_needs_the_confirmation_word(ups_client) -> None:
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
deep = "test.battery.start.deep"
|
|
refusal = ups_client.post("/api/ups/command", json={"ups": name, "command": deep})
|
|
assert refusal.status_code == 422
|
|
assert "DANGER" in refusal.json()["detail"]
|
|
assert ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": deep, "confirm": "DANGER"}).status_code == 200
|
|
|
|
|
|
def test_a_command_the_driver_does_not_advertise_is_refused_before_it_is_sent(
|
|
ups_client) -> None:
|
|
"""Being in the risk catalog is not permission; `LIST CMD` from the driver is (invariant 1).
|
|
|
|
`shutdown.return` is a real NUT command the catalog knows and this UPS does not offer,
|
|
so the refusal must come from the advertisement check and not from the confirmation gate.
|
|
"""
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "shutdown.return"})
|
|
assert response.status_code == 400
|
|
assert "does not advertise" in response.json()["detail"]
|
|
|
|
|
|
def test_cutting_the_output_is_refused_while_the_ups_is_already_on_battery(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Invariant 1, the one that matters most: no power cut while the pack is carrying the load.
|
|
|
|
Driven through the simulator's own state machine — a quick battery test puts the pack on
|
|
battery — so the 409 comes from the same poll the buttons are drawn from, with a 0.5s
|
|
interval to keep the wait short. Nothing here can touch real hardware: `NUT_SIMULATE=1`.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True, nut_interval=0.5) as test_client:
|
|
name = wait_for_payload(test_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
start = test_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.battery.start.quick"})
|
|
assert start.status_code == 200
|
|
deadline = time.time() + 6.0
|
|
card: dict[str, Any] = {}
|
|
while time.time() < deadline:
|
|
card = next(item for item in test_client.get("/api/ups/snapshot").json()["ups"]
|
|
if item["name"] == name)
|
|
if card["on_battery"]:
|
|
break
|
|
time.sleep(0.05)
|
|
assert card.get("on_battery") is True, "the quick test must put the simulated pack on battery"
|
|
refusal = test_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "load.off",
|
|
"confirm": "DANGER"})
|
|
assert refusal.status_code == 409
|
|
assert "on battery" in refusal.json()["detail"]
|
|
# and the same command is still offered once the pack is back on line power
|
|
assert not any(item["command"] == "load.off" and item["enabled"]
|
|
for item in card["commands"]), "the panel must not offer a blocked button"
|
|
|
|
|
|
class ScriptedNutClient:
|
|
"""`nut.NutClient` with the socket left out: the real code path, no battery attached.
|
|
|
|
`AGENTS.md` keeps live tests read-only, so this is where the one write endpoint's real
|
|
branch gets proven. The variable names are the ones `VAR_MAP` already pins from real
|
|
upsd captures; only the replies are scripted (a fake for behaviour, per the test policy).
|
|
"""
|
|
|
|
#: set per test: an exception instance to raise from `instcmd`, e.g. a refusal
|
|
refusal: Exception | None = None
|
|
sent: list[tuple[str, str]]
|
|
logged_in: list[str]
|
|
|
|
def __init__(self, host: str, port: int, *, timeout: float = 4.0,
|
|
username: str | None = None, password: str | None = None) -> None:
|
|
self.credentials = (username, password)
|
|
|
|
async def __aenter__(self) -> ScriptedNutClient:
|
|
return self
|
|
|
|
async def __aexit__(self, *_exc: object) -> None:
|
|
return None
|
|
|
|
async def list_ups(self) -> list[dict[str, str]]:
|
|
return [{"name": "apc", "description": "APC Back-UPS RS 1500MS"}]
|
|
|
|
async def list_vars(self, ups: str) -> dict[str, str]:
|
|
return {"ups.mfr": "APC", "ups.model": "Back-UPS RS 1500MS",
|
|
"ups.status": "OL", "battery.charge": "100", "ups.load": "18",
|
|
"battery.voltage": "13.6", "input.voltage": "230",
|
|
"battery.mfr.date": "2022-04", "ups.test.result": "Ok last test passed"}
|
|
|
|
async def list_cmds(self, ups: str) -> list[str]:
|
|
return ["test.panel.start", "beeper.mute", "load.off"]
|
|
|
|
async def list_rw_vars(self, ups: str) -> list[str]:
|
|
return []
|
|
|
|
async def login(self) -> None:
|
|
assert self.credentials[0], "a command send must authenticate as a named user"
|
|
ScriptedNutClient.logged_in.append(str(self.credentials[0]))
|
|
|
|
async def instcmd(self, ups: str, command: str) -> str:
|
|
if ScriptedNutClient.refusal is not None:
|
|
raise ScriptedNutClient.refusal
|
|
ScriptedNutClient.sent.append((ups, command))
|
|
return "OK Start Panel Test"
|
|
|
|
|
|
@pytest.fixture
|
|
def real_upsd_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[type[ScriptedNutClient]]:
|
|
"""Swap the module-level `NutClient` so the collector and the send use the same stand-in."""
|
|
ScriptedNutClient.sent = []
|
|
ScriptedNutClient.logged_in = []
|
|
ScriptedNutClient.refusal = None
|
|
monkeypatch.setattr(nut, "NutClient", ScriptedNutClient)
|
|
yield ScriptedNutClient
|
|
|
|
|
|
@pytest.fixture
|
|
def nut_client_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
real_upsd_client: type[ScriptedNutClient]) -> Iterator[TestClient]:
|
|
"""A non-simulated UPS app: real code paths, scripted upsd, 0.5s polls."""
|
|
with build_client(tmp_path, monkeypatch, nut_host="127.0.0.1", nut_user="dashboards",
|
|
nut_password="not-the-real-one", nut_allow_actions=True,
|
|
nut_interval=0.5) as test_client:
|
|
yield test_client
|
|
|
|
|
|
def test_a_real_upsd_send_authenticates_sends_and_audits(nut_client_app,
|
|
real_upsd_client) -> None:
|
|
body = wait_for_payload(nut_client_app, "/api/ups/snapshot")
|
|
assert body["simulated"] is False, "this test is about the branch that talks to upsd"
|
|
name = body["ups"][0]["name"]
|
|
response = nut_client_app.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.panel.start"})
|
|
assert response.status_code == 200, response.text
|
|
assert response.json()["simulated"] is False
|
|
assert real_upsd_client.sent == [(name, "test.panel.start")]
|
|
assert real_upsd_client.logged_in, "instcmd requires a LOGIN first (upsd refuses otherwise)"
|
|
events = nut_client_app.get("/api/events").json()["events"]
|
|
assert any(event["kind"] == "action" and name in event["message"] for event in events)
|
|
|
|
|
|
def test_a_refusal_from_upsd_is_a_502_and_an_error_event(nut_client_app,
|
|
real_upsd_client) -> None:
|
|
real_upsd_client.refusal = nut.NutError("Access denied", code="ACCESS-DENIED")
|
|
name = wait_for_payload(nut_client_app, "/api/ups/snapshot")["ups"][0]["name"]
|
|
response = nut_client_app.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 502
|
|
detail = response.json()["detail"]
|
|
assert "ACCESS-DENIED" in detail or "Access denied" in detail, detail
|
|
assert real_upsd_client.sent == [], "a refused command was never accepted"
|
|
events = nut_client_app.get("/api/events").json()["events"]
|
|
failures = [event for event in events if event["kind"] == "action"
|
|
and event["severity"] == "error"]
|
|
assert failures, "an upsd refusal has to leave a trace, not just an HTTP status"
|
|
|
|
|
|
def test_unknown_commands_are_refused_even_with_actions_on(ups_client) -> None:
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
for command in ("load.format", "rm -rf /", "INSTCMD x y"):
|
|
response = ups_client.post("/api/ups/command", json={"ups": name, "command": command})
|
|
assert response.status_code == 400, command
|
|
|
|
|
|
def test_commands_for_an_unknown_ups_are_404(ups_client) -> None:
|
|
wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": "not-real", "command": "beeper.mute"})
|
|
assert response.status_code == 404
|
|
|
|
|
|
def test_missing_fields_are_a_400(ups_client) -> None:
|
|
assert ups_client.post("/api/ups/command", json={"ups": ""}).status_code == 400
|
|
|
|
|
|
def test_actions_flag_off_blocks_every_command(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1") as client:
|
|
body = wait_for_payload(client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
assert body["actions_enabled"] is False
|
|
response = client.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 403
|
|
assert "NUT_ALLOW_ACTIONS" in response.json()["detail"]
|
|
|
|
|
|
def test_read_only_beats_the_actions_flag(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True, read_only=True) as client:
|
|
body = wait_for_payload(client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
response = client.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 403
|
|
commands = wait_for_payload(client, "/api/ups/commands")
|
|
assert commands["read_only"] is True
|
|
assert all(item["enabled"] is False for ups in commands["ups"]
|
|
for item in ups["commands"])
|
|
|
|
|
|
def test_ups_history_answers_for_simulated_ups(ups_client) -> None:
|
|
wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
time.sleep(1.2)
|
|
body = ups_client.get("/api/ups/history", params={"metrics": "ups.charge_pct",
|
|
"range": "1h"}).json()
|
|
assert body["ups"], "the UPS page needs a name to chart before any history exists"
|
|
|
|
|
|
def response_ok(response) -> bool:
|
|
assert response.status_code == 200, response.text
|
|
return True
|
|
|
|
|
|
def test_samples_written_by_the_poller_are_queryable(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The write path (poller -> store) and the read path (/api/history) must agree on names."""
|
|
db = str(tmp_path / "roundtrip.sqlite3")
|
|
store = SampleStore(db)
|
|
store.add_many([Sample("llm", "gufo", "llm.decode_tokens_per_second", 40.0, time.time() - 60),
|
|
Sample("llm", "gufo", "llm.decode_tokens_per_second", 45.0, time.time() - 30)])
|
|
app = create_app(Settings(db_path=db), store)
|
|
with TestClient(app) as client:
|
|
metrics = client.get("/api/metrics", params={"source": "llm"}).json()["metrics"]
|
|
assert "llm.decode_tokens_per_second" in metrics
|
|
points = client.get("/api/history", params={"source": "llm", "host": "gufo",
|
|
"metric": "llm.decode_tokens_per_second",
|
|
"range": "15m"}).json()["points"]
|
|
assert [p["value"] for p in points] == [40.0, 45.0]
|
|
store.close()
|