ducoterra
f44a617ef9
**Phase 04 final verification pass — all tasks already in `complete/`; verified + fixed 3 defects found in verification.**
- Verified `/llm` live (both engines, gufo-only instance on :8099, `llm_monitor` on :8095) in a real browser; dev server left running/healthy.
- Completed task-04's truncated layout fix: `#engines` now carries `.items-start` (the CSS rule existed but nothing used it) — gufo 544 px vs vLLM 715 px, no blank card; pinned by `test_llm_engine_panels_are_not_stretched_to_their_tallest_sibling`.
- Fixed: event rows said the subject twice ("gufo gufo is answering again"), and `/ups` mangled mid-message names ("sent to apc: OK" → "sent to OK") → shared `Dash.eventText()` (strips `UpstreamError: ` + a *leading* name only, mirroring `alerts._clean_error`) used by `/system`, `/llm`, `/ups`; new test.
- Fixed: count sparklines autoscaled rollup averages (queue max 0.1379) into full-height spikes → `sparkline({minSpan})`, `minSpan: 1` on sessions/queued (spike 30 px → 4 px); assertions added.
- Tests: `bash .agents/validate.sh` → **EXIT=0** (`uv run pytest --cov --cov-fail-under=94`: **485 passed, 7 skipped**, coverage **96.87%**; `ruff check src tests agent`: All checks passed; `pyright`: 0 errors; `node --check` all static JS clean; repo hygiene clean; `uv pip list --outdated` empty). `uv run pytest -m live --live` → **7 passed**.
- Completion criteria: heroes largest per engine ✔ (38.4 px vs 16.8 px row values); sparkline per cell + selected window ✔ (5/5 gufo, 7/7 vLLM; 6 h chip moved KV max 53 %→58 %); mean+p95 without hover ✔ (243 / 48.8 / 285 ms feet); provenance tag ✔ (`Δ10.0s`/`Δ8.0s` + `title`=exact source); single-engine ✔ (1 panel, totals hidden, zero red); screenshots ✔ `data/logs/llm-verify-{both-engines,charts-and-slots,single-engine,monitor-reference}.png` + `NOTES-chart-set.md`; pytest/cov/ruff/pyright ✔; earlier phases unchanged-behaviour ✔ (all 01–03 tests pass).
- Deviation (documented, not code): throughput heroes stay **neutral ink** — `severity.py` has no throughput row and 0 tok/s idle is healthy; colour comes from rated cells (KV) + reachability. Kept per `NOTES-chart-set`.
- Deviation: could not append the task-04 written verdict to `NOTES-chart-set.md` (it lives in `todo/`, which I must not edit). Verdict: ours reads better than `:8095` — bigger throughput numbers, per-cell shape over the selected window, percentiles inline; `llm_monitor` only sparklines decode+KV and has no window control.
- Docs: AGENTS.md updated (485 test count, `sparkline` `minSpan` rule, `Dash.eventText` event-row rule, `/llm` panel `items-start`); README reviewed in full — commands, config table (all `config.py` vars present), architecture bullets, page descriptions verified accurate; no other inconsistencies found.
- Note for the harness: `.agents/phases/todo/03…/NOTES-verify.md` and `todo/04…/NOTES-chart-set.md` are untracked notes left behind by their phases (task files moved, notes did not).
- Next pending phase: `05_ups_hero_battery_glyph`.
1575 lines
81 KiB
Python
1575 lines
81 KiB
Python
"""HTTP contract: what the pages may assume, and what must never leak.
|
|
|
|
The behaviours here are the ones a screenshot cannot show: an unconfigured source answering
|
|
404 `configured: false` instead of dialling a hostname that does not exist; a bearer token
|
|
that stops at the backend; the single write endpoint refusing on every gate; no external
|
|
asset the network can fail to deliver.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import time
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from fastapi import HTTPException
|
|
from fastapi.testclient import TestClient
|
|
|
|
from dashboards import alerts
|
|
from dashboards.api import RANGE_SECONDS, parse_range
|
|
from dashboards.app import create_app
|
|
from dashboards.config import Settings, Target
|
|
from dashboards.severity import classify
|
|
from dashboards.sources import nut
|
|
from dashboards.sources.llm import HISTORY_METRICS
|
|
from dashboards.store import Event, Sample, SampleStore
|
|
|
|
# refused immediately, and never dials out of the test environment
|
|
DEAD_URL = "http://127.0.0.1:1"
|
|
|
|
|
|
@contextmanager
|
|
def build_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, simulate: bool = False,
|
|
**overrides: Any) -> Iterator[TestClient]:
|
|
"""A TestClient whose lifespan runs, against a private DB and no real network.
|
|
|
|
The store is injected, so `create_app()` deliberately does not close it (production
|
|
owns its own); the caller here does, or every test leaks a SQLite handle and the
|
|
`--cov` log fills with ResourceWarnings that bury the warnings worth reading.
|
|
"""
|
|
if simulate:
|
|
monkeypatch.setenv("NUT_SIMULATE", "1")
|
|
else:
|
|
monkeypatch.delenv("NUT_SIMULATE", raising=False)
|
|
db = str(tmp_path / "api.sqlite3")
|
|
settings = Settings(db_path=db, **overrides)
|
|
store = SampleStore(db)
|
|
client = TestClient(create_app(settings, store))
|
|
client.__enter__()
|
|
try:
|
|
yield client
|
|
finally:
|
|
client.__exit__(None, None, None)
|
|
store.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
|
with build_client(tmp_path, monkeypatch) as test_client:
|
|
yield test_client
|
|
|
|
|
|
@pytest.fixture
|
|
def ups_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
|
"""NUT on the labelled simulator, actions allowed — the dev configuration."""
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True) as c:
|
|
yield c
|
|
|
|
|
|
def wait_for_payload(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
response = client.get(path)
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
time.sleep(0.1)
|
|
pytest.fail(f"{path} never became available")
|
|
|
|
|
|
def failing_host_payload(scrape: Any, agent_snapshot: dict[str, Any]) -> dict[str, Any]:
|
|
"""Two scrapes of the real driveripper capture, nudged onto known sides of the table.
|
|
|
|
Driven from the fixture rather than hand-written so a schema drift breaks here first.
|
|
"""
|
|
def nudge(snap: dict[str, Any]) -> None:
|
|
snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.95 # mem.used_pct -> fail
|
|
snap["cpu_temperature_celsius"] = 91.0 # cpu.temp_c -> fail
|
|
for fs in snap["filesystems"]:
|
|
fs["used_pct"] = 93.0 # disk -> fail
|
|
|
|
return scrape(agent_snapshot, cpu_busy=10.0, mutate=nudge)
|
|
|
|
|
|
@pytest.fixture
|
|
def data_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
agent_snapshot: dict[str, Any], host_scrape: Any) -> Iterator[TestClient]:
|
|
"""Overview-shaped app: one host payload in the hub plus the simulated UPSes.
|
|
|
|
The host job points at a dead port, so a poll can never overwrite the injected payload
|
|
(a failed poll keeps the last good one) — the numbers below are ours, not the fleet's.
|
|
"""
|
|
monkeypatch.setenv("NUT_SIMULATE", "1")
|
|
db = str(tmp_path / "summary.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, nut_host="127.0.0.1",
|
|
host_targets=(Target("driveripper", DEAD_URL),)), store)
|
|
test_client = TestClient(app)
|
|
test_client.__enter__()
|
|
try:
|
|
app.state.ctx.hub.jobs["host:driveripper"].state.payload = failing_host_payload(
|
|
host_scrape, agent_snapshot)
|
|
wait_for_payload(test_client, "/api/ups/snapshot")
|
|
yield test_client
|
|
finally:
|
|
test_client.__exit__(None, None, None)
|
|
store.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def healthy_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
agent_snapshot: dict[str, Any],
|
|
host_scrape: Any) -> Iterator[TestClient]:
|
|
"""The all-clear app: one host answering, nothing past a threshold, no UPS configured.
|
|
|
|
The job's collector is swapped for one that hands back the injected payload, so the
|
|
health record says `ok` truthfully instead of being overridden behind the poller.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
from dashboards.poller import Result # noqa: PLC0415 - only this fixture needs it
|
|
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
|
|
async def collect() -> Result:
|
|
return Result(payload=payload, samples=[])
|
|
|
|
job = test_client.app.state.ctx.hub.jobs["host:driveripper"] # type: ignore[attr-defined]
|
|
job.fn = collect
|
|
# let the poll already in flight against the dead port land first (it fails in
|
|
# microseconds), then take the health record over: the alerts must describe a host
|
|
# that is answering, not the half-second of startup where it was not
|
|
deadline = time.time() + 2.0
|
|
while time.time() < deadline and job.state.attempts == 0:
|
|
time.sleep(0.02)
|
|
time.sleep(0.05)
|
|
job.state.payload = payload
|
|
job.state.ok = True
|
|
job.state.error = None
|
|
job.state.consecutive_failures = 0
|
|
job.state.last_ok = time.time()
|
|
yield test_client
|
|
|
|
|
|
# ---- severity travels with the value --------------------------------------- #
|
|
|
|
def test_summary_carries_the_verdict_for_every_column_the_page_colours(data_client) -> None:
|
|
body = data_client.get("/api/summary").json()
|
|
row = body["system"]["hosts"][0]
|
|
assert row["severity"] == {"cpu_usage_pct": "ok", "mem_used_pct": "fail",
|
|
"disk_worst_used_pct": "fail", "temp_c": "fail"}
|
|
# the fleet card is judged by the same table, so amber means the same thing at both sizes
|
|
assert body["system"]["fleet"]["severity"] == {
|
|
"cpu_usage_pct_avg": "ok", "memory_used_pct": "fail",
|
|
"disk_worst_used_pct": "fail", "gpu_busy_pct_avg": "ok",
|
|
}
|
|
for ups in body["ups"]["ups"]:
|
|
assert set(ups["severity"]) == {"charge_pct", "load_pct", "runtime_seconds"}
|
|
assert ups["severity"]["charge_pct"] == classify("ups.charge_pct", ups["charge_pct"])
|
|
assert ups["severity"]["charge_pct"] == "ok", "the simulator starts at float charge"
|
|
|
|
|
|
def test_summary_shape_did_not_shrink(data_client) -> None:
|
|
"""Additive-only anchor: /api/summary gained `severity` and lost nothing.
|
|
|
|
The overview, the live tests, and any external consumer read the flat keys; this test is
|
|
the tripwire for a well-meaning refactor that "tidies" them away.
|
|
"""
|
|
body = data_client.get("/api/summary").json()
|
|
fleet_keys = {"hosts_total", "hosts_live", "cpu_cores", "cpu_usage_pct_avg",
|
|
"memory_total_bytes", "memory_used_bytes", "memory_used_pct",
|
|
"disk_worst_used_pct", "load1_total", "net_rx_bytes_per_s",
|
|
"net_tx_bytes_per_s", "gpu_count", "gpu_busy_pct_avg", "power_estimate_w"}
|
|
host_keys = {"name", "os", "cpu_usage_pct", "load1", "cores", "mem_used_pct",
|
|
"disk_worst_used_pct", "gpu_busy_pct", "temp_c", "uptime_seconds"}
|
|
ups_keys = {"name", "overall", "status", "charge_pct", "load_pct", "runtime_seconds"}
|
|
assert fleet_keys <= set(body["system"]["fleet"]), fleet_keys - set(body["system"]["fleet"])
|
|
assert host_keys <= set(body["system"]["hosts"][0]), host_keys - set(body["system"]["hosts"][0])
|
|
assert ups_keys <= set(body["ups"]["ups"][0]), ups_keys - set(body["ups"]["ups"][0])
|
|
assert set(body) == {"ts", "system", "llm", "ups", "recent_events", "alerts"}
|
|
# ...and the new keys are there beside the old ones, not instead of them
|
|
assert "severity" in body["system"]["fleet"]
|
|
assert "severity" in body["system"]["hosts"][0]
|
|
assert "severity" in body["ups"]["ups"][0]
|
|
|
|
|
|
# `/` paints two panels; the payload is still the whole-fleet contract. Every path here is
|
|
# read by something that is not the overview page: the live suite, an external probe, a
|
|
# notifier later. Deleting a card must never delete a key.
|
|
SUMMARY_CONTRACT = ("system.enabled", "system.fleet", "system.hosts",
|
|
"llm.engines", "ups.ups", "recent_events", "alerts")
|
|
|
|
|
|
def dig(body: dict[str, Any], path: str) -> Any:
|
|
"""`body["a"]["b"]` that names the missing path when it is not there."""
|
|
node: Any = body
|
|
for part in path.split("."):
|
|
assert isinstance(node, dict) and part in node, f"/api/summary lost `{path}`"
|
|
node = node[part]
|
|
return node
|
|
|
|
|
|
@pytest.mark.parametrize("fixture", ["client", "data_client"])
|
|
def test_summary_serves_every_path_another_consumer_reads(request, fixture: str) -> None:
|
|
"""The keys survive the overview cleanup whether or not anything is configured.
|
|
|
|
Run twice on purpose: `client` has no targets at all, so it proves the keys are present
|
|
as `None`/`[]` rather than omitted when a source is unconfigured (invariant 3 is about
|
|
the payload shape as much as about colour); `data_client` proves the same paths carry
|
|
the fleet once there is one.
|
|
"""
|
|
test_client = request.getfixturevalue(fixture)
|
|
body = test_client.get("/api/summary").json()
|
|
populated = fixture == "data_client"
|
|
for path in SUMMARY_CONTRACT:
|
|
dig(body, path) # presence is the assertion; a missing path raises with its name
|
|
assert dig(body, "system.enabled") is populated
|
|
assert (dig(body, "system.fleet") is not None) is populated
|
|
names = {host["name"] for host in dig(body, "system.hosts")}
|
|
assert names == ({"driveripper"} if populated else set())
|
|
batteries = {item["name"] for item in dig(body, "ups.ups")}
|
|
assert batteries == ({"apc", "cyberpower"} if populated else set())
|
|
assert dig(body, "llm.engines") == [], "neither fixture configures an engine"
|
|
assert isinstance(dig(body, "recent_events"), list)
|
|
assert isinstance(dig(body, "alerts"), list)
|
|
# and the payload as a whole still carries every top-level section it always did
|
|
assert set(body) >= {"ts", "system", "llm", "ups", "recent_events", "alerts"}
|
|
|
|
|
|
# ---- alerts: the landing page's answer to "what is wrong" ------------------- #
|
|
|
|
def test_summary_is_empty_of_alerts_when_everything_is_healthy(healthy_client) -> None:
|
|
"""`[]`, not a green banner: an empty list is the clearest possible all-clear."""
|
|
body = healthy_client.get("/api/summary").json()
|
|
assert body["alerts"] == []
|
|
assert body["system"]["fleet"]["hosts_live"] == 1
|
|
|
|
|
|
def test_summary_raises_one_alert_for_the_host_that_stopped_answering(data_client) -> None:
|
|
"""One dead host is one line naming it, with the link to the page that explains it."""
|
|
body = data_client.get("/api/summary").json()
|
|
down = [item for item in body["alerts"] if item["source"] == "driveripper"]
|
|
assert len(down) == 1, "its stale payload must not add amber lines on top of the red one"
|
|
assert down[0]["severity"] == "fail"
|
|
assert down[0]["href"] == "/system"
|
|
assert "unreachable" in down[0]["message"]
|
|
|
|
|
|
def test_a_host_that_has_never_answered_is_still_a_row_in_the_fleet(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A dead agent is a red row in the fleet, not a fleet that quietly got smaller.
|
|
|
|
`/api/system/snapshot` already stubs a job with no payload; `/api/summary` has to agree
|
|
with it, or a target that never answered since startup leaves the table at "2/2 hosts"
|
|
while the panel above says a third box is down (`AGENTS.md` invariants 3 and 4).
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
body = test_client.get("/api/summary").json()
|
|
rows = body["system"]["hosts"]
|
|
assert [row["name"] for row in rows] == ["driveripper"]
|
|
assert rows[0]["unreachable"] is True
|
|
assert rows[0]["cpu_usage_pct"] is None, "n/a, never a 0.0 nobody measured"
|
|
assert body["system"]["fleet"]["hosts_total"] == 1
|
|
assert body["system"]["fleet"]["hosts_live"] == 0
|
|
# and the alert list stays exactly one line for it, not a row plus a stale payload
|
|
assert [item["source"] for item in body["alerts"]] == ["driveripper"]
|
|
|
|
|
|
def test_summary_alerts_are_worst_first_and_carry_the_contract(data_client) -> None:
|
|
body = data_client.get("/api/summary").json()
|
|
alerts_list = body["alerts"]
|
|
assert alerts_list, "a dead host plus simulated UPS data cannot read as all-clear"
|
|
for item in alerts_list:
|
|
assert set(item) == {"id", "severity", "source", "message", "href", "at"}
|
|
assert item["severity"] in ("warn", "fail")
|
|
ranks = [0 if item["severity"] == "fail" else 1 for item in alerts_list]
|
|
assert ranks == sorted(ranks), "fails before warns"
|
|
assert any("simulat" in item["message"].lower() and item["href"] == "/ups"
|
|
for item in alerts_list), "NUT_SIMULATE must be said out loud on the landing page"
|
|
|
|
|
|
def test_summary_reuses_the_alert_builder_verbatim(data_client) -> None:
|
|
"""The endpoint is a thin call: what `alerts.build` says over the same state is the body."""
|
|
body = data_client.get("/api/summary").json()
|
|
hub = data_client.app.state.ctx.hub # type: ignore[attr-defined]
|
|
settings = data_client.app.state.ctx.settings # type: ignore[attr-defined]
|
|
hosts = [job.state.payload for job in hub.jobs.values()
|
|
if job.dashboard == "system" and job.state.payload]
|
|
ups_payload = hub.jobs["nut"].state.payload
|
|
expected = alerts.build(settings, hub.states(), hosts, [], ups_payload, [])
|
|
assert [item["id"] for item in body["alerts"]] == [item["id"] for item in expected]
|
|
|
|
|
|
def test_snapshot_payloads_carry_their_own_severity_sections(data_client) -> None:
|
|
"""/api/system/snapshot and /api/ups/snapshot ship the same verdicts the summary reuses."""
|
|
system = data_client.get("/api/system/snapshot").json()
|
|
host = system["hosts"][0]
|
|
assert set(host["cpu"]["severity"]) == {"usage_pct", "temp_c"}
|
|
assert set(host["memory"]["severity"]) == {"used_pct"}
|
|
assert set(host["disks"]["severity"]) == {"worst_used_pct"}
|
|
assert set(host["gpus"][0]["severity"]) == {"busy_pct", "mem_used_pct"}
|
|
ups = data_client.get("/api/ups/snapshot").json()
|
|
for card in ups["ups"]:
|
|
assert set(card["severity"]) == {"charge_pct", "load_pct", "battery_runtime"}
|
|
# numbers keep their own shape; severity is a sibling, never a replacement
|
|
assert isinstance(card["numbers"], dict) and "charge_pct" in card["numbers"]
|
|
|
|
|
|
# ---- unconfigured: say so, and do not dial -------------------------------- #
|
|
|
|
def test_config_reports_every_source_as_off(client) -> None:
|
|
body = client.get("/api/config").json()
|
|
assert body["system"]["enabled"] is False
|
|
assert body["system"]["targets"] == []
|
|
assert body["llm"]["gufo"]["enabled"] is False
|
|
assert body["llm"]["vllm"]["enabled"] is False
|
|
assert body["ups"]["enabled"] is False
|
|
assert body["ups"]["actions_enabled"] is False
|
|
assert body["ranges"], "the page builds its window buttons from this list"
|
|
|
|
|
|
def test_health_with_nothing_configured_is_not_a_failure(client) -> None:
|
|
"""Nothing configured is 'empty', not 'broken' — and the page must not go red."""
|
|
body = client.get("/api/health").json()
|
|
assert body["status"] in ("ok", "empty")
|
|
assert body["ok"] is True
|
|
assert body["sources"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("path", ["/api/system/snapshot", "/api/llm/snapshot",
|
|
"/api/ups/snapshot", "/api/ups/commands"])
|
|
def test_unconfigured_source_is_404_configured_false(client, path) -> None:
|
|
response = client.get(path)
|
|
assert response.status_code == 404
|
|
assert response.json()["configured"] is False
|
|
assert "detail" in response.json()
|
|
|
|
|
|
def test_summary_still_answers_with_no_sources(client) -> None:
|
|
body = client.get("/api/summary").json()
|
|
assert body["system"]["enabled"] is False
|
|
assert body["system"]["fleet"] is None
|
|
assert body["llm"]["enabled"] is False
|
|
assert body["ups"]["enabled"] is False
|
|
|
|
|
|
def test_unknown_route_is_json_not_html(client) -> None:
|
|
response = client.get("/api/nope")
|
|
assert response.status_code == 404
|
|
assert response.headers["content-type"].startswith("application/json")
|
|
assert "no such route" in response.json()["detail"]
|
|
|
|
|
|
# ---- tokens --------------------------------------------------------------- #
|
|
|
|
def test_agent_tokens_never_reach_the_browser(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("NUT_SIMULATE", "")
|
|
targets = [Target(name="driveripper", url="http://driveripper:9120",
|
|
headers={"Authorization": "Bearer SEKRIT-TOKEN"})]
|
|
db = str(tmp_path / "tokens.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, host_targets=tuple(targets)), store)
|
|
with TestClient(app) as client:
|
|
body = client.get("/api/config")
|
|
assert "SEKRIT-TOKEN" not in body.text
|
|
target = body.json()["system"]["targets"][0]
|
|
assert target["name"] == "driveripper"
|
|
assert "SEKRIT" not in target["url"] and "@" not in target["url"]
|
|
# and a host that never answered stays in the list, marked, with the fleet card intact
|
|
snapshot = wait_for_state(client, "/api/system/snapshot")
|
|
assert snapshot["hosts"][0]["name"] == "driveripper"
|
|
assert snapshot["hosts"][0].get("unreachable") is True
|
|
assert snapshot["fleet"]["hosts_total"] == 1
|
|
assert snapshot["fleet"]["hosts_live"] == 0
|
|
assert snapshot["fleet"]["memory_total_bytes"] is None
|
|
store.close()
|
|
|
|
|
|
def test_a_dead_host_never_writes_its_token_into_the_feed(tmp_path: Path) -> None:
|
|
"""The second path a credential can take to a browser: the failure message.
|
|
|
|
`parse_targets` moves a userinfo token out of the URL and into a bearer header, so the
|
|
documented `HOST_TARGETS` path never puts one in `Target.url` — this is the backstop for
|
|
any `Target` that does (a hand-built one, a future parser), and the message path is where
|
|
it would matter: the poller keeps an `UpstreamError` string in the health record and logs
|
|
it as an event, so a userinfo-bearing URL there would sit in SQLite and be served on
|
|
`/api/events` forever (AGENTS.md invariant 6). Asserted on every endpoint a page reads,
|
|
not just `/api/config`.
|
|
"""
|
|
db = str(tmp_path / "dead-host-token.sqlite3")
|
|
store = SampleStore(db)
|
|
settings = Settings(
|
|
db_path=db,
|
|
host_interval=0.05,
|
|
host_targets=(Target(name="driveripper", url="http://SEKRIT-AGENT-TOKEN@127.0.0.1:1"),),
|
|
)
|
|
with TestClient(create_app(settings, store)) as client:
|
|
deadline = time.time() + 10
|
|
while time.time() < deadline:
|
|
health = client.get("/api/health").json()
|
|
if health["sources"] and health["sources"][0]["error"]:
|
|
break
|
|
time.sleep(0.05)
|
|
else:
|
|
pytest.fail("the dead host never recorded an error")
|
|
for path in ("/api/health", "/api/events", "/api/system/snapshot", "/api/summary"):
|
|
body = client.get(path).text
|
|
assert "SEKRIT-AGENT-TOKEN" not in body, f"{path} leaked the agent token"
|
|
# and the outage is still legible — redaction is not the same as silence
|
|
assert "driveripper" in client.get("/api/health").text
|
|
assert "unreachable" in client.get("/api/events").text
|
|
store.close()
|
|
|
|
|
|
def test_engine_urls_are_redacted_the_same_way_hosts_are(tmp_path: Path) -> None:
|
|
"""`GUFO_BASE_URL`/`VLLM_BASE_URL` go to the browser verbatim in `/api/config`, and unlike
|
|
`HOST_TARGETS` nothing parses a userinfo token out of them into a header — so a URL that
|
|
happens to carry credentials has to be redacted where it is served (invariant 6).
|
|
"""
|
|
db = str(tmp_path / "engine-redact.sqlite3")
|
|
settings = Settings(db_path=db, gufo_url="http://SEKRIT-ENGINE-TOKEN@deskwork:8731",
|
|
vllm_url="http://driveripper:8000")
|
|
store = SampleStore(db)
|
|
with TestClient(create_app(settings, store)) as client:
|
|
body = client.get("/api/config")
|
|
assert "SEKRIT-ENGINE-TOKEN" not in body.text
|
|
llm = body.json()["llm"]
|
|
assert llm["gufo"]["base_url"] == "http://deskwork:8731"
|
|
assert llm["vllm"]["base_url"] == "http://driveripper:8000" # untouched when clean
|
|
store.close()
|
|
|
|
|
|
def wait_for_state(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
response = client.get(path)
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
time.sleep(0.1)
|
|
pytest.fail(f"{path} never answered")
|
|
|
|
|
|
def test_target_problems_are_surfaced_not_swallowed(client) -> None:
|
|
body = client.get("/api/config").json()
|
|
assert body["system"]["target_problems"] == []
|
|
|
|
|
|
# ---- pages ---------------------------------------------------------------- #
|
|
|
|
@pytest.mark.parametrize("path,title", [
|
|
("/", "overview"), ("/system", "system"), ("/llm", "llm"), ("/ups", "ups"),
|
|
])
|
|
def test_pages_serve_and_declare_themselves(client, path, title) -> None:
|
|
response = client.get(path)
|
|
assert response.status_code == 200
|
|
assert "text/html" in response.headers["content-type"]
|
|
assert title in response.text.lower()
|
|
|
|
|
|
def test_no_page_loads_anything_from_the_internet(client) -> None:
|
|
"""No CDN, no webfonts: this dashboard has to work with no route to the internet."""
|
|
for path in ("/", "/system", "/llm", "/ups"):
|
|
html = client.get(path).text
|
|
for fragment in ("//cdn.", "https://fonts.", "//unpkg.com", "//cdnjs"):
|
|
assert fragment not in html, f"{path} references {fragment}"
|
|
|
|
|
|
def test_static_assets_are_served(client) -> None:
|
|
for asset in ("/static/dash.css", "/static/app.js", "/static/charts.js",
|
|
"/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js",
|
|
"/static/page-index.js"):
|
|
assert client.get(asset).status_code == 200, asset
|
|
|
|
|
|
# ---- the severity language in the static assets --------------------------- #
|
|
#
|
|
# There is no JS test runner (a locked anchor: adding one would mean a build step).
|
|
# What *can* be asserted end-to-end, from the served bytes, is the contract the pages
|
|
# depend on: the hero style exists, its colour comes from the shared ok/warn/fail tokens,
|
|
# no parallel `.sev-*` vocabulary appeared, and `Dash.hero()` reads the verdict instead of
|
|
# comparing the number to a threshold in JavaScript.
|
|
|
|
ASSETS = ("/static/dash.css", "/static/app.js", "/static/charts.js",
|
|
"/static/page-index.js", "/static/page-system.js", "/static/page-llm.js",
|
|
"/static/page-ups.js")
|
|
|
|
|
|
def _css(client: TestClient) -> str:
|
|
return client.get("/static/dash.css").text
|
|
|
|
|
|
def _hero_css(css: str) -> str:
|
|
"""Just the `hero numbers` section of the stylesheet."""
|
|
return css.split("/* ---- hero numbers", 1)[1].split("/* ---- bars", 1)[0]
|
|
|
|
|
|
def test_css_ships_the_hero_number_style(client) -> None:
|
|
css = _css(client)
|
|
hero_block = _hero_css(css)
|
|
for rule in (".hero {", ".hero .label", ".hero .value", ".hero .unit", ".hero .sub",
|
|
".hero.ok .value", ".hero.warn .value", ".hero.fail .value",
|
|
".hero-bar", ".hero-bar.ok .bar"):
|
|
assert rule in hero_block, f"dash.css lost the {rule} rule"
|
|
# a display size with fixed-width digits, so the number does not jitter between polls
|
|
assert "2.4rem" in hero_block
|
|
assert "tabular-nums" in hero_block
|
|
|
|
|
|
def test_hero_colour_comes_from_the_shared_verdict_tokens(client) -> None:
|
|
"""`.hero` reuses `--ok/--warn/--fail`; it does not invent a palette of its own."""
|
|
hero_block = _hero_css(_css(client))
|
|
for token in ("var(--ok)", "var(--warn)", "var(--fail)", "var(--text)",
|
|
"var(--text-muted)", "var(--text-faint)"):
|
|
assert token in hero_block, f"the hero style no longer uses {token}"
|
|
|
|
|
|
def test_no_stylesheet_rule_uses_an_undeclared_token(client) -> None:
|
|
"""A typo'd `var(--x)` renders unstyled and reads as a broken panel, not an error."""
|
|
css = _css(client)
|
|
root = re.search(r":root\s*\{(.*?)\n\}", css, re.S)
|
|
assert root is not None
|
|
declared = set(re.findall(r"(--[a-z0-9-]+)\s*:", root.group(1)))
|
|
used = set(re.findall(r"var\(\s*(--[a-z0-9-]+)", css))
|
|
assert not used - declared
|
|
|
|
|
|
def _code(body: str) -> str:
|
|
"""An asset with its comments stripped — prose may *name* a class the code must not define."""
|
|
body = re.sub(r"/\*.*?\*/", " ", body, flags=re.S)
|
|
body = re.sub(r"<!--.*?-->", " ", body, flags=re.S)
|
|
return re.sub(r"^\s*//.*$", " ", body, flags=re.M)
|
|
|
|
|
|
def test_no_parallel_severity_vocabulary_exists_anywhere(client) -> None:
|
|
"""One word per colour: `ok|warn|fail`, as in severity.py and the existing components."""
|
|
for asset in (*ASSETS, "/", "/system", "/llm", "/ups"):
|
|
body = _code(client.get(asset).text)
|
|
assert ".sev-" not in body, f"{asset} invented a .sev-* class"
|
|
assert not re.search(r"classList\.(?:add|toggle)\(\s*[\"'](?:danger|good|warning)[\"']",
|
|
body), f"{asset} invented a synonym for fail"
|
|
|
|
|
|
def test_app_js_exposes_the_hero_helpers(client) -> None:
|
|
js = client.get("/static/app.js").text
|
|
assert "function hero(" in js
|
|
assert "function severityFrom(" in js
|
|
exports = js.rsplit("return {", 1)[1].split("}", 1)[0]
|
|
for name in ("hero", "severityFrom", "severityFor"):
|
|
assert re.search(rf"\b{name}\b", exports), f"Dash.{name} is not exported"
|
|
|
|
|
|
def test_hero_reads_the_verdict_instead_of_a_threshold(client) -> None:
|
|
"""`hero()` may only colour what the API rated — no thresholds re-derived in JS."""
|
|
js = client.get("/static/app.js").text
|
|
start = js.index("function hero(")
|
|
body = js[start:js.index("\n function setDot", start)]
|
|
assert "severityFrom(" in body
|
|
assert "severityFor(" not in body
|
|
# an unknown value gets n/a and no colour class at all
|
|
assert "naText" in body
|
|
assert re.search(r"raw != null", body)
|
|
|
|
|
|
def test_severity_for_is_kept_as_the_documented_fallback(client) -> None:
|
|
"""The old call sites still work until the remaining phases convert each page.
|
|
|
|
`/` was converted in phase 02, `/system` in phase 03 and `/llm` in phase 04: each
|
|
colours itself from the `severity` verdicts the API already sends, so it carries no
|
|
threshold of its own. `/ups` is phase 05's job and must keep compiling against the
|
|
fallback until it is converted — deleting it early would break that page.
|
|
"""
|
|
js = client.get("/static/app.js").text
|
|
assert "FALLBACK ONLY" in js
|
|
assert "severityFor" in client.get("/static/page-ups.js").text, \
|
|
"/ups still uses the fallback until phase 05 converts it"
|
|
|
|
|
|
def test_overview_colours_from_the_payload_not_from_js_thresholds(client) -> None:
|
|
"""`page-index.js` reads verdicts; it does not compare a number to a bound."""
|
|
js = client.get("/static/page-index.js").text
|
|
assert "severityFrom" in js
|
|
assert "severityFor" not in js
|
|
|
|
|
|
def test_system_page_colours_from_the_payload_not_from_js_thresholds(client) -> None:
|
|
"""`page-system.js` hero cards paint payload verdicts, not thresholds of their own.
|
|
|
|
Phase 03 task 02's promise: every colour on a host card comes from a `severity` dict
|
|
`severity.py` produced (`cpu.severity.usage_pct`, `memory.severity.used_pct`,
|
|
`disks.severity.worst_used_pct`, `gpus[].severity.busy_pct`, `cpu.severity.temp_c`).
|
|
A threshold written here could disagree with the overview's alerts; a hero with no
|
|
verdict prints `n/a` uncoloured, which is `Dash.hero()`'s contract.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
assert "severityFor(" not in js, "the system page re-invented a threshold"
|
|
assert "Dash.hero(" in js
|
|
for hook in ("cpu.severity.usage_pct", "memory.severity.used_pct",
|
|
"disks.severity.worst_used_pct", "gpus[].severity.busy_pct",
|
|
"cpu.severity.temp_c"):
|
|
assert hook in js, f"the card no longer names {hook} as a colour source"
|
|
|
|
|
|
def test_system_host_card_is_six_heroes_without_per_core(client) -> None:
|
|
"""Each host card leads with exactly six heroes: CPU, MEM, TEMP, DISK, GPU, NET.
|
|
|
|
Asserted on the served script because there is no JS test runner (a locked anchor).
|
|
The two dense `.metrics` rows and the per-core heat bars are exactly what made the
|
|
card unreadable, and "no per-core anywhere on /system" is a promise a stray
|
|
`renderCores` call would silently break — the series stays stored, the page just
|
|
stopped drawing it (PLAN.md *Dropped visuals*).
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
card = js[js.index("function hostCard("):js.index("function renderHostCard(")]
|
|
slots = re.findall(r'<div class="(?:hero|hero-slot)[^"]*" data-k="([^"]+)"', card)
|
|
assert slots == ["cpu", "mem", "temp", "disk", "gpus", "net"], \
|
|
"the hero row is no longer the six numbers in the agreed order"
|
|
# per-core visualisation is gone from the card code entirely
|
|
assert "cores" not in card, "a per-core visual came back to the card"
|
|
assert "renderCores" not in js and "core_usage" not in js
|
|
# the NET slot is a slot with two lines and no verdict: a byte rate has no good/bad
|
|
assert 'data-k="net"' in card
|
|
assert not re.search(r'data-k="net"[^>]*data-sev', card), "the NET slot invented a colour"
|
|
# what the glance leaves out stays reachable, collapsed by default
|
|
assert "More detail" in card
|
|
for key in ("load", "freq", "dr", "dw", "dutil", "uptime", "details"):
|
|
assert f'data-k="{key}"' in card, f"{key} left the page instead of the glance"
|
|
# the panel owns its charts: the slots live inside the card, not on the page
|
|
for chart in ("util", "net"):
|
|
assert f'data-chart="{chart}"' in card, f"the panel lost its {chart} chart slot"
|
|
|
|
|
|
def test_system_page_keeps_no_fleet_wide_chart_or_metric_picker(client) -> None:
|
|
"""Nothing on `/system` plots two hosts, and nothing needs a selection to make sense.
|
|
|
|
TODO.md L4 is the instruction ("The system page doesn't need shared charts for
|
|
anything") and the overview's alerts + fleet table are what made removing them safe
|
|
(phase 02). Asserted on the served bytes because the only thing that would otherwise
|
|
notice a resurrected `<select>` is a browser: there is no JS test runner (a locked
|
|
anchor), and `test_no_page_carries_an_id_nothing_reads` cannot see ids that were deleted
|
|
correctly from the page but are still queried by the script.
|
|
"""
|
|
html = _code(client.get("/system").text)
|
|
for gone in ('id="chart-cpu"', 'id="chart-mem"', 'id="chart-disk"', 'id="chart-net"',
|
|
'id="chart-gpu"', 'id="chart-temp"', 'id="chart-detail"', 'id="detail-host"',
|
|
'id="detail-metric"', 'id="f-cpu"', 'id="f-load"', 'id="f-gpu"',
|
|
"Fleet totals", "Fleet charts", "Series detail", "<select"):
|
|
assert gone not in html, f"/system still carries {gone}"
|
|
# the two panels that are left are the per-host grid and the event feed
|
|
assert html.count("<section") == 2, "/system grew a panel back on top of the host grid"
|
|
assert 'id="hosts"' in html and 'id="events"' in html
|
|
|
|
# and the script must not reach for the deleted endpoints/ids either — a `#detail-host`
|
|
# query against a removed node is a silent null, so the picker would just never appear
|
|
js = client.get("/static/page-system.js").text
|
|
for gone in ("#chart-", "detail-host", "detail-metric", "detail-note",
|
|
"renderFleet", "makeCharts", "refreshGpuChart", "populateDetailMetrics"):
|
|
assert gone not in js, f"page-system.js still references {gone}"
|
|
|
|
|
|
def test_every_system_chart_is_drawn_inside_its_own_host_panel(client) -> None:
|
|
"""One host per chart: the legends of `/system` can never list two machine names.
|
|
|
|
The construction is a factory that takes the card, so a chart cannot be built against a
|
|
page-level id; the metric list is per host, so the GPU streams cannot leak from one
|
|
panel into another; and a panel is torn down (with its charts) when its host leaves the
|
|
payload, which is the bug class a keyed-by-host chart map exists to avoid.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
builds = re.findall(r"new DashCharts\.LineChart\(([^,]+),", js)
|
|
assert builds, "the system page builds no charts at all"
|
|
# every chart is built from a `[data-chart]` slot inside the host card, never from a
|
|
# page-level id — which is what makes "one host per chart" structural, not a convention
|
|
assert all(re.search(r"(?:card|slot)\.querySelector\(.\[data-chart=", b) for b in builds), \
|
|
f"a chart is built outside a host panel: {builds}"
|
|
# three construction sites, one per slot (util, net, gpu) — never one per metric
|
|
assert len(builds) == 3, f"charts are not built one per panel slot: {builds}"
|
|
assert sorted(re.findall(r'data-chart=\\?"(\w+)"', " ".join(builds))) == ["gpu", "net", "util"]
|
|
|
|
factory = js[js.index("async function renderHostCharts("):js.index("function pruneHosts(")]
|
|
assert "hosts=${encodeURIComponent(host.name)}" in factory, \
|
|
"the per-host chart query is not scoped to one host"
|
|
assert "&metrics=" in factory and "state.range" in factory, \
|
|
"a host's charts need one metric list and the selected window"
|
|
# GPU series are named from the payload's own `gpus[].index`, never discovered/guessed
|
|
assert "gpu.busy_pct@" in js and "gpu.index" in js
|
|
# the GPU chart exists only while the host reports cards
|
|
gpu_slot = js[js.index("function hostChartSet("):js.index("function seriesFor(")]
|
|
assert re.search(r"if \(!wantsGpu && set\.gpu\)", gpu_slot), \
|
|
"a host that stops reporting GPUs keeps a GPU chart it has no data for"
|
|
|
|
pruner = js[js.index("function pruneHosts("):js.index("// ---- snapshot")]
|
|
assert "state.hosts.delete(name)" in pruner and "state.charts.delete(name)" in pruner, \
|
|
"a host that left the payload keeps a live panel and drawing charts"
|
|
|
|
|
|
def test_every_system_chart_names_itself_for_assistive_tech(client) -> None:
|
|
"""A 120px SVG is silence to a screen reader unless the page names it.
|
|
|
|
The phase-03 accessibility pass (task 04) found the per-host charts drawing anonymous
|
|
plots: the legend is a sibling element and labels nothing, so the `<svg>` itself must
|
|
carry `role="img"` + `aria-label` that says which host and which metrics. Pinned on the
|
|
served bytes because there is no JS test runner (a locked anchor).
|
|
"""
|
|
charts_js = client.get("/static/charts.js").text
|
|
# the shared LineChart applies the name to the svg, and only when the caller opted in
|
|
assert "if (this.opts.ariaLabel)" in charts_js, "LineChart stopped honouring ariaLabel"
|
|
assert "setAttribute('role', 'img')" in charts_js
|
|
assert "setAttribute('aria-label', String(this.opts.ariaLabel))" in charts_js
|
|
|
|
js = client.get("/static/page-system.js").text
|
|
# all three slots name themselves, and each name starts with the host the panel belongs
|
|
# to — an aria-label that cannot say *which machine* is not an accessible name here
|
|
assert js.count("ariaLabel: `") == 3, "a per-host chart slot lost its accessible name"
|
|
for slot in ("utilisation", "network traffic", "GPU busy"):
|
|
assert f"${{host.name}} \u2014 {slot}" in js or f"${{host.name}} — {slot}" in js, \
|
|
f"the {slot} chart has no host-scoped accessible name"
|
|
|
|
|
|
def test_system_event_feed_queries_the_source_transitions_are_stored_under(client) -> None:
|
|
"""Host transitions are stored under the *dashboard* name, not the sample source.
|
|
|
|
`poller.py` logs `Event(ts, self.dashboard, …)` — host jobs write `system`, the way
|
|
the UPS job writes `ups` — while *samples* use `host`. The `/system` feed queried
|
|
`source=host` since the first commit and therefore always rendered "no transitions
|
|
yet", even with a real outage (a gateway disconnect) sitting in the store. Pinned on
|
|
the served bytes because there is no JS test runner (a locked anchor).
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
feed = js[js.index("async function refreshEvents("):]
|
|
url = re.search(r"fetchJSON\('(/api/events[^']*)'\)", feed)
|
|
assert url, "the /system page no longer fetches the event feed"
|
|
assert "source=system" in url.group(1), \
|
|
"the /system event feed must ask for the source the poller stores events under"
|
|
assert "source=host" not in url.group(1), \
|
|
"source=host matches no stored event — the feed is empty forever"
|
|
|
|
|
|
def test_system_footnote_prints_the_window_it_actually_measured(client) -> None:
|
|
"""`Δ<window>` is the page's promise that a rate is a measurement (invariant 2), so the
|
|
number behind it has to come from the measurement.
|
|
|
|
The `window_seconds` beside the fleet is `HOST_INTERVAL_SECONDS`, while the window behind a
|
|
rate is the estimator's `RATE_WINDOW_SECONDS` — equal at the defaults, which is why the
|
|
footnote could quote the poll interval and look right. Set `RATE_WINDOW_SECONDS=30` and it
|
|
would have labelled every rate on the page with a window it never used. The footnote now
|
|
reads each host's own measured `window_seconds`, and prints a range when they disagree.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
assert "function windowLabel(" in js, "the footnote lost its measured-window helper"
|
|
footnote = js[js.index("$('#foot-note').textContent"):js.index("await Promise.all(chartJobs)")]
|
|
assert "windowLabel(" in footnote
|
|
assert "host.window_seconds" in footnote, \
|
|
"the footnote is quoting the poll interval as a measurement again"
|
|
# a footnote that says "window" and a page that says "poll" must stay two separate claims
|
|
assert "poll ${Math.round(state.pollMs / 1000)}s" in footnote
|
|
|
|
|
|
def test_no_per_core_visual_library_survives_the_page(client) -> None:
|
|
"""The per-core heat bars left `/system` and `charts.js` with them (PLAN.md *Dropped
|
|
visuals* is about what the pages draw, not what the store keeps).
|
|
|
|
`renderCores` had one caller — the hero-card per-core block deleted in task 02 — so
|
|
keeping it would be dead markup generating unstyled `.core` spans; `.cores` was the CSS
|
|
that block needed. Both are checked absent here so neither can come back as a
|
|
half-change, while `sparkline`/`LineChart` stay for phase 04 and the UPS page.
|
|
"""
|
|
assert "renderCores" not in client.get("/static/charts.js").text
|
|
css = client.get("/static/dash.css").text
|
|
assert ".cores" not in css
|
|
for asset in ASSETS:
|
|
assert "core_usage" not in _code(client.get(asset).text), f"{asset} plots per-core"
|
|
# what phase 04 and the other pages still chart from
|
|
for name in ("LineChart", "sparkline", "renderLegend"):
|
|
assert name in client.get("/static/charts.js").text
|
|
|
|
|
|
def test_sparkline_ships_the_fixed_scale_and_verdict_options(client) -> None:
|
|
"""Phase 04 turns `/llm` cells into sparkline rows, so the shared `sparkline()` must
|
|
behave like `llm_monitor`'s `drawSpark` (the task-02 contract, NOTES-chart-set).
|
|
|
|
Pinned on the served bytes because there is no JS test runner (a locked anchor). What
|
|
matters is not the option names but the semantics behind each one: a percent metric
|
|
that silently rescaled to its own data would make an 8 % KV cache and a 92 % spike the
|
|
same squiggle, and a stroke coloured from a page-invented threshold would break the
|
|
one-colour-language rule the same way.
|
|
"""
|
|
charts_js = client.get("/static/charts.js").text
|
|
body = charts_js[charts_js.index("function sparkline("):charts_js.index("window.DashCharts")]
|
|
|
|
# the phase grep criterion, as a test: the options are really in the shipped file
|
|
hits = [ln for ln in charts_js.splitlines()
|
|
if re.search(r"fixedMax|align|severity", ln)]
|
|
assert len(hits) >= 3, "the sparkline upgrade is not in the served charts.js"
|
|
for opt in ("fixedMax", "align", "labels", "severity", "minPoints", "span", "format",
|
|
"minSpan"):
|
|
assert opt in body, f"sparkline() lost its `{opt}` option"
|
|
|
|
# fixedMax: the data clamps to the axis, never the axis to the data
|
|
assert "Math.min(Math.max(v, lo), hi)" in body, \
|
|
"a fixedMax sparkline rescales to its own data again"
|
|
# align:'right' grows within a real time window (span seconds), not a sample count
|
|
assert "x1 - o.span" in body, "align:'right' forgot the window it grows inside"
|
|
# verdict colour comes from the theme tokens only; unrated is faint grey, never green
|
|
for token in ("cssVar('--ok')", "cssVar('--warn')", "cssVar('--fail')",
|
|
"cssVar('--text-faint')"):
|
|
assert token in body, f"sparkline() stopped taking its colour from {token}"
|
|
assert ".sev-" not in body, "sparkline invented a parallel severity vocabulary"
|
|
# the `now / max` foot llm_monitor prints, via the caller's format callback
|
|
assert "now ${fmt(last.value)}" in body and "max ${fmt(dataMax)}" in body
|
|
# never a confident line through one point, and the foot quotes data, not the padded axis
|
|
assert "minPoints: 2" in body and "no history" in body
|
|
# ASSUMPTION (task 02): no animation at all, so prefers-reduced-motion holds by design
|
|
assert "transition:" not in body and "animation:" not in body
|
|
|
|
|
|
# ---- the `/llm` layout (phase 04 task 03) ---------------------------------
|
|
#
|
|
# The drift this phase fixes was a 14-cell grid where everything was the same size and
|
|
# nothing showed recent shape. These tests pin the *shape* the page committed to on the
|
|
# served bytes (a locked anchor: no JS test runner), so a regression to "all cells equal"
|
|
# fails a test rather than a screenshot.
|
|
|
|
def test_llm_page_colours_from_the_payload_not_from_js_thresholds(client) -> None:
|
|
"""`page-llm.js` paints the KV verdict `severity.py` produced; it owns no threshold.
|
|
|
|
Both `severityFor(value, 75, 90)` call sites are gone (the task's explicit work item):
|
|
the KV row takes `severity` from the rated ratio cell, the fleet KV bar was deleted
|
|
with the summed card, and every unrated number — throughput above all, because 0 tok/s
|
|
on an idle server is healthy — draws no colour at all.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
assert "severityFor(" not in js, "the LLM page re-invented a threshold"
|
|
assert "severityFrom" in js
|
|
# the KV verdict comes from the payload cells `sources/llm.py` rates
|
|
kv = js[js.index("case 'cache':"):js.index("case 'ttft':")]
|
|
assert "kv_cache_usage_ratio" in kv and "gpu_cache_usage" in kv and "severity" in kv
|
|
|
|
|
|
def test_llm_engine_panel_leads_with_two_throughput_heroes(client) -> None:
|
|
"""Decode and prompt are the largest elements per engine; the rest is visibly secondary.
|
|
|
|
The hero row is `Dash.hero()` markup (2.4rem via `.hero`), built from the `hero: true`
|
|
flags in `CELLS`, and throughput carries **no** severity — `severity.py` has no
|
|
throughput row and the page invents none (the neutral-ink decision in NOTES-chart-set,
|
|
mirroring the `/system` NET precedent). The provenance tag stays in the hero's `.sub`
|
|
with the exact `source` as the tooltip: removing it to look cleaner is a regression.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
panel = js[js.index("function enginePanel("):js.index("function renderPanel(")]
|
|
assert 'data-hero="${cell.key}"' in panel, "the hero row is not flag-driven markup"
|
|
assert '<div class="heroes">' in panel, "the hero row left the engine panel"
|
|
heroes = re.findall(r"{ key: '(\w+)'[^}]*?hero: true", js)
|
|
assert heroes == ["decode", "prompt"], "the hero row is no longer the two throughputs"
|
|
# driven by flags, not fourteen identical cells
|
|
assert "SPARK_CELLS" in js and "ALARM_CELLS" in js
|
|
assert js.count("spark: '") >= 6, "the compact sparkline list shrank back to a grid"
|
|
# the hero call must not pass a severity — unrated by design
|
|
hero_call = js[js.index("const node = hero("):js.index("if (node) node.title")]
|
|
assert "severity" not in hero_call, "throughput invented a verdict"
|
|
# provenance intact: the tag is generated from the cell's `source`, never hardcoded
|
|
assert "provenance" in js and "Δ" in js
|
|
assert "node.title = value.source" in js
|
|
|
|
|
|
def test_every_llm_cell_carries_a_window_sparkline_with_percentile_feet(client) -> None:
|
|
"""Each compact cell draws its own sparkline over the *selected* window; latency cells
|
|
print mean **and** p95 without a hover.
|
|
|
|
The window is honest time (`align: 'right'` + `span`), percent-like metrics keep a
|
|
fixed 0-100 axis, and the p95 comes from the snapshot's histogram cells — p95 needs no
|
|
stored series. The `only` scopes are what keep a gufo panel from showing a latency
|
|
mystery and a vLLM panel from showing a spec-decode row it cannot fill.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
sparks = js[js.index("function drawSparks("):js.index("function renderFleet(")]
|
|
assert "DashCharts.sparkline(" in sparks
|
|
for opt in ("align: 'right'", "span: state.range", "fixedMax", "labels: { max: true }"):
|
|
assert opt in sparks, f"cell sparklines lost `{opt}`"
|
|
# percent cells pin the axis; ratio series are scaled to the printed percent
|
|
assert js.count("fixedMax: 100") >= 3, "KV / prefix-hit / spec-accept lost their 0-100 axis"
|
|
assert "scale: 1000" in js and "scale: 100" in js
|
|
# every spark cell declares its history series in the markup it is built from
|
|
assert 'data-spark data-metric="${cell.spark}"' in js
|
|
# percentile foot lines: mean big in the row, p95 beneath, n/a when nothing measured
|
|
assert "function p95Value(" in js
|
|
for name in ("time_to_first_token_seconds__p95", "time_per_output_token_seconds__p95",
|
|
"request_queue_time_seconds__p95"):
|
|
assert name in js, f"{name} no longer feeds a foot line"
|
|
assert "p95 n/a" in js
|
|
assert 'data-foot' in js
|
|
# engine-scoped rows: gufo never grows a TTFT row, vLLM never a spec-accept row
|
|
assert "only: ['vllm']" in js and "only: ['gufo']" in js
|
|
scoped = js[js.index("function sparkCellsFor("):js.index("function enginePanel(")]
|
|
assert "cell.only" in scoped
|
|
# a count row never gets an axis finer than one request: history arrives as rollups,
|
|
# so 8 s of a single queued request averages to 0.14 and a fitted axis draws a spike for
|
|
# a queue nobody was in — the twin of the rescale `fixedMax` fixes
|
|
assert _code(js).count("minSpan: 1") == 2, "sessions/queued lost their one-request axis floor"
|
|
assert "minSpan: cell.minSpan" in sparks, "cell minSpan never reaches the sparkline"
|
|
|
|
|
|
def test_llm_sparklines_and_charts_cost_one_batched_history_request(client) -> None:
|
|
"""One `/api/llm/history` call per refresh feeds charts *and* every sparkline row.
|
|
|
|
The endpoint answers a metric list for every engine, and unknown names come back as
|
|
empty series on purpose — so the page asks for the full cell set once instead of one
|
|
XHR per cell, and the two series phase 04 added (`tpot_seconds`, `spec_accept_rate`)
|
|
must be in that list or those rows are permanent "no history" slots.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
# comments aside, exactly one fetch hits the endpoint: one request, every metric
|
|
calls = re.findall(r"fetchJSON\(\s*`/api/llm/history", _code(js))
|
|
assert len(calls) == 1, "the page makes more than one history request again"
|
|
assert "HISTORY_SERIES.join" in js and "SPARK_CELLS.map((cell) => cell.spark)" in js
|
|
request = js[js.index("async function refreshHistory("):js.index("function drawSparks(")]
|
|
assert "metrics=${HISTORY_SERIES.join" in request
|
|
assert "range=${state.range}" in request
|
|
# and the endpoint really answers every name the page asks for
|
|
for name in ("tpot_seconds", "spec_accept_rate", "ttft_seconds", "queue_seconds",
|
|
"kv_cache_usage", "requests_running", "requests_waiting",
|
|
"prefix_cache_hit_ratio"):
|
|
assert name in HISTORY_METRICS, f"llm.{name} is not stored, so its spark is dead"
|
|
|
|
|
|
def test_llm_alarm_cells_exist_only_while_there_is_an_incident(client) -> None:
|
|
"""`preemptions` / `device_lost` are absent at zero and red chips above zero.
|
|
|
|
The rule is the *existence of an incident*, not a graded threshold — `sources/llm.py`
|
|
already treats any non-zero `gufo_device_lost_total` as one. The chip reuses the shared
|
|
`.chip.fail` vocabulary; nothing new is painted, and a container emptied at zero means
|
|
the row does not exist rather than showing a permanent ignored 0.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
alarms = js[js.index("function renderAlarms("):js.index("function renderSlots(")]
|
|
assert "class=\"chip fail\"" in alarms, "the alarm chip left the shared fail vocabulary"
|
|
assert "value.value > 0" in alarms
|
|
assert "host.hidden = chips.length === 0" in alarms
|
|
assert 'data-alarm="${cell.key}"' in alarms
|
|
assert js.count("alarm: true") == 2
|
|
|
|
|
|
def test_llm_totals_card_says_only_what_no_engine_panel_says(client) -> None:
|
|
"""Engines answering, active, queued — and hidden below two engines.
|
|
|
|
The summed decode/prompt and the un-rated fleet KV bar are gone: per-engine is the
|
|
honest view (the fleet KV bar had no `severity` to render, which is invariant-grade
|
|
trouble), and a cross-engine sum of one engine just duplicates that engine's own card
|
|
(the `llm_monitor` rule). `/api/llm/snapshot` keeps every field — this is a page
|
|
decision, not an API one.
|
|
"""
|
|
html = _code(client.get("/llm").text)
|
|
for gone in ("f-decode", "f-prompt", "f-kv", "f-tokens", "seam-note"):
|
|
assert gone not in html, f"/llm still totals {gone}"
|
|
for kept in ("f-live", "f-active", "f-waiting", 'id="totals"'):
|
|
assert kept in html, f"/llm lost {kept}"
|
|
js = client.get("/static/page-llm.js").text
|
|
assert "card.hidden = total < 2" in js, "the totals card no longer hides for one engine"
|
|
|
|
|
|
def test_llm_charts_are_secondary_and_named(client) -> None:
|
|
"""The four trend charts stay, at half height, *below* the engine panels, each named.
|
|
|
|
Half of the old 165px and moved after `#engines` is the "cells read first" decision;
|
|
every `LineChart` gets an `ariaLabel` because the legend is a sibling that labels
|
|
nothing (`/system` did this in phase 03, `/llm` never had).
|
|
"""
|
|
html = _code(client.get("/llm").text)
|
|
assert html.index('id="engines"') < html.index('id="chart-decode"'), \
|
|
"the charts are back on top of the engine panels"
|
|
for chart in ("chart-decode", "chart-prompt", "chart-kv", "chart-queue"):
|
|
assert f'id="{chart}"' in html, f"/llm lost the {chart} chart"
|
|
js = client.get("/static/page-llm.js").text
|
|
assert "height: 110" in js and "height: 165" not in js
|
|
charts = js[js.index("function makeCharts("):js.index("function pointsFor(")]
|
|
assert charts.count("ariaLabel:") == 4, "an /llm chart lost its accessible name"
|
|
|
|
|
|
def test_llm_engine_panels_are_not_stretched_to_their_tallest_sibling(client) -> None:
|
|
"""A 5-row gufo panel beside a 7-row vLLM panel keeps its own height.
|
|
|
|
Found while comparing against `llm_monitor`: the shorter engine card grew 172 px of
|
|
blank card, and a blank panel reads as "no data" — the exact misreading the unconfigured
|
|
rules exist to prevent. Pinned on the served markup so the `.grid.items-start` rule in
|
|
`dash.css` cannot silently become dead CSS (and the void cannot silently come back).
|
|
"""
|
|
html = _code(client.get("/llm").text)
|
|
engines_div = html[html.index('<div class="grid'):html.index('id="engines"') + 40]
|
|
assert "items-start" in engines_div, "the engine grid stretches its cards again"
|
|
css = client.get("/static/dash.css").text
|
|
assert ".grid.items-start" in css, "the rule the engine grid relies on is gone"
|
|
|
|
|
|
def test_overview_is_exactly_two_panels(client) -> None:
|
|
"""The landing page answers one question: what is wrong, and what is the fleet doing.
|
|
|
|
Asserted on the served bytes because there is no JS test runner (a locked anchor):
|
|
the summary cards and the raw event feed are what made this page busy, and once a
|
|
panel is gone a browser is the only thing that would notice its return.
|
|
"""
|
|
html = client.get("/").text
|
|
assert html.count("<section") == 2, "the overview grew a third panel"
|
|
assert 'id="alerts"' in html and 'id="alerts-clear"' in html
|
|
assert 'aria-live="polite"' in html
|
|
assert 'id="fleet-table"' in html
|
|
for gone in ("sys-cpu", "sys-mem", "sys-disk", "sys-gpus", "llm-decode", "llm-kv",
|
|
"ups-list", 'id="events"'):
|
|
assert gone not in html, f"the overview still renders {gone}"
|
|
# the header's source dots are chrome owned by app.js, not the deleted card dots
|
|
assert 'id="dot-system"' not in html and 'id="nav-status"' in html
|
|
|
|
|
|
def test_overview_alert_rows_link_to_the_dashboard_that_explains_them(client) -> None:
|
|
"""Every alert carries the `href` its row turns into a link; the page invents none."""
|
|
page_js = client.get("/static/page-index.js").text
|
|
assert "alert.href" in page_js
|
|
css = client.get("/static/dash.css").text
|
|
for selector in ("ul.alerts li.ok .src", "ul.alerts li.warn .src", "ul.alerts li.fail .src",
|
|
"table.data td.warn", "table.data td.fail"):
|
|
assert selector in css, f"dash.css lost {selector}"
|
|
# the swap that puts a live problem above the fleet table
|
|
assert 'data-state="alerting"' in css
|
|
|
|
|
|
def test_overview_dims_the_fleet_row_of_a_host_that_stopped_answering(client) -> None:
|
|
"""A dead host keeps its last good numbers, so the row has to say they are stale.
|
|
|
|
`AGENTS.md` invariant 4: keep the payload and mark it stale, never zero it. The
|
|
overview's table is the one place those last good numbers still print, so it reads the
|
|
per-source verdicts `/api/summary` sends (`system.states`) and applies the shared
|
|
`.stale` class the other pages already use — no new palette, no JS threshold.
|
|
"""
|
|
page_js = client.get("/static/page-index.js").text
|
|
assert "system.states" in page_js, "the fleet table stopped reading reachability"
|
|
assert 'class="stale"' in page_js
|
|
assert ".stale" in client.get("/static/dash.css").text
|
|
|
|
|
|
# Which script owns which page, for the orphan check below. `charts.js` and `app.js` are
|
|
# loaded by all four pages, so an id either page code or shared chrome may read.
|
|
PAGE_SCRIPTS = {
|
|
"/": "page-index.js",
|
|
"/system": "page-system.js",
|
|
"/llm": "page-llm.js",
|
|
"/ups": "page-ups.js",
|
|
}
|
|
SHARED_SCRIPTS = ("app.js", "charts.js")
|
|
|
|
|
|
def test_no_page_carries_an_id_nothing_reads(client) -> None:
|
|
"""An id no script queries, no stylesheet styles and no attribute points at is dead markup.
|
|
|
|
The summary cards were retired this way — a `<span id="sys-cpu">` left behind after its
|
|
writer was deleted is the same mistake in slower motion. Asserted on served bytes: the
|
|
browser console is otherwise the only thing that would notice.
|
|
"""
|
|
scripts = {name: client.get(f"/static/{name}").text for name in
|
|
(*SHARED_SCRIPTS, *PAGE_SCRIPTS.values(), "dash.css")}
|
|
for path, own_script in PAGE_SCRIPTS.items():
|
|
html = _code(client.get(path).text)
|
|
corpus = "".join(_code(scripts[name])
|
|
for name in (*SHARED_SCRIPTS, own_script, "dash.css"))
|
|
# ids the markup uses itself: labels, targets, `<label for>`
|
|
anchors = set(re.findall(
|
|
r'(?:aria-labelledby|aria-describedby|aria-controls|for)="([^"]+)"', html))
|
|
for element_id in re.findall(r'id="([^"]+)"', html):
|
|
quoted = (f'"{element_id}"' in corpus or f"'{element_id}'" in corpus
|
|
or f"#{element_id}" in corpus or element_id in anchors)
|
|
assert quoted, f"{path} has id=\"{element_id}\" that nothing reads"
|
|
|
|
|
|
def test_api_responses_are_not_cacheable(client) -> None:
|
|
for path in ("/api/config", "/api/health", "/api/summary"):
|
|
assert "no-store" in client.get(path).headers.get("cache-control", "")
|
|
|
|
|
|
# ---- history -------------------------------------------------------------- #
|
|
|
|
def test_history_returns_seeded_samples(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
db = str(tmp_path / "hist.sqlite3")
|
|
store = SampleStore(db)
|
|
now = time.time()
|
|
for i in range(30):
|
|
store.add("host", "driveripper", "cpu.usage_pct", 10.0 + i, ts=now - 300 + i)
|
|
app = create_app(Settings(db_path=db, host_targets=(Target("driveripper", DEAD_URL),)), store)
|
|
with TestClient(app) as client:
|
|
body = client.get("/api/history", params={"source": "host", "host": "driveripper",
|
|
"metric": "cpu.usage_pct", "range": "15m"})
|
|
assert response_ok(body)
|
|
points = body.json()["points"]
|
|
assert len(points) >= 25
|
|
assert points[-1]["value"] == pytest.approx(39.0)
|
|
store.close()
|
|
|
|
|
|
def test_history_rejects_an_unknown_range(client) -> None:
|
|
response = client.get("/api/history", params={"source": "host", "host": "h",
|
|
"metric": "m", "range": "banana"})
|
|
assert response.status_code == 400
|
|
|
|
|
|
def test_history_requires_the_series_coordinates(client) -> None:
|
|
assert client.get("/api/history").status_code == 422
|
|
|
|
|
|
@pytest.mark.parametrize(("raw", "seconds"), [
|
|
*[(key, float(seconds)) for key, seconds in RANGE_SECONDS.items()],
|
|
("45s", 45.0), ("13h", 46_800.0), ("7200", 7_200.0), (" 2D ", 172_800.0),
|
|
(None, 3_600.0), ("", 3_600.0),
|
|
])
|
|
def test_parse_range_reads_every_window_the_ui_offers(raw: Any, seconds: float) -> None:
|
|
"""Every chip key (5m…90d), any `<n><s|m|h|d>`, raw seconds, and the 1h default.
|
|
|
|
`parse_range` is the only thing between a typo'd query string and a chart of the wrong
|
|
window, and a wrong window is worse than an error: it still looks like data.
|
|
"""
|
|
assert parse_range(raw) == seconds
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["banana", "1w", "h", "-4h", "one"])
|
|
def test_parse_range_refuses_a_window_it_cannot_interpret(raw: str) -> None:
|
|
"""A 400 that names the value, never a silent fallback to 1h."""
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
parse_range(raw)
|
|
assert excinfo.value.status_code == 400
|
|
assert raw in str(excinfo.value.detail)
|
|
|
|
|
|
# ---- the routes the *other* pages chart from -------------------------------- #
|
|
#
|
|
# `/` renders two panels now; `/system`, `/llm` and `/ups` still need these, and so does
|
|
# anything outside the browser. Retiring overview code must not retire them, so the shapes
|
|
# are pinned here rather than discovered by a page three phases from now.
|
|
|
|
def test_system_history_charts_every_configured_host_in_one_request(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""N hosts x M metrics, one round trip.
|
|
|
|
`/system` no longer uses the N-hosts half of this (every chart belongs to one host and
|
|
the page issues one request per host, pinned by
|
|
`test_every_system_chart_is_drawn_inside_its_own_host_panel`); the shape is kept because
|
|
the metric-list half is exactly what the per-host query costs one request with, and
|
|
because the endpoint is a public interface for programmatic consumers.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),
|
|
Target("deskwork", DEAD_URL))) as test_client:
|
|
body = test_client.get("/api/system/history",
|
|
params={"metrics": "cpu.usage_pct,mem.used_pct",
|
|
"range": "1h"}).json()
|
|
assert body["hosts"] == ["driveripper", "deskwork"]
|
|
assert body["range_seconds"] == 3_600.0
|
|
assert len(body["series"]) == 4
|
|
assert {(item["host"], item["metric"]) for item in body["series"]} == {
|
|
("driveripper", "cpu.usage_pct"), ("driveripper", "mem.used_pct"),
|
|
("deskwork", "cpu.usage_pct"), ("deskwork", "mem.used_pct")}
|
|
# an explicit `hosts` list narrows it instead of adding to the default
|
|
narrowed = test_client.get("/api/system/history",
|
|
params={"metrics": "cpu.usage_pct",
|
|
"hosts": "deskwork"}).json()
|
|
assert narrowed["hosts"] == ["deskwork"]
|
|
|
|
|
|
def test_one_request_per_host_answers_every_per_host_chart(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The exact query `/system` issues per host returns every series that panel draws.
|
|
|
|
`page-system.js` sends one request per host carrying the utilisation trio
|
|
(`cpu.usage_pct`, `mem.used_pct`, `disk.used_pct`), both network rates, and one
|
|
`gpu.busy_pct@<index>` per card the payload reported — so a name missing here is a chart
|
|
that silently reads "no samples yet" in the browser, and a renamed series in
|
|
`sources/host.py` fails here rather than in a screenshot. This is the metric-name half of
|
|
the contract; `tests/test_system_contract.py` holds the payload-path half.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
store = test_client.app.state.ctx.store # type: ignore[attr-defined]
|
|
now = time.time()
|
|
metrics = ["cpu.usage_pct", "mem.used_pct", "disk.used_pct",
|
|
"net.rx_bytes_per_s", "net.tx_bytes_per_s", "gpu.busy_pct@0"]
|
|
for metric in metrics:
|
|
store.add("host", "driveripper", metric, 12.5, ts=now - 60)
|
|
store.add("host", "driveripper", metric, 25.0, ts=now - 30)
|
|
|
|
body = test_client.get("/api/system/history", params={
|
|
"hosts": "driveripper", "metrics": ",".join(metrics), "range": "1h"}).json()
|
|
assert body["hosts"] == ["driveripper"], "one panel, one host"
|
|
assert [item["metric"] for item in body["series"]] == metrics
|
|
assert [len(item["points"]) for item in body["series"]] == [2] * len(metrics)
|
|
|
|
# a second card the host does not have answers empty, never zero and never 404
|
|
gap = test_client.get("/api/system/history",
|
|
params={"hosts": "driveripper", "metrics": "gpu.busy_pct@1"}).json()
|
|
assert gap["series"][0]["points"] == []
|
|
|
|
|
|
def test_llm_snapshot_keeps_an_engine_that_never_answered_in_the_list(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A configured-but-dead engine is listed as unreachable, not dropped (invariant 4)."""
|
|
# a host target too, so the snapshot's job filter has to skip a non-LLM job
|
|
with build_client(tmp_path, monkeypatch, vllm_url=DEAD_URL,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
body = test_client.get("/api/llm/snapshot").json()
|
|
assert [engine["engine"] for engine in body["engines"]] == ["vLLM"]
|
|
assert body["engines"][0]["unreachable"] is True
|
|
assert body["fleet"]["engines_total"] == 1
|
|
assert body["fleet"]["engines_live"] == 0
|
|
assert [state["key"] for state in body["states"]] == ["llm:vllm"]
|
|
|
|
|
|
def test_llm_history_charts_both_engines_by_default(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, gufo_url=DEAD_URL,
|
|
vllm_url=DEAD_URL) as test_client:
|
|
body = test_client.get("/api/llm/history",
|
|
params={"metrics": "decode_tokens_per_second,kv_cache_usage_ratio",
|
|
"range": "15m"}).json()
|
|
assert body["engines"] == ["gufo", "vllm"]
|
|
assert body["range_seconds"] == 900.0
|
|
assert {(item["host"], item["metric"]) for item in body["series"]} == {
|
|
("gufo", "llm.decode_tokens_per_second"),
|
|
("gufo", "llm.kv_cache_usage_ratio"),
|
|
("vllm", "llm.decode_tokens_per_second"),
|
|
("vllm", "llm.kv_cache_usage_ratio")}
|
|
|
|
|
|
def test_one_batched_llm_history_request_answers_every_cell_series(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The exact query `/llm` issues per refresh returns every chart *and* sparkline series.
|
|
|
|
Phase 04 sends one request carrying the chart trio and the whole spark cell list —
|
|
including the two series the layout required (`tpot_seconds` for the vLLM TPOT row,
|
|
`spec_accept_rate` for the gufo spec row). A name missing here is a cell that reads
|
|
"no history" forever in the browser, which is the dead-cell anti-pattern the phase
|
|
wrote NOTES-chart-set to prevent; a rename in `sources/llm.py` fails here first.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, gufo_url=DEAD_URL,
|
|
vllm_url=DEAD_URL) as test_client:
|
|
store = test_client.app.state.ctx.store # type: ignore[attr-defined]
|
|
now = time.time()
|
|
metrics = ["decode_tokens_per_second", "prompt_tokens_per_second",
|
|
"kv_cache_usage", "requests_running", "requests_waiting",
|
|
"ttft_seconds", "tpot_seconds", "queue_seconds",
|
|
"prefix_cache_hit_ratio", "spec_accept_rate"]
|
|
for metric in metrics:
|
|
store.add("llm", "gufo", f"llm.{metric}", 1.5, ts=now - 60)
|
|
store.add("llm", "vllm", f"llm.{metric}", 2.5, ts=now - 30)
|
|
body = test_client.get("/api/llm/history", params={
|
|
"engines": "gufo,vllm", "metrics": ",".join(metrics), "range": "1h"}).json()
|
|
assert body["range_seconds"] == 3_600.0
|
|
by_key = {(item["host"], item["metric"]) for item in body["series"]}
|
|
assert by_key == {(engine, f"llm.{metric}")
|
|
for engine in ("gufo", "vllm") for metric in metrics}
|
|
points = {(item["host"], item["metric"]): item["points"] for item in body["series"]}
|
|
assert points[("gufo", "llm.spec_accept_rate")][0]["value"] == pytest.approx(1.5)
|
|
assert points[("vllm", "llm.tpot_seconds")][0]["value"] == pytest.approx(2.5)
|
|
|
|
|
|
def test_metrics_endpoint_lists_what_actually_has_history(client) -> None:
|
|
body = client.get("/api/metrics").json()
|
|
assert body["metrics"] == []
|
|
|
|
|
|
# ---- events --------------------------------------------------------------- #
|
|
|
|
def test_events_are_returned_in_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
db = str(tmp_path / "events.sqlite3")
|
|
store = SampleStore(db)
|
|
store.log_event(Event(ts=time.time(), source="host", host="deskwork", kind="down",
|
|
severity="error", message="deskwork is not answering"))
|
|
app = create_app(Settings(db_path=db), store)
|
|
with TestClient(app) as client:
|
|
events = client.get("/api/events").json()["events"]
|
|
assert events[0]["host"] == "deskwork"
|
|
assert events[0]["severity"] == "error"
|
|
store.close()
|
|
|
|
|
|
def test_event_rows_say_their_subject_once(client) -> None:
|
|
"""`poller.py` messages name their own subject, and the row prints a host chip too.
|
|
|
|
"gufo gufo is answering again after 1 failure" reads as two problems — the same defect
|
|
`alerts._clean_error()` already fixes for the overview's alert list. `/ups` had the
|
|
mirror-image bug: it `replace()`d `host + ':'` *anywhere*, so "Quick battery test sent
|
|
to apc: OK" printed as "Quick battery test sent to OK". All three feeds now share
|
|
`Dash.eventText()`, which strips the wrapper and a **leading** name only.
|
|
"""
|
|
app_js = client.get("/static/app.js").text
|
|
helper = app_js[app_js.index("function eventText("):app_js.index("// ---- DOM helpers")]
|
|
assert "eventText" in app_js[app_js.index("return {"):], "eventText is not exported"
|
|
assert "^UpstreamError:\\s+" in helper, "the wrapper `alerts.py` drops is back"
|
|
assert "new RegExp" in helper and "stripped.trim() ? stripped : out" in helper, \
|
|
"the host strip is no longer anchored to the front, or can empty a message"
|
|
for page in ("/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js"):
|
|
js = client.get(page).text
|
|
assert "eventText(event)" in js, f"{page} prints the raw message again"
|
|
assert "replace(event.host" not in client.get("/static/page-ups.js").text, \
|
|
"/ups is mangling the host name mid-message again"
|
|
|
|
|
|
# ---- the one write endpoint ------------------------------------------------ #
|
|
|
|
def test_command_endpoint_is_post_only(ups_client) -> None:
|
|
assert ups_client.post("/api/ups/command", json={}).status_code in (400, 403, 404)
|
|
assert ups_client.put("/api/ups/command", json={}).status_code == 405
|
|
assert ups_client.delete("/api/ups/command").status_code == 405
|
|
|
|
|
|
def test_commands_are_offered_when_actions_are_enabled(ups_client) -> None:
|
|
body = wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
assert body["simulated"] is True, "the simulator must label itself"
|
|
assert body["actions_enabled"] is True
|
|
ups = body["ups"][0]
|
|
enabled = [item for item in ups["commands"] if item["enabled"]]
|
|
assert enabled, "with NUT_ALLOW_ACTIONS=1 and a simulator, safe commands are sendable"
|
|
assert all(item["risk"] in ("safe", "caution", "dangerous") for item in ups["commands"])
|
|
|
|
|
|
def test_a_safe_command_actually_reaches_the_driver(ups_client) -> None:
|
|
body = wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.panel.start"})
|
|
assert response.status_code == 200, response.text
|
|
payload = response.json()
|
|
assert payload["ok"] is True and payload["simulated"] is True
|
|
events = ups_client.get("/api/events").json()["events"]
|
|
assert any("test.panel.start" in event["message"] or "Panel test" in event["message"]
|
|
for event in events), "every sent command must leave an audit trail"
|
|
|
|
|
|
def test_a_dangerous_command_needs_the_confirmation_word(ups_client) -> None:
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
deep = "test.battery.start.deep"
|
|
refusal = ups_client.post("/api/ups/command", json={"ups": name, "command": deep})
|
|
assert refusal.status_code == 422
|
|
assert "DANGER" in refusal.json()["detail"]
|
|
assert ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": deep, "confirm": "DANGER"}).status_code == 200
|
|
|
|
|
|
def test_a_command_the_driver_does_not_advertise_is_refused_before_it_is_sent(
|
|
ups_client) -> None:
|
|
"""Being in the risk catalog is not permission; `LIST CMD` from the driver is (invariant 1).
|
|
|
|
`shutdown.return` is a real NUT command the catalog knows and this UPS does not offer,
|
|
so the refusal must come from the advertisement check and not from the confirmation gate.
|
|
"""
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "shutdown.return"})
|
|
assert response.status_code == 400
|
|
assert "does not advertise" in response.json()["detail"]
|
|
|
|
|
|
def test_cutting_the_output_is_refused_while_the_ups_is_already_on_battery(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Invariant 1, the one that matters most: no power cut while the pack is carrying the load.
|
|
|
|
Driven through the simulator's own state machine — a quick battery test puts the pack on
|
|
battery — so the 409 comes from the same poll the buttons are drawn from, with a 0.5s
|
|
interval to keep the wait short. Nothing here can touch real hardware: `NUT_SIMULATE=1`.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True, nut_interval=0.5) as test_client:
|
|
name = wait_for_payload(test_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
start = test_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.battery.start.quick"})
|
|
assert start.status_code == 200
|
|
deadline = time.time() + 6.0
|
|
card: dict[str, Any] = {}
|
|
while time.time() < deadline:
|
|
card = next(item for item in test_client.get("/api/ups/snapshot").json()["ups"]
|
|
if item["name"] == name)
|
|
if card["on_battery"]:
|
|
break
|
|
time.sleep(0.05)
|
|
assert card.get("on_battery") is True, "the quick test must put the simulated pack on battery"
|
|
refusal = test_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "load.off",
|
|
"confirm": "DANGER"})
|
|
assert refusal.status_code == 409
|
|
assert "on battery" in refusal.json()["detail"]
|
|
# and the same command is still offered once the pack is back on line power
|
|
assert not any(item["command"] == "load.off" and item["enabled"]
|
|
for item in card["commands"]), "the panel must not offer a blocked button"
|
|
|
|
|
|
class ScriptedNutClient:
|
|
"""`nut.NutClient` with the socket left out: the real code path, no battery attached.
|
|
|
|
`AGENTS.md` keeps live tests read-only, so this is where the one write endpoint's real
|
|
branch gets proven. The variable names are the ones `VAR_MAP` already pins from real
|
|
upsd captures; only the replies are scripted (a fake for behaviour, per the test policy).
|
|
"""
|
|
|
|
#: set per test: an exception instance to raise from `instcmd`, e.g. a refusal
|
|
refusal: Exception | None = None
|
|
sent: list[tuple[str, str]]
|
|
logged_in: list[str]
|
|
|
|
def __init__(self, host: str, port: int, *, timeout: float = 4.0,
|
|
username: str | None = None, password: str | None = None) -> None:
|
|
self.credentials = (username, password)
|
|
|
|
async def __aenter__(self) -> ScriptedNutClient:
|
|
return self
|
|
|
|
async def __aexit__(self, *_exc: object) -> None:
|
|
return None
|
|
|
|
async def list_ups(self) -> list[dict[str, str]]:
|
|
return [{"name": "apc", "description": "APC Back-UPS RS 1500MS"}]
|
|
|
|
async def list_vars(self, ups: str) -> dict[str, str]:
|
|
return {"ups.mfr": "APC", "ups.model": "Back-UPS RS 1500MS",
|
|
"ups.status": "OL", "battery.charge": "100", "ups.load": "18",
|
|
"battery.voltage": "13.6", "input.voltage": "230",
|
|
"battery.mfr.date": "2022-04", "ups.test.result": "Ok last test passed"}
|
|
|
|
async def list_cmds(self, ups: str) -> list[str]:
|
|
return ["test.panel.start", "beeper.mute", "load.off"]
|
|
|
|
async def list_rw_vars(self, ups: str) -> list[str]:
|
|
return []
|
|
|
|
async def login(self) -> None:
|
|
assert self.credentials[0], "a command send must authenticate as a named user"
|
|
ScriptedNutClient.logged_in.append(str(self.credentials[0]))
|
|
|
|
async def instcmd(self, ups: str, command: str) -> str:
|
|
if ScriptedNutClient.refusal is not None:
|
|
raise ScriptedNutClient.refusal
|
|
ScriptedNutClient.sent.append((ups, command))
|
|
return "OK Start Panel Test"
|
|
|
|
|
|
@pytest.fixture
|
|
def real_upsd_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[type[ScriptedNutClient]]:
|
|
"""Swap the module-level `NutClient` so the collector and the send use the same stand-in."""
|
|
ScriptedNutClient.sent = []
|
|
ScriptedNutClient.logged_in = []
|
|
ScriptedNutClient.refusal = None
|
|
monkeypatch.setattr(nut, "NutClient", ScriptedNutClient)
|
|
yield ScriptedNutClient
|
|
|
|
|
|
@pytest.fixture
|
|
def nut_client_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
real_upsd_client: type[ScriptedNutClient]) -> Iterator[TestClient]:
|
|
"""A non-simulated UPS app: real code paths, scripted upsd, 0.5s polls."""
|
|
with build_client(tmp_path, monkeypatch, nut_host="127.0.0.1", nut_user="dashboards",
|
|
nut_password="not-the-real-one", nut_allow_actions=True,
|
|
nut_interval=0.5) as test_client:
|
|
yield test_client
|
|
|
|
|
|
def test_a_real_upsd_send_authenticates_sends_and_audits(nut_client_app,
|
|
real_upsd_client) -> None:
|
|
body = wait_for_payload(nut_client_app, "/api/ups/snapshot")
|
|
assert body["simulated"] is False, "this test is about the branch that talks to upsd"
|
|
name = body["ups"][0]["name"]
|
|
response = nut_client_app.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.panel.start"})
|
|
assert response.status_code == 200, response.text
|
|
assert response.json()["simulated"] is False
|
|
assert real_upsd_client.sent == [(name, "test.panel.start")]
|
|
assert real_upsd_client.logged_in, "instcmd requires a LOGIN first (upsd refuses otherwise)"
|
|
events = nut_client_app.get("/api/events").json()["events"]
|
|
assert any(event["kind"] == "action" and name in event["message"] for event in events)
|
|
|
|
|
|
def test_a_refusal_from_upsd_is_a_502_and_an_error_event(nut_client_app,
|
|
real_upsd_client) -> None:
|
|
real_upsd_client.refusal = nut.NutError("Access denied", code="ACCESS-DENIED")
|
|
name = wait_for_payload(nut_client_app, "/api/ups/snapshot")["ups"][0]["name"]
|
|
response = nut_client_app.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 502
|
|
detail = response.json()["detail"]
|
|
assert "ACCESS-DENIED" in detail or "Access denied" in detail, detail
|
|
assert real_upsd_client.sent == [], "a refused command was never accepted"
|
|
events = nut_client_app.get("/api/events").json()["events"]
|
|
failures = [event for event in events if event["kind"] == "action"
|
|
and event["severity"] == "error"]
|
|
assert failures, "an upsd refusal has to leave a trace, not just an HTTP status"
|
|
|
|
|
|
def test_unknown_commands_are_refused_even_with_actions_on(ups_client) -> None:
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
for command in ("load.format", "rm -rf /", "INSTCMD x y"):
|
|
response = ups_client.post("/api/ups/command", json={"ups": name, "command": command})
|
|
assert response.status_code == 400, command
|
|
|
|
|
|
def test_commands_for_an_unknown_ups_are_404(ups_client) -> None:
|
|
wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": "not-real", "command": "beeper.mute"})
|
|
assert response.status_code == 404
|
|
|
|
|
|
def test_missing_fields_are_a_400(ups_client) -> None:
|
|
assert ups_client.post("/api/ups/command", json={"ups": ""}).status_code == 400
|
|
|
|
|
|
def test_actions_flag_off_blocks_every_command(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1") as client:
|
|
body = wait_for_payload(client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
assert body["actions_enabled"] is False
|
|
response = client.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 403
|
|
assert "NUT_ALLOW_ACTIONS" in response.json()["detail"]
|
|
|
|
|
|
def test_read_only_beats_the_actions_flag(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True, read_only=True) as client:
|
|
body = wait_for_payload(client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
response = client.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 403
|
|
commands = wait_for_payload(client, "/api/ups/commands")
|
|
assert commands["read_only"] is True
|
|
assert all(item["enabled"] is False for ups in commands["ups"]
|
|
for item in ups["commands"])
|
|
|
|
|
|
def test_ups_history_answers_for_simulated_ups(ups_client) -> None:
|
|
wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
time.sleep(1.2)
|
|
body = ups_client.get("/api/ups/history", params={"metrics": "ups.charge_pct",
|
|
"range": "1h"}).json()
|
|
assert body["ups"], "the UPS page needs a name to chart before any history exists"
|
|
|
|
|
|
def response_ok(response) -> bool:
|
|
assert response.status_code == 200, response.text
|
|
return True
|
|
|
|
|
|
def test_samples_written_by_the_poller_are_queryable(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The write path (poller -> store) and the read path (/api/history) must agree on names."""
|
|
db = str(tmp_path / "roundtrip.sqlite3")
|
|
store = SampleStore(db)
|
|
store.add_many([Sample("llm", "gufo", "llm.decode_tokens_per_second", 40.0, time.time() - 60),
|
|
Sample("llm", "gufo", "llm.decode_tokens_per_second", 45.0, time.time() - 30)])
|
|
app = create_app(Settings(db_path=db), store)
|
|
with TestClient(app) as client:
|
|
metrics = client.get("/api/metrics", params={"source": "llm"}).json()["metrics"]
|
|
assert "llm.decode_tokens_per_second" in metrics
|
|
points = client.get("/api/history", params={"source": "llm", "host": "gufo",
|
|
"metric": "llm.decode_tokens_per_second",
|
|
"range": "15m"}).json()["points"]
|
|
assert [p["value"] for p in points] == [40.0, 45.0]
|
|
store.close()
|