**Phase 26 `ups_event_split` — final verification pass (all 3 tasks already complete; nothing missing from `complete/`).**
Verified
- `page-ups.js`: `POWER_EVENT_KINDS = {status,test,action}` split, unconditional `refreshEvents()`, no `state.events`/toggle-fetch, shared `eventRow`, `#ups-errors-card.hidden = errors.length === 0`; `ups.html` error card above the disclosure, `hidden`, outside every `<details>`. Kind set cross-checked against the real writers (`collectors.py:393/409`, `api.py:1043/1058`, `poller.py:127/157/164/190`) — classification is correct.
- Re-ran the phase's browser pass live: `node data/logs/26-task03-verify.cjs` → **31 checks, 0 skipped, ALL PASS** (`data/logs/26-final-verify.out`) — poll-failure rows readable with the disclosure shut, card hidden on a power-only feed and appearing on the first `unreachable`, lists paging independently, 360 px clean.
Tests / lint (all green)
- `uv run pytest` → **1545 passed, 15 skipped**, exit 0 · `bash .agents/validate.sh` → exit 0, coverage **98.94%** (floor 94%) · `uv run ruff check src tests agent` clean · `uv run pyright` 0 errors · `node --check` clean · `uv run pytest -m live --live` (with `.env.dev`) → **11 passed**, dev server reads all 9 sources `ok`.
Completion criteria: poll-failure visible + power collapsed ✅ (pinned + browser); hidden-when-empty card, one `/api/events?source=ups` fetch, shared pager ✅; stale-last-good via the empty `catch` pin ✅; pytest/coverage/ruff/pyright ✅; `validate.sh` ✅; no behaviour change to prior work — `dash.css`/`app.js` untouched, disclosure CSS / pager / row-shape pins pass unchanged ✅.
Docs
- Fixed (pre-existing + this phase's drift): AGENTS.md "one helper with four call sites" → **five over four pages** and the `row(event)` sentence (phase 26 added a second `/ups` call site); invariant 5's `web/static/charts.js` → `src/dashboards/web/static/charts.js`; recorded the completion-pass transcript. README: "one helper renders every feed" now notes five lists. Full sweep otherwise consistent — all 38 env vars match `config.py` exactly, all served routes/scripts/contracts referenced exist, suite/coverage/live counts and marker splits verified.
Deviations: `fastapi 0.153.0 → 0.159.1` advisory left alone (validate.sh declares it advisory-not-a-gate; this phase touched no dependencies, matching prior phases' precedent). Next phase: **`27_mobile_tables_and_audit`**.
6384 lines
355 KiB
Python
6384 lines
355 KiB
Python
"""HTTP contract: what the pages may assume, and what must never leak.
|
|
|
|
The behaviours here are the ones a screenshot cannot show: an unconfigured source answering
|
|
404 `configured: false` instead of dialling a hostname that does not exist; a bearer token
|
|
that stops at the backend; the single write endpoint refusing on every gate; no external
|
|
asset the network can fail to deliver.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
import json
|
|
import re
|
|
import time
|
|
from collections.abc import Iterator
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
from cryptography import x509
|
|
from cryptography.x509.oid import NameOID
|
|
from fastapi import HTTPException
|
|
from fastapi.testclient import TestClient
|
|
|
|
import store_perf
|
|
from auth_client import sign_in
|
|
from auth_contract import ADMIN_PASSWORD, FAST_PBKDF2
|
|
from card_contract import HOST_SHAPES, assert_card_contract
|
|
from dashboards import alerts, auth, certs
|
|
from dashboards import app as app_module
|
|
from dashboards.api import MIN_PASSWORD_LENGTH, RANGE_SECONDS, _disk_visible, parse_range
|
|
from dashboards.app import create_app
|
|
from dashboards.collectors import UpstreamError
|
|
from dashboards.config import Settings, Target
|
|
from dashboards.rates import MIN_WINDOW_SECONDS, RateEstimator
|
|
from dashboards.severity import classify
|
|
from dashboards.sources import nut
|
|
from dashboards.sources.disks import SMART_SERIES, DisksCollector, parse_disks_snapshot
|
|
from dashboards.sources.host import HostCollector, parse_host_snapshot
|
|
from dashboards.sources.llm import (
|
|
HISTORY_METRICS,
|
|
parse_gufo_metrics,
|
|
parse_vllm_metrics,
|
|
)
|
|
from dashboards.store import RES_RAW, Event, Sample, SampleStore
|
|
from rate_window_contract import LLM_WINDOW, delta_label
|
|
from slots_contract import (
|
|
CANONICAL_TASK_ID_FIELD,
|
|
OPTIONAL_NEXT_TOKEN_FIELDS,
|
|
OPTIONAL_ROW_FIELDS,
|
|
REQUIRED_NEXT_TOKEN_FIELDS,
|
|
REQUIRED_ROW_FIELDS,
|
|
TASK_ID_ALIASES,
|
|
USER_CONTENT_FIELDS,
|
|
load_slots_capture,
|
|
)
|
|
|
|
# refused immediately, and never dials out of the test environment
|
|
DEAD_URL = "http://127.0.0.1:1"
|
|
|
|
#: Every dashboard route, in nav order. The markup pins below (no CDN, no `.sev-*`, no
|
|
#: narration, no orphan `id`) run across this list rather than a hand-written four, so a
|
|
#: fifth page cannot join the masthead and skip the checks the others had to pass.
|
|
PAGES = ("/", "/system", "/disks", "/llm", "/ups")
|
|
|
|
#: `/settings` is a **page** and not a dashboard (phase 11): it carries the same masthead with
|
|
#: no `aria-current` tab, and the account link in every dashboard's masthead is how you reach
|
|
#: it. Kept out of `PAGES` because the pins over that tuple assert tab structure — one current
|
|
#: tab, a fixed tab order — which this page deliberately does not have. `MARKUP_PAGES` is the
|
|
#: list for the checks that are about the bytes rather than the nav.
|
|
SETTINGS_PAGE = "/settings"
|
|
#: `/certs` is the second admin page (phase 15 task 05), kept out of `PAGES` for exactly the same
|
|
#: reason: it is not a tab. It is in `MARKUP_PAGES` and `PAGE_SCRIPTS`, so the CDN, `.sev-`,
|
|
#: flavour-text and orphan-`id` audits cover it like the five dashboards.
|
|
CERTS_PAGE = "/certs"
|
|
MARKUP_PAGES = (*PAGES, SETTINGS_PAGE, CERTS_PAGE)
|
|
|
|
|
|
@contextmanager
|
|
def build_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, simulate: bool = False,
|
|
**overrides: Any) -> Iterator[TestClient]:
|
|
"""A TestClient whose lifespan runs, against a private DB and no real network.
|
|
|
|
The store is injected, so `create_app()` deliberately does not close it (production
|
|
owns its own); the caller here does, or every test leaks a SQLite handle and the
|
|
`--cov` log fills with ResourceWarnings that bury the warnings worth reading.
|
|
"""
|
|
if simulate:
|
|
monkeypatch.setenv("NUT_SIMULATE", "1")
|
|
else:
|
|
monkeypatch.delenv("NUT_SIMULATE", raising=False)
|
|
db = str(tmp_path / "api.sqlite3")
|
|
settings = Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2, **overrides)
|
|
store = SampleStore(db)
|
|
client = TestClient(create_app(settings, store))
|
|
client.__enter__()
|
|
try:
|
|
sign_in(client)
|
|
yield client
|
|
finally:
|
|
client.__exit__(None, None, None)
|
|
store.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
|
with build_client(tmp_path, monkeypatch) as test_client:
|
|
yield test_client
|
|
|
|
|
|
@pytest.fixture
|
|
def ups_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
|
"""NUT on the labelled simulator, actions allowed — the dev configuration."""
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True) as c:
|
|
yield c
|
|
|
|
|
|
def wait_for_source_error(client: TestClient, key: str, fragment: str,
|
|
*, timeout: float = 15.0) -> str:
|
|
"""Wait for one poll loop to record an error saying `fragment`, and hand back the sentence.
|
|
|
|
The poll loops run in the app's event loop, so an assertion about what a source said has to
|
|
wait for a poll rather than read the state at import time. The failure names the last error
|
|
seen, because "it never said that" and "it said something else entirely" are different bugs.
|
|
"""
|
|
deadline = time.time() + timeout
|
|
error = ""
|
|
while time.time() < deadline:
|
|
rows = {row["key"]: row for row in client.get("/api/health").json()["sources"]}
|
|
error = str((rows.get(key) or {}).get("error") or "")
|
|
if fragment in error:
|
|
return error
|
|
time.sleep(0.05)
|
|
pytest.fail(f"{key} never reported {fragment!r} (last error: {error!r})")
|
|
|
|
|
|
def wait_for_payload(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
response = client.get(path)
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
time.sleep(0.1)
|
|
pytest.fail(f"{path} never became available")
|
|
|
|
|
|
def failing_host_payload(scrape: Any, agent_snapshot: dict[str, Any]) -> dict[str, Any]:
|
|
"""Two scrapes of the real driveripper capture, nudged onto known sides of the table.
|
|
|
|
Driven from the fixture rather than hand-written so a schema drift breaks here first.
|
|
"""
|
|
def nudge(snap: dict[str, Any]) -> None:
|
|
snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.95 # mem.used_pct -> fail
|
|
snap["cpu_temperature_celsius"] = 91.0 # cpu.temp_c -> fail
|
|
for fs in snap["filesystems"]:
|
|
fs["used_pct"] = 93.0 # disk -> fail
|
|
|
|
return scrape(agent_snapshot, cpu_busy=10.0, mutate=nudge)
|
|
|
|
|
|
@pytest.fixture
|
|
def data_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
agent_snapshot: dict[str, Any], host_scrape: Any) -> Iterator[TestClient]:
|
|
"""Overview-shaped app: one host payload in the hub plus the simulated UPSes.
|
|
|
|
The host job points at a dead port, so a poll can never overwrite the injected payload
|
|
(a failed poll keeps the last good one) — the numbers below are ours, not the fleet's.
|
|
"""
|
|
monkeypatch.setenv("NUT_SIMULATE", "1")
|
|
db = str(tmp_path / "summary.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, nut_host="127.0.0.1",
|
|
admin_password=ADMIN_PASSWORD, pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=(Target("driveripper", DEAD_URL),)), store)
|
|
test_client = TestClient(app)
|
|
test_client.__enter__()
|
|
try:
|
|
sign_in(test_client)
|
|
app.state.ctx.hub.jobs["host:driveripper"].state.payload = failing_host_payload(
|
|
host_scrape, agent_snapshot)
|
|
wait_for_payload(test_client, "/api/ups/snapshot")
|
|
yield test_client
|
|
finally:
|
|
test_client.__exit__(None, None, None)
|
|
store.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def healthy_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
agent_snapshot: dict[str, Any],
|
|
host_scrape: Any) -> Iterator[TestClient]:
|
|
"""The all-clear app: one host answering on both of its jobs, nothing past a threshold.
|
|
|
|
The job's collector is swapped for one that hands back the injected payload, so the
|
|
health record says `ok` truthfully instead of being overridden behind the poller — and
|
|
that has to happen for the `disks` job too, which polls the same dead port: a host is
|
|
either answering or it is not, and a fixture that claims all-clear cannot leave one of
|
|
its two readers failing. Each reader gets **its own** projection of the same capture
|
|
(`parse_disks_snapshot` for the drive page), because a `disks` job holding a `system`
|
|
payload would make `/api/summary`'s drive block answer from the wrong shape and read as
|
|
a passing test that proved nothing.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
from dashboards.poller import Result # noqa: PLC0415 - only this fixture needs it
|
|
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
disks_payload = parse_disks_snapshot("driveripper", agent_snapshot,
|
|
DisksCollector()).payload
|
|
|
|
async def collect() -> Result:
|
|
return Result(payload=payload, samples=[])
|
|
|
|
async def collect_disks() -> Result:
|
|
return Result(payload=disks_payload, samples=[])
|
|
|
|
hub = test_client.app.state.ctx.hub # type: ignore[attr-defined]
|
|
job = hub.jobs["host:driveripper"]
|
|
disks_job = hub.jobs["disks:driveripper"]
|
|
job.fn = collect
|
|
disks_job.fn = collect_disks
|
|
# let the poll already in flight against the dead port land first (it fails in
|
|
# microseconds), then take the health record over: the alerts must describe a host
|
|
# that is answering, not the half-second of startup where it was not
|
|
deadline = time.time() + 2.0
|
|
while time.time() < deadline and job.state.attempts == 0:
|
|
time.sleep(0.02)
|
|
time.sleep(0.05)
|
|
for reader in hub.jobs.values():
|
|
if reader.dashboard not in ("system", "disks"):
|
|
continue
|
|
reader.state.payload = disks_payload if reader is disks_job else payload
|
|
reader.state.ok = True
|
|
reader.state.error = None
|
|
reader.state.consecutive_failures = 0
|
|
reader.state.last_ok = time.time()
|
|
yield test_client
|
|
|
|
|
|
# ---- severity travels with the value --------------------------------------- #
|
|
|
|
def test_summary_carries_the_verdict_for_every_column_the_page_colours(data_client) -> None:
|
|
body = data_client.get("/api/summary").json()
|
|
row = body["system"]["hosts"][0]
|
|
assert row["severity"] == {"cpu_usage_pct": "ok", "mem_used_pct": "fail",
|
|
"disk_worst_used_pct": "fail", "temp_c": "fail"}
|
|
# the fleet card is judged by the same table, so amber means the same thing at both
|
|
# sizes — and no GPU verdict, at fleet size or host size (phase 06: busy is the job)
|
|
assert body["system"]["fleet"]["severity"] == {
|
|
"cpu_usage_pct_avg": "ok", "memory_used_pct": "fail",
|
|
"disk_worst_used_pct": "fail",
|
|
}
|
|
for ups in body["ups"]["ups"]:
|
|
assert set(ups["severity"]) == {"charge_pct", "load_pct", "runtime_seconds"}
|
|
assert ups["severity"]["charge_pct"] == classify("ups.charge_pct", ups["charge_pct"])
|
|
assert ups["severity"]["charge_pct"] == "ok", "the simulator starts at float charge"
|
|
|
|
|
|
def test_summary_shape_did_not_shrink(data_client) -> None:
|
|
"""Additive-only anchor: /api/summary gained `severity` and lost nothing.
|
|
|
|
The overview, the live tests, and any external consumer read the flat keys; this test is
|
|
the tripwire for a well-meaning refactor that "tidies" them away.
|
|
"""
|
|
body = data_client.get("/api/summary").json()
|
|
fleet_keys = {"hosts_total", "hosts_live", "cpu_cores", "cpu_usage_pct_avg",
|
|
"memory_total_bytes", "memory_used_bytes", "memory_used_pct",
|
|
"disk_worst_used_pct", "load1_total", "net_rx_bytes_per_s",
|
|
"net_tx_bytes_per_s", "gpu_count", "gpu_busy_pct_avg", "power_estimate_w"}
|
|
host_keys = {"name", "os", "cpu_usage_pct", "load1", "cores", "mem_used_pct",
|
|
"disk_worst_used_pct", "gpu_busy_pct", "temp_c", "uptime_seconds"}
|
|
ups_keys = {"name", "overall", "status", "charge_pct", "load_pct", "runtime_seconds"}
|
|
assert fleet_keys <= set(body["system"]["fleet"]), fleet_keys - set(body["system"]["fleet"])
|
|
assert host_keys <= set(body["system"]["hosts"][0]), host_keys - set(body["system"]["hosts"][0])
|
|
assert ups_keys <= set(body["ups"]["ups"][0]), ups_keys - set(body["ups"]["ups"][0])
|
|
# `disks` joined in phase 09 and changed nothing else: the set grows, never shrinks
|
|
assert set(body) == {"ts", "system", "disks", "llm", "ups", "recent_events", "alerts"}
|
|
# ...and the new keys are there beside the old ones, not instead of them
|
|
assert "severity" in body["system"]["fleet"]
|
|
assert "severity" in body["system"]["hosts"][0]
|
|
assert "severity" in body["ups"]["ups"][0]
|
|
|
|
|
|
# `/` paints two panels; the payload is still the whole-fleet contract. Every path here is
|
|
# read by something that is not the overview page: the live suite, an external probe, a
|
|
# notifier later. Deleting a card must never delete a key.
|
|
SUMMARY_CONTRACT = ("system.enabled", "system.fleet", "system.hosts",
|
|
"disks.enabled", "disks.fleet", "disks.hosts",
|
|
"llm.engines", "ups.ups", "recent_events", "alerts")
|
|
|
|
|
|
def dig(body: dict[str, Any], path: str) -> Any:
|
|
"""`body["a"]["b"]` that names the missing path when it is not there."""
|
|
node: Any = body
|
|
for part in path.split("."):
|
|
assert isinstance(node, dict) and part in node, f"/api/summary lost `{path}`"
|
|
node = node[part]
|
|
return node
|
|
|
|
|
|
@pytest.mark.parametrize("fixture", ["client", "data_client"])
|
|
def test_summary_serves_every_path_another_consumer_reads(request, fixture: str) -> None:
|
|
"""The keys survive the overview cleanup whether or not anything is configured.
|
|
|
|
Run twice on purpose: `client` has no targets at all, so it proves the keys are present
|
|
as `None`/`[]` rather than omitted when a source is unconfigured (invariant 3 is about
|
|
the payload shape as much as about colour); `data_client` proves the same paths carry
|
|
the fleet once there is one.
|
|
"""
|
|
test_client = request.getfixturevalue(fixture)
|
|
body = test_client.get("/api/summary").json()
|
|
populated = fixture == "data_client"
|
|
for path in SUMMARY_CONTRACT:
|
|
dig(body, path) # presence is the assertion; a missing path raises with its name
|
|
assert dig(body, "system.enabled") is populated
|
|
assert (dig(body, "system.fleet") is not None) is populated
|
|
names = {host["name"] for host in dig(body, "system.hosts")}
|
|
assert names == ({"driveripper"} if populated else set())
|
|
# the drive-health block is configured exactly when the agents it reads are, and lists the
|
|
# same hosts: a `disks` job exists for every `HOST_TARGETS` entry and for no other reason
|
|
assert dig(body, "disks.enabled") is populated
|
|
assert (dig(body, "disks.fleet") is not None) is populated
|
|
assert {host["name"] for host in dig(body, "disks.hosts")} == names
|
|
batteries = {item["name"] for item in dig(body, "ups.ups")}
|
|
assert batteries == ({"apc", "cyberpower"} if populated else set())
|
|
assert dig(body, "llm.engines") == [], "neither fixture configures an engine"
|
|
assert isinstance(dig(body, "recent_events"), list)
|
|
assert isinstance(dig(body, "alerts"), list)
|
|
# and the payload as a whole still carries every top-level section it always did
|
|
assert set(body) >= {"ts", "system", "llm", "ups", "recent_events", "alerts"}
|
|
|
|
|
|
# ---- alerts: the landing page's answer to "what is wrong" ------------------- #
|
|
|
|
def test_summary_is_empty_of_alerts_when_everything_is_healthy(healthy_client) -> None:
|
|
"""`[]`, not a green banner: an empty list is the clearest possible all-clear."""
|
|
body = healthy_client.get("/api/summary").json()
|
|
assert body["alerts"] == []
|
|
assert body["system"]["fleet"]["hosts_live"] == 1
|
|
|
|
|
|
def test_summary_raises_one_alert_for_the_host_that_stopped_answering(data_client) -> None:
|
|
"""One dead host is one line naming it, with the link to the page that explains it."""
|
|
body = data_client.get("/api/summary").json()
|
|
down = [item for item in body["alerts"] if item["source"] == "driveripper"]
|
|
assert len(down) == 1, "its stale payload must not add amber lines on top of the red one"
|
|
assert down[0]["severity"] == "fail"
|
|
assert down[0]["href"] == "/system"
|
|
assert "unreachable" in down[0]["message"]
|
|
|
|
|
|
def test_a_host_that_has_never_answered_is_still_a_row_in_the_fleet(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A dead agent is a red row in the fleet, not a fleet that quietly got smaller.
|
|
|
|
`/api/system/snapshot` already stubs a job with no payload; `/api/summary` has to agree
|
|
with it, or a target that never answered since startup leaves the table at "2/2 hosts"
|
|
while the panel above says a third box is down (`AGENTS.md` invariants 3 and 4).
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
body = test_client.get("/api/summary").json()
|
|
rows = body["system"]["hosts"]
|
|
assert [row["name"] for row in rows] == ["driveripper"]
|
|
assert rows[0]["unreachable"] is True
|
|
assert rows[0]["cpu_usage_pct"] is None, "n/a, never a 0.0 nobody measured"
|
|
assert body["system"]["fleet"]["hosts_total"] == 1
|
|
assert body["system"]["fleet"]["hosts_live"] == 0
|
|
# and the alert list stays exactly one line for it, not a row plus a stale payload
|
|
assert [item["source"] for item in body["alerts"]] == ["driveripper"]
|
|
|
|
|
|
def test_summary_alerts_are_worst_first_and_carry_the_contract(data_client) -> None:
|
|
body = data_client.get("/api/summary").json()
|
|
alerts_list = body["alerts"]
|
|
assert alerts_list, "a dead host plus simulated UPS data cannot read as all-clear"
|
|
for item in alerts_list:
|
|
assert set(item) == {"id", "severity", "source", "message", "href", "at"}
|
|
assert item["severity"] in ("warn", "fail")
|
|
ranks = [0 if item["severity"] == "fail" else 1 for item in alerts_list]
|
|
assert ranks == sorted(ranks), "fails before warns"
|
|
assert any("simulat" in item["message"].lower() and item["href"] == "/ups"
|
|
for item in alerts_list), "NUT_SIMULATE must be said out loud on the landing page"
|
|
|
|
|
|
def test_summary_reuses_the_alert_builder_verbatim(data_client) -> None:
|
|
"""The endpoint is a thin call: what `alerts.build` says over the same state is the body."""
|
|
body = data_client.get("/api/summary").json()
|
|
hub = data_client.app.state.ctx.hub # type: ignore[attr-defined]
|
|
settings = data_client.app.state.ctx.settings # type: ignore[attr-defined]
|
|
hosts = [job.state.payload for job in hub.jobs.values()
|
|
if job.dashboard == "system" and job.state.payload]
|
|
ups_payload = hub.jobs["nut"].state.payload
|
|
expected = alerts.build(settings, hub.states(), hosts, [], ups_payload, [])
|
|
assert [item["id"] for item in body["alerts"]] == [item["id"] for item in expected]
|
|
|
|
|
|
def test_snapshot_payloads_carry_their_own_severity_sections(data_client) -> None:
|
|
"""/api/system/snapshot and /api/ups/snapshot ship the same verdicts the summary reuses."""
|
|
system = data_client.get("/api/system/snapshot").json()
|
|
host = system["hosts"][0]
|
|
assert set(host["cpu"]["severity"]) == {"usage_pct", "temp_c"}
|
|
assert set(host["memory"]["severity"]) == {"used_pct"}
|
|
assert set(host["disks"]["severity"]) == {"worst_used_pct"}
|
|
assert "severity" not in host["gpus"][0], (
|
|
"a GPU verdict came back on /api/system/snapshot — phase 06 removed it"
|
|
)
|
|
ups = data_client.get("/api/ups/snapshot").json()
|
|
for card in ups["ups"]:
|
|
assert set(card["severity"]) == {"charge_pct", "load_pct", "battery_runtime"}
|
|
# numbers keep their own shape; severity is a sibling, never a replacement
|
|
assert isinstance(card["numbers"], dict) and "charge_pct" in card["numbers"]
|
|
|
|
|
|
# ---- the self-test verdict on the two surfaces that were wrong (phase 12) --- #
|
|
|
|
#: The `ups.test.result` sentences as the hardware answers them. `Done and passed` is the
|
|
#: CyberPower's verbatim word from the 2026-10-19 capture (`tests/test_nut.py
|
|
#: ::LIVE_SELF_TEST_WORDS`); `FAILED` is what a driver says when a test ran and did not pass,
|
|
#: and is the reason the check exists at all. `tests/test_nut.py` pins the *word map*; what is
|
|
#: pinned here is the surface `TODO.md:3` names — a warning in the overview's Warnings panel —
|
|
#: because a verdict that grades correctly inside `assess()` and still reaches the alert feed
|
|
#: as a warning is the same bug wearing a different hat.
|
|
SELF_TEST_PASS_WORD = "Done and passed"
|
|
SELF_TEST_FAIL_WORD = "FAILED"
|
|
|
|
#: How `/api/summary` says the check out loud (`alerts._ups_alerts`): `"<check name>: <detail>"`.
|
|
SELF_TEST_ALERT_TEXT = "Last self-test"
|
|
|
|
|
|
def set_self_test_verdict(payload: dict[str, Any], status: str, detail: str) -> None:
|
|
"""Rewrite every card's `Last self-test` check in the hub's own UPS payload.
|
|
|
|
Mutating the payload the job actually serves (instead of hand-building a fake one) is what
|
|
makes this an end-to-end pin: `/api/ups/snapshot` and `alerts.build()` then read the *same*
|
|
object, so the test cannot pass by having the two surfaces disagree about the verdict.
|
|
"""
|
|
graded = 0
|
|
for card in payload["ups"]:
|
|
for check in card["health"]["checks"]:
|
|
if check["name"] == SELF_TEST_ALERT_TEXT:
|
|
check["status"] = status
|
|
check["detail"] = detail
|
|
graded += 1
|
|
assert graded, "no `Last self-test` check in the UPS payload — nothing was tested"
|
|
|
|
|
|
@contextmanager
|
|
def self_test_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
) -> Iterator[tuple[TestClient, dict[str, Any]]]:
|
|
"""A UPS app whose served payload cannot be replaced by a poll mid-assertion.
|
|
|
|
The simulated UPSes answer `PASSED`, so the pass direction would survive a race with the
|
|
poll loop and the *fail* direction would not — a test that flakes on a poll is a test of
|
|
timing. Hence `healthy_client`'s trick for the `nut` job: the collector hands back the very
|
|
dict this harness holds, so the health record says `ok` truthfully (invariant 4) and the
|
|
payload stays ours. The interval is shortened for the same reason — waiting a real
|
|
`NUT_INTERVAL_SECONDS` per test would only make the suite slower at proving the same thing.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_interval=0.5) as test_client:
|
|
from dashboards.poller import Result # noqa: PLC0415 - only this harness needs it
|
|
|
|
hub = test_client.app.state.ctx.hub # type: ignore[attr-defined]
|
|
job = hub.jobs["nut"]
|
|
wait_for_payload(test_client, "/api/ups/snapshot")
|
|
payload = job.state.payload
|
|
|
|
async def collect() -> Result:
|
|
return Result(payload=payload, samples=[])
|
|
|
|
job.fn = collect
|
|
# let one poll of *our* collector land before the caller mutates the payload, so the
|
|
# health record and the served bytes are both demonstrably ours
|
|
baseline = job.state.attempts
|
|
deadline = time.time() + 5.0
|
|
while time.time() < deadline and job.state.attempts <= baseline:
|
|
time.sleep(0.02)
|
|
assert job.state.payload is payload and job.state.ok, (
|
|
"the nut job is not serving the payload under test — the pin below would be fiction"
|
|
)
|
|
yield test_client, payload
|
|
|
|
|
|
def self_test_alerts(alerts_list: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
return [item for item in alerts_list
|
|
if SELF_TEST_ALERT_TEXT.lower() in item["message"].lower()
|
|
or SELF_TEST_ALERT_TEXT.lower() in item["id"].lower()]
|
|
|
|
|
|
def test_a_passed_self_test_is_quiet_on_the_overview_and_ok_on_the_ups_page(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The TODO's sentence, end to end: `Done and passed` is not a warning anywhere.
|
|
|
|
Both halves of the fix are asserted on one payload: `/api/ups/snapshot` carries the `ok`
|
|
entry the `/ups` check list renders verbatim, and `/api/summary` — the only call the
|
|
overview makes — carries **no** row naming it. The simulated card still contributes its
|
|
`NUT_SIMULATE` line, so the list is not empty and this cannot be a vacuous pass.
|
|
"""
|
|
with self_test_app(tmp_path, monkeypatch) as (test_client, payload):
|
|
set_self_test_verdict(payload, "ok", SELF_TEST_PASS_WORD)
|
|
cards = wait_for_payload(test_client, "/api/ups/snapshot")["ups"]
|
|
for card in cards:
|
|
check = next(c for c in card["health"]["checks"]
|
|
if c["name"] == SELF_TEST_ALERT_TEXT)
|
|
assert check["status"] == "ok", f"{card['name']} reports a pass as {check['status']}"
|
|
assert check["detail"] == SELF_TEST_PASS_WORD, "the driver's word is the useful half"
|
|
rows = self_test_alerts(test_client.get("/api/summary").json()["alerts"])
|
|
assert rows == [], f"a passed battery test is on the overview as {rows}"
|
|
assert any(item["source"] == "ups" for item in
|
|
test_client.get("/api/summary").json()["alerts"]), (
|
|
"the simulated-UPS line disappeared, so this assertion has nothing left to be quiet about"
|
|
)
|
|
|
|
|
|
def test_a_never_tested_battery_is_not_an_alert_either(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The APC's factory `No test initiated` reaches the overview as silence, not amber.
|
|
|
|
This is the state the live hardware is actually in on one of the two UPSes, so the
|
|
screenshot proves the absence of a warning for a *never-run* test while the payload pin
|
|
above proves it for a passed one. `unknown` is not a verdict an operator can act on, and
|
|
`alerts._RANK` admits only `fail` and `warn`.
|
|
"""
|
|
with self_test_app(tmp_path, monkeypatch) as (test_client, payload):
|
|
set_self_test_verdict(payload, "unknown", nut.NEVER_TESTED_DETAIL)
|
|
cards = wait_for_payload(test_client, "/api/ups/snapshot")["ups"]
|
|
assert all(next(c for c in card["health"]["checks"]
|
|
if c["name"] == SELF_TEST_ALERT_TEXT)["status"] == "unknown"
|
|
for card in cards)
|
|
assert self_test_alerts(test_client.get("/api/summary").json()["alerts"]) == []
|
|
|
|
|
|
def test_a_failed_self_test_still_reaches_the_overview(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The check keeps its reason for existing: a test that failed is a red line, once per UPS.
|
|
|
|
Without this direction the absence test above is satisfiable by deleting the check, which
|
|
is the failure mode the phase's completion criterion calls out.
|
|
"""
|
|
with self_test_app(tmp_path, monkeypatch) as (test_client, payload):
|
|
set_self_test_verdict(payload, "fail", SELF_TEST_FAIL_WORD)
|
|
names = {card["name"] for card in
|
|
wait_for_payload(test_client, "/api/ups/snapshot")["ups"]}
|
|
assert names, "no UPS card to fail"
|
|
rows = self_test_alerts(test_client.get("/api/summary").json()["alerts"])
|
|
assert {row["source"] for row in rows} == names
|
|
for row in rows:
|
|
assert row["severity"] == "fail", f"a failed battery test reads {row['severity']}"
|
|
assert row["href"] == "/ups", "the alert must link to the card that names the UPS"
|
|
assert SELF_TEST_ALERT_TEXT in row["message"] and SELF_TEST_FAIL_WORD in row["message"]
|
|
assert row["message"].count(SELF_TEST_ALERT_TEXT) == 1, (
|
|
"the row names the check twice — see alerts._clean_error")
|
|
assert all(row["id"] == f"{row['source']}:health:{SELF_TEST_ALERT_TEXT}" for row in rows)
|
|
|
|
|
|
def test_the_ups_check_list_needs_no_new_markup_for_a_passed_verdict(client) -> None:
|
|
"""Task 03 expected no page change, and this pins *why* there is none.
|
|
|
|
`page-ups.js` puts `check.status` straight on the `<li>`, and `dash.css` colours only the
|
|
`.mark` glyph from the shared `--ok`/`--warn`/`--fail` tokens while the check's name and
|
|
detail stay `--text`/`--text-muted` — so an `ok` row is quiet ink with a tick, and a verdict
|
|
fixed in `assess()` is a verdict rendered. Nothing on either asset names the self-test
|
|
check, which is the point: a special case added here would be a second opinion about the
|
|
driver's word, and the API already made it.
|
|
"""
|
|
js = _code(client.get("/static/page-ups.js").text)
|
|
assert '<li class="${check.status}">' in js, "the row no longer wears the payload verdict as its class"
|
|
for verdict in ("ok", "warn", "fail", "unknown"):
|
|
assert f"{verdict}: '" in js[js.index("const MARKS"):], f"MARKS lost the {verdict} glyph"
|
|
rows = js[js.index("const health = item.health"):js.index("const commands = item.commands")]
|
|
assert "self-test" not in rows.lower(), "the check list grew an opinion about one check by name"
|
|
css = _code(_css(client))
|
|
ok_rule = re.search(r"ul\.checks li\.ok \.mark \{([^}]*)\}", css)
|
|
assert ok_rule and "var(--ok)" in ok_rule.group(1), "the ok tick is not the shared token"
|
|
assert "ul.checks li.ok {" not in css, "the whole ok row is coloured — a pass is not a banner"
|
|
assert "self-test" not in css.lower(), "the stylesheet special-cases one check by name"
|
|
|
|
|
|
# ---- unconfigured: say so, and do not dial -------------------------------- #
|
|
|
|
def test_config_reports_every_source_as_off(client) -> None:
|
|
body = client.get("/api/config").json()
|
|
assert body["system"]["enabled"] is False
|
|
assert body["system"]["targets"] == []
|
|
assert body["disks"] == {"enabled": False, "interval_seconds": 300.0,
|
|
"timeout_seconds": 30.0}, (
|
|
"the drive-health page is configured exactly when the host agents are")
|
|
assert body["llm"]["gufo"]["enabled"] is False
|
|
assert body["llm"]["vllm"]["enabled"] is False
|
|
assert body["ups"]["enabled"] is False
|
|
assert body["ups"]["actions_enabled"] is False
|
|
assert body["ranges"], "the page builds its window buttons from this list"
|
|
|
|
|
|
def test_health_with_nothing_configured_is_not_a_failure(client) -> None:
|
|
"""Nothing configured is 'empty', not 'broken' — and the page must not go red."""
|
|
body = client.get("/api/health").json()
|
|
assert body["status"] in ("ok", "empty")
|
|
assert body["ok"] is True
|
|
assert body["sources"] == []
|
|
|
|
|
|
def test_the_disk_source_has_its_own_health_record_per_host(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""`disks` is a source of its own, not a field of `system` (phase 09).
|
|
|
|
Its own record is what gives the page its own dot and its own staleness, and it is
|
|
also what makes `alerts.py` need a rule: two jobs read the same agent, so one dead
|
|
host must still be one line on the overview (see `test_summary_raises_one_alert...`).
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
sources = {row["key"]: row for row in test_client.get("/api/health").json()["sources"]}
|
|
assert "disks:driveripper" in sources, "no separate record, no separate dot"
|
|
assert sources["disks:driveripper"]["dashboard"] == "disks"
|
|
assert sources["disks:driveripper"]["label"] == "driveripper"
|
|
assert sources["disks:driveripper"]["configured"] is True
|
|
assert sources["host:driveripper"]["dashboard"] == "system", (
|
|
"the system job is still its own source: one slow SMART poll must not "
|
|
"decide whether the host looks alive")
|
|
|
|
|
|
def test_no_targets_means_no_disk_job_that_could_look_broken(client) -> None:
|
|
"""No host agents, no `disks` task at all — so nothing red for an unconfigured page."""
|
|
assert client.get("/api/health").json()["sources"] == []
|
|
|
|
|
|
@pytest.mark.parametrize("path", ["/api/system/snapshot", "/api/llm/snapshot",
|
|
"/api/ups/snapshot", "/api/ups/commands"])
|
|
def test_unconfigured_source_is_404_configured_false(client, path) -> None:
|
|
response = client.get(path)
|
|
assert response.status_code == 404
|
|
assert response.json()["configured"] is False
|
|
assert "detail" in response.json()
|
|
|
|
|
|
def test_summary_still_answers_with_no_sources(client) -> None:
|
|
body = client.get("/api/summary").json()
|
|
assert body["system"]["enabled"] is False
|
|
assert body["system"]["fleet"] is None
|
|
assert body["llm"]["enabled"] is False
|
|
assert body["ups"]["enabled"] is False
|
|
|
|
|
|
def test_unknown_route_is_json_not_html(client) -> None:
|
|
response = client.get("/api/nope")
|
|
assert response.status_code == 404
|
|
assert response.headers["content-type"].startswith("application/json")
|
|
assert "no such route" in response.json()["detail"]
|
|
|
|
def test_an_upstream_failure_escaping_a_route_keeps_its_own_status(tmp_path: Path) -> None:
|
|
"""`collectors.UpstreamError` is the one error type a route may raise to say "upstream".
|
|
|
|
The handler turns it into `{ok: false, detail}` with the collector's status (502, or 504
|
|
for a timeout) instead of a 500 traceback, and the message is the collector's already
|
|
redacted sentence. Nothing dials here: the error is raised by the route under test.
|
|
"""
|
|
db = str(tmp_path / "upstream-error.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
|
|
@app.get("/api/_boom")
|
|
async def boom() -> None:
|
|
raise UpstreamError("driveripper: http://127.0.0.1:9120/snapshot timed out", status=504)
|
|
|
|
with TestClient(app) as test_client:
|
|
sign_in(test_client)
|
|
response = test_client.get("/api/_boom")
|
|
assert response.status_code == 504, "a timeout stays a timeout, not a 500"
|
|
assert response.json() == {"ok": False,
|
|
"detail": "driveripper: http://127.0.0.1:9120/snapshot "
|
|
"timed out"}
|
|
assert "no-store" in response.headers["cache-control"]
|
|
store.close()
|
|
|
|
|
|
# ---- tokens --------------------------------------------------------------- #
|
|
|
|
def test_agent_tokens_never_reach_the_browser(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("NUT_SIMULATE", "")
|
|
targets = [Target(name="driveripper", url="http://driveripper:9120",
|
|
headers={"Authorization": "Bearer SEKRIT-TOKEN"})]
|
|
db = str(tmp_path / "tokens.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=tuple(targets)), store)
|
|
with TestClient(app) as client:
|
|
sign_in(client)
|
|
body = client.get("/api/config")
|
|
assert "SEKRIT-TOKEN" not in body.text
|
|
target = body.json()["system"]["targets"][0]
|
|
assert target["name"] == "driveripper"
|
|
assert "SEKRIT" not in target["url"] and "@" not in target["url"]
|
|
# and a host that never answered stays in the list, marked, with the fleet card intact
|
|
snapshot = wait_for_state(client, "/api/system/snapshot")
|
|
assert snapshot["hosts"][0]["name"] == "driveripper"
|
|
assert snapshot["hosts"][0].get("unreachable") is True
|
|
assert snapshot["fleet"]["hosts_total"] == 1
|
|
assert snapshot["fleet"]["hosts_live"] == 0
|
|
assert snapshot["fleet"]["memory_total_bytes"] is None
|
|
store.close()
|
|
|
|
|
|
def test_a_dead_host_never_writes_its_token_into_the_feed(tmp_path: Path) -> None:
|
|
"""The second path a credential can take to a browser: the failure message.
|
|
|
|
`parse_targets` moves a userinfo token out of the URL and into a bearer header, so the
|
|
documented `HOST_TARGETS` path never puts one in `Target.url` — this is the backstop for
|
|
any `Target` that does (a hand-built one, a future parser), and the message path is where
|
|
it would matter: the poller keeps an `UpstreamError` string in the health record and logs
|
|
it as an event, so a userinfo-bearing URL there would sit in SQLite and be served on
|
|
`/api/events` forever (AGENTS.md invariant 6). Asserted on every endpoint a page reads,
|
|
not just `/api/config`.
|
|
"""
|
|
db = str(tmp_path / "dead-host-token.sqlite3")
|
|
store = SampleStore(db)
|
|
settings = Settings(
|
|
db_path=db,
|
|
admin_password=ADMIN_PASSWORD, pbkdf2_iterations=FAST_PBKDF2,
|
|
host_interval=0.05,
|
|
host_targets=(Target(name="driveripper", url="http://SEKRIT-AGENT-TOKEN@127.0.0.1:1"),),
|
|
)
|
|
with TestClient(create_app(settings, store)) as client:
|
|
sign_in(client)
|
|
deadline = time.time() + 10
|
|
while time.time() < deadline:
|
|
health = client.get("/api/health").json()
|
|
if health["sources"] and health["sources"][0]["error"]:
|
|
break
|
|
time.sleep(0.05)
|
|
else:
|
|
pytest.fail("the dead host never recorded an error")
|
|
for path in ("/api/health", "/api/events", "/api/system/snapshot",
|
|
"/api/disks/snapshot", "/api/summary"):
|
|
body = client.get(path).text
|
|
assert "SEKRIT-AGENT-TOKEN" not in body, f"{path} leaked the agent token"
|
|
# and the outage is still legible — redaction is not the same as silence
|
|
assert "driveripper" in client.get("/api/health").text
|
|
assert "unreachable" in client.get("/api/events").text
|
|
# the `disks` job dials the same token-bearing URL on its own cadence, so its
|
|
# sentence is a second copy of the same risk: `fetch_snapshot` is the one place
|
|
# either dashboard builds a failure message, and it uses the redacted URL
|
|
disks_state = next(row for row in client.get("/api/health").json()["sources"]
|
|
if row["key"] == "disks:driveripper")
|
|
assert disks_state["error"] and "SEKRIT-AGENT-TOKEN" not in disks_state["error"]
|
|
assert "127.0.0.1:1/snapshot" in disks_state["error"], "the failure still names where"
|
|
store.close()
|
|
|
|
|
|
def test_a_revoked_agent_fails_both_of_its_sources_without_a_dial(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The chain end to end, wiring included: revoke it, and the next poll stops talking to it.
|
|
|
|
`collectors.agent_tls_map` is what `build_hub` hands the jobs, so this is the proof that a
|
|
revocation reaches a *running* app through its own `meta` table — no restart, and no second
|
|
process telling the first. The spy replaces `httpx.AsyncClient.get` on the class, so it
|
|
catches the shared client and the per-dial TLS client alike: what must not happen is any
|
|
request at all, because a dial that then gets refused still put a socket on a host the
|
|
operator struck out.
|
|
"""
|
|
db = str(tmp_path / "revoked-agent.sqlite3")
|
|
store = SampleStore(db)
|
|
settings = Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_interval=0.05, disks_interval=0.05,
|
|
host_targets=(Target(name="deskwork", url="https://127.0.0.1:1"),))
|
|
with TestClient(create_app(settings, store)) as client:
|
|
sign_in(client)
|
|
# Before the revoke the same dead host fails as an unreachable upstream. That sentence is
|
|
# the control: it is what a dial produces, so the one after it has to mean something else.
|
|
wait_for_source_error(client, "host:deskwork", "unreachable")
|
|
state = certs.issue_agent_cert(store, certs.ensure_certs(store), "deskwork",
|
|
["127.0.0.1"])
|
|
certs.revoke(store, state, "deskwork")
|
|
|
|
calls: list[str] = []
|
|
|
|
async def spy_get(self: Any, url: Any, **kwargs: Any) -> Any:
|
|
calls.append(str(url))
|
|
raise AssertionError(f"a revoked agent was dialled at {url}")
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "get", spy_get)
|
|
for key in ("host:deskwork", "disks:deskwork"):
|
|
error = wait_for_source_error(client, key, "agent certificate revoked")
|
|
assert "unreachable" not in error, "the refusal replaced the dial's sentence"
|
|
assert calls == [], "the gate is in front of the network, not behind it"
|
|
# and the overview says the operator revoked it rather than that the box went down
|
|
messages = [row["message"] for row in client.get("/api/summary").json()["alerts"]]
|
|
assert any("certificate revoked" in text for text in messages), messages
|
|
store.close()
|
|
|
|
|
|
def test_engine_urls_are_redacted_the_same_way_hosts_are(tmp_path: Path) -> None:
|
|
"""`GUFO_BASE_URL`/`VLLM_BASE_URL` go to the browser verbatim in `/api/config`, and unlike
|
|
`HOST_TARGETS` nothing parses a userinfo token out of them into a header — so a URL that
|
|
happens to carry credentials has to be redacted where it is served (invariant 6).
|
|
"""
|
|
db = str(tmp_path / "engine-redact.sqlite3")
|
|
settings = Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
gufo_url="http://SEKRIT-ENGINE-TOKEN@deskwork:8731",
|
|
vllm_url="http://driveripper:8000")
|
|
store = SampleStore(db)
|
|
with TestClient(create_app(settings, store)) as client:
|
|
sign_in(client)
|
|
body = client.get("/api/config")
|
|
assert "SEKRIT-ENGINE-TOKEN" not in body.text
|
|
llm = body.json()["llm"]
|
|
assert llm["gufo"]["base_url"] == "http://deskwork:8731"
|
|
assert llm["vllm"]["base_url"] == "http://driveripper:8000" # untouched when clean
|
|
store.close()
|
|
|
|
|
|
@pytest.mark.parametrize(("llm_interval", "rate_window"), [(2.0, 10.0), (4.0, 20.0)])
|
|
def test_build_hub_gives_each_source_the_estimator_its_own_cadence_fills(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
llm_interval: float, rate_window: float) -> None:
|
|
"""Two rate estimators, one per measuring cadence (phase 23, `TODO.md:8`).
|
|
|
|
The spy is the point: the numbers themselves are `config` defaults and a test on the default
|
|
would pass even if `build_hub` handed one estimator to both sources. So this asserts the two
|
|
**objects** the jobs receive, and does it at a non-default cadence pair to prove the LLM span
|
|
tracks `LLM_INTERVAL_SECONDS` rather than a second knob nobody configured — one 10 s window
|
|
smoothed a slot start across the five polls the operator was watching (the ramp), and a 2 s
|
|
window over a 10 s `host` poll would answer `n/a` every poll.
|
|
|
|
The payload-level twin of this pin is the `Δ<window>` label each rate cell carries: this is
|
|
the wiring, that is what the browser prints (phase 23 task 02 pins the labels).
|
|
"""
|
|
seen: dict[str, RateEstimator] = {}
|
|
real_host_jobs = app_module.host_jobs
|
|
real_llm_jobs = app_module.llm_jobs
|
|
|
|
def spy_host_jobs(targets: Any, http: Any, host_collector: Any, interval: float,
|
|
**kwargs: Any) -> list[Any]:
|
|
seen["host"] = host_collector.rates
|
|
return real_host_jobs(targets, http, host_collector, interval, **kwargs)
|
|
|
|
def spy_llm_jobs(cfg: Any, http: Any, rates: RateEstimator) -> list[Any]:
|
|
seen["llm"] = rates
|
|
return real_llm_jobs(cfg, http, rates)
|
|
|
|
monkeypatch.setattr(app_module, "host_jobs", spy_host_jobs)
|
|
monkeypatch.setattr(app_module, "llm_jobs", spy_llm_jobs)
|
|
|
|
db = str(tmp_path / "rate-windows.sqlite3")
|
|
store = SampleStore(db)
|
|
settings = Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=(Target("deskwork", DEAD_URL),),
|
|
gufo_url="http://deskwork:8731", vllm_url="http://driveripper:8000",
|
|
llm_interval=llm_interval, rate_window=rate_window)
|
|
# `build_hub` only wires jobs; nothing here dials, so the shared client is never started
|
|
# and there is no event loop to close one in.
|
|
hub, _nut_monitor = app_module.build_hub(settings, store, httpx.AsyncClient())
|
|
store.close()
|
|
|
|
assert sorted(seen) == ["host", "llm"], "a source stopped being handed an estimator"
|
|
assert sorted(hub.jobs) == ["disks:deskwork", "host:deskwork", "llm:gufo", "llm:vllm"]
|
|
assert seen["llm"] is not seen["host"], \
|
|
"one estimator for both cadences is the bug this phase exists to fix"
|
|
assert seen["llm"].span == pytest.approx(llm_interval), \
|
|
"the LLM window must track LLM_INTERVAL_SECONDS, not a second knob"
|
|
assert seen["host"].span == pytest.approx(rate_window), \
|
|
"the host rates keep RATE_WINDOW_SECONDS: a 2 s span over a 10 s poll is `n/a` forever"
|
|
# …and the smaller window still carries the production floor, so the phase-13 refusal of the
|
|
# poll loop's ~0.2 s catch-up gap is not a consequence of the wider span.
|
|
assert seen["llm"].min_window == MIN_WINDOW_SECONDS
|
|
assert seen["host"].min_window == MIN_WINDOW_SECONDS
|
|
assert seen["llm"].span > MIN_WINDOW_SECONDS, "the LLM window would refuse every poll"
|
|
|
|
|
|
def test_config_reports_the_window_the_llm_rates_are_actually_measured_over(
|
|
client: TestClient) -> None:
|
|
"""`/api/config`'s llm window is a measurement claim, so it is the LLM's own span.
|
|
|
|
It used to echo `RATE_WINDOW_SECONDS` for both sources, which was harmless while one estimator
|
|
served them all. Since phase 23 that same field would have advertised a 10 s window beside
|
|
rates measured over 2 s — a number the page can print and no poll ever used (rule 2).
|
|
"""
|
|
llm = client.get("/api/config").json()["llm"]
|
|
assert llm["rate_window_seconds"] == llm["interval_seconds"] == 2.0
|
|
|
|
|
|
def wait_for_state(client: TestClient, path: str, *, timeout: float = 5.0) -> dict[str, Any]:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
response = client.get(path)
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
time.sleep(0.1)
|
|
pytest.fail(f"{path} never answered")
|
|
|
|
|
|
def test_target_problems_are_surfaced_not_swallowed(client) -> None:
|
|
body = client.get("/api/config").json()
|
|
assert body["system"]["target_problems"] == []
|
|
|
|
|
|
# ---- the credential the server refuses to start without (phase 11) --------- #
|
|
|
|
|
|
def test_no_verifier_and_no_seed_means_the_server_does_not_come_up(tmp_path: Path) -> None:
|
|
"""Fail closed, at the one place a mistake is unmissable: startup.
|
|
|
|
Task 03 is what demands a session on every route; this is the half before it — a deployment
|
|
with neither a stored `auth.verifier` row nor `DASH_ADMIN_PASSWORD` must not run open, so
|
|
the lifespan raises instead of publishing a context, starting the pollers, or serving a
|
|
page. Asserting that nothing was written is part of it: "refused" that seeded a row on the
|
|
way out would leave the next start half-configured.
|
|
"""
|
|
db = str(tmp_path / "no-credential.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=""), store)
|
|
try:
|
|
with pytest.raises(RuntimeError) as caught, TestClient(app):
|
|
pytest.fail("the lifespan yielded with no admin credential")
|
|
message = str(caught.value)
|
|
assert "DASH_ADMIN_PASSWORD" in message, "the operator's fix is in the exit output"
|
|
assert "auth.verifier" in message and "DELETE FROM meta" in message
|
|
assert not hasattr(app.state, "ctx"), "nothing is published, so nothing can serve"
|
|
assert store.get_meta("auth.verifier") is None
|
|
assert store.get_meta("auth.session_secret") is None
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
def test_the_same_server_starts_once_the_environment_seeds_a_password(tmp_path: Path) -> None:
|
|
"""The other side of the gate, so the failure above cannot be satisfied by a broken app.
|
|
|
|
With a seed the process comes up, the state hangs off the context (task 03 reads it on
|
|
every request), and both rows are durable — which is also what lets the operator delete
|
|
`DASH_ADMIN_PASSWORD` from the deployment after the first boot.
|
|
"""
|
|
db = str(tmp_path / "seeded.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
try:
|
|
with TestClient(app) as client:
|
|
assert client.get("/api/health").status_code == 200 # public: the healthcheck
|
|
state = app.state.ctx.auth
|
|
assert state is not None
|
|
assert state.seeded is True and state.env_ignored is False
|
|
assert state.iterations == FAST_PBKDF2
|
|
assert auth.verify_password(ADMIN_PASSWORD, state.verifier) is True
|
|
assert store.get_meta("auth.verifier") == state.verifier
|
|
assert store.get_meta("auth.session_secret") == state.secret
|
|
assert ADMIN_PASSWORD not in state.verifier + state.secret
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
# ---- pages ---------------------------------------------------------------- #
|
|
|
|
@pytest.mark.parametrize("path,title", [
|
|
("/", "overview"), ("/system", "system"), ("/disks", "disks"), ("/llm", "llm"),
|
|
("/ups", "ups"),
|
|
])
|
|
def test_pages_serve_and_declare_themselves(client, path, title) -> None:
|
|
response = client.get(path)
|
|
assert response.status_code == 200
|
|
assert "text/html" in response.headers["content-type"]
|
|
assert title in response.text.lower()
|
|
|
|
|
|
def test_no_page_loads_anything_from_the_internet(client) -> None:
|
|
"""No CDN, no webfonts: this dashboard has to work with no route to the internet.
|
|
|
|
`/login` is not in `MARKUP_PAGES` and cannot be: this `client` is signed in, a signed-in
|
|
`GET /login` is a `303`, and following it would audit `/`'s bytes under the wrong name. The
|
|
same fragments are checked against the real login page, unsigned, by
|
|
`tests/test_auth_api.py::test_the_login_page_loads_nothing_from_the_internet`.
|
|
"""
|
|
for path in MARKUP_PAGES:
|
|
html = client.get(path).text
|
|
for fragment in ("//cdn.", "https://fonts.", "//unpkg.com", "//cdnjs"):
|
|
assert fragment not in html, f"{path} references {fragment}"
|
|
|
|
|
|
def test_static_assets_are_served(client) -> None:
|
|
for asset in ASSETS:
|
|
assert client.get(asset).status_code == 200, asset
|
|
|
|
|
|
# ---- /settings: the account page (phase 11 task 06) ----------------------- #
|
|
#
|
|
# `/settings` is the operator's own surface: the session this browser holds, a sign-out, and the
|
|
# one form that changes the admin password. Its HTTP surface is tested twice on purpose — the
|
|
# route-enumeration test in `tests/test_auth_api.py` proves nothing outside the public list
|
|
# answers anonymously (which is how `/settings` is gated for free by being in `PAGES` in
|
|
# `app.py`), and the assertions here pin the **page**: the form fields a password manager needs,
|
|
# the outcome line that is a live region, the precedence sentence that stops the "I edited the
|
|
# env file and nothing happened" support call, and the masthead link that is the only way in.
|
|
|
|
def test_the_settings_page_carries_the_session_and_the_form(client) -> None:
|
|
response = client.get(SETTINGS_PAGE)
|
|
assert response.status_code == 200
|
|
assert "text/html" in response.headers["content-type"]
|
|
html = response.text
|
|
# the session panel and its way out
|
|
for fragment in ('id="sign-out"', 'id="session-user"', 'id="session-expires"',
|
|
'id="session-left"'):
|
|
assert fragment in html, f"/settings has no {fragment}"
|
|
# the change-password form: three fields, each labelled, each with the floor spelled out,
|
|
# and the autocomplete values a password manager needs to fill the right one
|
|
for fragment in ('id="current-password"', 'autocomplete="current-password"',
|
|
'id="new-password"', 'autocomplete="new-password"',
|
|
'id="confirm-password"', 'id="change-password"',
|
|
'<label for="current-password">', '<label for="new-password">',
|
|
'<label for="confirm-password">', 'aria-describedby="password-hint"'):
|
|
assert fragment in html, f"/settings' form is missing {fragment}"
|
|
assert "8 characters" in html, "the form stopped naming the length floor it enforces"
|
|
# the outcome line is announced, because the operator asked for it and nothing else will
|
|
assert 'id="settings-result"' in html and 'role="status"' in html
|
|
|
|
|
|
def test_the_settings_page_states_the_precedence_rule(client) -> None:
|
|
"""The sentence that makes the env file's losing half legible on the page that wins.
|
|
|
|
`README.md` documents the same rule, but the operator reading it there is deciding what to
|
|
write in a file; the operator reading *this* page just changed a password and needs to know
|
|
why a container env will not undo it.
|
|
"""
|
|
html = client.get(SETTINGS_PAGE).text
|
|
assert "DASH_ADMIN_PASSWORD" in html
|
|
assert "auth.verifier" in html
|
|
assert "never overwrites" in html
|
|
|
|
|
|
def test_the_settings_page_is_not_one_of_the_dashboards(client) -> None:
|
|
"""The tabs are there to leave with; nothing marks itself current, because nothing is.
|
|
|
|
A tab marked current on a page that is not in the tab list claims a location the app does not
|
|
have, and the four dashboards' own masthead pin (`aria-current` exactly once) relies on that
|
|
marker meaning something.
|
|
"""
|
|
html = client.get(SETTINGS_PAGE).text
|
|
assert '<nav class="tabs"' in html
|
|
assert 'aria-current="page"' not in html
|
|
for path in PAGES:
|
|
assert f'href="{path}"' in html, f"/settings cannot navigate to {path}"
|
|
# The fleet badges belong here too (`TODO.md:3`): the container is what made them appear on
|
|
# the five dashboards, and a page that renders the masthead without it is the page where they
|
|
# "disappear". `Dash.initChrome()` fills it from `/api/health` exactly as it fills the others.
|
|
assert 'id="nav-status"' in html, "/settings lost the fleet badges the operator asked for"
|
|
|
|
|
|
def test_the_settings_page_refuses_a_caller_with_no_session(tmp_path: Path) -> None:
|
|
"""One page-level assertion of the gate, not a second enumeration of it.
|
|
|
|
`tests/test_auth_api.py::test_every_route_that_is_not_public_requires_a_session` is the
|
|
general proof; this one exists so a regression in `/settings` names `/settings`.
|
|
"""
|
|
db = str(tmp_path / "settings-gate.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
try:
|
|
with TestClient(app, follow_redirects=False) as anon:
|
|
page = anon.get(SETTINGS_PAGE, headers={"accept": "text/html"})
|
|
assert page.status_code == 303
|
|
assert page.headers["location"] == f"/login?next={SETTINGS_PAGE}"
|
|
data = anon.get(SETTINGS_PAGE, headers={"accept": "application/json"})
|
|
assert data.status_code == 401
|
|
assert data.json() == {"ok": False, "detail": "sign in required",
|
|
"login": "/login"}
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
def test_every_page_links_to_both_admin_pages(client, tmp_path: Path) -> None:
|
|
"""`⚙ Account` and `⚷ Certs` are the only way to the two admin pages, so they are everywhere.
|
|
|
|
One loop over all seven signed-in pages (`MARKUP_PAGES`), because the control row is the *same*
|
|
on all of them: `⚷ Certs` → `⚙ Account` → theme toggle → clock, in `.masthead-right` and
|
|
**outside** `.tabs` (the tab strip's structure — five dashboards, one of them current — is pinned by
|
|
`test_the_disks_tab_is_in_the_masthead_of_every_page`, and a sixth entry with no
|
|
`aria-current` would break what that marker means).
|
|
|
|
Both links are on the pages they name, which **inverts** phase 15's "no self-link"
|
|
precedent on the operator's decision (`TODO.md:4` — "clicking the certs tab causes the certs
|
|
button to disappear … These should remain visible"): masthead chrome that changes shape
|
|
depending on where you already are reads as a broken control, while a link to the page you
|
|
are on is a no-op that answers with the page you were looking at.
|
|
|
|
The clock last is phase 25 task 03 (`TODO.md:5` — "The time should show up as the farthest
|
|
right item on the navbar"). It is pinned as **markup**, not as a CSS `order` property, because
|
|
that is what makes the claim true: task 02 left the control row `justify-content: flex-end`, so
|
|
the last child of `.masthead-right` *is* the farthest-right item, and an `order:` override the
|
|
pin could not see would leave this test green over a bar that says something else. (Below 900 px
|
|
the clock is hidden by the long-standing `.clock` media rule — a readout, not a control, so
|
|
hiding it is not the "no control is hidden to fit the width" rule this app keeps — and the
|
|
markup order this test reads is width-independent.)
|
|
"""
|
|
account_open = '<a class="ghost account" href="/settings"'
|
|
certs_open = '<a class="ghost account" href="/certs"'
|
|
# the opening tag **and** its label: the claim is the cluster does not change shape between
|
|
# pages, and `/certs` shipped a theme toggle missing its `aria-pressed` for exactly the
|
|
# reason nothing ever compared the seven markups
|
|
account = f'{account_open} title="Session and password">⚙ Account</a>'
|
|
certs_link = f'{certs_open} title="Agent certificates">⚷ Certs</a>'
|
|
theme = '<button class="ghost" id="theme-toggle" type="button" aria-pressed="false">'
|
|
# The other half of the clock claim, from the stylesheet side: no `order:` on anything in the
|
|
# bar, because a re-ordering property would leave this markup audit describing a bar that
|
|
# paints in some other order (`data/logs/25-task03-verify.cjs` measures the painted order at
|
|
# both widths; this is the served-byte guard under that measurement).
|
|
css = _css(client)
|
|
for selector in (".masthead", ".masthead-right", "#nav-status", ".clock"):
|
|
assert "order" not in _declarations(css, selector), \
|
|
f"{selector} grew an `order:` declaration — the clock's place in the row is markup"
|
|
for path in MARKUP_PAGES:
|
|
html = client.get(path).text
|
|
assert certs_link in html, f"{path} has no route to the certificate authority"
|
|
assert account in html, f"{path} has no route to the account page"
|
|
before_theme = html[:html.index('id="theme-toggle"')]
|
|
assert certs_link in before_theme, f"{path} put the Certs link after the theme toggle"
|
|
assert account in before_theme, f"{path} put the Account link after the theme toggle"
|
|
assert before_theme.index(certs_link) < before_theme.index(account), \
|
|
f"{path} orders the two admin controls differently from the dashboards"
|
|
# …and the clock after the toggle, i.e. last in `.masthead-right`: on the `flex-end`
|
|
# control row that is the farthest-right item on the bar (TODO.md:5).
|
|
assert html.index('id="clock"') > html.index('id="theme-toggle"'), \
|
|
f"{path} does not end its control row with the clock"
|
|
tabs = html[html.index('<nav class="tabs"'):html.index("</nav>")]
|
|
assert "/certs" not in tabs, f"{path} made Certs a dashboard tab"
|
|
assert "/settings" not in tabs, f"{path} made Account a dashboard tab"
|
|
assert theme in html, f"{path}'s theme toggle no longer matches the other seven"
|
|
|
|
# The one page that renders neither, and its own client: `client` is signed in, and a
|
|
# signed-in `GET /login` is a `303` to the dashboard, which would audit `/` and call it
|
|
# `/login` (the trap documented at `test_the_login_page_carries_no_nav_and_no_toggle`).
|
|
# Pre-auth there is nothing to administer, so the masthead stays brand link + theme toggle.
|
|
db = str(tmp_path / "admin-links-login.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
try:
|
|
with TestClient(app) as anon:
|
|
response = anon.get("/login")
|
|
assert response.status_code == 200, \
|
|
"a signed-out /login must be the form, not a bounce"
|
|
login_html = response.text
|
|
finally:
|
|
store.close()
|
|
# the opening tag, so an `href` under any other class or label still fails here
|
|
for link in (certs_open, account_open):
|
|
assert link not in login_html, f"/login offers {link} to a caller who cannot use it"
|
|
|
|
|
|
def test_the_certs_page_is_a_page_and_not_a_sixth_dashboard(client) -> None:
|
|
"""`/certs` borrows the dashboard chrome and none of the dashboard meaning (task 15.05).
|
|
|
|
Five tabs with nothing current, the four fleet badges like every other masthead (phase 25:
|
|
the badges report the fleet, and that report is the point of the site on this page too), and
|
|
the same head and stylesheet as every other page — this is the
|
|
`test_the_settings_page_is_not_one_of_the_dashboards` assertion for the second admin page,
|
|
and it is what a sixth tab in the strip would fail on.
|
|
"""
|
|
html = client.get(CERTS_PAGE).text
|
|
assert html.count('aria-current="page"') == 0
|
|
tabs = re.findall(r'<a href="([^"]+)"[^>]*aria-current="page"', html)
|
|
assert tabs == [], f"/certs marked a tab current: {tabs}"
|
|
assert 'id="nav-status"' in html, "/certs lost the fleet badges the operator asked for"
|
|
for path in PAGES:
|
|
assert f'href="{path}"' in html, f"/certs cannot navigate to {path}"
|
|
assert '/static/dash.css' in html
|
|
|
|
|
|
def test_the_certs_page_carries_the_authority_and_the_agents_table(client) -> None:
|
|
"""The three blocks the page is: the root, the client line, and one row per host.
|
|
|
|
Pinned on the served bytes rather than on a browser: the root and client panels and the table
|
|
are markup, and the two rows of the table body are built from `GET /api/certs` by
|
|
`page-certs.js` — so the per-row action labels are asserted on the served **script**, the way
|
|
`/ups` pins its command buttons. `tests/test_api.py` has no JS runner (a locked anchor), and a
|
|
label that shipped in neither file would be an invisible button either way.
|
|
"""
|
|
html = client.get(CERTS_PAGE).text
|
|
js = client.get("/static/page-certs.js").text
|
|
|
|
assert "Root CA" in html and 'id="root-serial"' in html
|
|
assert 'id="root-expires"' in html and 'id="root-left"' in html
|
|
assert "Rotate root CA" in html, "the CA-wide action disappeared"
|
|
assert "Dashboard client" in html and 'id="client-serial"' in html
|
|
assert 'id="client-expires"' in html, "/certs lost the client expiry"
|
|
assert 'id="agents-table"' in html and "<tbody></tbody>" in html
|
|
assert 'id="agents-empty"' in html and "hidden" in html
|
|
|
|
for label in ("Rotate", "Revoke", "bundle"):
|
|
assert f"'{label}'" in js, f"the {label} action lost its label"
|
|
for route in ("/rotate", "/revoke", "/bundle"):
|
|
assert route in js, f"the page no longer reaches {route}"
|
|
assert "/api/certs/root/rotate" in js
|
|
|
|
|
|
def test_the_certs_agents_table_scrolls_to_its_last_column(client) -> None:
|
|
"""`#agents-table` carries the `.scroll` modifier, and the stylesheet still styles it.
|
|
|
|
`TODO.md:9`: at phone width the serial, expiry, `san` and controls columns were clipped by
|
|
`.card { overflow: hidden }` with no way to reach them. `dash.css` turns `table.data` into
|
|
a scroll container **only** through the `scroll` class, so the class and the rule are one
|
|
mechanism and are pinned together — the phase-18 idiom of comparing the two places a
|
|
contract lives, because a rename on either side silently un-scrolls the table.
|
|
"""
|
|
html = client.get(CERTS_PAGE).text
|
|
assert '<table class="data scroll" id="agents-table">' in html, \
|
|
"the agents table lost its .scroll modifier — the controls column is unreachable at 360px"
|
|
|
|
css = client.get("/static/dash.css").text
|
|
start = css.index("table.data.scroll {")
|
|
rule = css[start:css.index("}", start)]
|
|
assert "overflow: auto" in rule, "table.data.scroll no longer scrolls — the class names nothing"
|
|
|
|
|
|
def test_the_certs_page_prints_the_names_the_authority_mints(client) -> None:
|
|
"""The CN of each certificate is markup, so the literal is pinned to `certs.py`.
|
|
|
|
`MIN_PASSWORD_LENGTH`'s idiom: there is no build step to share a Python constant with a page,
|
|
so a page that restates one says it in markup and a test compares the two. Printing a CN the
|
|
module did not sign with would be the page inventing a fact about a stored certificate.
|
|
"""
|
|
html = client.get(CERTS_PAGE).text
|
|
assert f">{certs.ROOT_CN}<" in html, "the root panel's CN is not what certs.py signs"
|
|
assert f">{certs.CLIENT_CN}<" in html, "the client line's CN is not what certs.py signs"
|
|
|
|
|
|
def test_the_certs_page_reads_only_fields_the_cert_api_publishes(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""End to end, in both directions: the page's column table and `GET /api/certs` are one shape.
|
|
|
|
The served script's `COLUMNS` entries name every payload key the page indexes, so this is the
|
|
`SLOT_FIELDS` device on the surface where a rename is worst: a `deployed_bundle_rev` that
|
|
quietly became `agent_bundle_rev` would print `n/a` beside a healthy fleet forever.
|
|
"""
|
|
script = "/static/page-certs.js"
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(3),
|
|
host_interval=3600.0) as client:
|
|
js = client.get(script).text
|
|
names: set[str] = set()
|
|
for listed in re.findall(r"fields: \[([^\]]*)\]", js):
|
|
names.update(re.findall(r"'([^']+)'", listed))
|
|
assert names == CERT_AGENT_KEYS, sorted(names ^ CERT_AGENT_KEYS)
|
|
|
|
client.post("/api/certs/deskwork/rotate")
|
|
for row in cert_agent_rows(client):
|
|
assert len(row) == 8
|
|
for name in names:
|
|
assert name in row, f"the page indexes `{name}` and the API does not publish it"
|
|
|
|
# …and the two authority blocks: the page prints a serial and an expiry for each, which is
|
|
# exactly the key set the API answers there.
|
|
body = cert_document(client)
|
|
for block in ("root", "client"):
|
|
assert set(body[block]) == CERT_TIMESERIAL_KEYS
|
|
for name in CERT_TIMESERIAL_KEYS:
|
|
assert f".{name}" in js, f"the page stopped reading `{name}`"
|
|
|
|
|
|
def test_the_certs_page_never_colours_a_certificate(client) -> None:
|
|
"""No severity vocabulary reaches this page at all — an expiry is a date, not a verdict.
|
|
|
|
`/api/certs` sends no `severity` anywhere (task 04, and `severity.py` rates no certificate),
|
|
so a page that coloured an expiring cert would be inventing a threshold in JavaScript. The one
|
|
colour on the page is the payload's `status` word through a word→class map, and `unissued`
|
|
maps to no class at all — a certificate that does not exist must not borrow green.
|
|
"""
|
|
js = _code(client.get("/static/page-certs.js").text)
|
|
assert "severity" not in js, "/certs grew a verdict the API does not send"
|
|
assert "severityFrom" not in js and "severityFor" not in js
|
|
table = js[js.index("const STATUS_CHIP = {"):js.index("};")]
|
|
assert "active: 'ok'" in table and "revoked: 'fail'" in table
|
|
assert "unissued: ''" in table, "an unissued host would be coloured"
|
|
assert "not_after" not in table and "serial" not in table, "an expiry became a verdict"
|
|
|
|
|
|
def test_the_certs_page_asks_before_it_writes_and_offers_nothing_the_api_refuses(
|
|
client) -> None:
|
|
"""Every write is confirmed, and no row offers a control its own payload rules out.
|
|
|
|
`confirm()` and not the typed `DANGER` word: that gate belongs to commands that cut power,
|
|
and this page has none — the worst outcome here is a host that is not dialled until somebody
|
|
installs a bundle. The guards are the API's, restated: a row outside `HOST_TARGETS` gets no
|
|
buttons at all (`404 {"configured": false}` is all three routes' first line), a never-issued
|
|
host has nothing to revoke and no bundle to download, and the bundle download is a plain
|
|
`GET` link because a session cookie is all a signed `GET` needs.
|
|
"""
|
|
js = _code(client.get("/static/page-certs.js").text)
|
|
assert "window.confirm(" in js
|
|
assert "DANGER" not in js, "the power-cut word migrated to a page with no power cut"
|
|
assert "entry.configured" in js and "not in HOST_TARGETS" in js
|
|
assert "entry.status !== 'unissued'" in js, "a row could revoke nothing"
|
|
assert "entry.serial" in js
|
|
assert "bundle.href = '/api/certs/'" in js
|
|
assert "method: 'POST'" in js
|
|
# …and the refusal is text, never markup: these are server sentences
|
|
assert "innerHTML" not in js
|
|
|
|
|
|
def test_the_certs_page_prints_the_restart_sentence_the_api_spoke(client) -> None:
|
|
"""`restart_required` prints the route's `hint`, not a sentence the page wrote for itself.
|
|
|
|
The restart fact lives in `api.py` (the CA and the client pair are startup material, so the
|
|
table is new before the socket is). A page copy of it is the version that quietly stops
|
|
saying "and every bundle needs re-installing".
|
|
"""
|
|
js = client.get("/static/page-certs.js").text
|
|
assert "restart_required" in js
|
|
assert "body.hint" in js
|
|
assert "restart the dashboard" not in js.lower(), "the page invented the restart sentence"
|
|
|
|
|
|
def test_the_password_floor_the_form_checks_is_the_one_the_api_enforces(client) -> None:
|
|
"""The client-side length check is a courtesy; the server's is the rule.
|
|
|
|
There is no build step to share the constant (a locked anchor), so the two literals are
|
|
pinned together here: a floor raised in `api.py` alone would leave the form promising eight
|
|
characters to a request the server answers `422`.
|
|
"""
|
|
js = client.get("/static/page-settings.js").text
|
|
found = re.search(r"const MIN_PASSWORD_LENGTH = (\d+);", js)
|
|
assert found, "page-settings.js no longer names its length floor"
|
|
assert int(found.group(1)) == MIN_PASSWORD_LENGTH
|
|
|
|
|
|
def test_a_rotated_secret_clears_the_cached_session_every_page_uses(client) -> None:
|
|
"""`Dash.resetAuth()` is the difference between a password change and a 403 dashboard.
|
|
|
|
`POST /api/auth/password` rotates the signing secret on purpose (`auth.update_verifier`),
|
|
which invalidates the CSRF token `ensureAuth()` memoised. Every page writes through
|
|
`fetchJSON`, which reads that cache — so unless the change clears it, the operator changes
|
|
the password, clicks *beeper disable* on `/ups`, and gets `missing or invalid CSRF token`
|
|
until they reload the page. A bounce on a `401` clears it for the same reason: the session it
|
|
described no longer exists.
|
|
"""
|
|
app_js = client.get("/static/app.js").text
|
|
assert "function resetAuth(" in app_js
|
|
exports = app_js.rsplit("return {", 1)[1].split("}", 1)[0]
|
|
assert re.search(r"\bresetAuth\b", exports), "Dash.resetAuth is not exported"
|
|
reset = app_js[app_js.index("function resetAuth("):app_js.index("function bounceToLogin(")]
|
|
for field in ("loaded: false", "authenticated: false", "csrf: null"):
|
|
assert field in reset, f"resetAuth() left `{field}` cached"
|
|
assert "authPromise = null" in reset, \
|
|
"resetAuth() left an in-flight status call memoised — the fresh write would reuse it"
|
|
bounce = app_js[app_js.index("function bounceToLogin("):app_js.index("const READ_METHODS")]
|
|
assert "resetAuth()" in bounce, "a 401 bounce kept the dead session's token cached"
|
|
|
|
js = client.get("/static/page-settings.js").text
|
|
assert js.count("Dash.resetAuth()") >= 2, \
|
|
"the settings page must reset the cached session after both a change and a sign-out"
|
|
assert "textContent" in js and "innerHTML" not in js, \
|
|
"/settings renders a server sentence as markup"
|
|
|
|
|
|
# ---- the severity language in the static assets --------------------------- #
|
|
#
|
|
# There is no JS test runner (a locked anchor: adding one would mean a build step).
|
|
# What *can* be asserted end-to-end, from the served bytes, is the contract the pages
|
|
# depend on: the hero style exists, its colour comes from the shared ok/warn/fail tokens,
|
|
# no parallel `.sev-*` vocabulary appeared, and `Dash.hero()` reads the verdict instead of
|
|
# comparing the number to a threshold in JavaScript.
|
|
|
|
ASSETS = ("/static/dash.css", "/static/app.js", "/static/charts.js",
|
|
"/static/page-index.js", "/static/page-disks.js", "/static/page-system.js",
|
|
"/static/page-llm.js", "/static/page-ups.js", "/static/page-login.js",
|
|
"/static/page-settings.js", "/static/page-certs.js")
|
|
|
|
|
|
def _css(client: TestClient) -> str:
|
|
return client.get("/static/dash.css").text
|
|
|
|
|
|
def _css_section(css: str, name: str) -> str:
|
|
"""One `/* ---- name … */` section of the stylesheet, up to the **next** section header.
|
|
|
|
Slicing to a named neighbour worked while the sheet had five sections. Every page has since
|
|
added a component section, and a slice that runs across someone else's rules accuses the
|
|
fleet block of colouring numbers it never coloured — so the boundary is structural: the end
|
|
of *this* section, whoever comes next.
|
|
"""
|
|
start = css.index(f"/* ---- {name}")
|
|
return css[start:css.index("/* ----", start + 10)]
|
|
|
|
|
|
def _hero_css(css: str) -> str:
|
|
"""Just the `hero numbers` section of the stylesheet."""
|
|
return css.split("/* ---- hero numbers", 1)[1].split("/* ---- bars", 1)[0]
|
|
|
|
|
|
def test_css_ships_the_hero_number_style(client) -> None:
|
|
css = _css(client)
|
|
hero_block = _hero_css(css)
|
|
for rule in (".hero {", ".hero .label", ".hero .value", ".hero .unit", ".hero .sub",
|
|
".hero.ok .value", ".hero.warn .value", ".hero.fail .value",
|
|
".hero-bar", ".hero-bar.ok .bar"):
|
|
assert rule in hero_block, f"dash.css lost the {rule} rule"
|
|
# a display size with fixed-width digits, so the number does not jitter between polls
|
|
assert "2.4rem" in hero_block
|
|
assert "tabular-nums" in hero_block
|
|
|
|
|
|
def test_hero_colour_comes_from_the_shared_verdict_tokens(client) -> None:
|
|
"""`.hero` reuses `--ok/--warn/--fail`; it does not invent a palette of its own."""
|
|
hero_block = _hero_css(_css(client))
|
|
for token in ("var(--ok)", "var(--warn)", "var(--fail)", "var(--text)",
|
|
"var(--text-muted)", "var(--text-faint)"):
|
|
assert token in hero_block, f"the hero style no longer uses {token}"
|
|
|
|
|
|
def test_no_stylesheet_rule_uses_an_undeclared_token(client) -> None:
|
|
"""A typo'd `var(--x)` renders unstyled and reads as a broken panel, not an error."""
|
|
css = _css(client)
|
|
root = re.search(r":root\s*\{(.*?)\n\}", css, re.S)
|
|
assert root is not None
|
|
declared = set(re.findall(r"(--[a-z0-9-]+)\s*:", root.group(1)))
|
|
used = set(re.findall(r"var\(\s*(--[a-z0-9-]+)", css))
|
|
assert not used - declared
|
|
|
|
|
|
def _code(body: str) -> str:
|
|
"""An asset with its comments stripped — prose may *name* a class the code must not define."""
|
|
body = re.sub(r"/\*.*?\*/", " ", body, flags=re.S)
|
|
body = re.sub(r"<!--.*?-->", " ", body, flags=re.S)
|
|
return re.sub(r"^\s*//.*$", " ", body, flags=re.M)
|
|
|
|
|
|
# The `fleet` object paths `/llm`'s aggregate block reads. `engines_total` drives the
|
|
# "hidden below two engines" rule and the tag; `engines_live` is what makes a partial sum
|
|
# visible; the rest are the row itself.
|
|
FLEET_BLOCK_PATHS = frozenset({
|
|
"engines_total", "engines_live", "requests_active", "requests_waiting",
|
|
"prompt_tokens_per_second", "decode_tokens_per_second", "tokens_total",
|
|
})
|
|
# The summed half of that list — the ones that must be null, not 0, with nothing answering.
|
|
FLEET_SUM_PATHS = ("requests_active", "requests_waiting", "prompt_tokens_per_second",
|
|
"decode_tokens_per_second", "tokens_total")
|
|
|
|
|
|
def _fleet_tag(html: str) -> str:
|
|
"""The aggregate block's opening tag — the one place its `hidden` default lives."""
|
|
start = html.index('<section class="card flat fleet"')
|
|
return html[start:html.index(">", start)]
|
|
|
|
|
|
def _fleet_render_path(js: str) -> str:
|
|
"""Just the body of `renderFleet()` — the aggregate block's whole code path, comments out.
|
|
|
|
Everything the block writes goes through here, so it is the slice that can be asserted
|
|
verdict-free without tripping over the engine panels' rated KV sparkline next door.
|
|
"""
|
|
js = _code(js)
|
|
return js[js.index("function renderFleet("):js.index("async function refresh(")]
|
|
|
|
|
|
def _fleet_field_metrics(js: str) -> list[str]:
|
|
"""The `fleet` keys the block's rows are bound to, in the order the block prints them."""
|
|
js = _code(js)
|
|
start = js.index("const FLEET_FIELDS = [")
|
|
table = js[start:js.index("];", start) + 2]
|
|
return re.findall(r"metric: '([a-z_]+)'", table)
|
|
|
|
|
|
def test_no_parallel_severity_vocabulary_exists_anywhere(client) -> None:
|
|
"""One word per colour: `ok|warn|fail`, as in severity.py and the existing components."""
|
|
for asset in (*ASSETS, *MARKUP_PAGES):
|
|
body = _code(client.get(asset).text)
|
|
assert ".sev-" not in body, f"{asset} invented a .sev-* class"
|
|
assert not re.search(r"classList\.(?:add|toggle)\(\s*[\"'](?:danger|good|warning)[\"']",
|
|
body), f"{asset} invented a synonym for fail"
|
|
|
|
|
|
def test_app_js_exposes_the_hero_helpers(client) -> None:
|
|
js = client.get("/static/app.js").text
|
|
assert "function hero(" in js
|
|
assert "function severityFrom(" in js
|
|
exports = js.rsplit("return {", 1)[1].split("}", 1)[0]
|
|
for name in ("hero", "severityFrom", "severityFor"):
|
|
assert re.search(rf"\b{name}\b", exports), f"Dash.{name} is not exported"
|
|
|
|
|
|
def test_hero_reads_the_verdict_instead_of_a_threshold(client) -> None:
|
|
"""`hero()` may only colour what the API rated — no thresholds re-derived in JS."""
|
|
js = client.get("/static/app.js").text
|
|
start = js.index("function hero(")
|
|
body = js[start:js.index("\n function setDot", start)]
|
|
assert "severityFrom(" in body
|
|
assert "severityFor(" not in body
|
|
# an unknown value gets n/a and no colour class at all
|
|
assert "naText" in body
|
|
assert re.search(r"raw != null", body)
|
|
|
|
|
|
def test_severity_for_is_kept_as_the_documented_fallback(client) -> None:
|
|
"""Every page call site is gone now that phase 05 converted `/ups`.
|
|
|
|
`/` was converted in phase 02, `/system` in phase 03, `/llm` in phase 04 and `/ups` in
|
|
phase 05, so no page colours anything from a threshold it wrote itself. The function
|
|
itself stays: it is the documented fallback `setBar()` uses when a caller names no
|
|
verdict, and a page that ever needs a verdict the API does not send should add it to
|
|
`severity.py` rather than here — which is what the docstring on it says.
|
|
"""
|
|
js = client.get("/static/app.js").text
|
|
assert "FALLBACK ONLY" in js
|
|
assert "severityFor" in js, "the documented fallback itself was deleted"
|
|
for page in PAGE_SCRIPTS.values():
|
|
assert "severityFor" not in client.get(f"/static/{page}").text, \
|
|
f"{page} colours something from a threshold of its own"
|
|
|
|
|
|
def test_overview_colours_from_the_payload_not_from_js_thresholds(client) -> None:
|
|
"""`page-index.js` reads verdicts; it does not compare a number to a bound."""
|
|
js = client.get("/static/page-index.js").text
|
|
assert "severityFrom" in js
|
|
assert "severityFor" not in js
|
|
|
|
|
|
def test_system_page_colours_from_the_payload_not_from_js_thresholds(client) -> None:
|
|
"""`page-system.js` hero cards paint payload verdicts, not thresholds of their own.
|
|
|
|
Phase 03 task 02's promise: every colour on a host card comes from a `severity` dict
|
|
`severity.py` produced (`cpu.severity.usage_pct`, `memory.severity.used_pct`,
|
|
`disks.severity.worst_used_pct`, `cpu.severity.temp_c`).
|
|
A threshold written here could disagree with the overview's alerts; a hero with no
|
|
verdict prints `n/a` uncoloured, which is `Dash.hero()`'s contract.
|
|
|
|
The GPU slot names no verdict on purpose (phase 06): `severity.THRESHOLDS` has no
|
|
`gpu.*` row, the payload carries no `gpus[].severity`, and the hero is built without
|
|
`severity` or `bar` — a bar under a verdictless number would be a verdict by shape.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
assert "severityFor(" not in js, "the system page re-invented a threshold"
|
|
assert "Dash.hero(" in js
|
|
for hook in ("cpu.severity.usage_pct", "memory.severity.used_pct",
|
|
"disks.severity.worst_used_pct", "cpu.severity.temp_c"):
|
|
assert hook in js, f"the card no longer names {hook} as a colour source"
|
|
# and the GPU hero grades nothing: no verdict hook, no utilisation bar
|
|
assert "gpus[].severity" not in js, "the GPU slot invented a colour source again"
|
|
gpu_slot = js[js.index("function renderGpuSlot("):js.index("function renderGpuExtra(")]
|
|
assert "severity:" not in gpu_slot, "the GPU hero passes a severity option again"
|
|
assert "bar:" not in gpu_slot, "the GPU hero grew a bar back (a verdict by shape)"
|
|
assert "hero-bar" not in gpu_slot, "the GPU hero rebuilt the bar variant of .hero"
|
|
|
|
|
|
#: Every cell of the `/system` GPU card's value grid, in render order, with the `gpus[]`
|
|
#: key(s) it prints. This is the `SLOT_FIELDS` device applied to the four values phase 21
|
|
#: task 02 created: the table is read out of the served `GPU_VALUE_FIELDS` and resolved
|
|
#: against every captured host shape *and* against `/api/system/snapshot`, so a rename on
|
|
#: either side fails a test instead of printing `n/a` beside a healthy fleet — and so the
|
|
#: dense ` · `-joined sub-line the TODO complained about cannot come back.
|
|
GPU_VALUE_CELLS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
("Model", ("model",)),
|
|
("VRAM", ("vram_used_bytes", "vram_total_bytes")),
|
|
("Pool", ("mem_used_pct",)),
|
|
("Temp", ("temp_c",)),
|
|
)
|
|
|
|
|
|
def _gpu_value_fields(js: str) -> list[tuple[str, tuple[str, ...]]]:
|
|
"""The served `GPU_VALUE_FIELDS` table as `(label, payload keys)` rows."""
|
|
table = re.search(r"const GPU_VALUE_FIELDS = \[(.*?)\n\];", js, re.S)
|
|
assert table, "page-system.js no longer has a GPU_VALUE_FIELDS table"
|
|
rows: list[tuple[str, tuple[str, ...]]] = []
|
|
for entry in re.findall(r"\{[^{}]*\}", table.group(1)):
|
|
label = re.search(r"label: '(\w+)'", entry)
|
|
field = re.search(r"field: '(\w+)'", entry)
|
|
assert label and field, f"a GPU value cell names no label or field: {entry!r}"
|
|
total = re.search(r"total: '(\w+)'", entry)
|
|
rows.append((label.group(1),
|
|
(field.group(1), total.group(1)) if total else (field.group(1),)))
|
|
return rows
|
|
|
|
|
|
def test_gpu_values_are_four_labelled_cells_not_a_dense_sub_line(client) -> None:
|
|
"""`TODO.md:12` — "… is too much info for such small text and should be its own values."
|
|
|
|
The GPU slot used to print `Radeon 8060S Graphics · VRAM 180.2 MiB / 512.0 MiB · pool
|
|
79% · 69 °C` as one `.sub` line in 11px mono. It is now four labelled cells, and the
|
|
pin is bidirectional: the table has the four rows in order, and the joined-sentence
|
|
builder that produced the TODO's quoted string is gone (`gpuSub` and its
|
|
`bits.join(' · ')` are the thing that made the card unreadable).
|
|
|
|
Two invariants ride on the same slice. The model string is upstream text (a PCI device
|
|
name), so it leaves through `DashCharts.escapeHtml`; and the temperature keeps its
|
|
null guard — `0 °C` would be a cool GPU that does not exist (phase 06).
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
assert _gpu_value_fields(js) == list(GPU_VALUE_CELLS), \
|
|
"the GPU value grid is no longer Model / VRAM / Pool / Temp over those payload keys"
|
|
# the °C rides in the value (the label is the word, as on the /disks card and the GPU
|
|
# detail table), and it appears exactly once in the table: one cell, one unit.
|
|
assert js.count("unit: ' °C'") == 1, "the Temp cell's unit is not the one °C on the card"
|
|
|
|
body = js[js.index("function gpuValues("):js.index("function renderGpuSlot(")]
|
|
assert "GPU_VALUE_FIELDS.map(" in body, "gpuValues() stopped rendering the table"
|
|
assert "escapeHtml" in body, "the Model cell prints an upstream string unescaped"
|
|
assert "== null" in body, "a cell lost its null guard (missing must print n/a, not 0)"
|
|
|
|
# the dense sub-line the TODO quoted is gone, container and all
|
|
assert "function gpuSub(" not in js, "gpuSub() came back"
|
|
assert "join(' · ')" not in js, "the ` · `-joined GPU footnote came back"
|
|
|
|
# "More detail" keeps the per-label breakdown: a surface change, not a de-duplication
|
|
extra = js[js.index("function renderGpuExtra("):js.index("function renderHostCard(")]
|
|
assert "Temp" in extra and "gpu.temp_c" in extra, "GPU detail lost its temperature column"
|
|
|
|
|
|
def test_gpu_value_cells_resolve_against_every_captured_host(
|
|
agent_snapshots: dict[str, dict[str, Any]]) -> None:
|
|
"""Every key the four cells print exists on every captured card — discrete GPU and APU.
|
|
|
|
`tests/card_contract.py` owns the payload table; this resolves the *page's* table
|
|
against the same three captures, which is where a rename would otherwise be caught by
|
|
an operator looking at `n/a` next to a healthy machine. `temp_c` may legitimately be
|
|
`None` (a driver with no hwmon input) — present is the contract, and the cell prints
|
|
`n/a` for it, never `0 °C`.
|
|
"""
|
|
for name in HOST_SHAPES:
|
|
payload = parse_host_snapshot(name, agent_snapshots[name], HostCollector(
|
|
RateEstimator(span=60.0)), now=1_000.0).payload
|
|
assert_card_contract(payload)
|
|
for gpu in payload["gpus"]:
|
|
for label, keys in GPU_VALUE_CELLS:
|
|
for key in keys:
|
|
assert key in gpu, f"{name}: the {label!r} cell has no {key!r} key"
|
|
if not payload["gpus"]:
|
|
# gateway: the omission's input. An empty list is a *fact* about the box, and
|
|
# `renderGpuSlot()` turns that fact into an absent slot rather than an `n/a`.
|
|
assert name == "gateway", "a capture lost its GPU row"
|
|
|
|
|
|
def test_the_gpu_value_cells_are_served_for_the_hosts_that_have_a_card(
|
|
tmp_path: Path, agent_snapshots: dict[str, dict[str, Any]]) -> None:
|
|
"""The same four keys over HTTP, on the body `/system` actually renders.
|
|
|
|
Parsing in-process proves the source; this proves the endpoint. One app with all three
|
|
hosts injected (dead ports, so no poll can overwrite what the test put there) answers
|
|
with the discrete-GPU card, the APU card, and gateway's empty list — the exact union
|
|
`renderGpuSlot()` branches on: cards present means four labelled values per card, `[]`
|
|
means no slot at all.
|
|
"""
|
|
db = str(tmp_path / "gpu-cells.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=tuple(
|
|
Target(name, DEAD_URL) for name in HOST_SHAPES)), store)
|
|
test_client = TestClient(app)
|
|
test_client.__enter__()
|
|
try:
|
|
sign_in(test_client)
|
|
hub = app.state.ctx.hub
|
|
for name in HOST_SHAPES:
|
|
hub.jobs[f"host:{name}"].state.payload = parse_host_snapshot(
|
|
name, agent_snapshots[name], HostCollector(RateEstimator(span=60.0)),
|
|
now=1_000.0).payload
|
|
hosts = {row["name"]: row for row in test_client.get("/api/system/snapshot").json()["hosts"]}
|
|
assert set(hosts) == set(HOST_SHAPES)
|
|
for name, row in hosts.items():
|
|
for gpu in row["gpus"]:
|
|
for label, keys in GPU_VALUE_CELLS:
|
|
for key in keys:
|
|
assert key in gpu, f"{name}: /api/system/snapshot has no {key!r} " \
|
|
f"for the {label!r} cell"
|
|
assert hosts["gateway"]["gpus"] == [], \
|
|
"gateway must answer [] — that is what makes the page omit the slot"
|
|
assert hosts["driveripper"]["gpus"] and hosts["deskwork"]["gpus"]
|
|
finally:
|
|
test_client.__exit__(None, None, None)
|
|
store.close()
|
|
|
|
|
|
def test_a_host_with_no_gpu_omits_the_gpu_slot_instead_of_printing_n_a(client) -> None:
|
|
"""`TODO.md:12` — "Boxes without a GPU should not report N/A but instead should omit
|
|
these values."
|
|
|
|
gateway has no card, and the sixth hero slot used to say so with a big faint `n/a` plus
|
|
the sentence "No GPU reported by this agent" — invariant 3's complaint in the one place
|
|
the page had not learned it. `renderGpuSlot()` now empties the slot and sets `hidden`,
|
|
and `[hidden] { display: none !important }` at the end of `dash.css` is what takes the
|
|
wrapper out of the `.heroes` grid so the row reflows to five. Pinned on the served
|
|
bytes (no JS runner — a locked anchor) and measured in a browser by
|
|
`data/logs/21-task02-gpu-values.cjs`.
|
|
|
|
The wrapper element itself stays in `hostCard()`: the hero row remains one static list
|
|
of six `data-k` slots (pinned by the test below), a host whose agent starts reporting a
|
|
card has something to grow into, and an empty hidden element is zero footprint.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
slot = js[js.index("function renderGpuSlot("):js.index("function renderGpuExtra(")]
|
|
assert "cell.hidden = gpus.length === 0" in slot, \
|
|
"the GPU slot no longer hides itself when the host has no card"
|
|
assert "No GPU reported" not in js, "the no-GPU n/a sentence came back"
|
|
assert "n/a" not in slot, "the GPU slot grew its own n/a fallback again"
|
|
# and the row itself keeps the six static slots, so a GPU host still has somewhere to go
|
|
assert "display: none !important" in client.get("/static/dash.css").text, \
|
|
"[hidden] stopped beating .hero-slot's display, so a hidden slot would still " \
|
|
"occupy a hero track"
|
|
|
|
|
|
def test_gpu_value_grid_ships_no_verdict_and_wraps(client) -> None:
|
|
"""The four new cells colour nothing and let a long model string wrap.
|
|
|
|
Phase 06's rule extends to them: `severity.THRESHOLDS` has no `gpu.*` row, the payload
|
|
carries no `gpus[].severity`, so neither the markup nor the grid may reach for
|
|
`--ok`/`--warn`/`--fail` or a `severity` option (a busy GPU on an LLM server is the job).
|
|
`overflow-wrap` is the other half of `TODO.md:12` — the readability complaint *is* the
|
|
20-character model string, and truncating it would keep the complaint.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
# word-boundary on purpose: the docstring in this slice is allowed to *say* "severity",
|
|
# the code is not allowed to pass one (or a bar, or a verdict class) to anything
|
|
markup = js[js.index("const GPU_VALUE_FIELDS"):js.index("function renderGpuSlot(")]
|
|
verdicts = re.findall(r"\b(ok|warn|fail|severity:|bar:|hero-bar)\b", markup)
|
|
assert not verdicts, f"the GPU value grid grew a verdict: {verdicts}"
|
|
|
|
css = client.get("/static/dash.css").text
|
|
rule = re.search(r"\.gpu-values \{([^}]*)\}", css)
|
|
assert rule, "/system lost its .gpu-values grid"
|
|
assert "auto-fit" in rule.group(1), ".gpu-values stopped adapting to the hero track"
|
|
wrap = re.search(r"\.gpu-values \.value \{([^}]*)\}", css)
|
|
assert wrap and "overflow-wrap" in wrap.group(1), \
|
|
"a long GPU model string truncates instead of wrapping"
|
|
for token in ("--ok", "--warn", "--fail"):
|
|
assert token not in rule.group(1) + (wrap.group(1) if wrap else ""), \
|
|
f"the GPU value grid coloured itself with {token} (severity.py rates no GPU)"
|
|
|
|
|
|
def test_system_host_card_is_six_heroes_without_per_core(client) -> None:
|
|
"""Each host card ships six hero slots in order: CPU, MEM, TEMP, DISK, GPU, NET.
|
|
|
|
Asserted on the served script because there is no JS test runner (a locked anchor).
|
|
The two dense `.metrics` rows and the per-core heat bars are exactly what made the
|
|
card unreadable, and "no per-core anywhere on /system" is a promise a stray
|
|
`renderCores` call would silently break — the series stays stored, the page just
|
|
stopped drawing it (PLAN.md *Dropped visuals*).
|
|
|
|
Six **slots**, not six visible heroes: the GPU slot ships `hidden` and
|
|
`renderGpuSlot()` un-hides it only for a host whose payload has a card, so gateway's
|
|
card renders five (pinned by the test above and measured in the browser by
|
|
`data/logs/21-task02-gpu-values.cjs`). The static order below is what makes that a
|
|
one-line rule instead of a re-flowing template.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
card = js[js.index("function hostCard("):js.index("function renderHostCard(")]
|
|
slots = re.findall(r'<div class="(?:hero|hero-slot)[^"]*" data-k="([^"]+)"', card)
|
|
assert slots == ["cpu", "mem", "temp", "disk", "gpus", "net"], \
|
|
"the hero row is no longer the six numbers in the agreed order"
|
|
assert re.search(r'data-k="gpus"[^>]*hidden', card), \
|
|
"the GPU slot is visible before the payload says this host has a card"
|
|
# per-core visualisation is gone from the card code entirely
|
|
assert "cores" not in card, "a per-core visual came back to the card"
|
|
assert "renderCores" not in js and "core_usage" not in js
|
|
# the NET slot is a slot with two lines and no verdict: a byte rate has no good/bad
|
|
assert 'data-k="net"' in card
|
|
assert not re.search(r'data-k="net"[^>]*data-sev', card), "the NET slot invented a colour"
|
|
# what the glance leaves out stays reachable, collapsed by default
|
|
assert "More detail" in card
|
|
for key in ("load", "freq", "dr", "dw", "dutil", "uptime", "details"):
|
|
assert f'data-k="{key}"' in card, f"{key} left the page instead of the glance"
|
|
# the panel owns its charts: the slots live inside the card, not on the page
|
|
for chart in ("util", "net"):
|
|
assert f'data-chart="{chart}"' in card, f"the panel lost its {chart} chart slot"
|
|
|
|
|
|
def test_system_page_keeps_no_fleet_wide_chart_or_metric_picker(client) -> None:
|
|
"""Nothing on `/system` plots two hosts, and nothing needs a selection to make sense.
|
|
|
|
TODO.md L4 is the instruction ("The system page doesn't need shared charts for
|
|
anything") and the overview's alerts + fleet table are what made removing them safe
|
|
(phase 02). Asserted on the served bytes because the only thing that would otherwise
|
|
notice a resurrected `<select>` is a browser: there is no JS test runner (a locked
|
|
anchor), and `test_no_page_carries_an_id_nothing_reads` cannot see ids that were deleted
|
|
correctly from the page but are still queried by the script.
|
|
"""
|
|
html = _code(client.get("/system").text)
|
|
for gone in ('id="chart-cpu"', 'id="chart-mem"', 'id="chart-disk"', 'id="chart-net"',
|
|
'id="chart-gpu"', 'id="chart-temp"', 'id="chart-detail"', 'id="detail-host"',
|
|
'id="detail-metric"', 'id="f-cpu"', 'id="f-load"', 'id="f-gpu"',
|
|
"Fleet totals", "Fleet charts", "Series detail", "<select"):
|
|
assert gone not in html, f"/system still carries {gone}"
|
|
# the two panels that are left are the per-host grid and the event feed
|
|
assert html.count("<section") == 2, "/system grew a panel back on top of the host grid"
|
|
assert 'id="hosts"' in html and 'id="events"' in html
|
|
|
|
# and the script must not reach for the deleted endpoints/ids either — a `#detail-host`
|
|
# query against a removed node is a silent null, so the picker would just never appear
|
|
js = client.get("/static/page-system.js").text
|
|
for gone in ("#chart-", "detail-host", "detail-metric", "detail-note",
|
|
"renderFleet", "makeCharts", "refreshGpuChart", "populateDetailMetrics"):
|
|
assert gone not in js, f"page-system.js still references {gone}"
|
|
|
|
|
|
def test_every_system_chart_is_drawn_inside_its_own_host_panel(client) -> None:
|
|
"""One host per chart: the legends of `/system` can never list two machine names.
|
|
|
|
The construction is a factory that takes the card, so a chart cannot be built against a
|
|
page-level id; the metric list is per host, so the GPU streams cannot leak from one
|
|
panel into another; and a panel is torn down (with its charts) when its host leaves the
|
|
payload, which is the bug class a keyed-by-host chart map exists to avoid.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
builds = re.findall(r"new DashCharts\.LineChart\(([^,]+),", js)
|
|
assert builds, "the system page builds no charts at all"
|
|
# every chart is built from a `[data-chart]` slot inside the host card, never from a
|
|
# page-level id — which is what makes "one host per chart" structural, not a convention
|
|
assert all(re.search(r"(?:card|slot)\.querySelector\(.\[data-chart=", b) for b in builds), \
|
|
f"a chart is built outside a host panel: {builds}"
|
|
# three construction sites, one per slot (util, net, gpu) — never one per metric
|
|
assert len(builds) == 3, f"charts are not built one per panel slot: {builds}"
|
|
assert sorted(re.findall(r'data-chart=\\?"(\w+)"', " ".join(builds))) == ["gpu", "net", "util"]
|
|
|
|
factory = js[js.index("async function renderHostCharts("):js.index("function pruneHosts(")]
|
|
assert "hosts=${encodeURIComponent(host.name)}" in factory, \
|
|
"the per-host chart query is not scoped to one host"
|
|
assert "&metrics=" in factory and "state.range" in factory, \
|
|
"a host's charts need one metric list and the selected window"
|
|
# GPU series are named from the payload's own `gpus[].index`, never discovered/guessed
|
|
assert "gpu.busy_pct@" in js and "gpu.index" in js
|
|
# the GPU chart exists only while the host reports cards
|
|
gpu_slot = js[js.index("function hostChartSet("):js.index("function seriesFor(")]
|
|
assert re.search(r"if \(!wantsGpu && set\.gpu\)", gpu_slot), \
|
|
"a host that stops reporting GPUs keeps a GPU chart it has no data for"
|
|
|
|
pruner = js[js.index("function pruneHosts("):js.index("// ---- snapshot")]
|
|
assert "state.hosts.delete(name)" in pruner and "state.charts.delete(name)" in pruner, \
|
|
"a host that left the payload keeps a live panel and drawing charts"
|
|
|
|
|
|
def test_system_history_charts_are_collapsed_behind_a_toggle(client) -> None:
|
|
"""TODO.md L4: "The historical charts on the system resources page should only be visible
|
|
if expanded. I only want to see current stats to keep things clean."
|
|
|
|
The default `/system` view is the hero row. Pinned on the served script because the card
|
|
markup is generated in JS and there is no JS test runner (a locked anchor), and the two
|
|
ways this regresses are invisible otherwise: a `<details open>` (charts visible by
|
|
default, i.e. the task un-done but the page still pretty) and a paint pass that runs
|
|
while the stack is shut, which builds three `LineChart`s per host inside a
|
|
`display:none` container — measured 0 px wide, never drawn, and rebuilt every poll.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
card = js[js.index("function hostCard("):js.index("function memorySub(")]
|
|
|
|
# the stack is *wrapped*, not merely preceded by a heading: it lives inside the <details>
|
|
box = card.index('<details class="charts-box" data-k="charts">')
|
|
stack = card.index('<div class="host-charts">')
|
|
closed = card.index("</details>")
|
|
assert box < stack < closed, "the chart stack is not inside the toggle"
|
|
assert closed < card.index("More detail"), "the charts toggle swallowed the detail row too"
|
|
assert not re.search(r"<details[^>]*\bopen\b", card), \
|
|
"a panel ships its charts open — the default view is not current stats"
|
|
assert re.search(r"<summary[^>]*>\s*Charts", card), "the toggle no longer reads Charts"
|
|
# it also says which window the expanded stack draws (the chips stay in the page head)
|
|
assert 'data-k="charts-window"' in card, "the toggle does not carry the window"
|
|
# …and the toggle is what turns the paint pass on
|
|
assert "addEventListener('toggle'" in card and "chartsToggled(" in card
|
|
|
|
setter = js[js.index("function setChartsWindow("):js.index("function memorySub(")]
|
|
assert "state.rangeKey" in setter, \
|
|
"the toggle re-derives its window label instead of using the chips' own"
|
|
handler = js[js.index("Dash.rangeControl("):js.index("state.poller = poll(")]
|
|
assert "state.rangeKey = key" in handler and "setChartsWindow(" in handler, \
|
|
"changing the window leaves every toggle labelling the old one"
|
|
|
|
# nothing is built or drawn while the stack is shut, in either path that could paint
|
|
paint = js[js.index("function drawHostCharts("):js.index("async function renderHostCharts(")]
|
|
assert "if (!chartsOpen(card)) return" in paint, "a closed panel gets painted anyway"
|
|
assert "hostChartSet(card, host)" in paint, \
|
|
"the LineChart instances are no longer built by the open pass"
|
|
toggle = js[js.index("function chartsToggled("):js.index("function drawHostCharts(")]
|
|
assert "if (!chartsOpen(card)) return" in toggle, "closing the stack still paints"
|
|
assert "renderHostCharts(card, host)" in toggle and "state.range" in toggle, \
|
|
"a cached response from another window is drawn under this window's label"
|
|
|
|
# the request discipline is unchanged: one per host per refresh, every series named
|
|
fetch = js[js.index("async function renderHostCharts("):js.index("function windowLabel(")]
|
|
assert "hostChartSet(" not in fetch, "the poll path still builds charts for closed panels"
|
|
assert "state.history.set(" in fetch, \
|
|
"the response is not kept, so the first open is an empty box until the next poll"
|
|
assert "if (range === state.range) drawHostCharts(" in fetch, \
|
|
"a response for a window the operator just left is painted under the new label"
|
|
assert "drawHostCharts(" in fetch, "an open panel is no longer repainted on refresh"
|
|
|
|
|
|
def test_a_pruned_host_takes_its_charts_toggle_with_it(client) -> None:
|
|
"""A host that leaves the payload must leave no orphan control in the grid.
|
|
|
|
The toggle is a child of the panel, which is why `card.remove()` is enough — but that is
|
|
only true while the markup is built inside `hostCard()`, so the placement is pinned here
|
|
along with the three per-host Maps the panel owns.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
card = js[js.index("function hostCard("):js.index("function memorySub(")]
|
|
assert 'data-k="charts"' in card, "the toggle is no longer part of the panel"
|
|
pruner = js[js.index("function pruneHosts("):js.index("// ---- snapshot")]
|
|
assert "state.hosts.get(name).remove()" in pruner
|
|
for owned in ("state.charts", "state.payloads", "state.history"):
|
|
assert f"{owned}.delete(name)" in pruner, f"{owned} outlives the panel it belongs to"
|
|
|
|
|
|
def test_the_charts_toggle_is_a_quiet_disclosure_not_a_new_component(client) -> None:
|
|
"""The collapse adds no palette and no second vocabulary (UI conventions).
|
|
|
|
Open/closed is carried by a caret *shape* as well as a colour step, the summary is muted
|
|
rather than accented, and it introduces no `.sev-*` family — the same rule the hero
|
|
numbers live by, checked here because a `<summary>` is the first control on the page a
|
|
designer might be tempted to grade.
|
|
"""
|
|
css = client.get("/static/dash.css").text
|
|
block = css[css.index("details.charts-box { "):css.index(".host-charts {")]
|
|
assert "details.charts-box > summary::before" in block, "the toggle has no caret shape"
|
|
assert 'details.charts-box[open] > summary::before' in block, \
|
|
"open and closed differ only in colour"
|
|
assert "var(--text-faint)" in block, "the closed toggle is not muted"
|
|
assert ".sev-" not in block and "var(--accent)" not in block, "the toggle invented a palette"
|
|
assert 'class="charts-box"' in client.get("/static/page-system.js").text, \
|
|
"the stylesheet styles a class the page no longer renders"
|
|
|
|
|
|
def test_an_expanded_chart_stack_does_not_grow_its_collapsed_siblings(client) -> None:
|
|
"""One host open must not stretch the hosts beside it into blank cards.
|
|
|
|
`/system` is the same shape `/llm` had to fix: an expanded panel beside a collapsed one
|
|
grows the collapsed card to match, and a blank panel reads as "no data" (the misreading
|
|
the unconfigured rules exist to prevent). `.grid.items-start` already exists for exactly
|
|
this, so the page uses it rather than growing a rule of its own.
|
|
"""
|
|
html = _code(client.get("/system").text)
|
|
hosts_class = re.findall(r'<div class="([^"]*)" id="hosts">', html)
|
|
assert hosts_class == ["grid auto items-start"], \
|
|
f"the host grid stretches its cards again: {hosts_class}"
|
|
assert ".grid.items-start" in client.get("/static/dash.css").text, \
|
|
"the rule the host grid relies on is gone"
|
|
|
|
|
|
def test_every_system_chart_names_itself_for_assistive_tech(client) -> None:
|
|
"""A 120px SVG is silence to a screen reader unless the page names it.
|
|
|
|
The phase-03 accessibility pass (task 04) found the per-host charts drawing anonymous
|
|
plots: the legend is a sibling element and labels nothing, so the `<svg>` itself must
|
|
carry `role="img"` + `aria-label` that says which host and which metrics. Pinned on the
|
|
served bytes because there is no JS test runner (a locked anchor).
|
|
"""
|
|
charts_js = client.get("/static/charts.js").text
|
|
# the shared LineChart applies the name to the svg, and only when the caller opted in
|
|
assert "if (this.opts.ariaLabel)" in charts_js, "LineChart stopped honouring ariaLabel"
|
|
assert "setAttribute('role', 'img')" in charts_js
|
|
assert "setAttribute('aria-label', String(this.opts.ariaLabel))" in charts_js
|
|
|
|
js = client.get("/static/page-system.js").text
|
|
# all three slots name themselves, and each name starts with the host the panel belongs
|
|
# to — an aria-label that cannot say *which machine* is not an accessible name here
|
|
assert js.count("ariaLabel: `") == 3, "a per-host chart slot lost its accessible name"
|
|
for slot in ("utilisation", "network traffic", "GPU busy"):
|
|
assert f"${{host.name}} \u2014 {slot}" in js or f"${{host.name}} — {slot}" in js, \
|
|
f"the {slot} chart has no host-scoped accessible name"
|
|
|
|
|
|
# ---- event feeds: a bounded, paginated window ----------------------------- #
|
|
#
|
|
# Every `ul.events` feed is one component (`Dash.eventsPager`) because a pager that
|
|
# works on only one page is the bug this shape prevents: the `event` table grows for as long
|
|
# as the fleet runs, and an unbounded feed turns `/system` into a scroll to nowhere
|
|
# (`TODO.md` L6 — "All the events tables need to be scrollable and paginated"). There is no
|
|
# JS test runner (a locked anchor), so what is pinned here is the shape every feed must keep:
|
|
# one shared renderer, a fetch bigger than one page, a footer that counts and labels itself,
|
|
# and a container that scrolls instead of growing the page.
|
|
|
|
# page -> (its script, the `source=` the poller stores that page's events under)
|
|
EVENT_FEEDS = {
|
|
"/system": ("page-system.js", "system"),
|
|
"/disks": ("page-disks.js", "disks"),
|
|
"/llm": ("page-llm.js", "llm"),
|
|
"/ups": ("page-ups.js", "ups"),
|
|
}
|
|
|
|
|
|
def _event_feed_path(js: str) -> str:
|
|
"""Just the `refreshEvents()` half of a page script — the feed's whole code path, comments out."""
|
|
js = _code(js)
|
|
return js[js.index("async function refreshEvents("):]
|
|
|
|
|
|
def _pager_call(feed: str, selector: str) -> str:
|
|
"""The options object of one `Dash.eventsPager($('<selector>'), …)` call inside the feed path.
|
|
|
|
Sliced per list so an assertion can name *which* feed lost its renderer, its accessible name
|
|
or its empty sentence. `/ups` is the page with two of them, so a whole-file assertion there
|
|
cannot tell one list from the other.
|
|
"""
|
|
head = f"Dash.eventsPager($('{selector}')"
|
|
assert head in feed, f"this feed is no longer rendered through Dash.eventsPager ({selector})"
|
|
start = feed.index(head)
|
|
end = feed.find("});", start)
|
|
assert end != -1, f"the {selector} pager call is never closed"
|
|
return feed[start:end]
|
|
|
|
|
|
def _feed_fetch_url(js: str) -> str:
|
|
"""The `/api/events` URL a feed asks for, quoted or templated."""
|
|
found = re.search(r"fetchJSON\([`'](/api/events[^`']*)[`']\)", _event_feed_path(js))
|
|
assert found, "this page no longer fetches the event feed"
|
|
return found.group(1)
|
|
|
|
|
|
def test_the_event_feeds_are_one_shared_pager_not_one_per_page(client) -> None:
|
|
"""`Dash.eventsPager()` renders every feed, and no page grows its own pager.
|
|
|
|
One copy per page is one chance per page to keep the old behaviour — and `/system` is the
|
|
one whose feed was already too tall when this was written down. `/disks` joined as a
|
|
fourth call site of the same helper, not as a fifth implementation of it.
|
|
"""
|
|
app_js = _code(client.get("/static/app.js").text)
|
|
assert app_js.count("function eventsPager(") == 1
|
|
exports = client.get("/static/app.js").text.rsplit("return {", 1)[1].split("}", 1)[0]
|
|
assert "eventsPager" in exports, "Dash.eventsPager is not exported"
|
|
|
|
for path, (script, _) in EVENT_FEEDS.items():
|
|
js = _code(client.get(f"/static/{script}").text)
|
|
assert "Dash.eventsPager($('#events')" in js, f"{path} does not render its feed through the pager"
|
|
assert "function eventsPager(" not in js, f"{path} invented its own pager"
|
|
assert "events.slice(" not in js, f"{path} windows its own events instead of asking the pager"
|
|
assert "row:" in _event_feed_path(js), f"{path} passes no row renderer"
|
|
|
|
|
|
def test_the_pager_pages_50_events_and_fetches_more_than_one_page(client) -> None:
|
|
"""The two numbers that make a client-side pager work, and the relation between them.
|
|
|
|
`EVENT_PAGE_SIZE` is the 50 the operator asked for; `EVENT_FEED_LIMIT` is what the feed
|
|
asks `/api/events` for. A limit at or below the page size is a pager that silently never
|
|
shows — there is never a second page to reach — and the feeds used to fetch 20/25/40.
|
|
"""
|
|
app_js = client.get("/static/app.js").text
|
|
page_size = re.search(r"const EVENT_PAGE_SIZE = (\d+);", app_js)
|
|
feed_limit = re.search(r"const EVENT_FEED_LIMIT = (\d+);", app_js)
|
|
assert page_size and int(page_size.group(1)) == 50, "a feed no longer pages 50 events"
|
|
assert feed_limit, "the feeds' event limit is no longer a named constant"
|
|
assert int(feed_limit.group(1)) > int(page_size.group(1)), \
|
|
"a fetch at or below one page means page 2 can never exist"
|
|
|
|
for path, (script, source) in EVENT_FEEDS.items():
|
|
url = _feed_fetch_url(client.get(f"/static/{script}").text)
|
|
assert "EVENT_FEED_LIMIT" in url, f"{path} hardcodes its own event limit"
|
|
assert f"source={source}" in url, f"{path} asks for the wrong event source"
|
|
|
|
|
|
def test_the_pager_footer_counts_and_labels_its_controls(client) -> None:
|
|
"""The footer counts (`1\u201350 of 137`), and prev/next are labelled and disabled at the ends.
|
|
|
|
The count is what tells the operator the window is a window; a control that stays live at
|
|
the last page claims there is more history than there is.
|
|
"""
|
|
app_js = _code(client.get("/static/app.js").text)
|
|
assert "${first}\u2013${last} of ${total}" in app_js, "the footer no longer counts"
|
|
assert 'data-pager="prev"' in app_js and 'data-pager="next"' in app_js
|
|
assert "prev.setAttribute('aria-label'" in app_js
|
|
assert "next.setAttribute('aria-label'" in app_js
|
|
assert "prev.disabled = pager.page === 0" in app_js
|
|
assert "next.disabled = pager.page >= pager.pages - 1" in app_js
|
|
# pagination only applies when there is more than one page of data
|
|
assert "foot.hidden = !paginating" in app_js
|
|
|
|
css = client.get("/static/dash.css").text
|
|
assert ".events-pager {" in css, "the pager footer has no style"
|
|
assert "button.ghost:disabled" in css, "a disabled pager control looks live"
|
|
|
|
|
|
def test_the_event_feeds_scroll_instead_of_growing_the_page(client) -> None:
|
|
"""The `<ul>` is the scrollable element, bounded in `dash.css`; the page stays short.
|
|
|
|
Pinned on the served bytes because there is no JS test runner (a locked anchor): a
|
|
stylesheet edit that drops `max-height` is invisible until the feed is long, and by then
|
|
the page is massive again.
|
|
"""
|
|
css = client.get("/static/dash.css").text
|
|
block = css[css.index("ul.events {"):css.index("}", css.index("ul.events {"))]
|
|
assert "max-height" in block and "overflow-y: auto" in block, \
|
|
"the event feed is unbounded again — the page grows with history"
|
|
|
|
for path in EVENT_FEEDS:
|
|
html = client.get(path).text
|
|
assert '<ul class="events" id="events"' in html, f"{path} lost its feed's id"
|
|
|
|
|
|
def _details_blocks(html: str) -> list[str]:
|
|
"""Every `<details>…</details>` block on a served page, in document order.
|
|
|
|
Nesting is not a thing on these pages (one disclosure per card), so a flat scan of pairs is
|
|
enough — and it is what lets a pin say *no disclosure contains this*, rather than only
|
|
"not the one disclosure I happen to know about". Phase 26 needs exactly that: the operator's
|
|
complaint was that a connection error was reachable only by opening something.
|
|
"""
|
|
blocks = []
|
|
for found in re.finditer(r"<details\b[^>]*>", html):
|
|
end = html.find("</details>", found.end())
|
|
assert end != -1, "a served <details> is never closed"
|
|
blocks.append(html[found.start():end])
|
|
return blocks
|
|
|
|
|
|
def test_ups_power_events_feed_sits_behind_a_collapsed_disclosure(client) -> None:
|
|
"""The `/ups` power feed is *expandable*: one `details.events-box`, shut by default.
|
|
|
|
`TODO.md:4` asked for an expandable list of incidents and phase 26 kept that shape — what it
|
|
overruled was *what* was behind it. A collapsed `<details>` renders no content, so the shut
|
|
card is still the resting state; since phase 26 it gates **visibility only** (the feed is
|
|
fetched on every poll, which the next test pins), and it holds only the power kinds.
|
|
|
|
Pinned on the served bytes because there is no JS test runner (a locked anchor): an `open`
|
|
attribute would undo the phase silently, the `#events` id has to stay *inside* the disclosure
|
|
or the pager and the bounded 320px scroll lose their target, and the **error** half
|
|
(`#events-errors`) has to be outside *every* disclosure and ship `hidden` rather than open —
|
|
a `NUT server NutError: cannot reach upsd …` line an operator has to click to read is the
|
|
bug this phase exists to fix (`TODO.md:6`).
|
|
"""
|
|
html = _code(client.get("/ups").text)
|
|
box = re.search(r'<details class="events-box">(.*?)</details>', html, re.S)
|
|
assert box, "the Power events card is no longer a details.events-box disclosure"
|
|
assert not re.search(r"<details[^>]*\bopen\b", html), \
|
|
"the disclosure ships open — the list is tall again, not expandable"
|
|
inner = box.group(1)
|
|
assert "<summary>Power events</summary>" in inner, "the summary lost its label"
|
|
assert '<ul class="events" id="events"' in inner, "the feed moved outside the disclosure"
|
|
assert "loading…" in inner, "the first-expand placeholder is gone"
|
|
assert "<h3>Power events</h3>" not in html, "the card heads its own body twice"
|
|
|
|
# The error half is on the page **and not collapsible**: no disclosure anywhere holds it.
|
|
# The card-local pin in `test_the_ups_connection_errors_are_a_card_of_their_own_above_the_disclosure`
|
|
# checks the card does not *contain* a `<details>`; this is the stronger direction — moving
|
|
# that card inside a new disclosure somewhere else would pass there and fail here.
|
|
assert 'id="events-errors"' in html, "the connection-error list left the served page"
|
|
assert not any('id="events-errors"' in block for block in _details_blocks(html)), \
|
|
"the error list is inside a disclosure — reading it needs a click again"
|
|
card_open = html[html.index('<section class="card" id="ups-errors-card"'):
|
|
html.index(">", html.index('<section class="card" id="ups-errors-card"'))]
|
|
assert "hidden" in card_open, \
|
|
"the error card's resting state is visible — an empty card on every quiet day"
|
|
|
|
# the rest of the page is untouched by the disclosure
|
|
for still_there in ('id="ups-cards"', 'id="chart-charge"', 'id="chart-load"',
|
|
'id="confirm-dialog"'):
|
|
assert still_there in client.get("/ups").text, f"/ups lost {still_there}"
|
|
|
|
|
|
# The event kinds `/ups` keeps behind the collapsed disclosure, and the writers that store
|
|
# them. `collectors.py` writes `status` (an `ups.status` transition, power loss included)
|
|
# and `test` (a self-test verdict); `api.py` writes `action` (a command this page sent).
|
|
# Everything else in the feed is a connection/error event (`unreachable` / `reachable` /
|
|
# `recording` / `store-error`) and belongs in the list that is never hidden.
|
|
UPS_POWER_EVENT_KINDS = ("status", "test", "action")
|
|
|
|
|
|
def _power_event_kinds(js: str) -> frozenset[str]:
|
|
"""The kinds `page-ups.js` keeps behind the disclosure, read out of its own set literal.
|
|
|
|
Parsing beats matching one exact line. The claim is *these three and only these* — a kind
|
|
added to the hidden list moves an operator-visible row out of sight, and a kind dropped from
|
|
it hides a power event — so an exact-string assert that happens to survive a reformat is not
|
|
the pin. An empty set is refused rather than returned, because it would make every event an
|
|
error row and look like a passing classification.
|
|
"""
|
|
found = re.search(r"const POWER_EVENT_KINDS = new Set\(\[(.*?)]\)", js, re.S)
|
|
assert found, "page-ups.js no longer defines POWER_EVENT_KINDS as a set literal of kind words"
|
|
kinds = re.findall(r"['\"]([^'\"]+)['\"]", found.group(1))
|
|
assert kinds, "POWER_EVENT_KINDS is empty — every event would land in the error list"
|
|
return frozenset(kinds)
|
|
|
|
|
|
def test_ups_events_are_fetched_on_every_poll(client) -> None:
|
|
"""Phase 19's lazy gate is out, and the feed splits into power events and errors.
|
|
|
|
`TODO.md:6` is not a complaint about the disclosure existing, it is a complaint about what
|
|
was behind it: a poll failure (`NUT server NutError: cannot reach upsd … (timed out after
|
|
4s)`) sat in the same collapsed list as the power events, so the one thing an operator
|
|
needs without opening anything was the one thing they had to open something for. The fix
|
|
is page-side and keyed on the stored `kind`: `status` / `test` / `action` stay behind the
|
|
disclosure, **everything else** is the always-visible error list — and because that list
|
|
is visible, `refresh()` fetches on every poll (a visible list that ages is worse than one
|
|
more 200-event fetch per interval). `/api/events` grew no vocabulary for any of this.
|
|
|
|
Pinned on the served script because there is no JS test runner (a locked anchor), and
|
|
pinned in both directions: the gate's absence *and* the kind set, so a silent rename on
|
|
the writer's side fails here instead of moving errors back into the hidden list.
|
|
"""
|
|
js = _code(client.get("/static/page-ups.js").text)
|
|
|
|
# The kind set, read out of the literal instead of matched as one line: exactly these three.
|
|
assert _power_event_kinds(js) == frozenset(UPS_POWER_EVENT_KINDS), \
|
|
(f"the hidden list's kind set is {sorted(_power_event_kinds(js))}, not "
|
|
f"{sorted(UPS_POWER_EVENT_KINDS)} — that classification is the operator's, not the page's")
|
|
|
|
# The lazy cache is gone: with an always-visible list there is nothing left to stamp.
|
|
decl = js[js.index("const state = {"):js.index("const MARKS")]
|
|
assert "events" not in decl, "the disclosure's lazy-fetch state came back"
|
|
assert "state.events" not in js, "something still reads the deleted state.events object"
|
|
|
|
# The poll path asks every interval, gated on nothing.
|
|
refresh = js[js.index("async function refresh("):js.index("async function refreshEvents(")]
|
|
assert re.search(r"^\s*await refreshEvents\(\);", refresh, flags=re.M), \
|
|
"the poll loop no longer fetches the feed on every poll"
|
|
assert "state.events.open" not in refresh and "eventsBox" not in refresh, \
|
|
"the fetch is gated on the disclosure again"
|
|
|
|
# The toggle handler's fetch logic is gone with it: the native `<details>` now gates only
|
|
# visibility, so no `toggle` listener may exist to decide whether to fetch.
|
|
boot = js[js.index("document.addEventListener('DOMContentLoaded'"):]
|
|
assert "addEventListener('toggle'" not in boot, \
|
|
"a toggle handler is deciding whether to fetch again"
|
|
assert "eventsBox" not in boot, "the disclosure's fetch-on-expand handler came back"
|
|
|
|
# One fetch, two lists, split on `kind` — and the URL is the phase-19 one verbatim.
|
|
feed = _event_feed_path(js)
|
|
assert _feed_fetch_url(js) == "/api/events?limit=${Dash.EVENT_FEED_LIMIT}&source=ups", \
|
|
"the feed's fetch is no longer the unchanged /api/events call"
|
|
for vocabulary in ("kind=", "page=", "severity="):
|
|
assert vocabulary not in _feed_fetch_url(js), \
|
|
f"/api/events grew {vocabulary[:-1]} vocabulary for a page-side split"
|
|
assert "const power = data.events.filter((e) => POWER_EVENT_KINDS.has(e.kind));" in feed, \
|
|
"the power list is no longer the filter's positive arm"
|
|
assert "const errors = data.events.filter((e) => !POWER_EVENT_KINDS.has(e.kind));" in feed, \
|
|
"the error list is no longer the filter's default arm (an unknown kind must be visible)"
|
|
|
|
# Stale-last-good: the catch clause does *nothing*, so a poll that never answered leaves
|
|
# both `<ul>`s holding the rows they already had (the pager rewrites a list only when its
|
|
# markup changed — invariant 4's shape on the page side).
|
|
opened = "} catch (err) {"
|
|
at = feed.index(opened)
|
|
catch_body = feed[at + len(opened):feed.index("}", at + len(opened))].strip()
|
|
assert catch_body == "", \
|
|
f"the failure path does something ({catch_body!r}) instead of leaving the last good rows"
|
|
|
|
|
|
def test_the_ups_event_rows_keep_the_date_time_and_severity_shape(client) -> None:
|
|
"""`TODO.md:4`'s other half: an expandable list is only *clear* if each row says when.
|
|
|
|
Phase 19's disclosure and phase 26's split are the new parts; this is the half that had to
|
|
survive both untouched. Every row still opens with a `<time>` carrying a month/day plus a
|
|
24-hour clock, then the UPS name inside the `sev <severity>` chip the stylesheet colours from
|
|
the payload's own verdict, then the message through `eventText()` (subject named once) with
|
|
the runtime footnote only when the event actually carries one. Pinned on the served script
|
|
because there is no JS test runner (a locked anchor): a "tidy-up" of the row template that
|
|
drops the timestamp, prints the raw message, or escapes nothing would ship silently.
|
|
|
|
Since phase 26 task 02 the renderer is **one const read by both lists**, so the two halves
|
|
of one feed cannot grow two shapes — which is what this reads, rather than a `row:` option
|
|
that could differ per list.
|
|
"""
|
|
js = _code(client.get("/static/page-ups.js").text)
|
|
feed = _event_feed_path(js)
|
|
at = js.index("const eventRow = (event) =>")
|
|
row = js[at:js.index("async function refreshEvents(", at)]
|
|
|
|
# the date/time: `<time>` first, month + day + hour + minute, 24-hour (the operator reads
|
|
# an incident log in the order it happened, and "14:05" on its own is not that)
|
|
assert "<time>" in row and "</time>" in row, "the row lost its timestamp element"
|
|
assert "new Date(event.ts * 1000).toLocaleString(" in row, \
|
|
"the row formats something other than the event's own timestamp"
|
|
for option in ("month: 'short'", "day: 'numeric'", "hour: '2-digit'", "minute: '2-digit'",
|
|
"hour12: false"):
|
|
assert option in row, f"the timestamp no longer asks for {option}"
|
|
assert row.index("<time>") < row.index('class="sev'), "the row no longer leads with the date"
|
|
|
|
# who: the host inside the verdict chip, escaped, and the severity is the API's word
|
|
assert 'class="sev ${event.severity}"' in row, \
|
|
"the row stopped taking its colour word from the stored severity"
|
|
assert "${DashCharts.escapeHtml(event.host)}</span>" in row, \
|
|
"the row no longer names the UPS it belongs to, escaped"
|
|
|
|
# what: the message through the shared subject-said-once cleaner, and escaped
|
|
assert "${DashCharts.escapeHtml(eventText(event))}" in row, \
|
|
"the message is no longer cleaned and escaped (raw HTML from an upstream is a bug)"
|
|
|
|
# the runtime footnote is conditional: an event with no runtime says nothing about one
|
|
assert "event.data && event.data.runtime_seconds" in row, \
|
|
"the runtime footnote is no longer gated on the event carrying a runtime"
|
|
assert "duration(event.data.runtime_seconds)} runtime)" in row, \
|
|
"the runtime footnote stopped printing the stored seconds as a duration"
|
|
|
|
# and the renderer is still handed to the shared pager over the bounded list, labelled.
|
|
# Per **call**, not per file: `#events-errors` is a list of its own now, and the claim is
|
|
# that it is drawn, named and emptied exactly like the power list — a count of `row:` in the
|
|
# file would still pass if one call lost its renderer to a default.
|
|
for selector, label, empty in (("#events", "Power events", "No power events yet"),
|
|
("#events-errors", "Connection & errors",
|
|
"No connection errors")):
|
|
call = _pager_call(feed, selector)
|
|
assert "row: eventRow," in call, f"the {selector} list is no longer drawn by the shared row"
|
|
assert f"label: '{label}'" in call, f"the {selector} list lost its accessible name"
|
|
assert empty in call, f"the {selector} list no longer says which feed is empty"
|
|
|
|
# one renderer, two lists: a template the two lists do not share would be two shapes for one
|
|
# event, and the row may not branch on which list it landed in — the only split is the kind set
|
|
assert js.count("const eventRow = (event) =>") == 1, \
|
|
"the /ups lists no longer share one row template"
|
|
assert "event.kind" not in row and "POWER_EVENT_KINDS" not in row, \
|
|
"the row template reads the kind — the two lists are growing two shapes"
|
|
|
|
# The card's visibility is the one state this page claims about a list it cannot see, it is
|
|
# decided by the **error** half alone, and only after both lists are painted (earlier would
|
|
# flash the `loading…` seed on a first fetch with no error rows). The markup's own `hidden`
|
|
# default is pinned in test_ups_power_events_feed_sits_behind_a_collapsed_disclosure.
|
|
hidden_at = feed.index("$('#ups-errors-card').hidden")
|
|
assert feed[:hidden_at].count("Dash.eventsPager(") == 2, \
|
|
"the card's visibility is decided before both lists are drawn"
|
|
assert feed[hidden_at:feed.index(";", hidden_at)].endswith("= errors.length === 0"), \
|
|
"the error card's visibility no longer follows the error half of the feed"
|
|
|
|
|
|
def test_the_ups_connection_errors_are_a_card_of_their_own_above_the_disclosure(client) -> None:
|
|
"""Phase 26 task 02: the error half of the feed is a card, not a second list in a `<details>`.
|
|
|
|
`TODO.md:6` is that a `NUT server NutError: cannot reach upsd …` line is not something to
|
|
open a summary for. So the error rows get their own `section.card` **above** the Power
|
|
events disclosure, built from the vocabulary every other card on this page uses
|
|
(`.card > header`, `.card > .body`, `ul.events`, the pager's strip) — **no new stylesheet
|
|
selector**, because a second style for a second list is a second thing to keep in sync with
|
|
the bounded 320px scroll and the `1\u201350 of N` footer.
|
|
|
|
`hidden` is the resting state in the markup and the script keeps it true while the feed has
|
|
no non-power kinds: a permanent empty "Connection & errors" card is exactly the ink the
|
|
page's empty-state rules exist to avoid, and the first `unreachable` row is what shows it.
|
|
"""
|
|
html = _code(client.get("/ups").text)
|
|
|
|
card_at = html.index('<section class="card" id="ups-errors-card"')
|
|
card_end = html.index("</section>", card_at)
|
|
card = html[card_at:card_end]
|
|
opening = html[card_at:html.index(">", card_at)]
|
|
assert "hidden" in opening, "the error card ships visible with nothing in it"
|
|
assert '<ul class="events" id="events-errors"' in card, "the error card has no bounded feed"
|
|
assert "<h3>Connection & errors</h3>" in card, "the card lost its title"
|
|
# the whole point: this list is *not* behind a summary
|
|
assert "<details" not in card, "the error list is inside a disclosure again"
|
|
assert card_end < html.index('<details class="events-box">'), \
|
|
"the error card sits below the power-events disclosure it is meant to lead"
|
|
assert 'id="events"' in html[html.index('<details class="events-box">'):], \
|
|
"the power list left its disclosure"
|
|
|
|
js = _code(client.get("/static/page-ups.js").text)
|
|
feed = _event_feed_path(js)
|
|
assert "Dash.eventsPager($('#events-errors'), errors, {" in feed, \
|
|
"the error list is not rendered through the shared pager"
|
|
assert "label: 'Connection & errors'" in feed, "the error list has no accessible name"
|
|
assert "$('#ups-errors-card').hidden = errors.length === 0;" in feed, \
|
|
"the error card no longer appears and disappears with its rows"
|
|
# and the pager state is per `<ul>` (`app.js`: a `WeakMap` keyed by the list), so the two
|
|
# lists page independently: this is the contract that makes a second call site free
|
|
app_js = _code(client.get("/static/app.js").text)
|
|
assert "const feedPagers = new WeakMap();" in app_js
|
|
assert "feedPagers.get(list)" in app_js and "feedPagers.set(list, pager)" in app_js
|
|
|
|
|
|
def test_the_ups_error_card_adds_no_stylesheet_selector(client) -> None:
|
|
"""The second feed is markup + the existing vocabulary, and `dash.css` must stay that way.
|
|
|
|
Pinned as the *absence* of a rule, because the temptation this writes back into is a
|
|
`#ups-errors-card` block restating the feed's scroll or the pager's strip: `ul.events {` and
|
|
`.events-pager {` are each declared exactly once in the sheet (the disclosure pin already
|
|
holds that), and the error card carries its own gap as an inline `margin-top` like every
|
|
other section on this page rather than through an id rule.
|
|
"""
|
|
css = _css(client)
|
|
for selector in ("#ups-errors-card", "#events-errors"):
|
|
assert selector not in css, f"dash.css grew a rule for {selector}"
|
|
assert css.count("ul.events {") == 1 and css.count(".events-pager {") == 1
|
|
|
|
html = client.get("/ups").text
|
|
assert 'id="ups-errors-card" style="margin-top:.85rem"' in html, \
|
|
"the error card no longer carries the page's own section gap inline"
|
|
|
|
|
|
def test_the_ups_events_disclosure_is_a_quiet_disclosure_not_a_new_component(client) -> None:
|
|
"""The `/ups` summary borrows the card header's rhythm and no new palette (task 03).
|
|
|
|
Pinned on the served stylesheet because a CSS edit is invisible until someone looks: the
|
|
`\\25B8`/`\\25BE` caret pair says open/closed in a **shape** as well as a colour, the webkit
|
|
marker is hidden so the browser's own triangle does not double it, and this `<summary>`
|
|
carries the card header's ground and bottom rule because it *replaces* that header instead of
|
|
sitting inside the body like `charts-box`'s toggle does. The `> .body` padding restatement is
|
|
the quiet one: `.card > .body` is a **child** selector, so a body moved inside a disclosure
|
|
silently loses its padding and the feed renders flush to the card edge.
|
|
|
|
What the block must not do is grow a verdict colour (open/closed is not a grading) or
|
|
re-state the shared feed rules — `ul.events`' bounded scroll and `.events-pager`'s strip are
|
|
class-based and resolve inside the disclosure unchanged, so a second copy here would be a
|
|
second thing to keep in sync. The undeclared-token audit itself stays the existing
|
|
`test_no_stylesheet_rule_uses_an_undeclared_token`'s job, untouched.
|
|
"""
|
|
css = _css(client)
|
|
start = css.index("details.events-box { ")
|
|
block = css[start:css.index(".host-charts {", start)]
|
|
|
|
assert 'details.events-box > summary::before { content: "\\25B8"' in block, \
|
|
"the toggle has no caret shape"
|
|
assert 'details.events-box[open] > summary::before { content: "\\25BE"; }' in block, \
|
|
"open and closed differ only in colour"
|
|
assert "details.events-box > summary::-webkit-details-marker { display: none; }" in block, \
|
|
"the browser's own marker doubles the caret"
|
|
assert "cursor: pointer" in block and "list-style: none" in block, \
|
|
"the summary no longer reads as a control"
|
|
# it replaces the card header, so it carries the header's ground and bottom rule
|
|
assert "border-bottom: 1px solid var(--border)" in block, \
|
|
"a closed card no longer reads as its heading"
|
|
assert "color-mix(in srgb, var(--bg-raised) 70%, var(--bg-sunken))" in block, \
|
|
"the summary lost the header's sunken ground"
|
|
# open: the label steps back, and no margin — the `.body` below carries the spacing
|
|
assert "details.events-box[open] > summary { color: var(--text-faint); }" in block, \
|
|
"an open summary does not de-emphasise"
|
|
assert "margin-bottom" not in block, "the open summary stacks a gap on top of the body's"
|
|
# the child-selector gotcha that task 01's markup introduced
|
|
assert "details.events-box > .body { padding: .8rem; }" in block, \
|
|
"the feed body lost its padding inside the disclosure"
|
|
|
|
for token in ("var(--ok)", "var(--warn)", "var(--fail)", "var(--accent)"):
|
|
assert token not in block, f"the disclosure graded open/closed with {token}"
|
|
assert ".sev-" not in block, "the disclosure invented a verdict family"
|
|
|
|
# one copy of each shared feed rule, class-based, resolving inside the disclosure as elsewhere
|
|
assert css.count("ul.events {") == 1, "ul.events is styled twice — the bounded scroll drifted"
|
|
assert css.count(".events-pager {") == 1, "the pager strip is styled twice"
|
|
# the class this block styles is the class the page renders
|
|
assert 'class="events-box"' in client.get("/ups").text, \
|
|
"the stylesheet styles a disclosure no page renders"
|
|
|
|
# Phase 26's guard, here because it is the disclosure's own claim: the *second* feed this
|
|
# page renders borrowed `section.card` + `ul.events` + `.events-pager` and nothing else, so
|
|
# this block may not have grown a rule for it either. A second style for a second list is a
|
|
# second copy of the bounded scroll and the pager strip to keep in sync — the phase-19
|
|
# anti-pattern, one task later. (`test_the_ups_error_card_adds_no_stylesheet_selector` holds
|
|
# the same guard for the card's own section, plus the inline-gap pin.)
|
|
for selector in ("#ups-errors-card", "#events-errors"):
|
|
assert selector not in block and selector not in css, \
|
|
f"dash.css styles the second feed through {selector} instead of the shared classes"
|
|
|
|
|
|
def test_the_pager_returns_to_page_1_when_the_events_change(client) -> None:
|
|
"""New events reset the window; the same events do not.
|
|
|
|
Resetting on every poll would snap `next` back within one interval, so page 2 could never
|
|
be read; never resetting lands an operator on page 3 of a list that changed underneath
|
|
them. The page key is the set of events, not the clock.
|
|
"""
|
|
app_js = _code(client.get("/static/app.js").text)
|
|
body = app_js[app_js.index("function eventsPager("):]
|
|
assert "signature !== pager.signature" in body, "the pager no longer notices new events"
|
|
reset = body[body.index("signature !== pager.signature"):]
|
|
assert reset.index("pager.page = 0") < reset.index("paintEventFeed"), \
|
|
"new events do not reset the page before the feed is drawn"
|
|
assert "if (pager.markup !== markup)" in app_js, \
|
|
"every poll rewrites 50 rows and throws away the reader's scroll position"
|
|
|
|
|
|
def test_system_event_feed_queries_the_source_transitions_are_stored_under(client) -> None:
|
|
"""Host transitions are stored under the *dashboard* name, not the sample source.
|
|
|
|
`poller.py` logs `Event(ts, self.dashboard, …)` — host jobs write `system`, the way
|
|
the UPS job writes `ups` — while *samples* use `host`. The `/system` feed queried
|
|
`source=host` since the first commit and therefore always rendered "no transitions
|
|
yet", even with a real outage (a gateway disconnect) sitting in the store. Pinned on
|
|
the served bytes because there is no JS test runner (a locked anchor).
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
feed = _event_feed_path(js)
|
|
url = re.search(r"fetchJSON\([`'](/api/events[^`']*)[`']\)", feed)
|
|
assert url, "the /system page no longer fetches the event feed"
|
|
assert "source=system" in url.group(1), \
|
|
"the /system event feed must ask for the source the poller stores events under"
|
|
assert "source=host" not in url.group(1), \
|
|
"source=host matches no stored event — the feed is empty forever"
|
|
|
|
|
|
def test_system_footnote_prints_the_window_it_actually_measured(client) -> None:
|
|
"""`Δ<window>` is the page's promise that a rate is a measurement (invariant 2), so the
|
|
number behind it has to come from the measurement.
|
|
|
|
The `window_seconds` beside the fleet is `HOST_INTERVAL_SECONDS`, while the window behind a
|
|
rate is the estimator's `RATE_WINDOW_SECONDS` — equal at the defaults, which is why the
|
|
footnote could quote the poll interval and look right. Set `RATE_WINDOW_SECONDS=30` and it
|
|
would have labelled every rate on the page with a window it never used. The footnote now
|
|
reads each host's own measured `window_seconds`, and prints a range when they disagree.
|
|
"""
|
|
js = client.get("/static/page-system.js").text
|
|
assert "function windowLabel(" in js, "the footnote lost its measured-window helper"
|
|
footnote = js[js.index("$('#foot-note').textContent"):js.index("await Promise.all(chartJobs)")]
|
|
assert "windowLabel(" in footnote
|
|
assert "host.window_seconds" in footnote, \
|
|
"the footnote is quoting the poll interval as a measurement again"
|
|
# a footnote that says "window" and a page that says "poll" must stay two separate claims
|
|
assert "poll ${Math.round(state.pollMs / 1000)}s" in footnote
|
|
|
|
|
|
def test_no_per_core_visual_library_survives_the_page(client) -> None:
|
|
"""The per-core heat bars left `/system` and `charts.js` with them (PLAN.md *Dropped
|
|
visuals* is about what the pages draw, not what the store keeps).
|
|
|
|
`renderCores` had one caller — the hero-card per-core block deleted in task 02 — so
|
|
keeping it would be dead markup generating unstyled `.core` spans; `.cores` was the CSS
|
|
that block needed. Both are checked absent here so neither can come back as a
|
|
half-change, while `sparkline`/`LineChart` stay for phase 04 and the UPS page.
|
|
"""
|
|
assert "renderCores" not in client.get("/static/charts.js").text
|
|
css = client.get("/static/dash.css").text
|
|
assert ".cores" not in css
|
|
for asset in ASSETS:
|
|
assert "core_usage" not in _code(client.get(asset).text), f"{asset} plots per-core"
|
|
# what phase 04 and the other pages still chart from
|
|
for name in ("LineChart", "sparkline", "renderLegend"):
|
|
assert name in client.get("/static/charts.js").text
|
|
|
|
|
|
def test_sparkline_ships_the_fixed_scale_and_verdict_options(client) -> None:
|
|
"""Phase 04 turns `/llm` cells into sparkline rows, so the shared `sparkline()` must
|
|
behave like `llm_monitor`'s `drawSpark` (the task-02 contract, NOTES-chart-set).
|
|
|
|
Pinned on the served bytes because there is no JS test runner (a locked anchor). What
|
|
matters is not the option names but the semantics behind each one: a percent metric
|
|
that silently rescaled to its own data would make an 8 % KV cache and a 92 % spike the
|
|
same squiggle, and a stroke coloured from a page-invented threshold would break the
|
|
one-colour-language rule the same way.
|
|
"""
|
|
charts_js = client.get("/static/charts.js").text
|
|
body = charts_js[charts_js.index("function sparkline("):charts_js.index("window.DashCharts")]
|
|
|
|
# the phase grep criterion, as a test: the options are really in the shipped file
|
|
hits = [ln for ln in charts_js.splitlines()
|
|
if re.search(r"fixedMax|align|severity", ln)]
|
|
assert len(hits) >= 3, "the sparkline upgrade is not in the served charts.js"
|
|
for opt in ("fixedMax", "align", "labels", "severity", "minPoints", "span", "format",
|
|
"minSpan"):
|
|
assert opt in body, f"sparkline() lost its `{opt}` option"
|
|
|
|
# fixedMax: the data clamps to the axis, never the axis to the data
|
|
assert "Math.min(Math.max(v, lo), hi)" in body, \
|
|
"a fixedMax sparkline rescales to its own data again"
|
|
# align:'right' grows within a real time window (span seconds), not a sample count
|
|
assert "x1 - o.span" in body, "align:'right' forgot the window it grows inside"
|
|
# verdict colour comes from the theme tokens only; unrated is faint grey, never green
|
|
for token in ("cssVar('--ok')", "cssVar('--warn')", "cssVar('--fail')",
|
|
"cssVar('--text-faint')"):
|
|
assert token in body, f"sparkline() stopped taking its colour from {token}"
|
|
assert ".sev-" not in body, "sparkline invented a parallel severity vocabulary"
|
|
# the `now / max` foot llm_monitor prints, via the caller's format callback
|
|
assert "now ${fmt(last.value)}" in body and "max ${fmt(dataMax)}" in body
|
|
# never a confident line through one point, and the foot quotes data, not the padded axis
|
|
assert "minPoints: 2" in body and "no history" in body
|
|
# ASSUMPTION (task 02): no animation at all, so prefers-reduced-motion holds by design
|
|
assert "transition:" not in body and "animation:" not in body
|
|
|
|
|
|
# ---- the battery glyph (phase 05 task 02) ---------------------------------
|
|
#
|
|
# `DashCharts.battery()` is the one visual in this app whose *shape* is the data: the fill
|
|
# is the charge. That makes the honesty rules heavier here than anywhere else — a fill
|
|
# rounded up to look reassuring, a colour invented for an unrated cell, or a state only
|
|
# readable as green-vs-red would each turn a glance at the battery into a wrong claim.
|
|
# There is no JS test runner (a locked anchor), so the construction is pinned on the served
|
|
# bytes and shown in a browser (`data/logs/ups-glyph-*.png`).
|
|
|
|
def _battery_js(client: TestClient) -> str:
|
|
"""The glyph's code as served (state table + body), comments stripped.
|
|
|
|
Prose cannot satisfy these assertions, and the phrase table the accessible name is
|
|
built from is part of the component, so the slice starts above `function battery`.
|
|
"""
|
|
charts = client.get("/static/charts.js").text
|
|
body = charts[charts.index("const BATTERY_STATES"):charts.index("window.DashCharts")]
|
|
assert "function battery(container, opts)" in body
|
|
return _code(body)
|
|
|
|
|
|
def test_battery_glyph_is_exported_from_the_shared_chart_library(client) -> None:
|
|
"""The phase's own grep criterion, as a test: exactly one glyph, and it is public."""
|
|
charts = client.get("/static/charts.js").text
|
|
assert charts.count("function battery") == 1
|
|
exports = charts[charts.index("window.DashCharts"):]
|
|
assert re.search(r"\bbattery\b", exports), "DashCharts.battery is not exported"
|
|
# the wrapper the stylesheet sizes, added by the component rather than each caller
|
|
assert ".battery {" in _css(client) and ".battery svg" in _css(client)
|
|
|
|
|
|
def test_battery_fill_is_the_charge_and_is_never_padded(client) -> None:
|
|
"""2 % must look nearly empty, and 0 % must leave an empty shell, not a stray pixel."""
|
|
body = _battery_js(client)
|
|
# clamped to the scale, and the clamp is the only thing that touches the number: a
|
|
# minimum visible width would be a reassuring battery over a flat one
|
|
assert "Math.max(0, Math.min(100," in body
|
|
assert "(pctValue / 100) * inner.w" in body, "the fill stopped being a fraction of charge"
|
|
assert not re.search(r"Math\.max\(\s*[0-9.]*[1-9]\s*,", body), \
|
|
"the charge fill got a minimum width — a small charge would look bigger than it is"
|
|
# nothing is drawn for a cell at zero, and the plate it fills is a clip, so a low fill
|
|
# cannot leak outside the rounded interior
|
|
assert "pctValue > 0" in body
|
|
assert "<clipPath" in body and 'clip-path="url(#' in body
|
|
# an empty cell is the sunken plate, never a transparent hole in the card
|
|
assert "cssVar('--bg-sunken')" in body
|
|
|
|
|
|
def test_battery_colours_come_from_the_verdict_tokens_only(client) -> None:
|
|
"""The verdict paints; an unrated cell draws a grey shape and no colour at all."""
|
|
body = _battery_js(client)
|
|
for token in ("cssVar('--ok')", "cssVar('--warn')", "cssVar('--fail')"):
|
|
assert token in body, f"the glyph stopped taking its colour from {token}"
|
|
# no verdict → muted outline, and a measured charge in faint grey at half strength: a
|
|
# shape, not a judgement (an unknown number must never look green)
|
|
assert "cssVar('--text-muted')" in body and "cssVar('--text-faint')" in body
|
|
assert ' opacity=".5"' in body, "an unrated charge is drawn with the same weight as a rated one"
|
|
assert ".sev-" not in body, "the glyph invented a parallel severity vocabulary"
|
|
assert "severityFor(" not in body, "the glyph compares the charge to a threshold itself"
|
|
|
|
|
|
def test_battery_states_are_readable_without_colour(client) -> None:
|
|
"""Every state has a *shape*, so the glyph survives monochrome and colour-blindness.
|
|
|
|
on battery = dashed outline, charging = bolt, unknown = a dash pattern through the
|
|
body, online = a plain solid shell. A word the page did not intend falls back to
|
|
`unknown` rather than to a confident "on line".
|
|
"""
|
|
body = _battery_js(client)
|
|
assert 'BATTERY_STATES.indexOf(o.state) >= 0' in body, \
|
|
"an unrecognised state would be drawn as some other state"
|
|
assert 'state === \'onbattery\'' in body and 'stroke-dasharray="7 5"' in body, \
|
|
"the on-battery outline is no longer dashed"
|
|
assert "state === 'charging'" in body and 'd="M13 2 L3 14' in body, "the charging bolt is gone"
|
|
assert "state === 'unknown'" in body and 'stroke-dasharray="5 6"' in body, \
|
|
"an unknown charge no longer draws a dash through the body"
|
|
# and the unknown case really is *no fill*: the shape has to say it on its own
|
|
assert "pctValue != null" in body
|
|
|
|
|
|
def test_battery_glyph_names_itself_and_can_say_it_once(client) -> None:
|
|
"""`role="img"` + `<title>` built from the inputs, with an explicit `aria-hidden` opt-out."""
|
|
body = _battery_js(client)
|
|
assert 'role="img"' in body and "<title>" in body and "aria-label=" in body
|
|
for phrase in ("Charge ", " percent", "Charge unknown", "on line", "on battery",
|
|
"charging", "state unknown"):
|
|
assert phrase in body, f"the accessible name lost {phrase!r}"
|
|
# the label is a UPS name from upsd, so it goes through the one escaper
|
|
assert body.count("escapeHtml(name)") == 2, "<title> or aria-label stopped escaping the label"
|
|
assert 'aria-hidden="true"' in body, "the decorative opt-out for a duplicate hero number is gone"
|
|
|
|
|
|
def test_battery_glyph_scales_with_its_container_and_never_animates(client) -> None:
|
|
"""One `viewBox` + `preserveAspectRatio` for a phone and a 27" monitor; no motion at all."""
|
|
body = _battery_js(client)
|
|
assert 'viewBox="0 0 ${W} ${H}"' in body
|
|
assert 'preserveAspectRatio="xMidYMid meet"' in body, "the glyph stopped scaling with its box"
|
|
assert "ResizeObserver" not in body, "the glyph redraws on resize instead of scaling"
|
|
assert "transition" not in body and "animation" not in body
|
|
css = _code(_css(client))
|
|
wrapper = re.search(r"\.battery \{([^}]*)\}", css)
|
|
inner = re.search(r"\.battery svg \{([^}]*)\}", css)
|
|
assert wrapper and inner, "dash.css no longer sizes the glyph box"
|
|
assert "width: 100%" in wrapper.group(1) and "max-width" in wrapper.group(1), \
|
|
"the glyph is not capped inside its container"
|
|
assert "width: 100%" in inner.group(1) and "height: auto" in inner.group(1), \
|
|
"the svg would stretch instead of keeping the viewBox ratio"
|
|
both = wrapper.group(1) + inner.group(1)
|
|
assert "transition" not in both and "animation" not in both
|
|
|
|
|
|
# ---- the `/ups` hero row (phase 05 task 03) ------------------------------
|
|
#
|
|
# The card used to be nine equal `.metric` cells in which charge, load and runtime were the
|
|
# same size as the battery amperage. These pin the shape that replaced it on the served
|
|
# bytes (a locked anchor: no JS test runner), because the whole point of the row is a size
|
|
# and colour hierarchy a later "tidy up" can quietly flatten back to nine equal cells.
|
|
|
|
def _ups_card_js(client: TestClient) -> str:
|
|
"""The card markup as served, from `upsCard()` to `renderCard()`."""
|
|
js = client.get("/static/page-ups.js").text
|
|
return js[js.index("function upsCard("):js.index("function renderCard(")]
|
|
|
|
|
|
def test_ups_card_opens_with_the_glyph_and_three_heroes(client) -> None:
|
|
"""Charge / Load / Runtime are the card's three largest elements, next to the glyph.
|
|
|
|
`data-sev` names the verdict path each slot colours itself from, the same convention the
|
|
`/system` host card uses — so "which number is coloured by what" is readable in the
|
|
markup, and a rename that disconnects the page from `/api/ups/snapshot` fails here
|
|
instead of shipping a card of `n/a`.
|
|
"""
|
|
card = _ups_card_js(client)
|
|
assert '<div class="ups-hero">' in card, "the hero row left the top of the card"
|
|
assert 'data-k="glyph"' in card, "the battery glyph is not part of the hero row"
|
|
slots = re.findall(r'<div class="hero[^"]*" data-k="([^"]+)"', card)
|
|
assert slots == ["charge", "load", "runtime"], \
|
|
"the hero row is no longer charge / load / runtime in the agreed order"
|
|
for hook in ("severity.charge_pct", "severity.load_pct", "severity.battery_runtime"):
|
|
assert hook in card, f"the card no longer names {hook} as a colour source"
|
|
# the hero row is the first thing in the card body, above the health checks and commands
|
|
assert card.index("ups-hero") < card.index('data-k="checks"')
|
|
assert card.index("ups-hero") < card.index('data-k="commands"')
|
|
|
|
|
|
def test_ups_heroes_take_their_colour_from_the_payload(client) -> None:
|
|
"""Three `Dash.hero()` calls, three payload verdicts, no JS threshold.
|
|
|
|
The charge hero carries **no** `.bar`: the glyph beside it is that same value drawn as a
|
|
shape, and one value drawn twice in one row is the pattern this app avoids (load, which
|
|
has no other shape, keeps its bar). Runtime prose comes from `runtimeText()` alone, and
|
|
a runtime nobody estimated is passed as `null` so `hero()` renders the faint, uncoloured
|
|
unknown rather than the string `n/a` styled like a measurement.
|
|
"""
|
|
js = client.get("/static/page-ups.js").text
|
|
body = js[js.index("function renderCard("):js.index("// ---- command dialog")]
|
|
assert body.count("Dash.hero(q(") == 3, "the hero row is not three hero numbers"
|
|
for verdict in ("verdicts.charge_pct", "verdicts.load_pct", "verdicts.battery_runtime"):
|
|
assert verdict in body, f"{verdict} never reaches a hero"
|
|
charge = body[body.index("Dash.hero(q('charge')"):body.index("Dash.hero(q('load')")]
|
|
assert "bar" not in charge, "charge grew a second shape when the glyph already is one"
|
|
load = body[body.index("Dash.hero(q('load')"):body.index("Dash.hero(q('runtime')")]
|
|
assert "bar: true" in load, "load lost its bar"
|
|
assert "runtimeText(n.battery_runtime)" in body, "runtime prose bypasses runtimeText()"
|
|
runtime = body[body.index("Dash.hero(q('runtime')"):]
|
|
assert "n.battery_runtime == null ? null" in runtime, \
|
|
"an unmeasured runtime is rendered as a measurement"
|
|
assert "severityFor(" not in body
|
|
|
|
|
|
def test_ups_secondary_electrical_row_is_demoted_not_deleted(client) -> None:
|
|
"""Line in, output, battery volts and amps, battery °C: still there, visibly second.
|
|
|
|
These are the numbers you reach for *while* the UPS is on battery, so they were demoted
|
|
rather than folded into the disclosure — and they stay one step down (smaller, muted) so
|
|
the three verdicts remain the largest elements, which is also what keeps the card from
|
|
growing so tall that two stop fitting side by side on a laptop.
|
|
"""
|
|
card = _ups_card_js(client)
|
|
assert 'class="metrics ups-electrical"' in card
|
|
for key in ("line", "out", "batt", "amps", "temp"):
|
|
assert f'data-k="{key}"' in card, f"the electrical row lost {key}"
|
|
assert card.index("ups-electrical") > card.index("ups-hero")
|
|
css = _code(_css(client))
|
|
rule = re.search(r"\.ups-electrical \.metric \.value\.sm \{([^}]*)\}", css)
|
|
assert rule, "dash.css no longer styles the secondary row"
|
|
assert "var(--text-muted)" in rule.group(1), "the secondary row is not muted"
|
|
sizes = re.findall(r"font-size: ([\d.]+)rem", rule.group(1))
|
|
assert sizes and float(sizes[0]) < 1.3, "the secondary row competes with the heroes"
|
|
# `n/a` is fainter than a muted reading — neither real driver reports a battery temperature
|
|
null_rule = re.search(r"\.ups-electrical \.metric \.value\.sm\.is-null \{([^}]*)\}", css)
|
|
assert null_rule and "var(--text-faint)" in null_rule.group(1), \
|
|
"an unmeasured secondary value is not faint, so it reads like a number"
|
|
|
|
|
|
def test_ups_glyph_state_comes_from_the_api_fields(client) -> None:
|
|
"""`glyphState()` maps `on_battery` + `charger_status` — and `full` is **not** charging.
|
|
|
|
The mapping is presentation, so it lives in the page (task 02's assumption), but it is
|
|
still a mapping the real drivers exercise: neither UPS here exports
|
|
`battery.charger.status`, so the fallback that reads a state the API rated `ok` is what
|
|
draws a solid shell on both of them. Nothing here re-reads `ups.status` —
|
|
`test_the_page_does_not_re_parse_nut_status_flags` holds that line.
|
|
"""
|
|
js = _code(client.get("/static/page-ups.js").text)
|
|
glyph = js[js.index("function glyphState("):js.index("function drawGlyph(")]
|
|
assert "item.on_battery" in glyph and "'onbattery'" in glyph
|
|
assert "charger === 'chg'" in glyph and "'charging'" in glyph
|
|
assert "charger === 'dchg'" in glyph, "a discharging charger token is on battery too"
|
|
assert "charger === 'full'" in glyph and "'online'" in glyph, \
|
|
"a full, online battery must not be drawn as charging"
|
|
assert "item.states" in glyph, "a driver with no rich status lost its state entirely"
|
|
assert "'unknown'" in glyph, "an unknown state now degrades to a confident 'on line'"
|
|
|
|
|
|
def test_ups_glyph_repaints_only_when_something_visible_changes(client) -> None:
|
|
"""Charge jitters by a tenth of a percent every poll; the `<svg>` does not rebuild on it.
|
|
|
|
Rebuilding would re-resolve the theme tokens and re-announce the accessible name every
|
|
`NUT_INTERVAL_SECONDS` for a difference no eye can see. `decorative: true` is the
|
|
component's documented opt-out — the charge hero and the state chips already say
|
|
"Charge 100 percent, on line", and a screen reader should not hear it twice.
|
|
"""
|
|
js = _code(client.get("/static/page-ups.js").text)
|
|
draw = js[js.index("function drawGlyph("):js.index("function upsCard(")]
|
|
assert "DashCharts.battery(" in draw
|
|
assert "cell.dataset.drawn === drawn" in draw, "the glyph rebuilds on every poll"
|
|
assert "decorative: true" in draw, "the glyph duplicates the hero number in a11y"
|
|
assert "glyphState(item)" in draw
|
|
|
|
|
|
def test_ups_hero_row_makes_room_for_the_glyph(client) -> None:
|
|
"""The glyph gets a real column, and the row stacks rather than clipping on a phone.
|
|
|
|
The track floor is the widest runtime prose the page prints (`9h 59m`) — a number that
|
|
cannot wrap needs the floor to be its own width, or the row grows a clipped `1h 28`.
|
|
"""
|
|
css = _code(_css(client))
|
|
row = re.search(r"\.ups-hero \{([^}]*)\}", css)
|
|
assert row, "dash.css lost the UPS hero row"
|
|
assert "minmax(112px, 156px) minmax(0, 1fr)" in row.group(1), "the glyph has no column"
|
|
assert "align-items: center" in row.group(1)
|
|
heroes = re.search(r"\.ups-hero \.heroes \{([^}]*)\}", css)
|
|
assert heroes and "minmax(min(140px, 100%), 1fr)" in heroes.group(1), \
|
|
"a hero track can now clip the widest runtime prose"
|
|
at = css.index("@media (max-width: 620px)")
|
|
block = css[at:at + 240]
|
|
assert ".ups-hero" in block and "minmax(0, 1fr)" in block, "the row never stacks"
|
|
|
|
|
|
def test_every_ups_chart_names_itself_for_assistive_tech(client) -> None:
|
|
"""The four trend plots on `/ups` carry an accessible name, like every other chart here.
|
|
|
|
`AGENTS.md` states the convention as *every* chart names itself: `LineChart` puts
|
|
`role="img"` + `aria-label` on the `<svg>` when the caller passes `ariaLabel`, because
|
|
the legend is a sibling that labels nothing and an unlabelled plot is silence. `/system`
|
|
(phase 03) and `/llm` (phase 04) were pinned; these four were the stragglers, so the
|
|
rule is now pinned on every page that draws one.
|
|
"""
|
|
js = client.get("/static/page-ups.js").text
|
|
charts = js[js.index("function makeCharts("):js.index("function points(")]
|
|
assert charts.count("ariaLabel:") == 4, "a `/ups` chart lost its accessible name"
|
|
for phrase in ("Battery charge", "Battery runtime", "Battery and line voltage", "UPS load"):
|
|
assert phrase in charts, f"the {phrase!r} chart names nothing readable"
|
|
|
|
|
|
# ---- the `/llm` layout (phase 04 task 03) ---------------------------------
|
|
#
|
|
# The drift this phase fixes was a 14-cell grid where everything was the same size and
|
|
# nothing showed recent shape. These tests pin the *shape* the page committed to on the
|
|
# served bytes (a locked anchor: no JS test runner), so a regression to "all cells equal"
|
|
# fails a test rather than a screenshot.
|
|
|
|
def test_llm_page_colours_from_the_payload_not_from_js_thresholds(client) -> None:
|
|
"""`page-llm.js` paints the KV verdict `severity.py` produced; it owns no threshold.
|
|
|
|
Both `severityFor(value, 75, 90)` call sites are gone (the task's explicit work item):
|
|
the KV row takes `severity` from the rated ratio cell, the fleet KV bar was deleted
|
|
with the summed card, and every unrated number — throughput above all, because 0 tok/s
|
|
on an idle server is healthy — draws no colour at all.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
assert "severityFor(" not in js, "the LLM page re-invented a threshold"
|
|
assert "severityFrom" in js
|
|
# the KV verdict comes from the payload cells `sources/llm.py` rates
|
|
kv = js[js.index("case 'cache':"):js.index("case 'ttft':")]
|
|
assert "kv_cache_usage_ratio" in kv and "gpu_cache_usage" in kv and "severity" in kv
|
|
|
|
|
|
def test_llm_engine_panel_leads_with_two_throughput_heroes(client) -> None:
|
|
"""Decode and prompt are the largest elements per engine; the rest is visibly secondary.
|
|
|
|
The hero row is `Dash.hero()` markup (2.4rem via `.hero`), built from the `hero: true`
|
|
flags in `CELLS`, and throughput carries **no** severity — `severity.py` has no
|
|
throughput row and the page invents none (the neutral-ink decision in NOTES-chart-set,
|
|
mirroring the `/system` NET precedent). The provenance tag stays in the hero's `.sub`
|
|
with the exact `source` as the tooltip: removing it to look cleaner is a regression.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
panel = js[js.index("function enginePanel("):js.index("function renderPanel(")]
|
|
assert 'data-hero="${cell.key}"' in panel, "the hero row is not flag-driven markup"
|
|
assert '<div class="heroes">' in panel, "the hero row left the engine panel"
|
|
heroes = re.findall(r"{ key: '(\w+)'[^}]*?hero: true", js)
|
|
assert heroes == ["decode", "prompt"], "the hero row is no longer the two throughputs"
|
|
# driven by flags, not fourteen identical cells. Phase 07 task 03 lifted the summary rows
|
|
# out of the panel into the stats strip above it, so what is left here is the pair plus
|
|
# the latency rows the strip does not print — never a fourteen-cell grid again.
|
|
assert "SPARK_CELLS" in js and "ALARM_CELLS" in js
|
|
assert js.count("spark: '") >= 2, "the panel lost the latency sparkline rows it kept"
|
|
assert js.count("hero: true") == 2, "the hero row is no longer exactly two numbers"
|
|
# the hero call must not pass a severity — unrated by design
|
|
hero_call = js[js.index("const node = hero("):js.index("if (node) node.title")]
|
|
assert "severity" not in hero_call, "throughput invented a verdict"
|
|
# provenance intact: the tag is generated from the cell's `source`, never hardcoded
|
|
assert "provenance" in js and "Δ" in js
|
|
assert "node.title = value.source" in js
|
|
|
|
|
|
def test_llm_provenance_tag_prints_the_window_the_payload_sent(client: TestClient) -> None:
|
|
"""`Δ2.0s` on `/llm` is the cell's own `window`, never a config echo (phase 23).
|
|
|
|
The window moved from 10 s to one poll interval without touching the page: `provenance()`
|
|
reads `cell.window`, which is the number `tests/test_llm_source.py` pins to the LLM span, and
|
|
both spell the label with the same one-decimal rounding (`delta_label` in
|
|
`tests/rate_window_contract.py` is that rounding, used by both payload pins). A page printing
|
|
the configured interval instead would advertise a window no poll measured (rule 2), and a
|
|
literal `Δ10.0s` in the script is the same lie with the number typed in.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
prov = js[js.index("function provenance("):js.index("function humanNumber(")]
|
|
assert "`Δ${cell.window.toFixed(1)}s`" in prov, \
|
|
f"the Δ tag is no longer built from the window the cell carries: {prov}"
|
|
assert delta_label(LLM_WINDOW) == "Δ2.0s", "the rounding the page and the pins share moved"
|
|
assert "rate_window_seconds" not in js, "the page echoes a config window beside a measurement"
|
|
assert "Δ10.0s" not in js, "a hard-coded Δ is an invented window (rule 2)"
|
|
|
|
|
|
def test_every_llm_cell_carries_a_window_sparkline_with_percentile_feet(client) -> None:
|
|
"""Each panel row draws its own sparkline over the *selected* window and prints mean
|
|
**and** p95 without a hover.
|
|
|
|
The window is honest time (`align: 'right'` + `span`) and the p95 comes from the
|
|
snapshot's histogram cells — p95 needs no stored series. The `only` scopes are what keep
|
|
a gufo panel from showing a latency mystery and a vLLM strip card from showing a
|
|
spec-decode row it cannot fill.
|
|
|
|
Phase 07 task 03 moved the percent and count rows to the stats strip, so the plotted
|
|
percent series left the cell list and the axis guarantee moved with it: the KV trend
|
|
chart pins `yMax: 100` (an 8 % window that once held a 92 % spike must not read flat),
|
|
and the strip converts its 0-1 ratios to per cent in `cellValue()` — a bar cannot
|
|
autoscale, so it cannot lie that way either. The `fixedMax`/`minSpan` options stay
|
|
plumbed through `drawSparks()` and are pinned on the component itself by
|
|
`test_sparkline_ships_the_fixed_scale_and_verdict_options`: a count sparkline that comes
|
|
back must not get an axis finer than one request.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
sparks = js[js.index("function drawSparks("):js.index("function renderFleet(")]
|
|
assert "DashCharts.sparkline(" in sparks
|
|
for opt in ("align: 'right'", "span: state.range", "fixedMax", "labels: { max: true }"):
|
|
assert opt in sparks, f"cell sparklines lost `{opt}`"
|
|
# the one percent series still plotted owns a fixed 0-100 axis; seconds stay ms
|
|
charts = js[js.index("function makeCharts("):js.index("function pointsFor(")]
|
|
assert "yMax: 100" in charts, "the KV trend chart no longer pins its 0-100 axis"
|
|
assert "scale: 1000" in js and "cell.scale || 1" in sparks
|
|
# every spark cell declares its history series in the markup it is built from
|
|
assert 'data-spark data-metric="${cell.spark}"' in js
|
|
# percentile foot lines: mean big in the row, p95 beneath, n/a when nothing measured
|
|
assert "function p95Value(" in js
|
|
for name in ("time_to_first_token_seconds__p95", "time_per_output_token_seconds__p95",
|
|
"request_queue_time_seconds__p95"):
|
|
assert name in js, f"{name} no longer feeds a foot line"
|
|
assert "p95 n/a" in js
|
|
assert 'data-foot' in js
|
|
# engine-scoped rows: gufo never grows a TTFT row, vLLM never a spec-accept row
|
|
assert "only: ['vllm']" in js and "only: ['gufo']" in js
|
|
scoped = js[js.index("function sparkCellsFor("):js.index("function enginePanel(")]
|
|
assert "cell.only" in scoped
|
|
assert "minSpan: cell.minSpan" in sparks, "cell minSpan never reaches the sparkline"
|
|
|
|
|
|
def test_llm_sparklines_and_charts_cost_one_batched_history_request(client) -> None:
|
|
"""One `/api/llm/history` call per refresh feeds charts *and* every sparkline row.
|
|
|
|
The endpoint answers a metric list for every engine, and unknown names come back as
|
|
empty series on purpose — so the page asks for the full cell set once instead of one
|
|
XHR per cell, and the two series phase 04 added (`tpot_seconds`, `spec_accept_rate`)
|
|
must be in that list or those rows are permanent "no history" slots.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
# comments aside, exactly one fetch hits the endpoint: one request, every metric
|
|
calls = re.findall(r"fetchJSON\(\s*`/api/llm/history", _code(js))
|
|
assert len(calls) == 1, "the page makes more than one history request again"
|
|
assert "HISTORY_SERIES.join" in js and "SPARK_CELLS.map((cell) => cell.spark)" in js
|
|
request = js[js.index("async function refreshHistory("):js.index("function drawSparks(")]
|
|
assert "metrics=${HISTORY_SERIES.join" in request
|
|
assert "range=${state.range}" in request
|
|
# and the endpoint really answers every name the page asks for
|
|
for name in ("tpot_seconds", "spec_accept_rate", "ttft_seconds", "queue_seconds",
|
|
"kv_cache_usage", "requests_running", "requests_waiting",
|
|
"prefix_cache_hit_ratio"):
|
|
assert name in HISTORY_METRICS, f"llm.{name} is not stored, so its spark is dead"
|
|
|
|
|
|
def test_llm_alarm_cells_exist_only_while_there_is_an_incident(client) -> None:
|
|
"""`preemptions` / `device_lost` are absent at zero and red chips above zero.
|
|
|
|
The rule is the *existence of an incident*, not a graded threshold — `sources/llm.py`
|
|
already treats any non-zero `gufo_device_lost_total` as one. The chip reuses the shared
|
|
`.chip.fail` vocabulary; nothing new is painted, and a container emptied at zero means
|
|
the row does not exist rather than showing a permanent ignored 0.
|
|
|
|
Phase 07 task 03 renders them on the engine's strip card (the summary you read first)
|
|
instead of inside the panel, so the two `alarm: true` rows live in `STRIP_ROWS` and
|
|
`ALARM_CELLS` filters them out of it — counted with the comments stripped, because the
|
|
docstring on that table explains the flag by name.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
alarms = js[js.index("function renderAlarms("):js.index("function makeCharts(")]
|
|
assert "class=\"chip fail\"" in alarms, "the alarm chip left the shared fail vocabulary"
|
|
assert "value.value > 0" in alarms
|
|
assert "host.hidden = chips.length === 0" in alarms
|
|
assert 'data-alarm="${cell.key}"' in alarms
|
|
assert _code(js).count("alarm: true") == 2
|
|
assert "ALARM_CELLS = STRIP_ROWS.filter" in js, "the alarm rows are not strip rows"
|
|
# the panel no longer carries a second container for the same incident
|
|
panel = js[js.index("function enginePanel("):js.index("function renderPanel(")]
|
|
assert 'data-k="alarms"' not in panel, "the alarm chips are drawn twice"
|
|
strip = js[js.index("function stripCard("):js.index("function renderStripCard(")]
|
|
assert 'data-k="alarms"' in strip, "the alarm chips left the strip card"
|
|
|
|
|
|
# ---- the `/llm` stats strip (phase 07 task 03) ---------------------------
|
|
#
|
|
# `llm_monitor`'s shared strip (`index.html` ~L743), ported: one card per configured engine
|
|
# between the fleet block and the panels. Two things make it a summary instead of a second
|
|
# set of panels, and both are pinned below: every row is a cell `/api/llm/snapshot` really
|
|
# publishes, and no row is printed in the strip *and* in the panel under it.
|
|
|
|
#: row key -> the cells `cellValue()` resolves it from (`sessions.*` is the gufo slot count,
|
|
#: which is not a metrics cell). The served `STRIP_ROWS` table must match these keys exactly.
|
|
STRIP_ROW_PATHS: dict[str, tuple[str, ...]] = {
|
|
"prompt": ("prompt_tokens_per_second",),
|
|
"decode": ("decode_tokens_per_second",),
|
|
"tokens_total": ("tokens_total",),
|
|
"slots_busy": ("sessions.list", "sessions.active"),
|
|
"running": ("requests_processing", "num_requests_running"),
|
|
"waiting": ("requests_deferred", "num_requests_waiting"),
|
|
"cache_hit": ("prefix_cache_hit_ratio", "prompt_cache_hit_ratio"),
|
|
"spec_accept": ("spec_accept_rate",),
|
|
"queue_mean": ("request_queue_time_seconds__mean",),
|
|
"preemptions": ("num_preemptions_total",),
|
|
"device_lost": ("device_lost_total",),
|
|
}
|
|
|
|
#: rows that are incidents, not stats: absent where the engine does not count them.
|
|
ALARM_ROW_KEYS = ("preemptions", "device_lost")
|
|
|
|
|
|
def _strip_rows(js: str) -> dict[str, str | None]:
|
|
"""The served `STRIP_ROWS` table: row key -> its `only` scope (`None` = every engine)."""
|
|
body = _code(js)
|
|
start = body.index("const STRIP_ROWS = [")
|
|
table = body[start:body.index("\n];", start)]
|
|
rows: dict[str, str | None] = {}
|
|
for chunk in table.split("{ key: '")[1:]:
|
|
scoped = re.search(r"only: \[([^\]]*)\]", chunk)
|
|
rows[chunk.split("'")[0]] = scoped.group(1) if scoped else None
|
|
return rows
|
|
|
|
|
|
def _strip_render_path(js: str) -> str:
|
|
"""Just `renderStripCard()` — the whole code path that paints one strip card."""
|
|
js = _code(js)
|
|
return js[js.index("function renderStripCard("):js.index("function pruneEngines(")]
|
|
|
|
|
|
def test_llm_stats_strip_sits_between_the_fleet_block_and_the_panels(client) -> None:
|
|
"""The strip is there, ships hidden, names itself, and appears only with engines.
|
|
|
|
An unconfigured engine gets no card and no strip (invariant 3) — the section is empty in
|
|
the markup and `page-llm.js` unhides it only when the payload has an engine to draw.
|
|
"""
|
|
html = _code(client.get("/llm").text)
|
|
assert 'id="strip"' in html, "/llm lost the per-engine stats strip"
|
|
assert html.index('id="fleet"') < html.index('id="strip"') < html.index('id="engines"'), \
|
|
"the strip is no longer between the fleet block and the engine panels"
|
|
start = html.index('<section class="strip"')
|
|
tag = html[start:html.index(">", start)]
|
|
assert "hidden" in tag, "the strip no longer ships hidden"
|
|
assert 'aria-label="Engine summary"' in tag, "the strip is a region that names nothing"
|
|
js = client.get("/static/page-llm.js").text
|
|
assert "$('#strip').hidden = payloads.length === 0" in js
|
|
|
|
|
|
def test_llm_strip_card_is_one_card_per_engine_of_shared_vocabulary(client) -> None:
|
|
"""`stripCard()` = `.card` + `.kv` + `.bar` + `.chip.fail`: a layout, not a new component.
|
|
|
|
Dot, name and age in the head (reachability, the same distinction the panels and the
|
|
fleet block draw), the KV bar, the rows keyed by payload field, the alarm container.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
card = js[js.index("function stripCard("):js.index("function renderStripCard(")]
|
|
assert "card.className = 'card stat'" in card
|
|
assert "card.dataset.engine = engine" in card, "the card stopped naming its engine"
|
|
assert '<span class="dot idle"></span>' in card
|
|
assert '<span class="faint mono updated"></span>' in card, "the card lost its timestamp"
|
|
assert '<span class="bar" data-kv-bar><i></i></span>' in card, "the KV bar left `.bar`"
|
|
assert '<dl class="kv">' in card, "the strip rows left the shared `.kv` vocabulary"
|
|
assert 'data-row="${row.key}"' in card
|
|
assert 'data-k="alarms"' in card
|
|
# the wrapper is why the shared `.kv` grid has to move *inside* it: applied to the
|
|
# wrappers, `max-content 1fr` lays two rows side by side and the card reads as a
|
|
# column of orphaned labels (found in the browser, not in a test failure)
|
|
css = client.get("/static/dash.css").text
|
|
section = css[css.index("/* ---- engine stats strip"):css.index("/* ---- fleet totals")]
|
|
assert ".strip .kv { margin: 0; grid-template-columns: 1fr; }" in section
|
|
assert ".strip .kv > div {" in section
|
|
assert "grid-template-columns: max-content minmax(0, 1fr)" in section
|
|
refresh = js[js.index("async function refresh("):js.index("async function refreshEvents(")]
|
|
assert "state.strips.get(payload.engine)" in refresh, "the strip rebuilds its cards per poll"
|
|
assert "$('#strip').appendChild(strip)" in refresh
|
|
assert "renderStripCard(strip, payload, stateRow)" in refresh
|
|
prune = js[js.index("function pruneEngines("):js.index("function makeCharts(")]
|
|
assert "state.strips" in prune and "state.panels" in prune, \
|
|
"an engine that leaves the payload leaves a card behind"
|
|
|
|
|
|
def test_llm_strip_scopes_its_rows_to_the_engines_that_publish_them(client) -> None:
|
|
"""gufo gets slots and speculative decoding; vLLM gets running and queue time."""
|
|
rows = _strip_rows(client.get("/static/page-llm.js").text)
|
|
assert rows["slots_busy"] == "'gufo'" and rows["spec_accept"] == "'gufo'"
|
|
assert rows["running"] == "'vllm'" and rows["queue_mean"] == "'vllm'"
|
|
assert rows["waiting"] is None and rows["cache_hit"] is None and rows["tokens_total"] is None
|
|
js = client.get("/static/page-llm.js").text
|
|
scoped = js[js.index("function stripRowsFor("):js.index("function stripCard(")]
|
|
assert "row.only" in scoped and "row.only.indexOf(engine)" in scoped
|
|
|
|
|
|
def test_llm_strip_rows_name_cells_the_engines_publish(client, metrics_text) -> None:
|
|
"""The strip renders what the payload sends — no row whose cell no engine produces.
|
|
|
|
`STRIP_ROW_PATHS` mirrors the JS table; every metric cell is checked against the
|
|
capture-parsed metrics of each engine that would render that row, so a rename in
|
|
`sources/llm.py` fails here instead of printing a permanent `n/a` in the browser, and a
|
|
row scoped to one engine cannot leak into the other's card.
|
|
"""
|
|
rows = _strip_rows(client.get("/static/page-llm.js").text)
|
|
assert set(rows) == set(STRIP_ROW_PATHS), \
|
|
f"the served strip rows drifted: {set(rows) ^ set(STRIP_ROW_PATHS)}"
|
|
parsed = {"gufo": parse_gufo_metrics(metrics_text("gufo")),
|
|
"vllm": parse_vllm_metrics(metrics_text("vllm"))}
|
|
for key, only in rows.items():
|
|
paths = STRIP_ROW_PATHS[key]
|
|
if paths[0].startswith("sessions."):
|
|
continue # its own payload object; pinned on the JS below
|
|
shown = {engine: metrics for engine, metrics in parsed.items()
|
|
if not only or engine in only}
|
|
present = [engine for engine in shown if any(path in shown[engine] for path in paths)]
|
|
if key in ALARM_ROW_KEYS:
|
|
# an alarm the engine does not count simply is not there: the chip renders only
|
|
# while non-zero, so absence *is* the zero state rather than a dead row
|
|
assert present, f"no engine publishes {'/'.join(paths)} for the {key} alarm"
|
|
continue
|
|
assert len(present) == len(shown), (
|
|
f"the {sorted(set(shown) - set(present))} card would print `{key}` as n/a forever "
|
|
f"({'/'.join(paths)} is not published)")
|
|
|
|
|
|
def test_llm_strip_slot_count_counts_published_slots_and_never_invents_zero(client) -> None:
|
|
"""`slots busy` is a count of `/slots` rows; when /slots did not answer it is `n/a`.
|
|
|
|
Counting rows the server published is a count of published facts. An empty list from a
|
|
server that never answered is not a measurement of an idle engine, so the row falls back
|
|
to the published `sessions.active` carrying the API's own `detail` string (rule 2).
|
|
"""
|
|
js = _code(client.get("/static/page-llm.js").text)
|
|
slots = js[js.index("case 'slots_busy':"):js.index("case 'cache':")]
|
|
assert "sessions.source" in slots and "sessions.list" in slots
|
|
assert "is_processing" in slots, "the count ignores the authoritative busy verdict"
|
|
assert "sessions.detail" in slots, "the fallback lost the API's own provenance"
|
|
|
|
|
|
def test_llm_strip_kv_bar_is_coloured_by_the_payload_verdict_only(client) -> None:
|
|
"""The strip's KV bar is the page's one rated cell: `severity` in, class out.
|
|
|
|
`sources/llm.py` rates `llm.kv_cache_usage_pct`, the ratio→percent conversion the panel
|
|
cell used to make rides with it, and `Dash.severityFrom` is the only thing that decides a
|
|
colour — no threshold in the page, and the stylesheet adds no palette (`ok` stays quiet
|
|
ink, exactly like the `.cell` rows and the fleet table).
|
|
"""
|
|
body = _strip_render_path(client.get("/static/page-llm.js").text)
|
|
assert "severityFrom(kv.severity)" in body
|
|
assert "cellValue(engine, metrics, sessions, 'cache')" in body
|
|
assert "Math.max(0, Math.min(100, kv.value))" in body, "the fill stopped clamping"
|
|
assert body.count("severityFrom") == 1, "a second strip row invented a verdict"
|
|
css = client.get("/static/dash.css").text
|
|
section = css[css.index("/* ---- engine stats strip"):css.index("/* ---- fleet totals")]
|
|
assert ".stat-bar.warn .value { color: var(--warn); }" in section
|
|
assert ".stat-bar.fail .value { color: var(--fail); }" in section
|
|
assert ".stat-bar.ok" not in section, "a healthy KV reading must stay quiet ink"
|
|
assert ".prov:empty" in section, "an empty provenance tag draws a pill of nothing"
|
|
|
|
|
|
def test_llm_strip_throughput_carries_provenance_without_colour(client) -> None:
|
|
"""`prompt tok/s` and `decode tok/s` keep the `Δ<window>` tag and stay ink.
|
|
|
|
The tag is generated from the cell's `source` (never hardcoded), the exact source rides
|
|
in the row's tooltip, and `severity.py` rates no throughput — so nothing here may paint
|
|
it (0 tok/s on an idle engine is healthy).
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
body = _strip_render_path(js)
|
|
assert "row.rate ? provenance(value) : ''" in body
|
|
assert "notes.join" in body and "value.source" in body
|
|
assert "severityFor" not in js
|
|
|
|
|
|
def test_llm_strip_and_panel_print_no_value_twice(client) -> None:
|
|
"""One value, one home — the UPS hero row's rule, applied across two components.
|
|
|
|
Task 03 lifted sessions, queue depth, KV pressure, the cache-hit and spec-accept percents,
|
|
the lifetime token counter and the alarm chips out of the panel cell list rather than
|
|
drawing them twice. The two throughput heroes are the exemption the task names: the panel
|
|
keeps its hero row (so does `llm_monitor`, which carries the pair in both places), and
|
|
everything else appears once. Queue time moved with its p95 foot, so the move lost nothing.
|
|
"""
|
|
js = _code(client.get("/static/page-llm.js").text)
|
|
start = js.index("const CELLS = [")
|
|
panel_keys = set(re.findall(r"\{ key: '(\w+)'", js[start:js.index("\n];", start)]))
|
|
assert panel_keys == {"decode", "prompt", "ttft", "tpot"}, \
|
|
f"the panel cell list changed shape: {sorted(panel_keys)}"
|
|
shared = set(_strip_rows(js)) & panel_keys
|
|
assert shared == {"decode", "prompt"}, f"printed in the strip and the panel: {sorted(shared)}"
|
|
body = _strip_render_path(js)
|
|
assert "p95Value(metrics, row.key)" in body, "queue time lost its p95 in the move"
|
|
|
|
|
|
def test_llm_strip_prints_no_peak_context_number(client) -> None:
|
|
"""The phase's assumption, pinned: `llm_monitor`'s peak-ctx row does not come over.
|
|
|
|
Its cell is `llamacpp:n_tokens_max`, a since-start maximum rather than a glance number,
|
|
and the task says this strip gets no peak-context cell. The series stays stored and
|
|
queryable from `/api/llm/history` — not plotting it is not removing it.
|
|
"""
|
|
js = _code(client.get("/static/page-llm.js").text)
|
|
assert "n_tokens_max" not in js, "/llm reads the peak-context gauge again"
|
|
assert not [key for key in _strip_rows(js) if "peak" in key]
|
|
|
|
|
|
def test_llm_fleet_block_leads_the_page_with_the_engine_sums(client) -> None:
|
|
"""The page opens with llm_monitor's aggregate block: the sums no panel can say.
|
|
|
|
Phase 04 retired these numbers to counts; the operator asked for the aggregate
|
|
throughput back (`TODO.md:5`), and phase 07 restores it in `llm_monitor`'s shape —
|
|
status dot, "all engines", an `N of M engines` tag, the note that the sums cover only
|
|
the engines answering now, then running / waiting / prompt / decode / total tokens.
|
|
The counts-only card it replaces (`#totals`, `f-live`, `f-active`) is gone, ids and all:
|
|
an element nothing writes is the orphan this repo pins against.
|
|
"""
|
|
html = _code(client.get("/llm").text)
|
|
assert 'id="fleet"' in html, "/llm lost the aggregate block"
|
|
assert html.index('id="fleet"') < html.index('id="engines"'), \
|
|
"the fleet block is no longer above the engine panels"
|
|
for part in ("fleet-dot", "fleet-count", "f-running", "f-waiting", "f-prompt",
|
|
"f-decode", "f-tokens"):
|
|
assert f'id="{part}"' in html, f"/llm lost the fleet block's {part}"
|
|
for words in ("all engines", "sums only engines answering now"):
|
|
assert words in html, f"/llm lost the fleet block's `{words}` line"
|
|
for gone in ('id="totals"', "f-live", "f-total", "f-active", "f-kv", "seam-note"):
|
|
assert gone not in html, f"/llm still carries {gone} from the retired counts card"
|
|
|
|
js = client.get("/static/page-llm.js").text
|
|
body = _fleet_render_path(js)
|
|
assert "block.hidden = total < 2" in body, "the fleet block no longer hides for one engine"
|
|
assert "hidden" in _fleet_tag(html), \
|
|
"the block no longer ships hidden, so a single engine would show a fleet row"
|
|
|
|
|
|
def test_llm_fleet_block_reads_only_paths_the_api_publishes(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Every number the block prints is a `fleet` key of `/api/llm/snapshot`.
|
|
|
|
The page sums nothing itself — `fleet_totals()` does — so the field table in
|
|
`page-llm.js` is a list of API paths, and a rename in `sources/llm.py` must fail here
|
|
rather than print a permanent `n/a` in the browser. Pinned with both engines configured
|
|
(the keys exist, empty-handed because nothing answered) and with neither (404
|
|
`configured: false` without dialling, invariant 3 — and the shipped block stays hidden).
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, gufo_url=DEAD_URL,
|
|
vllm_url=DEAD_URL) as test_client:
|
|
fleet = test_client.get("/api/llm/snapshot").json()["fleet"]
|
|
assert set(fleet) >= FLEET_BLOCK_PATHS, FLEET_BLOCK_PATHS - set(fleet)
|
|
# the served field table is exactly the sum rows, in llm_monitor's order
|
|
js = test_client.get("/static/page-llm.js").text
|
|
table = _fleet_field_metrics(js)
|
|
assert table == ["requests_active", "requests_waiting",
|
|
"prompt_tokens_per_second", "decode_tokens_per_second",
|
|
"tokens_total"], "the fleet row changed shape without the API"
|
|
assert set(FLEET_SUM_PATHS) == set(table), "the block totals something the API does not"
|
|
# and the tag's two numbers are API paths too, not arithmetic the page invented
|
|
body = _fleet_render_path(js)
|
|
assert "fleet.engines_total" in body and "fleet.engines_live" in body
|
|
|
|
with build_client(tmp_path, monkeypatch) as bare:
|
|
response = bare.get("/api/llm/snapshot")
|
|
assert response.status_code == 404
|
|
assert response.json()["configured"] is False
|
|
html = _code(bare.get("/llm").text)
|
|
assert 'id="fleet"' in html and "hidden" in _fleet_tag(html), \
|
|
"an unconfigured engine must not leave a fleet row on the page"
|
|
|
|
|
|
def test_llm_fleet_totals_drop_an_engine_that_stopped_answering(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A total presented as current is current: a dead engine contributes nothing.
|
|
|
|
`fleet_totals()`'s docstring is the contract and the block is where it is read. The
|
|
injected payload survives the dead-port polls (a failed poll keeps the last good one), so
|
|
these are the numbers a browser would get with gufo answering and vLLM down — the live
|
|
engine's sums verbatim, `engines_live` one of two, and not one zero where a number the
|
|
dead engine never published would sit (rule 2).
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, gufo_url=DEAD_URL,
|
|
vllm_url=DEAD_URL) as test_client:
|
|
hub = test_client.app.state.ctx.hub # type: ignore[attr-defined]
|
|
hub.jobs["llm:gufo"].state.payload = {
|
|
"engine": "gufo",
|
|
"metrics": {"decode_tokens_per_second": {"value": 41.0},
|
|
"prompt_tokens_per_second": {"value": 17.5},
|
|
"requests_processing": {"value": 2.0},
|
|
"requests_deferred": {"value": 1.0},
|
|
"tokens_total": {"value": 512.0}},
|
|
"sessions": {"active": 2, "list": []}}
|
|
fleet = test_client.get("/api/llm/snapshot").json()["fleet"]
|
|
assert fleet["engines_total"] == 2 and fleet["engines_live"] == 1
|
|
assert fleet["decode_tokens_per_second"] == pytest.approx(41.0)
|
|
assert fleet["prompt_tokens_per_second"] == pytest.approx(17.5)
|
|
assert fleet["requests_active"] == pytest.approx(2.0)
|
|
assert fleet["requests_waiting"] == pytest.approx(1.0)
|
|
assert fleet["tokens_total"] == pytest.approx(512.0)
|
|
|
|
# and with neither engine answering, every sum is null, never a confident 0
|
|
hub.jobs["llm:gufo"].state.payload = {"engine": "gufo", "metrics": {},
|
|
"sessions": {"active": None, "list": []}}
|
|
empty = test_client.get("/api/llm/snapshot").json()["fleet"]
|
|
assert empty["engines_live"] == 0
|
|
assert [empty[key] for key in FLEET_SUM_PATHS] == [None] * len(FLEET_SUM_PATHS)
|
|
|
|
|
|
def test_llm_fleet_block_proves_where_its_sums_come_from(client) -> None:
|
|
"""The seam tooltip, the excluded-engine tooltip, and an `n/a` with no unit.
|
|
|
|
Two provenance rules the rest of the app holds to, on the one block that adds engines
|
|
together: the prompt sum carries `fleet.seams` — the API's own words on why gufo's and
|
|
vLLM's prompt counters are not the same quantity — and the tag's tooltip names the
|
|
engines `fleet_totals()` left out, so a partial total says so in words and not only by
|
|
being dimmed. And a sum the API did not publish prints `n/a` with its unit cleared: "n/a
|
|
tok/s" would read as a measurement of nothing.
|
|
"""
|
|
js = _code(client.get("/static/page-llm.js").text)
|
|
body = _fleet_render_path(js)
|
|
assert "fleet.seams" in body, "the prompt sum lost the API's seam note"
|
|
assert "not answering — excluded from every sum below" in body, \
|
|
"the tag no longer names the engines left out of the totals"
|
|
assert "value == null ? ''" in body, "an n/a sum can still print a unit"
|
|
|
|
assert "value == null ? ''" in body, "an n/a sum can still print a unit"
|
|
|
|
|
|
def test_llm_fleet_block_colours_no_number(client) -> None:
|
|
"""No verdict on the aggregate row — and no dimming of anything but the tag.
|
|
|
|
`severity.py` rates no throughput and no queue depth (0 tok/s on an idle fleet is
|
|
healthy), so the block renders ink: the render path touches no `severity`, calls no
|
|
`Dash.hero()`, and the stylesheet section for the block reaches for no verdict token.
|
|
The one coloured thing on it is the shared **reachability** dot — the same distinction
|
|
the engine panels draw between their numbers and their header dot — and the partial-fleet
|
|
signal is the shared `.stale` dim on the count tag, which is opacity, not a verdict.
|
|
"""
|
|
js = _code(client.get("/static/page-llm.js").text)
|
|
body = _fleet_render_path(js)
|
|
assert "severity" not in body, "the fleet block invented a verdict"
|
|
assert "hero(" not in body, "the fleet block drew a hero number it cannot rate"
|
|
assert "setDot(" in body and "markStale(tag" in body, \
|
|
"a dead engine no longer dims the tag or darkens the block's dot"
|
|
|
|
css = client.get("/static/dash.css").text
|
|
section = _css_section(css, "fleet totals")
|
|
for token in ("var(--ok)", "var(--warn)", "var(--fail)"):
|
|
assert token not in section, f"the fleet block coloured a number with {token}"
|
|
assert ".fleet-head" in section and ".fleet-note" in section, \
|
|
"the fleet block's head row lost its styling"
|
|
|
|
|
|
#: Elements the browser treats as empty, so they never open a nesting level.
|
|
_VOID_TAGS = {"meta", "link", "br", "img", "input", "hr", "source", "track", "wbr"}
|
|
|
|
|
|
def _depth_at(html: str, needle: str) -> int:
|
|
"""How deeply the `needle` sits inside `<main>` (1 = a direct child of it).
|
|
|
|
The `~` in `#banners:empty ~ #fleet` only resolves between **siblings**, so the gap rule
|
|
depends on the markup as much as on the stylesheet. Counting tags is enough here and keeps
|
|
the check on the served bytes: no HTML parser, no dependency, and comments are already
|
|
gone by the time `_code` hands the page over.
|
|
"""
|
|
body = html[html.index("<main>"):html.index(needle, html.index("<main>"))]
|
|
depth = 0
|
|
for closing, tag, _attrs, self_closed in re.findall(
|
|
r"<(/?)([a-zA-Z][\w-]*)([^>]*?)(/?)>", body):
|
|
if self_closed or tag.lower() in _VOID_TAGS:
|
|
continue
|
|
depth += -1 if closing else 1
|
|
return depth
|
|
|
|
|
|
def test_llm_section_stack_keeps_the_shared_vertical_rhythm(client) -> None:
|
|
"""Fleet block → engine summaries → engine panels sit the shared `.85rem` apart.
|
|
|
|
`TODO.md:4`: the fleet section had no spacing between it and the engine summaries. Every
|
|
boundary below it was already handled inline — the section under `#strip` carries
|
|
`margin-top:.85rem` — so `#fleet` and `#strip` skipped the rhythm and the top of the page
|
|
read as one unbroken wall. Measured in Chromium at the 16px root (`data/logs/`
|
|
`llm-fleet-spacing*.png`), before → after:
|
|
|
|
| gap | before | after |
|
|
|-----|--------|-------|
|
|
| `#fleet` → `#strip` | **0 (flush)** | 13.59px |
|
|
| `#strip` → engine panels | 13.59px | 13.59px |
|
|
| banner slot → `#fleet`, no banner | 13.59px | 13.59px |
|
|
| banner slot → `#fleet`, one banner up | 13.59px | 13.59px (`margin-top: 0px`) |
|
|
|
|
One rendered distance moves, and it is the one the operator named. What the *shape* of the
|
|
two rules buys is that no other state can put it back:
|
|
|
|
* `#strip`'s gap is **unconditional**, because below two engines `#fleet` is `[hidden]`
|
|
(`display:none !important`, so it contributes nothing) and the strip is then the first
|
|
visible section: the 13.59px below the banner slot in that state is now the strip's own
|
|
margin rather than one borrowed from a slot above it, which is what `#fleet ~ #strip`
|
|
would have been;
|
|
* `#fleet`'s gap is guarded by `#banners:empty`, so the block owns its top gap exactly when
|
|
nothing occupies the slot above it, and a banner's own `margin-bottom:.85rem` stays the
|
|
single gap under it. An unguarded `#fleet { margin-top }` is a second rule for one gap:
|
|
margin collapsing hides it at `.85rem`/`.85rem` today, and it doubles the moment either
|
|
margin is changed. The empty slot keeps its inline bottom margin, so head → fleet stays
|
|
the 1rem + .85rem the other two banner pages (`/`, `/ups`) already render.
|
|
|
|
Pinned on the served bytes because a stylesheet "cleanup" that drops either rule is
|
|
invisible in a diff and only shows up as a page that reads worse.
|
|
"""
|
|
raw = _css(client)
|
|
section = _css_section(raw, "fleet totals")
|
|
assert "#strip { margin-top: .85rem; }" in section, \
|
|
"the fleet block and the engine summaries render flush again (TODO.md:4)"
|
|
assert "#banners:empty ~ #fleet { margin-top: .85rem; }" in section, (
|
|
"the gap above the fleet block stopped belonging to the block: a banner-less page "
|
|
"would then have it only as long as the empty banner slot keeps its inline margin")
|
|
|
|
# one rule per element: a second one (or an unguarded `#fleet`) is the double gap
|
|
css = _code(raw)
|
|
for element, expected in (("#strip", {"#strip"}),
|
|
("#fleet", {"#banners:empty ~ #fleet"})):
|
|
selectors = {s.strip() for s in re.findall(rf"([^{{}}]*{element}[^{{}}]*)\{{", css)}
|
|
assert selectors == expected, (
|
|
f"dash.css styles {element} with {sorted(selectors)} — exactly one rule, the "
|
|
"guarded one, or the page grows a double gap or loses the single one")
|
|
|
|
html = _code(client.get("/llm").text)
|
|
for element in ("banners", "fleet", "strip"):
|
|
assert _depth_at(html, f'id="{element}"') == 1, (
|
|
f"`#{element}` is no longer a direct child of <main>, so the `~` gap rule cannot "
|
|
"match: rewrite the selectors for the new structure instead of dropping the gap")
|
|
assert html.index('id="banners"') < html.index('id="fleet"') < html.index('id="strip"') \
|
|
< html.index('id="engines"'), "the section stack was re-ordered"
|
|
|
|
# the inline half of the rhythm, which the two rules are written around
|
|
assert 'style="margin-bottom:.85rem"' in html[:html.index('id="fleet"')], (
|
|
"#banners lost the inline bottom margin that is the banner → fleet gap, and today the "
|
|
"only thing spacing the page head from the block")
|
|
after_strip = html[html.index('id="strip"'):]
|
|
assert after_strip.count('style="margin-top:.85rem"') == 3, \
|
|
"a section below the strip lost the inline gap the strip's rule hands off to"
|
|
|
|
|
|
# ---- the `/llm` gufo slot grid (phase 07 task 04) -------------------------
|
|
#
|
|
# `llm_monitor`'s slot cards (`index.html` ~L1179), rebuilt against the capture instead of a
|
|
# memory: one keyed card per gufo slot, every cell naming **one** key of
|
|
# `tests/fixtures/slots_gufo.json`. Two things are pinned below, because they are the two ways
|
|
# this component failed before: a card that reads a key the server does not publish (dashes
|
|
# forever), and a card that is rebuilt every poll so a busy slot flickers in front of you.
|
|
|
|
#: The keys a slot card reads, and where they live: `field` (and its optional `sub` /
|
|
#: `ratioAgainst` partners) index the row, or its first `next_token` entry when `path` says so.
|
|
SLOT_ROW_PATHS = ("field", "sub", "ratioAgainst")
|
|
|
|
|
|
def _slot_table(js: str) -> list[dict[str, str | None]]:
|
|
"""The served `SLOT_FIELDS` table, one dict per printed cell."""
|
|
body = _code(js)
|
|
start = body.index("const SLOT_FIELDS = [")
|
|
table = body[start:body.index("\n];", start)]
|
|
cells: list[dict[str, str | None]] = []
|
|
for chunk in table.split("{ key: '")[1:]:
|
|
cell: dict[str, str | None] = {"key": chunk.split("'")[0]}
|
|
for option in (*SLOT_ROW_PATHS, "path"):
|
|
found = re.search(rf"{option}: '([a-z_0-9]+)'", chunk)
|
|
cell[option] = found.group(1) if found else None
|
|
cells.append(cell)
|
|
return cells
|
|
|
|
|
|
def _slot_js(js: str) -> str:
|
|
"""The slot grid's whole code path, comments out: `slotValue()` to the end of the block."""
|
|
body = _code(js)
|
|
return body[body.index("function slotValue("):body.index("function makeCharts(")]
|
|
|
|
|
|
def _slot_markup(js: str) -> str:
|
|
"""Just `slotSection()` — the markup the grid, its label and its empty states are built from."""
|
|
body = _code(js)
|
|
return body[body.index("function slotSection("):body.index("function renderPanel(")]
|
|
|
|
|
|
def test_llm_gufo_panel_holds_the_slot_grid_under_its_label(client) -> None:
|
|
"""`sessions (GET /slots)`, then the keyed grid, then the note about what it cannot contain.
|
|
|
|
The label is `llm_monitor`'s, the grid ships `hidden` (the empty state is a sentence, never
|
|
a card of nothing), and the note stays because a queued (deferred) request is not in
|
|
`/slots` at all — without that line an idle grid reads as "nothing waiting".
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
markup = _slot_markup(js)
|
|
assert "sessions (GET /slots)" in markup, "/llm lost the slot grid's label"
|
|
# label, grid, empty state, note — in that order, and the grid and the empty state both
|
|
# ship hidden so nothing paints before the payload says which of them is true
|
|
order = ['class="section-label" data-slots-label', "sessions (GET /slots)",
|
|
'<div class="slots" data-slots role="list" hidden>',
|
|
'class="slots-empty" data-slots-empty hidden',
|
|
'class="slots-note" data-slots-note hidden']
|
|
position = -1
|
|
for needle in order:
|
|
found = markup.index(needle)
|
|
assert found > position, f"`{needle}` is out of order in the slot section"
|
|
position = found
|
|
assert "queued (deferred) requests are not in /slots" in js, "the queued-request note is gone"
|
|
assert "renderSlotGrid(card, enginePayload)" in js, "nothing draws the grid any more"
|
|
# the label is a heading and the grid a list: a screen reader gets the block's name and
|
|
# its item count, which a stack of divs does not say
|
|
assert '<h4 class="section-label" data-slots-label>' in markup, \
|
|
"the slot section's label stopped being a heading"
|
|
assert 'data-slots role="list"' in markup, "the grid lost its list semantics"
|
|
# the grid's own markup: a keyed card, the state pill, and one dd per printed cell
|
|
card = js[js.index("function slotCard("):js.index("function slotSub(")]
|
|
assert "setAttribute('role', 'listitem')" in card, "the slot cards left the list"
|
|
assert 'data-slot-cell="${cell.key}"' in card, "the card cells are not table-driven"
|
|
assert 'data-pill' in card and 'data-mark' in card, "the card lost its state pill or glyph"
|
|
|
|
|
|
def test_llm_slot_grid_style_is_a_marker_and_not_a_verdict(client) -> None:
|
|
"""A busy slot is a border, a glyph and a word — `severity.py` rates no slot.
|
|
|
|
Phase 06 settled this argument for GPU utilisation on an LLM server: a slot generating
|
|
tokens is the job, so the accent colour says *this is the live one* and no `--ok/--warn/
|
|
--fail` token appears anywhere in the section. The glyph (`○`/`●`) and the pill's word mean
|
|
the marker survives monochrome, which is the battery glyph's rule applied here.
|
|
"""
|
|
css = client.get("/static/dash.css").text
|
|
section = _css_section(css, "engine slot grid")
|
|
for selector in (".slots {", ".slot {", ".slot.busy", ".slot-top", ".slot-id",
|
|
".slot-model", ".slot dt", ".slot dd", ".slots-empty", ".slots-note",
|
|
".section-label"):
|
|
assert selector in section, f"dash.css lost the {selector} rule"
|
|
assert "var(--accent)" in section, "the busy marker stopped using the non-verdict accent"
|
|
for token in ("var(--ok)", "var(--warn)", "var(--fail)"):
|
|
assert token not in section, f"the slot grid coloured a state with {token}"
|
|
js = client.get("/static/page-llm.js").text
|
|
assert "'\\u25cf'" in js or "'●'" in js, "the busy glyph left the card"
|
|
assert "processing" in _slot_js(js), "the pill stopped saying the state in a word"
|
|
|
|
|
|
def test_llm_slot_cells_each_name_one_key_the_capture_publishes(client) -> None:
|
|
"""Every printed cell traces to a key of `tests/fixtures/slots_gufo.json`.
|
|
|
|
One `field` per cell and no `a ?? b`: the deleted table's `slot.n_prompt ??
|
|
slot.prompt_tokens` guessing is what made it print dashes and then plausible noise. A key
|
|
the capture does not have fails here, and the alias the capture answers beside `id_task`
|
|
must not appear in the page at all — the contract names the canonical one.
|
|
"""
|
|
# comments out: this test bans names the prose is allowed to explain
|
|
js = _code(client.get("/static/page-llm.js").text)
|
|
cells = _slot_table(js)
|
|
assert cells, "the slot grid has no cell table to read"
|
|
assert [cell["key"] for cell in cells] == ["req", "prompt", "cache", "generated",
|
|
"remain", "ctx"], \
|
|
"the slot card's cell set changed without the capture"
|
|
for cell in cells:
|
|
assert cell["field"], f"the {cell['key']} cell names no row key"
|
|
|
|
rows = load_slots_capture()
|
|
assert rows, "the capture is empty, so this test would prove nothing"
|
|
for cell in cells:
|
|
for option in SLOT_ROW_PATHS:
|
|
key = cell[option]
|
|
if key is None:
|
|
continue
|
|
in_next = cell["path"] == "next_token"
|
|
required = REQUIRED_NEXT_TOKEN_FIELDS if in_next else REQUIRED_ROW_FIELDS
|
|
optional = OPTIONAL_NEXT_TOKEN_FIELDS if in_next else OPTIONAL_ROW_FIELDS
|
|
assert key in required or key in optional, (
|
|
f"the {cell['key']} cell reads {key!r}, which no contract table declares — the "
|
|
"page would be reading a field nobody pinned")
|
|
for row in rows:
|
|
source = row["next_token"][0] if in_next else row
|
|
if key in required:
|
|
assert key in source, (
|
|
f"the {cell['key']} cell reads {key!r}, which "
|
|
f"{'next_token[0]' if in_next else 'the row'} does not publish — this is "
|
|
"how the deleted table rendered dashes")
|
|
elif key in source: # optional: `prob` is on no captured row, and that is legal
|
|
assert isinstance(source[key], (int, float)), (
|
|
f"{key!r} is {source[key]!r}, which the cell cannot format")
|
|
|
|
assert cells[0]["field"] == CANONICAL_TASK_ID_FIELD, \
|
|
"the request cell stopped reading the canonical task id"
|
|
for alias in TASK_ID_ALIASES:
|
|
assert alias not in js, f"page-llm.js renders the {alias} alias the contract does not name"
|
|
# the old table's invented names stay invented, and so does its per-slot speed
|
|
for gone in ("n_predict", "decode_per_second", "cache_usable", "n_ctx_used"):
|
|
assert gone not in js, f"page-llm.js reads the invented {gone} again"
|
|
|
|
|
|
def test_llm_slot_card_prints_no_speed_it_cannot_measure(client) -> None:
|
|
"""No per-slot speed cell: the capture publishes no per-slot rate key.
|
|
|
|
`llm_monitor`'s grid has one, and so did the deleted table (`decode_per_second`) — which is
|
|
why it was dashes. A page-side Δn_decoded/Δt would be a measurement with no estimator, no
|
|
`Δ<window>` label and no `source` string, i.e. rule 2 with a confident number printed on it,
|
|
so the number stays on the engine's strip card and panel, where `rates.py` owns it.
|
|
"""
|
|
js = _code(client.get("/static/page-llm.js").text)
|
|
assert not [cell for cell in _slot_table(js)
|
|
if cell["key"] in {"speed", "tok_s", "decode"}], "a slot cell grew a rate"
|
|
assert "decode_per_second" not in js
|
|
|
|
|
|
def test_llm_slot_sentinels_print_a_word_or_na_never_a_negative_number(client) -> None:
|
|
"""`-1` is llama.cpp's "nothing in this slot", and it never becomes a countdown.
|
|
|
|
An idle slot answers `id_task: -1` and `next_token[0].n_remain: -1` (both in the capture).
|
|
The request cell says `idle` in a word; the remaining cell prints `n/a`. Neither prints a
|
|
negative number, and neither prints the `0` that would read as "one token from done".
|
|
"""
|
|
body = _slot_js(client.get("/static/page-llm.js").text)
|
|
assert "value < 0 ? 'idle'" in body, "the request cell prints the -1 sentinel again"
|
|
assert "value >= 0 ? value : null" in body, "the remaining cell prints the -1 sentinel"
|
|
|
|
|
|
def test_llm_slot_card_header_reads_only_fields_the_contract_declares(client) -> None:
|
|
"""The card top prints `model`, `speculative`, `id` and the state pair — and nothing else.
|
|
|
|
`id` keys the card (escaped once at build time), `is_processing` is the authoritative pill,
|
|
`state` labels a number the boolean cannot express, and `model` + `speculative` are the
|
|
footnote beside them. Each is declared in a contract table, so a rename fails there. The
|
|
user-content field a busy row carries (`prompt`) is the one key this page must never read.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
header = js[js.index("function renderSlotCard("):js.index("function renderSlotGrid(")]
|
|
read = {"is_processing", "state", "model", "speculative"}
|
|
for key in read:
|
|
assert f"row.{key}" in header, f"the slot card header stopped reading {key}"
|
|
declared = set(REQUIRED_ROW_FIELDS) | set(OPTIONAL_ROW_FIELDS)
|
|
assert read <= declared, "the header reads a field no contract table declares"
|
|
for field in USER_CONTENT_FIELDS:
|
|
assert field not in header, f"the slot card prints the user's {field}"
|
|
assert "escapeHtml(id)" in js, "the slot id reached the markup unescaped"
|
|
|
|
|
|
def test_llm_slot_cards_are_keyed_created_once_and_patched_in_place(client) -> None:
|
|
"""A busy slot does not get rebuilt every poll — and a slot that leaves loses its card.
|
|
|
|
`appendChild` of the existing node keeps DOM order == slot order; everything else is a text
|
|
patch, so a poll cannot flicker a card or re-announce it to assistive tech for a difference
|
|
of a few tokens. An engine that leaves the payload loses its card map with its panel, or the
|
|
refs of removed nodes would be re-appended when it came back.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
body = _slot_js(js)
|
|
assert "state.slots.get(enginePayload.engine)" in body, "the grid stopped keying its cards"
|
|
assert "if (!refs) {" in body and "refs = slotCard(id)" in body
|
|
assert "grid.appendChild(refs.root)" in body, "slot order is no longer kept by moving nodes"
|
|
assert "byId.delete(id)" in body, "a slot that left keeps its card forever"
|
|
assert "byId.forEach((refs) => refs.root.remove())" in body, \
|
|
"the empty state left cards on the page"
|
|
assert "setTxt(box.value" in body, "the cell values stopped being patched in place"
|
|
assert "innerHTML" not in body.split("function renderSlotCard(")[1], \
|
|
"a card is rebuilt from markup on a poll"
|
|
prune = js[js.index("function pruneEngines("):js.index("/* ---- the gufo slot grid")]
|
|
assert "state.slots.delete(engine)" in prune, "a pruned engine kept its slot cards"
|
|
|
|
|
|
def test_llm_slot_grid_says_why_there_are_no_rows(client) -> None:
|
|
"""Two honest empty states, and neither of them is a card or a 0-count claim.
|
|
|
|
`/slots` did not answer → the API's own `sessions.detail` sentence (the same words the
|
|
strip's session row carries). It answered with no rows → "engine reports 0 slots", which is
|
|
a fact about that answer. `sessions.source` is the gate that tells the two apart.
|
|
"""
|
|
body = _slot_js(client.get("/static/page-llm.js").text)
|
|
assert "Boolean(sessions.source)" in body, "the grid cannot tell no-answer from no-rows"
|
|
assert "engine reports 0 slots" in body
|
|
assert "sessions.detail ||" in body, "the no-answer state lost the API's own sentence"
|
|
assert "no /slots answer from this engine" in body, "the no-answer state can print undefined"
|
|
|
|
|
|
def test_llm_vllm_panel_says_there_is_no_per_slot_json(client) -> None:
|
|
"""vLLM gets one line and no grid — no fake rows for a server that cannot answer.
|
|
|
|
`sources/llm.py`'s `_sessions()` publishes counts for vLLM and says so in `detail`; the
|
|
panel turns that into the `llm_monitor` line, with the API's sentence as the tooltip. The
|
|
gate is the engine's name (`SLOT_ENGINES`), not `sessions.list.length`, so an engine that
|
|
has not answered yet still shows its label and the honest empty sentence.
|
|
"""
|
|
js = client.get("/static/page-llm.js").text
|
|
markup = _slot_markup(js)
|
|
branch = markup.split("if (SLOT_ENGINES.indexOf(engine) < 0) {")[1].split("}")[0]
|
|
assert 'class="slots-note" data-slots-note' in branch, "the vLLM line lost its element"
|
|
assert "NO_SLOT_JSON" in branch and "vLLM exposes no per-slot JSON" in js
|
|
assert '<div class="slots"' not in branch, "the vLLM panel grew a grid it cannot fill"
|
|
body = _slot_js(js)
|
|
assert body.startswith("function slotValue("), "the grid's code path moved"
|
|
assert "if (!grid) {" in body, "renderSlotGrid no longer leaves a panel with no grid alone"
|
|
assert "note.title = enginePayload.sessions.detail" in body, \
|
|
"the vLLM line stopped carrying the API's own words"
|
|
|
|
|
|
def test_the_captured_slot_rows_fill_the_cells_the_grid_prints(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The capture, through the API, into the keys the served cell table reads.
|
|
|
|
The fixture rows are injected as a live job payload (the `/slots` passthrough verbatim), so
|
|
this is the render path's input as the browser would receive it: `sessions.source` set (the
|
|
grid's gate), every row present, and every key `SLOT_FIELDS` names resolvable in it. A
|
|
rename in `sources/llm.py` or in gufo fails here rather than leaving a card of `n/a`s.
|
|
"""
|
|
rows = load_slots_capture()
|
|
with build_client(tmp_path, monkeypatch, gufo_url=DEAD_URL,
|
|
vllm_url=DEAD_URL) as test_client:
|
|
hub = test_client.app.state.ctx.hub # type: ignore[attr-defined]
|
|
hub.jobs["llm:gufo"].state.payload = {
|
|
"engine": "gufo", "metrics": {"requests_processing": {"value": 1.0}},
|
|
"sessions": {"active": 1, "list": rows, "source": "gufo /slots", "detail": None},
|
|
}
|
|
engines = test_client.get("/api/llm/snapshot").json()["engines"]
|
|
gufo = next(row for row in engines if row["engine"] == "gufo")
|
|
sessions = gufo["sessions"]
|
|
assert sessions["source"] == "gufo /slots", "the grid's gate is gone from the payload"
|
|
assert sessions["list"] == rows, "the rows are no longer the passthrough the grid reads"
|
|
|
|
cells = _slot_table(test_client.get("/static/page-llm.js").text)
|
|
for cell in cells:
|
|
for option in SLOT_ROW_PATHS:
|
|
key = cell[option]
|
|
if key is None:
|
|
continue
|
|
in_next = cell["path"] == "next_token"
|
|
required = REQUIRED_NEXT_TOKEN_FIELDS if in_next else REQUIRED_ROW_FIELDS
|
|
for row in sessions["list"]:
|
|
source = row["next_token"][0] if in_next else row
|
|
if key not in required and key not in source:
|
|
continue # an optional sub-line this build does not publish
|
|
value = source[key]
|
|
assert isinstance(value, (int, float)) and not isinstance(value, bool), (
|
|
f"the {cell['key']} cell reads {key!r} = {value!r}, which the number "
|
|
"formatter cannot print")
|
|
|
|
|
|
def test_llm_charts_are_secondary_and_named(client) -> None:
|
|
"""The four trend charts stay, at half height, *below* the engine panels, each named.
|
|
|
|
Half of the old 165px and moved after `#engines` is the "cells read first" decision;
|
|
every `LineChart` gets an `ariaLabel` because the legend is a sibling that labels
|
|
nothing (`/system` did this in phase 03, `/llm` never had).
|
|
"""
|
|
html = _code(client.get("/llm").text)
|
|
assert html.index('id="engines"') < html.index('id="chart-decode"'), \
|
|
"the charts are back on top of the engine panels"
|
|
for chart in ("chart-decode", "chart-prompt", "chart-kv", "chart-queue"):
|
|
assert f'id="{chart}"' in html, f"/llm lost the {chart} chart"
|
|
js = client.get("/static/page-llm.js").text
|
|
assert "height: 110" in js and "height: 165" not in js
|
|
charts = js[js.index("function makeCharts("):js.index("function pointsFor(")]
|
|
assert charts.count("ariaLabel:") == 4, "an /llm chart lost its accessible name"
|
|
|
|
|
|
def test_llm_engine_panels_are_not_stretched_to_their_tallest_sibling(client) -> None:
|
|
"""A 5-row gufo panel beside a 7-row vLLM panel keeps its own height.
|
|
|
|
Found while comparing against `llm_monitor`: the shorter engine card grew 172 px of
|
|
blank card, and a blank panel reads as "no data" — the exact misreading the unconfigured
|
|
rules exist to prevent. Pinned on the served markup so the `.grid.items-start` rule in
|
|
`dash.css` cannot silently become dead CSS (and the void cannot silently come back).
|
|
"""
|
|
html = _code(client.get("/llm").text)
|
|
engines_div = html[html.index('<div class="grid'):html.index('id="engines"') + 40]
|
|
assert "items-start" in engines_div, "the engine grid stretches its cards again"
|
|
css = client.get("/static/dash.css").text
|
|
assert ".grid.items-start" in css, "the rule the engine grid relies on is gone"
|
|
|
|
|
|
def test_overview_is_exactly_two_panels(client) -> None:
|
|
"""The landing page answers one question: what is wrong, and what is the fleet doing.
|
|
|
|
Asserted on the served bytes because there is no JS test runner (a locked anchor):
|
|
the summary cards and the raw event feed are what made this page busy, and once a
|
|
panel is gone a browser is the only thing that would notice its return.
|
|
"""
|
|
html = client.get("/").text
|
|
assert html.count("<section") == 2, "the overview grew a third panel"
|
|
assert 'id="alerts"' in html and 'id="alerts-clear"' in html
|
|
assert 'aria-live="polite"' in html
|
|
assert 'id="fleet-table"' in html
|
|
for gone in ("sys-cpu", "sys-mem", "sys-disk", "sys-gpus", "llm-decode", "llm-kv",
|
|
"ups-list", 'id="events"'):
|
|
assert gone not in html, f"the overview still renders {gone}"
|
|
# the header's source dots are chrome owned by app.js, not the deleted card dots
|
|
assert 'id="dot-system"' not in html and 'id="nav-status"' in html
|
|
|
|
|
|
def test_overview_alert_rows_link_to_the_dashboard_that_explains_them(client) -> None:
|
|
"""Every alert carries the `href` its row turns into a link; the page invents none."""
|
|
page_js = client.get("/static/page-index.js").text
|
|
assert "alert.href" in page_js
|
|
css = client.get("/static/dash.css").text
|
|
for selector in ("ul.alerts li.ok .src", "ul.alerts li.warn .src", "ul.alerts li.fail .src",
|
|
"table.data td.warn", "table.data td.fail"):
|
|
assert selector in css, f"dash.css lost {selector}"
|
|
# the swap that puts a live problem above the fleet table
|
|
assert 'data-state="alerting"' in css
|
|
|
|
|
|
def test_overview_dims_the_fleet_row_of_a_host_that_stopped_answering(client) -> None:
|
|
"""A dead host keeps its last good numbers, so the row has to say they are stale.
|
|
|
|
`AGENTS.md` invariant 4: keep the payload and mark it stale, never zero it. The
|
|
overview's table is the one place those last good numbers still print, so it reads the
|
|
per-source verdicts `/api/summary` sends (`system.states`) and applies the shared
|
|
`.stale` class the other pages already use — no new palette, no JS threshold.
|
|
"""
|
|
page_js = client.get("/static/page-index.js").text
|
|
assert "system.states" in page_js, "the fleet table stopped reading reachability"
|
|
assert 'class="stale"' in page_js
|
|
assert ".stale" in client.get("/static/dash.css").text
|
|
|
|
|
|
# Which script owns which page, for the orphan check below. `charts.js` and `app.js` are
|
|
# loaded by every page, so an id either page code or shared chrome may read.
|
|
PAGE_SCRIPTS = {
|
|
"/": "page-index.js",
|
|
"/system": "page-system.js",
|
|
"/disks": "page-disks.js",
|
|
"/llm": "page-llm.js",
|
|
"/ups": "page-ups.js",
|
|
# `/settings` is audited like a dashboard even though it is not one: an id nothing reads is
|
|
# dead markup whatever page it sits on, and this page has five new ones.
|
|
SETTINGS_PAGE: "page-settings.js",
|
|
# `/certs` too (phase 15 task 05): an authority panel and an agents table are markup, and
|
|
# every id in them is written by this page's own script.
|
|
CERTS_PAGE: "page-certs.js",
|
|
}
|
|
# `/login` is deliberately **not** in this table even though it is audited (phase 11): a
|
|
# signed-in client asking for it gets a 303 to the dashboard, so this helper's own
|
|
# redirect-following `get` would audit `/`'s markup and call it `/login`. The login page's
|
|
# orphan-`id`, CDN and masthead checks live in `tests/test_auth_api.py`, fetched unsigned, and
|
|
# `/static/page-login.js` is in `ASSETS` above.
|
|
|
|
SHARED_SCRIPTS = ("app.js", "charts.js")
|
|
|
|
|
|
def test_no_page_carries_an_id_nothing_reads(client) -> None:
|
|
"""An id no script queries, no stylesheet styles and no attribute points at is dead markup.
|
|
|
|
The summary cards were retired this way — a `<span id="sys-cpu">` left behind after its
|
|
writer was deleted is the same mistake in slower motion. Asserted on served bytes: the
|
|
browser console is otherwise the only thing that would notice.
|
|
"""
|
|
scripts = {name: client.get(f"/static/{name}").text for name in
|
|
(*SHARED_SCRIPTS, *PAGE_SCRIPTS.values(), "dash.css")}
|
|
for path, own_script in PAGE_SCRIPTS.items():
|
|
html = _code(client.get(path).text)
|
|
corpus = "".join(_code(scripts[name])
|
|
for name in (*SHARED_SCRIPTS, own_script, "dash.css"))
|
|
# ids the markup uses itself: labels, targets, `<label for>`
|
|
anchors = set(re.findall(
|
|
r'(?:aria-labelledby|aria-describedby|aria-controls|for)="([^"]+)"', html))
|
|
for element_id in re.findall(r'id="([^"]+)"', html):
|
|
quoted = (f'"{element_id}"' in corpus or f"'{element_id}'" in corpus
|
|
or f"#{element_id}" in corpus or element_id in anchors)
|
|
assert quoted, f"{path} has id=\"{element_id}\" that nothing reads"
|
|
|
|
|
|
# ---- the flavour-text audit (phase 08 task 02) ----------------------------
|
|
#
|
|
# `TODO.md:7`: "Get rid of most of the flavor text and add a note to AGENTS.md to not add it
|
|
# unless necessary. It just comes off as LLM generated slop, not helpful information." The
|
|
# test is one question per string — *would the operator act differently without it?* — and
|
|
# the answer decided the deletion. Copy that explains how the app builds its numbers is not
|
|
# helpful information; a provenance tag, a measured Δ window, and the sentence beside an
|
|
# empty panel are. `AGENTS.md` → *UI conventions* now carries the rule; the pins below are
|
|
# what stop the deleted sentences crawling back, because a screenshot only shows them once
|
|
# somebody already knows to look.
|
|
|
|
#: Fragments deleted from the served pages. The first two are the lines the operator quoted.
|
|
DELETED_FLAVOUR = (
|
|
# the overview's footer
|
|
"every number is scraped from the box",
|
|
"Sources the operator never configured are not problems",
|
|
# /system's "Per host" sub-line and its footer
|
|
"agent-reported values",
|
|
"each panel's charts cover only that host",
|
|
"numbers are measured, not inferred",
|
|
# /llm's footer, the one half-sentence of it that lived in the script too
|
|
"each throughput number carries its provenance",
|
|
"rates over the Δ window in each tag",
|
|
"a server you do not run",
|
|
# /ups's footer and its empty state
|
|
"commands are the only write path in this app",
|
|
"The dashboard speaks the NUT protocol",
|
|
# the three event feeds' header captions, all duplicated by an empty-state sentence
|
|
"unreachable / reachable / reboots",
|
|
"engine up / down",
|
|
"state changes, self-test results, commands sent",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("fragment", DELETED_FLAVOUR)
|
|
def test_no_page_narrates_its_own_pipeline(client, fragment: str) -> None:
|
|
"""The deleted copy is gone from every served byte, markup and script alike.
|
|
|
|
Asserted across all four pages and all five scripts rather than the file it came from:
|
|
the same sentence migrating into another page's card is the same slop, and a pin that
|
|
only guards its original file would let that through.
|
|
"""
|
|
for path in (*MARKUP_PAGES, *ASSETS):
|
|
body = client.get(path).text
|
|
assert fragment not in body, f"{path} narrates the app again: {fragment!r}"
|
|
|
|
|
|
def test_pages_leave_no_empty_narration_container(client) -> None:
|
|
"""A deleted sub-line removes its container, not just its words.
|
|
|
|
An empty `<span class="sub">` is markup that reserves a line for nothing — the page head
|
|
renders a gap where the sentence used to be. Elements the scripts fill (`#sub`,
|
|
`#alerts-count`) carry an `id` and are not in this category.
|
|
"""
|
|
for path in MARKUP_PAGES:
|
|
html = _code(client.get(path).text)
|
|
for cls in ("sub", "faint", "mono"):
|
|
assert f'<span class="{cls}"></span>' not in html, \
|
|
f"{path} keeps an empty .{cls} container"
|
|
|
|
|
|
def test_the_copy_the_operator_acts_on_survives_the_audit(client) -> None:
|
|
"""The other half of the deletion: functional copy stays.
|
|
|
|
Stripping flavour must not become stripping information. Every string here answers a
|
|
question the operator is asking while looking at the page — why is this panel empty,
|
|
is this number a measurement, is this battery real, is this host's data last-good —
|
|
and each one is the reason the audit deletes sentences and not labels.
|
|
"""
|
|
pages = {path: client.get(path).text for path in PAGES}
|
|
scripts = {name: client.get(f"/static/{name}").text for name in PAGE_SCRIPTS.values()}
|
|
|
|
# an empty panel says why, in terms the operator can act on
|
|
assert "No hosts configured" in pages["/"]
|
|
assert "HOST_TARGETS" in pages["/system"]
|
|
# …including the fourth page's: a `n/a` drive card is only actionable if the page names
|
|
# the two things that make SMART readable (the binary and the device-read grant).
|
|
assert "smartmontools" in pages["/disks"] + scripts["page-disks.js"]
|
|
assert "disk group" in scripts["page-disks.js"]
|
|
assert "No engines configured" in pages["/llm"]
|
|
assert "No UPS reporting" in pages["/ups"]
|
|
assert "instcmds" in pages["/ups"], "the UPS setup note stopped naming the grant that matters"
|
|
assert "3493" in pages["/ups"], "the page stopped warning that upsd's port is not always 3493"
|
|
# …and the setup copy names the grant NUT actually checks. `README.md` is explicit: a
|
|
# dashboard user gets `instcmds`, never `actions = -`, so a page that told the operator
|
|
# to write that was sending them to fix the wrong line in `upsd.users`.
|
|
assert "actions = -" not in pages["/ups"] + scripts["page-ups.js"] + scripts["page-index.js"]
|
|
assert "instcmds" in scripts["page-ups.js"] and "instcmds" in scripts["page-index.js"]
|
|
|
|
# a Δ label is a measurement, and the rate's provenance travels with the rate
|
|
assert "windowLabel(" in scripts["page-system.js"]
|
|
assert "Δ" in scripts["page-llm.js"] and "function provenance(" in scripts["page-llm.js"]
|
|
assert "at the current load" in scripts["page-ups.js"]
|
|
|
|
# modelled data is labelled, and so is a feed that cannot contain what you are hunting
|
|
assert "SIMULATED" in scripts["page-index.js"] and "SIMULATED" in scripts["page-ups.js"]
|
|
assert "queued (deferred) requests are not in /slots" in scripts["page-llm.js"]
|
|
assert "Last good values" in scripts["page-index.js"]
|
|
|
|
|
|
def test_api_responses_are_not_cacheable(client) -> None:
|
|
for path in ("/api/config", "/api/health", "/api/summary"):
|
|
assert "no-store" in client.get(path).headers.get("cache-control", "")
|
|
|
|
|
|
# ---- history -------------------------------------------------------------- #
|
|
|
|
def test_history_returns_seeded_samples(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
db = str(tmp_path / "hist.sqlite3")
|
|
store = SampleStore(db)
|
|
now = time.time()
|
|
for i in range(30):
|
|
store.add("host", "driveripper", "cpu.usage_pct", 10.0 + i, ts=now - 300 + i)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=(Target("driveripper", DEAD_URL),)), store)
|
|
with TestClient(app) as client:
|
|
sign_in(client)
|
|
body = client.get("/api/history", params={"source": "host", "host": "driveripper",
|
|
"metric": "cpu.usage_pct", "range": "15m"})
|
|
assert response_ok(body)
|
|
points = body.json()["points"]
|
|
assert len(points) >= 25
|
|
assert points[-1]["value"] == pytest.approx(39.0)
|
|
store.close()
|
|
|
|
|
|
def test_history_rejects_an_unknown_range(client) -> None:
|
|
response = client.get("/api/history", params={"source": "host", "host": "h",
|
|
"metric": "m", "range": "banana"})
|
|
assert response.status_code == 400
|
|
|
|
|
|
def test_history_requires_the_series_coordinates(client) -> None:
|
|
assert client.get("/api/history").status_code == 422
|
|
|
|
|
|
@pytest.mark.parametrize(("raw", "seconds"), [
|
|
*[(key, float(seconds)) for key, seconds in RANGE_SECONDS.items()],
|
|
("45s", 45.0), ("13h", 46_800.0), ("7200", 7_200.0), (" 2D ", 172_800.0),
|
|
(None, 3_600.0), ("", 3_600.0),
|
|
])
|
|
def test_parse_range_reads_every_window_the_ui_offers(raw: Any, seconds: float) -> None:
|
|
"""Every chip key (5m…90d), any `<n><s|m|h|d>`, raw seconds, and the 1h default.
|
|
|
|
`parse_range` is the only thing between a typo'd query string and a chart of the wrong
|
|
window, and a wrong window is worse than an error: it still looks like data.
|
|
"""
|
|
assert parse_range(raw) == seconds
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["banana", "1w", "h", "-4h", "one"])
|
|
def test_parse_range_refuses_a_window_it_cannot_interpret(raw: str) -> None:
|
|
"""A 400 that names the value, never a silent fallback to 1h."""
|
|
with pytest.raises(HTTPException) as excinfo:
|
|
parse_range(raw)
|
|
assert excinfo.value.status_code == 400
|
|
assert raw in str(excinfo.value.detail)
|
|
|
|
|
|
# ---- the routes the *other* pages chart from -------------------------------- #
|
|
#
|
|
# `/` renders two panels now; `/system`, `/llm` and `/ups` still need these, and so does
|
|
# anything outside the browser. Retiring overview code must not retire them, so the shapes
|
|
# are pinned here rather than discovered by a page three phases from now.
|
|
|
|
def test_system_history_charts_every_configured_host_in_one_request(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""N hosts x M metrics, one round trip.
|
|
|
|
`/system` no longer uses the N-hosts half of this (every chart belongs to one host and
|
|
the page issues one request per host, pinned by
|
|
`test_every_system_chart_is_drawn_inside_its_own_host_panel`); the shape is kept because
|
|
the metric-list half is exactly what the per-host query costs one request with, and
|
|
because the endpoint is a public interface for programmatic consumers.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),
|
|
Target("deskwork", DEAD_URL))) as test_client:
|
|
body = test_client.get("/api/system/history",
|
|
params={"metrics": "cpu.usage_pct,mem.used_pct",
|
|
"range": "1h"}).json()
|
|
assert body["hosts"] == ["driveripper", "deskwork"]
|
|
assert body["range_seconds"] == 3_600.0
|
|
assert len(body["series"]) == 4
|
|
assert {(item["host"], item["metric"]) for item in body["series"]} == {
|
|
("driveripper", "cpu.usage_pct"), ("driveripper", "mem.used_pct"),
|
|
("deskwork", "cpu.usage_pct"), ("deskwork", "mem.used_pct")}
|
|
# an explicit `hosts` list narrows it instead of adding to the default
|
|
narrowed = test_client.get("/api/system/history",
|
|
params={"metrics": "cpu.usage_pct",
|
|
"hosts": "deskwork"}).json()
|
|
assert narrowed["hosts"] == ["deskwork"]
|
|
|
|
|
|
def test_one_request_per_host_answers_every_per_host_chart(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The exact query `/system` issues per host returns every series that panel draws.
|
|
|
|
`page-system.js` sends one request per host carrying the utilisation trio
|
|
(`cpu.usage_pct`, `mem.used_pct`, `disk.used_pct`), both network rates, and one
|
|
`gpu.busy_pct@<index>` per card the payload reported — so a name missing here is a chart
|
|
that silently reads "no samples yet" in the browser, and a renamed series in
|
|
`sources/host.py` fails here rather than in a screenshot. This is the metric-name half of
|
|
the contract; `tests/test_system_contract.py` holds the payload-path half.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
store = test_client.app.state.ctx.store # type: ignore[attr-defined]
|
|
now = time.time()
|
|
metrics = ["cpu.usage_pct", "mem.used_pct", "disk.used_pct",
|
|
"net.rx_bytes_per_s", "net.tx_bytes_per_s", "gpu.busy_pct@0"]
|
|
for metric in metrics:
|
|
store.add("host", "driveripper", metric, 12.5, ts=now - 60)
|
|
store.add("host", "driveripper", metric, 25.0, ts=now - 30)
|
|
|
|
body = test_client.get("/api/system/history", params={
|
|
"hosts": "driveripper", "metrics": ",".join(metrics), "range": "1h"}).json()
|
|
assert body["hosts"] == ["driveripper"], "one panel, one host"
|
|
assert [item["metric"] for item in body["series"]] == metrics
|
|
assert [len(item["points"]) for item in body["series"]] == [2] * len(metrics)
|
|
|
|
# a second card the host does not have answers empty, never zero and never 404
|
|
gap = test_client.get("/api/system/history",
|
|
params={"hosts": "driveripper", "metrics": "gpu.busy_pct@1"}).json()
|
|
assert gap["series"][0]["points"] == []
|
|
|
|
|
|
def test_every_history_endpoint_answers_exactly_what_the_per_series_path_did(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Phase 14 task 02: the batched store door moved the cost of an N x M request, not its answer.
|
|
|
|
All four history routes now ask `store.history_many` for the whole batch instead of looping
|
|
`store.history` per pair. That is only safe if the JSON is the same document the loop built,
|
|
so this compares every route against points read straight out of the store with `query()` —
|
|
a third path, in neither the loop nor the batch — and pins the series order the pages parse
|
|
(host-major, metric-minor) plus the exact key set of one series entry.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
host_targets=(Target("driveripper", DEAD_URL),
|
|
Target("deskwork", DEAD_URL)),
|
|
gufo_url=DEAD_URL) as test_client:
|
|
store = test_client.app.state.ctx.store # type: ignore[attr-defined]
|
|
now = time.time()
|
|
writes = [
|
|
("host", "driveripper", "cpu.usage_pct", [1.0, 2.0, 3.0]),
|
|
("host", "driveripper", "mem.used_pct", [10.0, 20.0]),
|
|
("host", "deskwork", "cpu.usage_pct", [4.0, 5.0, 6.0]),
|
|
("host", "deskwork", "mem.used_pct", [30.0]),
|
|
("disks", "driveripper", "disk.temp_c@nvme0n1", [38.0, 39.0]),
|
|
("llm", "gufo", "llm.decode_tokens_per_second", [40.0, 45.0]),
|
|
# a UPS name the simulator does not export, so its 5-second poll cannot add rows to
|
|
# the series this assertion is about
|
|
("ups", "notaserial", "ups.charge_pct", [99.0, 100.0]),
|
|
]
|
|
for source, host, metric, values in writes:
|
|
store.add_many([Sample(source, host, metric, value, now - (len(values) - index) * 60)
|
|
for index, value in enumerate(values)])
|
|
|
|
def expected(source: str, host: str, metric: str) -> list[dict[str, float]]:
|
|
rows = store.query(source, host, metric, res=RES_RAW, start=now - 3_600.0, tail=True)
|
|
return [{"ts": ts, "value": value} for ts, value in rows]
|
|
|
|
routes = [
|
|
("/api/system/history", {"hosts": "driveripper,deskwork",
|
|
"metrics": "cpu.usage_pct,mem.used_pct", "range": "1h"},
|
|
[("host", host, metric) for host in ("driveripper", "deskwork")
|
|
for metric in ("cpu.usage_pct", "mem.used_pct")]),
|
|
("/api/disks/history", {"hosts": "driveripper",
|
|
"metrics": "disk.temp_c@nvme0n1", "range": "1h"},
|
|
[("disks", "driveripper", "disk.temp_c@nvme0n1")]),
|
|
("/api/llm/history", {"engines": "gufo", "metrics": "decode_tokens_per_second",
|
|
"range": "1h"},
|
|
[("llm", "gufo", "llm.decode_tokens_per_second")]),
|
|
("/api/ups/history", {"ups": "notaserial", "metrics": "ups.charge_pct",
|
|
"range": "1h"},
|
|
[("ups", "notaserial", "ups.charge_pct")]),
|
|
]
|
|
for path, params, pairs in routes:
|
|
body = test_client.get(path, params=params).json()
|
|
assert len(body["series"]) == len(pairs), path
|
|
for series, (source, host, metric) in zip(body["series"], pairs, strict=True):
|
|
assert set(series) == {"source", "host", "metric", "range_seconds", "points"}
|
|
assert (series["source"], series["host"], series["metric"]) == (source, host, metric)
|
|
assert series["range_seconds"] == 3_600.0
|
|
assert series["points"] == expected(source, host, metric), f"{path} {metric}"
|
|
|
|
# and an empty answer is still an answer: a name with no rows, in a batch with names that
|
|
# do, comes back as an empty series rather than a missing entry or a 404
|
|
body = test_client.get("/api/system/history", params={
|
|
"hosts": "driveripper", "metrics": "cpu.usage_pct,gpu.busy_pct@7",
|
|
"range": "1h"}).json()
|
|
assert [series["points"] for series in body["series"]] == [expected(
|
|
"host", "driveripper", "cpu.usage_pct"), []]
|
|
|
|
# no metric named is no series asked for: the batch door is never opened with an empty
|
|
# list (SQL `IN ()` is a syntax error waiting for that, and 500 is not the answer anyway)
|
|
empty = test_client.get("/api/system/history",
|
|
params={"hosts": "driveripper", "metrics": ""}).json()
|
|
assert empty["hosts"] == ["driveripper"] and empty["series"] == []
|
|
no_hosts = test_client.get("/api/system/history",
|
|
params={"hosts": "", "metrics": "cpu.usage_pct"}).json()
|
|
assert no_hosts["hosts"] == ["driveripper", "deskwork"], "empty means every configured host"
|
|
assert len(no_hosts["series"]) == 2
|
|
|
|
|
|
def test_llm_snapshot_keeps_an_engine_that_never_answered_in_the_list(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A configured-but-dead engine is listed as unreachable, not dropped (invariant 4)."""
|
|
# a host target too, so the snapshot's job filter has to skip a non-LLM job
|
|
with build_client(tmp_path, monkeypatch, vllm_url=DEAD_URL,
|
|
host_targets=(Target("driveripper", DEAD_URL),)) as test_client:
|
|
body = test_client.get("/api/llm/snapshot").json()
|
|
assert [engine["engine"] for engine in body["engines"]] == ["vLLM"]
|
|
assert body["engines"][0]["unreachable"] is True
|
|
assert body["fleet"]["engines_total"] == 1
|
|
assert body["fleet"]["engines_live"] == 0
|
|
assert [state["key"] for state in body["states"]] == ["llm:vllm"]
|
|
|
|
|
|
def test_llm_history_charts_both_engines_by_default(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, gufo_url=DEAD_URL,
|
|
vllm_url=DEAD_URL) as test_client:
|
|
body = test_client.get("/api/llm/history",
|
|
params={"metrics": "decode_tokens_per_second,kv_cache_usage_ratio",
|
|
"range": "15m"}).json()
|
|
assert body["engines"] == ["gufo", "vllm"]
|
|
assert body["range_seconds"] == 900.0
|
|
assert {(item["host"], item["metric"]) for item in body["series"]} == {
|
|
("gufo", "llm.decode_tokens_per_second"),
|
|
("gufo", "llm.kv_cache_usage_ratio"),
|
|
("vllm", "llm.decode_tokens_per_second"),
|
|
("vllm", "llm.kv_cache_usage_ratio")}
|
|
|
|
|
|
def test_one_batched_llm_history_request_answers_every_cell_series(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The exact query `/llm` issues per refresh returns every chart *and* sparkline series.
|
|
|
|
Phase 04 sends one request carrying the chart trio and the whole spark cell list —
|
|
including the two series the layout required (`tpot_seconds` for the vLLM TPOT row,
|
|
`spec_accept_rate` for the gufo spec row). A name missing here is a cell that reads
|
|
"no history" forever in the browser, which is the dead-cell anti-pattern the phase
|
|
wrote NOTES-chart-set to prevent; a rename in `sources/llm.py` fails here first.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, gufo_url=DEAD_URL,
|
|
vllm_url=DEAD_URL) as test_client:
|
|
store = test_client.app.state.ctx.store # type: ignore[attr-defined]
|
|
now = time.time()
|
|
metrics = ["decode_tokens_per_second", "prompt_tokens_per_second",
|
|
"kv_cache_usage", "requests_running", "requests_waiting",
|
|
"ttft_seconds", "tpot_seconds", "queue_seconds",
|
|
"prefix_cache_hit_ratio", "spec_accept_rate"]
|
|
for metric in metrics:
|
|
store.add("llm", "gufo", f"llm.{metric}", 1.5, ts=now - 60)
|
|
store.add("llm", "vllm", f"llm.{metric}", 2.5, ts=now - 30)
|
|
body = test_client.get("/api/llm/history", params={
|
|
"engines": "gufo,vllm", "metrics": ",".join(metrics), "range": "1h"}).json()
|
|
assert body["range_seconds"] == 3_600.0
|
|
by_key = {(item["host"], item["metric"]) for item in body["series"]}
|
|
assert by_key == {(engine, f"llm.{metric}")
|
|
for engine in ("gufo", "vllm") for metric in metrics}
|
|
points = {(item["host"], item["metric"]): item["points"] for item in body["series"]}
|
|
assert points[("gufo", "llm.spec_accept_rate")][0]["value"] == pytest.approx(1.5)
|
|
assert points[("vllm", "llm.tpot_seconds")][0]["value"] == pytest.approx(2.5)
|
|
|
|
|
|
def test_metrics_endpoint_lists_what_actually_has_history(client) -> None:
|
|
body = client.get("/api/metrics").json()
|
|
assert body["metrics"] == []
|
|
|
|
|
|
# ---- the `/disks` API (phase 09 task 03) ---------------------------------- #
|
|
#
|
|
# The drive-health surface answers like every other dashboard's: GETs only, the same
|
|
# stale/stub semantics for a host that is not answering, the same history mechanism for the
|
|
# `disk.*@dev` series, and the same 404-`configured: false` sentence when no agent is
|
|
# configured at all — the last one *without dialling*, because "not deployed here" must not
|
|
# cost a connect timeout or a red dot (invariant 3).
|
|
|
|
@pytest.fixture
|
|
def disks_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
disk_snapshot: Any) -> Iterator[TestClient]:
|
|
"""Two hosts configured; one of them answering SMART, through the production parser.
|
|
|
|
The payload and the stored samples are what `parse_disks_snapshot` makes of the
|
|
fixture-shaped capture (the agent's whole-disk rows plus the recorded `smartctl` shapes),
|
|
so every path a `/disks` card will read is a path the source really emits. The other host
|
|
keeps its `unreachable` stub, which is the shape a dead agent must keep (invariant 4).
|
|
Both targets point at a dead port, so nothing dials.
|
|
"""
|
|
db = str(tmp_path / "disks.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=(Target("driveripper", DEAD_URL),
|
|
Target("deskwork", DEAD_URL))), store)
|
|
with TestClient(app) as test_client:
|
|
sign_in(test_client)
|
|
now = time.time()
|
|
snapshot = disk_snapshot("driveripper")
|
|
snapshot["ts"] = now # "the agent answered just now", so charts are in range
|
|
result = parse_disks_snapshot("driveripper", snapshot, DisksCollector(), now=now)
|
|
store.add_many(result.samples)
|
|
app.state.ctx.hub.jobs["disks:driveripper"].state.payload = result.payload
|
|
yield test_client
|
|
store.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def mixed_disks_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
disk_snapshot: Any) -> Iterator[TestClient]:
|
|
"""Four configured hosts: one per answer an agent can give about `smartctl`, plus a stub.
|
|
|
|
Phase 22's visibility gate needs all three arms of `smartctl_available` in **one** fleet,
|
|
because "hidden" only means something against what stays:
|
|
|
|
* `driveripper` — every drive read: `smartctl_available: true`
|
|
* `deskwork` — every drive `null` with no error line: `null` (`--no-smart`, or an agent
|
|
predating the SMART pass — it answered, it just cannot say)
|
|
* `gateway` — every drive `null` plus the agent's own "not found" line: `false`, which is
|
|
the one arm that hides a host (`api._disk_visible`)
|
|
* `deadhost` — never answered: the `unreachable` stub, whose flag is `null` by shape
|
|
|
|
Each payload is `parse_disks_snapshot`'s own output over the fixture captures, and every
|
|
target points at a dead port, so nothing dials and no poll can overwrite what is set here.
|
|
"""
|
|
db = str(tmp_path / "mixed-disks.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=(Target("driveripper", DEAD_URL),
|
|
Target("deskwork", DEAD_URL),
|
|
Target("gateway", DEAD_URL),
|
|
Target("deadhost", DEAD_URL))), store)
|
|
with TestClient(app) as test_client:
|
|
sign_in(test_client)
|
|
now = time.time()
|
|
for name, mode in (("driveripper", "healthy"), ("deskwork", "silent"),
|
|
("gateway", "no_binary")):
|
|
snapshot = disk_snapshot(name, mode=mode)
|
|
snapshot["ts"] = now # "the agent answered just now"
|
|
result = parse_disks_snapshot(name, snapshot, DisksCollector(), now=now)
|
|
app.state.ctx.hub.jobs[f"disks:{name}"].state.payload = result.payload
|
|
yield test_client
|
|
store.close()
|
|
|
|
|
|
#: (`smart` field, severity metric) for the two drive numbers the table rates — the same
|
|
#: pair `sources/disks.py:SEVERITY_FIELDS` declares, restated here so a rename that only
|
|
#: touches one side fails rather than colouring nothing.
|
|
RATED_CELLS: tuple[tuple[str, str], ...] = (
|
|
("temp_c", "disk.temp_c"),
|
|
("percentage_used", "disk.percentage_used"),
|
|
)
|
|
|
|
|
|
def disk_rows(body: dict[str, Any], host: str = "driveripper") -> list[dict[str, Any]]:
|
|
"""The disk rows of one host out of a `/api/disks/snapshot` body."""
|
|
return next(row for row in body["hosts"] if row["name"] == host)["disks"]
|
|
|
|
|
|
def flat_routes(app: Any) -> list[Any]:
|
|
"""Every route the app serves, with `include_router()` results flattened to their routes.
|
|
|
|
The FastAPI this repo pins keeps an included router as an `_IncludedRouter` wrapper with
|
|
no `path` of its own, so walking `app.routes` alone sees five anonymous objects instead of
|
|
the API surface — and the verb assertion below is only as good as this walk.
|
|
"""
|
|
out: list[Any] = []
|
|
for route in app.routes:
|
|
inner = getattr(route, "original_router", None)
|
|
out.extend(inner.routes if inner is not None else [route])
|
|
return out
|
|
|
|
|
|
def series_source(body: dict[str, Any]) -> set[str]:
|
|
"""Which sample family a history answer came from: `disks`, never the 10-second `host`."""
|
|
return {series["source"] for series in body["series"]}
|
|
|
|
|
|
def test_disks_snapshot_answers_one_entry_per_host_with_its_physical_disks(
|
|
disks_client) -> None:
|
|
"""`GET /api/disks/snapshot` is per host: rows + `smart` + `smartctl_available`.
|
|
|
|
"Per host" means per **visible** host since phase 22 — this fleet has no `false` host in
|
|
it, so both stub and answering host are listed; the hidden case is pinned on a fleet that
|
|
has one (`mixed_disks_client`).
|
|
"""
|
|
body = disks_client.get("/api/disks/snapshot")
|
|
assert response_ok(body)
|
|
payload = body.json()
|
|
assert [host["name"] for host in payload["hosts"]] == ["deskwork", "driveripper"], (
|
|
"every configured host is a row, in name order, answering or not")
|
|
rows = disk_rows(payload)
|
|
assert rows, "the capture has 16 whole-disk devices, so the page has 16 cards"
|
|
assert all("/" not in row["device"] for row in rows), "a mount leaked into the disk list"
|
|
first = rows[0]
|
|
assert first["path"] == f"/dev/{first['device']}"
|
|
assert set(first) >= {"device", "path", "model", "size_bytes", "rotational",
|
|
"health", "smart", "severity"}
|
|
assert first["smart"]["power_on_hours"] > 0, "the drive's own counters travel verbatim"
|
|
live = next(host for host in payload["hosts"] if host["name"] == "driveripper")
|
|
assert live["smartctl_available"] is True
|
|
assert live["smart_errors"] == []
|
|
assert live["summary"]["disks"] == len(rows)
|
|
assert "unreachable" not in live, "answering is the absence of the flag, as on /system"
|
|
assert [state["key"] for state in payload["states"]] == ["disks:driveripper",
|
|
"disks:deskwork"], (
|
|
"the page's dots come from the disks jobs, never the system ones")
|
|
|
|
|
|
def test_disks_snapshot_carries_the_verdicts_the_page_will_paint(disks_client) -> None:
|
|
"""Colour is in the payload: each row's `severity` is what `severity.classify` says.
|
|
|
|
This is the pin that keeps the threshold out of the page — `page-disks.js` reads these
|
|
and never compares a drive temperature to a bound.
|
|
"""
|
|
rows = disk_rows(disks_client.get("/api/disks/snapshot").json())
|
|
assert "ok" in {row["severity"]["temp_c"] for row in rows}, "the capture is healthy"
|
|
for row in rows:
|
|
smart = row["smart"] or {}
|
|
assert set(row["severity"]) == {field for field, _metric in RATED_CELLS} | {"health"}
|
|
for field, metric in RATED_CELLS:
|
|
assert row["severity"][field] == classify(metric, smart.get(field)), (
|
|
f"{row['device']}.{field} is coloured by something other than the one table")
|
|
assert not {key for row in rows for key in row["severity"]} & {
|
|
"reallocated_sector_ct", "pending", "uncorrectable", "power_on_hours"}, (
|
|
"a counter is an alarm chip on presence, never a graded cell")
|
|
|
|
|
|
def test_a_configured_host_with_no_payload_yet_is_a_row_of_nas(disks_client) -> None:
|
|
"""The stub shape `/api/system/snapshot` hands out, so `n/a` is never a missing row.
|
|
|
|
A host in `HOST_TARGETS` that has not answered is still in the fleet (invariants 3 and
|
|
4) with its counts `None` rather than `0`: "unknown" and "no drives" are different
|
|
sentences (rule 2), and the fleet header counts it without summing it.
|
|
"""
|
|
payload = disks_client.get("/api/disks/snapshot").json()
|
|
stub = next(host for host in payload["hosts"] if host["name"] == "deskwork")
|
|
assert stub["unreachable"] is True
|
|
assert stub["disks"] == [], "nothing to draw, but the host is still listed"
|
|
assert stub["smartctl_available"] is None, "an agent that never answered says nothing"
|
|
assert stub["smart_errors"] == []
|
|
assert set(stub["summary"]) == {"disks", "with_smart", "health_failed", "alarm_disks"}
|
|
assert all(value is None for value in stub["summary"].values()), "no invented zeros"
|
|
fleet = payload["fleet"]
|
|
assert (fleet["hosts_total"], fleet["hosts_live"]) == (2, 1)
|
|
assert fleet["disks"] == len(disk_rows(payload))
|
|
|
|
|
|
def test_the_disks_fleet_header_is_summed_by_the_api_not_the_page(disks_client) -> None:
|
|
"""`fleet` is the header's only arithmetic source, and it carries no verdict."""
|
|
payload = disks_client.get("/api/disks/snapshot").json()
|
|
live = next(host for host in payload["hosts"] if host["name"] == "driveripper")
|
|
fleet = payload["fleet"]
|
|
assert fleet["disks"] == live["summary"]["disks"]
|
|
assert fleet["alarm_disks"] == live["summary"]["alarm_disks"] > 0, (
|
|
"the recorded ATA shape carries reallocated sectors, so the fleet has chips to draw")
|
|
assert "severity" not in fleet, "a count of drives is not a graded number"
|
|
|
|
|
|
# ---- the `/disks` visibility gate (phase 22 task 01) ----------------------- #
|
|
#
|
|
# `TODO.md:13`: "VMs that don't have smartctl enabled disks should not show up. Gateway
|
|
# shouldn't even be a host on the disks page." One predicate (`api._disk_visible`) over the
|
|
# three arms of `smartctl_available`, applied to the assembled host list in both endpoints, so
|
|
# the page's rows, that block's rows and the fleet sums cannot describe three fleets. The
|
|
# source, the store, the `disks:<host>` job and its health record still see every host.
|
|
|
|
def test_the_disk_visibility_gate_reads_only_the_explicit_false() -> None:
|
|
"""Three arms, one hide condition — and the test is `is False`, not falsy.
|
|
|
|
`None` means "nobody knows" (an agent predating the SMART pass, `--no-smart`, or a payload
|
|
with no such key at all) and an agent that cannot say is not an agent that said *no*; a
|
|
dead agent's stub stays too, because a machine that stopped answering must never read as
|
|
"never configured" (invariant 3).
|
|
"""
|
|
assert _disk_visible({"smartctl_available": True}) is True
|
|
assert _disk_visible({"smartctl_available": None}) is True
|
|
assert _disk_visible({}) is True, "a payload that never said anything is not a 'no'"
|
|
assert _disk_visible({"unreachable": True, "smartctl_available": None}) is True, (
|
|
"a dead agent is not a host without smartmontools")
|
|
assert _disk_visible({"smartctl_available": 0}) is True, (
|
|
"falsy is not the predicate — only the value the agent answered with")
|
|
assert _disk_visible({"smartctl_available": False}) is False
|
|
|
|
|
|
def test_a_host_whose_agent_has_no_smartctl_is_not_a_host_on_the_disks_page(
|
|
mixed_disks_client) -> None:
|
|
"""The absent host and the two `unknown` hosts, pinned in the same response.
|
|
|
|
gateway's VM answers "`smartctl` not found", so it has no card and no row; deskwork's
|
|
agent cannot say either way and deadhost's never answered, and both keep their dimmed
|
|
`n/a` rows exactly as before this gate existed.
|
|
"""
|
|
payload = mixed_disks_client.get("/api/disks/snapshot").json()
|
|
visible = {host["name"]: host for host in payload["hosts"]}
|
|
assert list(visible) == ["deadhost", "deskwork", "driveripper"], (
|
|
"only the host whose agent said 'no smartctl' is gone")
|
|
assert visible["driveripper"]["smartctl_available"] is True
|
|
assert visible["driveripper"]["disks"], "the host that stayed kept every drive row"
|
|
assert visible["deskwork"]["smartctl_available"] is None, (
|
|
"an agent that cannot say is still a card, with its own why-line")
|
|
assert "unreachable" not in visible["deskwork"], "it answered — silently, but it answered"
|
|
assert visible["deadhost"]["unreachable"] is True
|
|
assert visible["deadhost"]["smartctl_available"] is None
|
|
|
|
|
|
def test_the_disks_fleet_header_counts_only_the_hosts_the_page_lists(
|
|
mixed_disks_client) -> None:
|
|
"""`hosts_total` is the page's row count, and a hidden host lends it no drives."""
|
|
payload = mixed_disks_client.get("/api/disks/snapshot").json()
|
|
fleet = payload["fleet"]
|
|
rows_by_host = {host["name"]: host["disks"] for host in payload["hosts"]}
|
|
assert fleet["hosts_total"] == len(payload["hosts"]) == 3
|
|
assert fleet["hosts_live"] == 2, "the stub host is counted and never summed"
|
|
assert fleet["disks"] == len(rows_by_host["driveripper"]) + len(rows_by_host["deskwork"]), (
|
|
"the hidden host's `vda` is in no fleet count")
|
|
assert fleet["with_smart"] == len(rows_by_host["driveripper"])
|
|
assert "severity" not in fleet
|
|
|
|
|
|
def test_hiding_a_diskless_host_leaves_its_health_record_and_the_fleet_alone(
|
|
mixed_disks_client) -> None:
|
|
"""The gate reads a payload; it does not unwatch the machine.
|
|
|
|
`states` still lists `disks:gateway` (the job polls, and healthy is healthy), `/system`
|
|
still lists gateway as a host, and `/api/health` still answers for the job — a box with no
|
|
smartmontools is missing from the **drive** page and from nothing else (invariant 3).
|
|
"""
|
|
snapshot = mixed_disks_client.get("/api/disks/snapshot").json()
|
|
summary = mixed_disks_client.get("/api/summary").json()
|
|
assert {state["key"] for state in snapshot["states"]} == {
|
|
"disks:deadhost", "disks:deskwork", "disks:driveripper", "disks:gateway"}
|
|
assert "gateway" in {row["name"] for row in summary["system"]["hosts"]}, (
|
|
"the system page lost a machine over a missing binary")
|
|
health = {row["key"]: row for row in
|
|
mixed_disks_client.get("/api/health").json()["sources"]}
|
|
assert "disks:gateway" in health
|
|
|
|
|
|
def test_summary_and_the_disks_page_hide_the_same_hosts(mixed_disks_client) -> None:
|
|
"""One predicate, two endpoints: `/` and `/disks` may not list different machines."""
|
|
body = mixed_disks_client.get("/api/summary").json()
|
|
snapshot = mixed_disks_client.get("/api/disks/snapshot").json()
|
|
disks = body["disks"]
|
|
# the **set**, not the list: `/api/disks/snapshot` sorts by name, `/api/summary` answers
|
|
# rows in hub order and always has
|
|
assert {row["name"] for row in disks["hosts"]} == {
|
|
host["name"] for host in snapshot["hosts"]} == {
|
|
"deadhost", "deskwork", "driveripper"}
|
|
assert disks["fleet"] == snapshot["fleet"], "the header would disagree with the page"
|
|
assert disks["fleet"]["hosts_total"] == 3
|
|
assert disks["enabled"] is True
|
|
assert disks["fleet"]["hosts_total"] != len(body["system"]["hosts"]), (
|
|
"four machines configured, three drive rows — the two numbers are different facts")
|
|
# the rows that stayed carry the shape they always did, `unknown` and all (rule 2)
|
|
rows = {row["name"]: row for row in disks["hosts"]}
|
|
assert rows["deskwork"]["smartctl_available"] is None
|
|
assert rows["deskwork"]["unreachable"] is False
|
|
assert rows["deadhost"]["unreachable"] is True and rows["deadhost"]["disks"] is None
|
|
|
|
|
|
def test_the_disk_visibility_gate_applies_with_only_one_host_configured(
|
|
tmp_path: Path, disk_snapshot: Any) -> None:
|
|
"""A fleet of nothing but a `false` host is an empty drive page, not a 404.
|
|
|
|
Unconfigured still means "no `HOST_TARGETS`" (a 404 that dials nothing, invariant 3);
|
|
configured-and-hidden answers with an empty list and a zero fleet, which is what the page
|
|
renders as its empty state.
|
|
"""
|
|
db = str(tmp_path / "one-hidden-host.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=(Target("gateway", DEAD_URL),)), store)
|
|
with TestClient(app) as test_client:
|
|
sign_in(test_client)
|
|
now = time.time()
|
|
snapshot = disk_snapshot("gateway", mode="no_binary")
|
|
snapshot["ts"] = now
|
|
result = parse_disks_snapshot("gateway", snapshot, DisksCollector(), now=now)
|
|
assert result.payload["smartctl_available"] is False, "the arm this test is about"
|
|
app.state.ctx.hub.jobs["disks:gateway"].state.payload = result.payload
|
|
body = test_client.get("/api/disks/snapshot")
|
|
assert body.status_code == 200, "configured-and-hidden is not unconfigured"
|
|
payload = body.json()
|
|
assert payload["hosts"] == []
|
|
assert payload["fleet"]["hosts_total"] == 0
|
|
assert payload["fleet"]["disks"] == 0
|
|
assert [state["key"] for state in payload["states"]] == ["disks:gateway"]
|
|
summary = test_client.get("/api/summary").json()["disks"]
|
|
assert summary["enabled"] is True
|
|
assert summary["hosts"] == []
|
|
assert summary["fleet"] == payload["fleet"], (
|
|
"an all-hidden fleet must look the same from both endpoints")
|
|
store.close()
|
|
|
|
|
|
def test_the_disks_surface_is_gets_only_and_the_write_stays_the_ups_one(
|
|
disks_client) -> None:
|
|
"""Invariant 1: the app writes to an **upstream** from exactly one route."""
|
|
routes = flat_routes(disks_client.app)
|
|
disks = [route for route in routes
|
|
if str(getattr(route, "path", "")).startswith("/api/disks")]
|
|
assert {route.path for route in disks} == {"/api/disks/snapshot", "/api/disks/history"}
|
|
verbs = {verb for route in disks for verb in (getattr(route, "methods", None) or set())}
|
|
assert verbs == {"GET"}, "a write appeared on the drive-health surface"
|
|
for verb in ("POST", "PUT", "PATCH", "DELETE"):
|
|
response = disks_client.request(verb, "/api/disks/snapshot", json={})
|
|
assert response.status_code == 405, f"{verb} /api/disks/snapshot was accepted"
|
|
writes = sorted(str(route.path) for route in routes
|
|
if "POST" in (getattr(route, "methods", None) or set()))
|
|
# Invariant 1 is about the fleet. The `/auth/` writes create, re-credential, and clear a
|
|
# session in this process; the `/certs/` writes (phase 15) mint, reissue, and revoke a
|
|
# certificate in this process's `meta` table. Neither dials anything — and a revocation's whole
|
|
# effect is that the *next poll* refuses to dial — so they are named here rather than filtered
|
|
# out of sight, because the failure this assertion catches is a POST arriving quietly.
|
|
authority = [path for path in writes if path.startswith("/api/certs/")]
|
|
session = [path for path in writes if path.startswith("/api/auth/")]
|
|
upstream = [path for path in writes
|
|
if not path.startswith(("/api/auth/", "/api/certs/"))]
|
|
assert upstream == ["/api/ups/command"], "one write endpoint, four gates, unchanged"
|
|
assert session == ["/api/auth/login", "/api/auth/logout", "/api/auth/password"]
|
|
assert authority == ["/api/certs/root/rotate", "/api/certs/{name}/revoke",
|
|
"/api/certs/{name}/rotate"], "the CA surface grew a write nobody named"
|
|
# …and nothing anywhere grew the verbs this app has no use for, session included.
|
|
assert not sorted(path for route in routes
|
|
for path in [str(route.path)]
|
|
if set(getattr(route, "methods", None) or ()) & {"PUT", "PATCH", "DELETE"})
|
|
# The session writes are not upstream writes, but they are still the only ones: `/login`
|
|
# is the page, not a POST target, and `/api/auth/status` stays a read.
|
|
pages = sorted(str(route.path) for route in routes
|
|
if "GET" in (getattr(route, "methods", None) or set())
|
|
and not str(route.path).startswith("/api/"))
|
|
# `/settings` (phase 11) is a fifth dashboard-shaped **page** and it grew no verb: a page
|
|
# route is a GET, so the write surface this test guards stays the one UPS command.
|
|
assert pages == ["/", "/certs", "/disks", "/llm", "/login", "/settings", "/system",
|
|
"/ups"]
|
|
|
|
|
|
def test_config_prints_the_disks_cadence_the_page_explains_itself_with(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A card four minutes old is behaving, not broken — and the interval is the proof.
|
|
|
|
`/disks` has no upstream of its own, so `enabled` follows `HOST_TARGETS` and the cadence
|
|
is its own. Both numbers come from config rather than a constant in the page, so changing
|
|
`DISKS_INTERVAL_SECONDS` moves the sentence and the poller together.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=(Target("driveripper", DEAD_URL),),
|
|
disks_interval=120.0, disks_timeout=45.0) as test_client:
|
|
disks = test_client.get("/api/config").json()["disks"]
|
|
assert disks["enabled"] is True
|
|
assert disks["interval_seconds"] == 120.0
|
|
assert disks["timeout_seconds"] == 45.0
|
|
|
|
|
|
def test_disks_snapshot_404s_without_dialling_a_host_that_was_never_configured(
|
|
client, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The gate runs before anything that could reach the network (invariant 3).
|
|
|
|
Both spies raise, so a route that constructed a client or fetched a snapshot before
|
|
checking `HOST_TARGETS` fails here instead of costing a timeout and a red dot. The hub
|
|
assertion is the structural half: with no targets there is no `disks` job to dial with.
|
|
"""
|
|
calls: list[str] = []
|
|
|
|
def spy_get(self: Any, url: Any, **kwargs: Any) -> Any:
|
|
calls.append(f"GET {url}")
|
|
raise AssertionError(f"the unconfigured disks route dialled {url}")
|
|
|
|
def spy_fetch(*args: Any, **kwargs: Any) -> Any:
|
|
calls.append("fetch_snapshot")
|
|
raise AssertionError("the unconfigured disks route fetched an agent snapshot")
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "get", spy_get)
|
|
monkeypatch.setattr("dashboards.collectors.fetch_snapshot", spy_fetch)
|
|
|
|
response = client.get("/api/disks/snapshot")
|
|
assert response.status_code == 404
|
|
body = response.json()
|
|
assert body["configured"] is False
|
|
assert "HOST_TARGETS" in body["detail"], "the sentence names the fix"
|
|
assert calls == [], "an unconfigured page must not touch the network"
|
|
assert client.app.state.ctx.hub.jobs == {}, (
|
|
"no targets means no disks job, so nothing can age out or go red")
|
|
|
|
|
|
def test_disks_history_serves_the_dev_series_the_page_charts(disks_client) -> None:
|
|
"""One request, every series a host's cards could draw, points from the stored samples."""
|
|
rows = disk_rows(disks_client.get("/api/disks/snapshot").json())
|
|
metrics = [f"{base}@{row['device']}" for row in rows for _field, base in SMART_SERIES]
|
|
body = disks_client.get("/api/disks/history", params={
|
|
"hosts": "driveripper", "metrics": ",".join(metrics), "range": "1h"}).json()
|
|
assert body["hosts"] == ["driveripper"], "one host, one panel"
|
|
assert body["range_seconds"] == 3_600.0
|
|
assert [series["metric"] for series in body["series"]] == metrics
|
|
assert series_source(body) == {"disks"}, "the disks family, never the 10-second host one"
|
|
points = {(series["metric"]): series["points"] for series in body["series"]}
|
|
assert points["disk.temp_c@nvme0n1"][0]["value"] == pytest.approx(38.0)
|
|
assert points["disk.power_on_hours@nvme0n1"][0]["value"] == pytest.approx(8_760.0)
|
|
|
|
|
|
def test_disks_history_answers_empty_for_a_series_a_drive_does_not_publish(
|
|
disks_client) -> None:
|
|
"""Unknown names are an empty series, not a 404 — the /system and /llm contract.
|
|
|
|
The recorded ATA shape reports no `percentage_used`, so that drive's wear plot has
|
|
nothing to draw, and a device name the agent never listed behaves the same way. The card
|
|
asks for every series it could draw and reads `n/a` where there is nothing, instead of
|
|
the request failing.
|
|
"""
|
|
rows = disk_rows(disks_client.get("/api/disks/snapshot").json())
|
|
unworn = next(row["device"] for row in rows
|
|
if (row["smart"] or {}).get("percentage_used") is None)
|
|
body = disks_client.get("/api/disks/history", params={
|
|
"hosts": "driveripper",
|
|
"metrics": f"disk.percentage_used@{unworn},disk.temp_c@nosuchdisk,"
|
|
"disk.temp_c@nvme0n1",
|
|
"range": "1h"}).json()
|
|
by_metric = {series["metric"]: series["points"] for series in body["series"]}
|
|
assert by_metric[f"disk.percentage_used@{unworn}"] == [], "no measurement, no line"
|
|
assert by_metric["disk.temp_c@nosuchdisk"] == []
|
|
assert by_metric["disk.temp_c@nvme0n1"], "the series that does exist still answers"
|
|
|
|
|
|
def test_disks_history_charts_every_configured_host_in_one_request(disks_client) -> None:
|
|
"""With no `hosts` the answer covers every configured agent, dead ones included (empty)."""
|
|
body = disks_client.get("/api/disks/history",
|
|
params={"metrics": "disk.temp_c@nvme0n1,disk.temp_c@sda",
|
|
"range": "24h"}).json()
|
|
assert body["hosts"] == ["driveripper", "deskwork"]
|
|
assert body["range_seconds"] == 86_400.0
|
|
assert {(series["host"], series["metric"]) for series in body["series"]} == {
|
|
("driveripper", "disk.temp_c@nvme0n1"), ("driveripper", "disk.temp_c@sda"),
|
|
("deskwork", "disk.temp_c@nvme0n1"), ("deskwork", "disk.temp_c@sda")}
|
|
assert [series for series in body["series"] if series["host"] == "deskwork"], (
|
|
"a host with no history is an empty series, never a missing one")
|
|
|
|
|
|
def test_the_metrics_endpoint_answers_for_the_disks_family(disks_client) -> None:
|
|
""""Why is this chart empty" has an answer per page: /api/metrics?source=disks."""
|
|
body = disks_client.get("/api/metrics", params={"source": "disks"}).json()
|
|
assert "disk.temp_c@nvme0n1" in body["metrics"]
|
|
assert not [name for name in body["metrics"] if not name.startswith("disk.")], (
|
|
"the disks family holds only disk series")
|
|
assert not [name for name in disks_client.get(
|
|
"/api/metrics", params={"source": "host"}).json()["metrics"]
|
|
if name.startswith("disk.temp_c")], "SMART never mixes into the system family"
|
|
|
|
|
|
# ---- the `/disks` page (phase 09 task 04) ---------------------------------- #
|
|
#
|
|
# The fourth dashboard, asserted the way the other three are: on the served bytes, because a
|
|
# JS test runner would be a build step (a locked anchor). What is pinned is not that the page
|
|
# exists but that it keeps the four promises the phase wrote down — one card per **physical**
|
|
# drive and never one per partition, alarm chips that exist only while a counter is non-zero,
|
|
# a host without `smartctl` dimmed rather than reddened, and charts that belong to one host,
|
|
# stay collapsed until opened, and cost one history request per host per refresh.
|
|
|
|
def _disks_js(client: TestClient) -> str:
|
|
return client.get("/static/page-disks.js").text
|
|
|
|
|
|
def test_the_disks_tab_is_in_the_masthead_of_every_page(client) -> None:
|
|
"""Four tabs became five, on every page, with exactly one marking itself current.
|
|
|
|
The masthead is the app's only "which dashboard do I need" surface since phase 02 deleted
|
|
the overview's table, so a tab missing from one page is a page you cannot get to, and a
|
|
tab marked current on the wrong page is a lie about where you are.
|
|
"""
|
|
for path in PAGES:
|
|
html = client.get(path).text
|
|
assert '<a href="/disks"' in html, f"{path}'s nav has no Disks tab"
|
|
assert html.count('aria-current="page"') == 1, f"{path} marks more than one tab current"
|
|
assert f'<a href="{path}" aria-current="page">' in html, f"{path} does not mark itself"
|
|
assert (html.index('href="/system"') < html.index('href="/disks"')
|
|
< html.index('href="/llm"')), "tab order is machine, drive, engine, power"
|
|
|
|
|
|
# ---- the collapsing masthead nav (phase 18) -------------------------------- #
|
|
#
|
|
# At ≤640px the five tab links leave the bar and come back as a dropdown under it, opened by
|
|
# one `☰` button. There is no JS test runner (a locked anchor), so the phase is pinned the way
|
|
# every page surface here is: on the served bytes. Three questions, in the order the pieces
|
|
# were built — is the button on every tabbed page and on no others, does the stylesheet still
|
|
# hide it above the breakpoint, and do the stylesheet and the binding still agree on the
|
|
# breakpoint itself? That last one is the only failure mode no browser check catches: a number
|
|
# that drifts in one file opens a menu into a hidden nav on the operator's phone.
|
|
|
|
#: The attributes that make `#nav-toggle` a button and not a mystery glyph: it is not a form
|
|
#: control (`type`), it says what it toggles (`aria-controls`), it says what state it is in
|
|
#: (`aria-expanded`, which `app.js` keeps honest), and its only content is a `☰`.
|
|
NAV_TOGGLE_ATTRS = ('class="ghost nav-toggle"', 'type="button"', 'aria-expanded="false"',
|
|
'aria-controls="nav-tabs"', 'aria-label="Toggle navigation"')
|
|
|
|
|
|
def _masthead(html: str) -> str:
|
|
"""The served page's own masthead — the block the toggle and the nav have to live in.
|
|
|
|
Slicing rather than searching the whole document is the point: a button in the footer would
|
|
satisfy `id="nav-toggle" in html` and anchor to nothing, because the dropdown's
|
|
`top: 100%` resolves against `.masthead`.
|
|
"""
|
|
start = html.index('<header class="masthead"')
|
|
return html[start:html.index("</header>", start)]
|
|
|
|
|
|
def test_the_nav_toggle_is_in_the_masthead_of_every_tabbed_page(client) -> None:
|
|
"""Seven pages, one identical button, and the nav it controls still carrying five tabs.
|
|
|
|
The dropdown is the only way to change dashboard on a phone, so a page missing the button is
|
|
a page you can enter and leave the tab bar on, and a page whose nav lost its `id` is a button
|
|
that toggles nothing. Pinned on all seven (`MARKUP_PAGES`) rather than on `PAGES` because the
|
|
two admin pages have the same masthead and therefore the same contract.
|
|
"""
|
|
for path in MARKUP_PAGES:
|
|
header = _masthead(client.get(path).text)
|
|
button = re.search(r'<button[^>]*id="nav-toggle"[^>]*>', header, re.S)
|
|
assert button, f"{path}'s masthead has no #nav-toggle button"
|
|
for attr in NAV_TOGGLE_ATTRS:
|
|
assert attr in button.group(0), f"{path}'s nav toggle is missing {attr}"
|
|
assert header.count('id="nav-toggle"') == 1, f"{path} has more than one nav toggle"
|
|
|
|
# The nav is the menu: the id the button names, and all five dashboards inside it.
|
|
assert '<nav class="tabs" id="nav-tabs" aria-label="Dashboards">' in header, \
|
|
f"{path}'s tab nav has no #nav-tabs for the toggle to control"
|
|
nav = header[header.index('id="nav-tabs"'):header.index("</nav>",
|
|
header.index('id="nav-tabs"'))]
|
|
for tab in PAGES:
|
|
assert f'href="{tab}"' in nav, f"{path} collapsed a nav that lost the {tab} tab"
|
|
|
|
# Order inside the masthead: brand, toggle, nav — and the cluster after them, because
|
|
# the ≤640px block right-aligns the button and wraps the cluster onto its own row.
|
|
assert (header.index('id="nav-toggle"') < header.index('id="nav-tabs"')
|
|
< header.index('class="masthead-right"')), \
|
|
f"{path} put the toggle or the nav outside the brand → tabs → cluster row"
|
|
|
|
|
|
def test_the_login_page_carries_no_nav_and_no_toggle(tmp_path: Path) -> None:
|
|
"""`/login` is brand link + theme toggle, and the shared binding must be inert there.
|
|
|
|
No `client` fixture, and that is the whole shape of the test: the signed-in one cannot see
|
|
this page at all (a signed-in `GET /login` is a `303` to the dashboard, and following it
|
|
audits `/`'s markup under the name `/login`), which is the same redirect trap
|
|
`tests/test_auth_api.py::test_the_login_page_carries_no_id_nothing_reads_and_no_dashboard_nav`
|
|
documents for the orphan-`id` audit.
|
|
|
|
Every tab on this page bounces straight back to it, so the page renders no tabs at all; a
|
|
stray hamburger would be a control that opens a menu of redirects, and the `640px` block
|
|
would reveal it. The last assertion is the other half of the contract: `app.js` still loads
|
|
here, which is only safe because `if (navToggle && appNav)` no-ops without the elements.
|
|
"""
|
|
db = str(tmp_path / "login-nav.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
try:
|
|
with TestClient(app) as anon:
|
|
response = anon.get("/login")
|
|
assert response.status_code == 200, \
|
|
"a signed-out /login must be the form, not a bounce"
|
|
html = response.text
|
|
finally:
|
|
store.close()
|
|
|
|
for absent in ('id="nav-toggle"', 'id="nav-tabs"', 'class="tabs"',
|
|
'aria-controls="nav-tabs"', 'aria-label="Dashboards"'):
|
|
assert absent not in html, f"/login renders the collapsed nav's {absent}"
|
|
assert '<a class="brand" href="/">' in html and 'id="theme-toggle"' in html, \
|
|
"/login lost the two pieces of chrome that do belong on it"
|
|
assert 'src="/static/app.js"' in html, (
|
|
"/login stopped loading the shared chrome whose guard is its no-toggle contract")
|
|
|
|
|
|
# ---- the fleet badges on every masthead (phase 25 task 01) ------------------ #
|
|
#
|
|
# `TODO.md:3-4` — "hosts disks engines UPS disappear when in the certs/account page" and the
|
|
# operator's correction: "I did want hosts disks engines UPS badges". `Dash.initChrome()`
|
|
# builds the four badges whenever `#nav-status` exists, so the **container** is the whole
|
|
# difference between a page that has them and a page that does not — the two admin pages shipped
|
|
# without one, which is why they read as the badges vanishing rather than as two odd pages. The
|
|
# fix is the container and nothing else, so the pin is the container: on every page that renders
|
|
# the masthead, in the position the five dashboards already used (first child of
|
|
# `.masthead-right`, which is what puts it on its own row — task 02 re-justifies that row and this
|
|
# pin deliberately says nothing about which way the badges face). `tests/test_auth_api.py` keeps
|
|
# its own assertion that `/login` has none: a stranger has no fleet to be told about.
|
|
|
|
#: The container exactly as all seven signed-in pages write it (`index.html` is the reference).
|
|
NAV_STATUS_ELEMENT = '<div id="nav-status" class="gap" aria-live="polite"></div>'
|
|
|
|
#: The four badges `initChrome()` renders, in its own order.
|
|
NAV_STATUS_BADGES = ("hosts", "disks", "engines", "UPS")
|
|
|
|
|
|
def test_every_signed_in_page_carries_the_fleet_badges(client) -> None:
|
|
"""`#nav-status` is the first child of `.masthead-right` on all seven signed-in pages.
|
|
|
|
One loop over `MARKUP_PAGES`, because the control is the same on all of them: the five
|
|
dashboards had it, `/certs` and `/settings` grew it. The `app.js` half of the assertion is
|
|
what makes the markup mean something — the badges are built from the container's existence,
|
|
so a script that stopped reading `#nav-status` would turn seven `<div>`s into dead markup and
|
|
the orphan-`id` audit alone would not say so (it also matches `#nav-status` inside the CSS).
|
|
"""
|
|
js = _code(client.get("/static/app.js").text)
|
|
assert "const nav = $('#nav-status');" in js, \
|
|
"initChrome() no longer builds the badges from #nav-status, so the container is inert"
|
|
assert "if (nav) {" in js, \
|
|
"the container guard is gone — a page without #nav-status (only /login) would throw"
|
|
for label in NAV_STATUS_BADGES:
|
|
assert f"label: '{label}'" in js, f"the badge table lost {label}"
|
|
assert "setInterval(refreshStatus, 15000)" in js, \
|
|
"the badges' refresh cadence moved — this phase changes placement, not polling"
|
|
|
|
for path in MARKUP_PAGES:
|
|
header = _masthead(client.get(path).text)
|
|
assert NAV_STATUS_ELEMENT in header, \
|
|
f"{path} has no fleet badges in its masthead (TODO.md:3)"
|
|
after_cluster = header[header.index('<div class="masthead-right">')
|
|
+ len('<div class="masthead-right">'):]
|
|
assert after_cluster.lstrip().startswith(NAV_STATUS_ELEMENT), \
|
|
f"{path} does not render #nav-status as the first child of .masthead-right "\
|
|
"(the five dashboards' position is what puts the badges on their own row)"
|
|
assert header.count('id="nav-status"') == 1, f"{path} renders the badge container twice"
|
|
|
|
|
|
def test_the_nav_toggle_css_and_binding_carry_the_same_breakpoint(client) -> None:
|
|
"""Hidden by default, revealed at one breakpoint, in two files that must agree on its value.
|
|
|
|
The base rule is *not* a literal `.nav-toggle { display: none }` and pinning that literal
|
|
would be pinning a bug: the element also carries the shared `ghost` class, whose
|
|
`button.ghost { display: inline-flex }` out-ranks a single-class rule and ships a visible
|
|
button into the desktop masthead. The shipped shape is an id rule reading a token
|
|
(`--nav-toggle-display`, `none` in `:root`), which the ≤640px block flips by re-declaring the
|
|
token on the element — so that is what is asserted here, plus the dropdown and the four close
|
|
paths the binding owns.
|
|
"""
|
|
css = _code(_css(client))
|
|
assert re.search(r"--nav-toggle-display:\s*none", css), \
|
|
"the nav toggle is no longer hidden by default — the desktop masthead grew a ☰"
|
|
assert re.search(r"#nav-toggle\s*\{\s*display:\s*var\(--nav-toggle-display\)", css), \
|
|
"#nav-toggle stopped reading the token that keeps its reveal order-independent"
|
|
|
|
nav_css = _code(_css_section(_css(client), "mobile nav"))
|
|
media = re.search(r"@media\s*\(max-width:\s*(\d+)px\)", nav_css)
|
|
assert media, "the mobile nav block lost its media query"
|
|
for rule in ("--nav-toggle-display: inline-flex", # the reveal, through task 01's token
|
|
"nav.tabs.is-open", # the links leave the row as the menu
|
|
"position: absolute",
|
|
"top: 100%", # pinned under the whole masthead
|
|
"z-index: 41", # one above the masthead's 40
|
|
"background: var(--bg-raised)",
|
|
"flex-wrap: wrap", # the controls wrap, none is hidden
|
|
# the safe-area inset keeps the first-row #nav-toggle out of the system
|
|
# top-edge zone a real phone's status bar owns (taps there never reach
|
|
# the page; headless emulation has no such zone, so only this pin says
|
|
# the bar still pads for it):
|
|
"padding-top: calc(.5rem + env(safe-area-inset-top, 0px))"):
|
|
assert rule in nav_css, f"the ≤640px nav block lost `{rule}`"
|
|
# Nothing on the phone masthead may be hidden to make the width work: the cluster wraps.
|
|
assert ".masthead-right" in nav_css and "#nav-status" in nav_css, \
|
|
"the mobile masthead hid the status cluster instead of wrapping it"
|
|
|
|
js = _code(client.get("/static/app.js").text)
|
|
binding = js[js.index("const navToggle ="):js.index("function rangeControl(")]
|
|
assert "if (navToggle && appNav)" in binding, \
|
|
"the binding lost the guard that makes it inert on /login"
|
|
assert binding.count("classList.toggle('is-open'") == 1, \
|
|
"a state change now bypasses setNavMenu(), so is-open and aria-expanded can diverge"
|
|
for path in ("navToggle.setAttribute('aria-expanded'",
|
|
"navToggle.addEventListener('click'", # close (and open) on the button
|
|
"event.target.closest('a')", # close on a tab click, delegated
|
|
"event.key === 'Escape'", # close on Esc…
|
|
"navToggle.focus()", # …with focus back on the opener
|
|
"if (!mq.matches) setNavMenu(false)"): # close on resize back to desktop
|
|
assert path in binding, f"the nav binding lost a close path: {path}"
|
|
assert "mq.addEventListener" in binding, "the resize-back close was never registered"
|
|
|
|
js_bp = re.search(r"matchMedia\('\(max-width: (\d+)px\)'\)", js)
|
|
assert js_bp, "app.js no longer reads the nav breakpoint through matchMedia"
|
|
assert js_bp.group(1) == media.group(1), (
|
|
f"the nav breakpoint drifted: CSS hides the tabs below {media.group(1)}px, "
|
|
f"app.js closes the menu below {js_bp.group(1)}px")
|
|
assert media.group(1) == "640", "the nav breakpoint moved off the 640px the phase shipped"
|
|
|
|
|
|
# ---- the three-row masthead (phase 20 task 01) ------------------------------ #
|
|
#
|
|
# `TODO.md:5-6` asked for the per-source indicators and the `⚷ Certs ⚙ Account ☾ Dark` cluster
|
|
# each on their own line (phase 25 task 02 moved both of those lines flush right; the *rows* and
|
|
# the mechanism below are phase 20's). The markup could not change — `#nav-status` is a child
|
|
# of `.masthead-right`, and the tab structure is pinned by
|
|
# `test_the_disks_tab_is_in_the_masthead_of_every_page` — so the rows are made of line breaks:
|
|
# in a `flex-wrap: wrap` container an item whose *base* size fills the line is alone on it (the
|
|
# break happens before shrink), so `flex-basis: 100%` on `.masthead-right` starts row 2 of the
|
|
# bar and the same declaration on the dots starts row 2 *inside* it, pushing the controls to
|
|
# row 3. The alternative the task allowed — `flex-direction: column` — would have put the brand
|
|
# and the tabs on separate lines, which is why the fallback shipped. A browser proves the
|
|
# geometry at 1280 px and 360 px (`data/logs/20-task01-rows.cjs`, `masthead-rows*.png`); these
|
|
# pins keep a later re-flow from putting the buttons back on the dots' line in silence.
|
|
|
|
def _rule_body(css: str, selector: str) -> str:
|
|
"""One rule's declaration block, with the comments stripped.
|
|
|
|
Anchored on the selector itself rather than a section slice: `.masthead-right {` must not
|
|
satisfy a lookup for `.masthead {`, and the ≤640px block restates both selectors.
|
|
"""
|
|
match = re.search(rf"{re.escape(selector)}\s*\{{([^}}]*)\}}", _code(css))
|
|
assert match, f"dash.css no longer has a `{selector} {{ … }}` rule"
|
|
return match.group(1)
|
|
|
|
|
|
def _declarations(css: str, selector: str) -> dict[str, str]:
|
|
"""That block as `{property: value}`, so a pin can ask whether a property exists at all."""
|
|
found: dict[str, str] = {}
|
|
for declaration in _rule_body(css, selector).split(";"):
|
|
prop, sep, value = declaration.partition(":")
|
|
if sep:
|
|
found[prop.strip()] = value.strip()
|
|
return found
|
|
|
|
|
|
def _cascade(css: str, *selectors: str) -> dict[str, str]:
|
|
"""The declarations of several rules merged in cascade order, later winning.
|
|
|
|
Needed for the ≤640px menu: since phase 20 task 03 the dropdown's *geometry* lives on the
|
|
closed `nav.tabs` rule and `.is-open` carries only the visible counterpart, because a box
|
|
that does not exist cannot animate. A pin that asks "does the open menu sit at `top: 100%`
|
|
and paint above the bar?" is asking about the element, so it has to read both rules the way
|
|
the browser does — one duplicated `position` in two rules is the drift this avoids.
|
|
"""
|
|
merged: dict[str, str] = {}
|
|
for selector in selectors:
|
|
merged.update(_declarations(css, selector))
|
|
return merged
|
|
|
|
|
|
def test_the_masthead_is_three_rows_right_justified_at_every_width(client) -> None:
|
|
"""Brand + tabs, then the indicators, then the controls — three rows, two of them flush right.
|
|
|
|
Every assertion here is one declaration the row layout depends on, read out of the
|
|
**served** stylesheet: `flex-wrap` plus the two `flex-basis: 100%` rules are the rows, the
|
|
two `justify-content` values are the right justification (phase 25 task 02, after the
|
|
operator overruled phase 20's `flex-start`), and the absence of `margin-inline-start` on the
|
|
cluster keeps the *mechanism* the alignment rather than the old single-row right-pinning.
|
|
"""
|
|
css = _css(client)
|
|
bar = _declarations(css, ".masthead")
|
|
assert bar.get("flex-wrap") == "wrap" or bar.get("flex-direction") == "column", \
|
|
"the masthead can no longer break into rows: indicators and controls share a line again"
|
|
assert bar.get("flex-direction", "row") == "row", \
|
|
"flex-direction: column puts the brand and the tabs on separate rows"
|
|
assert "height" not in bar, \
|
|
".masthead has a fixed height again — three rows do not fit in --header-h and clip"
|
|
assert bar.get("min-height") == "var(--header-h)", \
|
|
".masthead lost the min-height that keeps a dots-less page from collapsing to one line"
|
|
assert bar.get("gap", "").startswith(".35rem"), \
|
|
"the row gap the three rows need is gone from the masthead's shorthand gap"
|
|
|
|
cluster = _declarations(css, ".masthead-right")
|
|
assert cluster.get("flex-basis") == "100%", \
|
|
"the control cluster no longer starts its own row under the indicators"
|
|
assert cluster.get("justify-content") == "flex-end", \
|
|
"the controls are no longer right-justified in their own row"
|
|
assert "margin-inline-start" not in cluster, \
|
|
"margin-inline-start: auto pushes the cluster back to the right edge of its own row"
|
|
|
|
dots = _declarations(css, "#nav-status")
|
|
assert dots.get("flex-basis") == "100%", \
|
|
"the status indicators no longer get a row to themselves"
|
|
assert dots.get("justify-content") == "flex-end", \
|
|
"the indicators are no longer right-justified"
|
|
|
|
|
|
def test_the_mobile_masthead_block_declares_no_second_bar_height(client) -> None:
|
|
"""One `min-height` in the file, and the safe-area padding still on the phone bar.
|
|
|
|
The ≤640px block used to own `height: auto` + `min-height` because the base rule was a
|
|
fixed `--header-h`; task 01 moved the growth to the base rule, and a mobile restatement is
|
|
exactly the pair that drifts (one says `auto`, the other a number, and the bar clips on a
|
|
phone only). The safe-area line and the wrapping cluster are the phase-18 contract and stay.
|
|
"""
|
|
css = _css(client)
|
|
nav_css = _code(_css_section(css, "mobile nav"))
|
|
block = re.search(r"\.masthead\s*\{([^}]*)\}", nav_css)
|
|
assert block, "the ≤640px block no longer styles the masthead at all"
|
|
for prop in ("height", "min-height", "flex-wrap"):
|
|
assert not re.search(rf"(^|;)\s*{prop}\s*:", block.group(1)), \
|
|
f"the ≤640px masthead re-declares `{prop}` — two rules now decide the bar's height"
|
|
assert "row-gap: .4rem" in block.group(1), "the phone lost its tighter row gap"
|
|
assert "padding-top: calc(.5rem + env(safe-area-inset-top, 0px))" in block.group(1), \
|
|
"the phone bar stopped padding for the system top-edge zone the toggle sits in"
|
|
# The dots stay one line and the cluster still wraps rather than hiding a control.
|
|
assert "#nav-status { flex-wrap: nowrap; }" in nav_css
|
|
assert re.search(r"\.masthead-right\s*\{[^}]*flex-wrap: wrap", nav_css)
|
|
|
|
|
|
# ---- the masthead that scrolls away (phase 20 task 02) ---------------------- #
|
|
#
|
|
# `TODO.md:10` — "The navbar is only sticky part way down certain pages on mobile? It shouldn't
|
|
# be sticky at all. We want to maximize the amount of screen space for data viewing." The quirk
|
|
# and the footprint had the same cause, so both went with the stick. What has to stay is the
|
|
# bar's second job: it is the containing block the ≤640px dropdown's `top: 100%` resolves
|
|
# against, which is why the declaration is `relative` and not the default `static`. These pins
|
|
# read the **served** stylesheet, so a re-stick or a `static` regression fails here rather than
|
|
# in a phone's hands. `data/logs/20-task02-unsticky.cjs` is the browser half: scrolled bars at
|
|
# 1280 px and 360 px, and the menu still landing under the bar.
|
|
|
|
def test_the_masthead_scrolls_away_and_stays_the_dropdown_anchor(client) -> None:
|
|
"""Not `sticky`, not `static` either: a relative bar scrolls and still anchors the menu.
|
|
|
|
Every assertion is one half of that trade. `sticky` (with its `top: 0`) is the pinning the
|
|
task removes and a `top` on a relative bar would shift the whole masthead down instead;
|
|
`position: relative` is the positioned ancestor the dropdown needs; the two `z-index`es are
|
|
the stacking sentence "the menu paints one above the bar", which survives the un-stick.
|
|
"""
|
|
css = _css(client)
|
|
bar = _declarations(css, ".masthead")
|
|
assert bar.get("position") == "relative", (
|
|
"the masthead stopped being the positioned ancestor the ≤640px dropdown resolves "
|
|
"against — `sticky` pins the bar to the viewport, `static` moves the menu to the top "
|
|
"of the document")
|
|
assert "sticky" not in bar.values(), \
|
|
".masthead is sticky again: the bar must scroll away for data viewing (TODO.md:10)"
|
|
assert "top" not in bar, \
|
|
"`top` belongs to a sticky bar; on a relative one it offsets the whole masthead"
|
|
assert bar.get("z-index") == "40", \
|
|
".masthead lost its z-index, so the dropdown's 41 no longer means one above the bar"
|
|
|
|
# The mobile half of the contract: the menu still opens directly under the bar and still
|
|
# paints above the content it overlaps — with the bar itself now free to scroll. Read as a
|
|
# cascade: task 03 moved the box onto the closed rule so the slide has something to animate,
|
|
# and `.is-open` is the visibility flip on top of it.
|
|
nav_css = _code(_css_section(css, "mobile nav"))
|
|
dropdown = _cascade(nav_css, "nav.tabs", "nav.tabs.is-open")
|
|
assert dropdown.get("position") == "absolute" and dropdown.get("top") == "100%", \
|
|
"the ≤640px menu no longer opens directly under the masthead"
|
|
assert int(dropdown["z-index"]) > int(bar["z-index"]), \
|
|
"the dropdown no longer stacks above the masthead it hangs from"
|
|
|
|
|
|
# ---- the menu that slides (phase 20 task 03) -------------------------------- #
|
|
#
|
|
# `TODO.md:11` — "I want a nice animation for the hamburger menu - a slide out and slide back in
|
|
# would be perfect. See ~/Project/Personal/brain-of-reese for an example." (The project is
|
|
# `brain_of_reese`; the pattern is its `.app-nav` closed/open transition pair plus the phase-46
|
|
# reduced-motion block.) It is pure CSS over the class flip `setNavMenu()` already performs,
|
|
# which is why `app.js` and its four close paths are untouched. What that pattern *does* force
|
|
# is a re-cut of the two rules: a `display: none` box has no intermediate state, so the closed
|
|
# menu is `visibility: hidden` now and carries the dropdown geometry the transition animates,
|
|
# while `.is-open` carries only the visible counterpart. Three questions are pinned here, in the
|
|
# order that makes the pair readable: does the closed state name the invisible, non-interactive
|
|
# start of the slide; does the open state name its visible end with the `visibility` delay
|
|
# dropped; and does `prefers-reduced-motion` name **both** states, because an open rule left out
|
|
# of that block keeps animating. `data/logs/20-task03-slide.cjs` is the browser half: the
|
|
# computed closed state, a mid-transition frame, the four close paths, the no-phantom-scroll
|
|
# check, and the snap under `prefers-reduced-motion: reduce`.
|
|
|
|
#: The one duration the slide uses, quoted into both states so a pin can demand they agree.
|
|
NAV_SLIDE_MS = "180ms ease"
|
|
|
|
|
|
def test_the_closed_phone_menu_is_the_start_of_the_slide(client) -> None:
|
|
"""Invisible, 8 px high, unhittable — and a box that exists, so it can animate.
|
|
|
|
Every declaration is one half of the closed state the reference pattern needs. `display:
|
|
none` is the one thing the block must **not** contain: it removes the box, and a transition
|
|
on a box that is not rendered never runs — which is exactly why the menu used to snap, and
|
|
why the last assertion is a regression guard and not a style preference.
|
|
"""
|
|
nav_css = _code(_css_section(_css(client), "mobile nav"))
|
|
closed = _declarations(nav_css, "nav.tabs")
|
|
assert closed.get("visibility") == "hidden", \
|
|
"the closed menu is not hidden by a property that can animate"
|
|
assert closed.get("opacity") == "0", "the slide lost its fade: the menu opens at full ink"
|
|
assert closed.get("transform") == "translateY(-8px)", \
|
|
"the slide lost its 8 px rise (`TODO.md:11`): the menu appears in place"
|
|
assert closed.get("pointer-events") == "none", \
|
|
"an invisible menu that takes clicks steals taps from the content under it"
|
|
assert closed.get("transition") == (
|
|
f"opacity {NAV_SLIDE_MS}, transform {NAV_SLIDE_MS}, visibility 0s linear 180ms"), (
|
|
"the closed state's transition is what runs on the way OUT; `visibility` must be "
|
|
f"delayed by exactly the {NAV_SLIDE_MS} slide or the box vanishes before it fades")
|
|
assert "display: none" not in nav_css, (
|
|
"a `display: none` in the ≤640px nav block is a box that cannot animate — the closed "
|
|
"menu became `visibility: hidden` in phase 20 task 03 for exactly this reason")
|
|
|
|
|
|
def test_the_open_phone_menu_is_the_visible_end_of_the_slide(client) -> None:
|
|
"""`transform: none`, and no `visibility` delay — or the menu is invisible for its own 180 ms.
|
|
|
|
`.is-open` out-ranks `nav.tabs` (two classes to one), so these four declarations are what a
|
|
click on `#nav-toggle` actually applies, and the transition line is restated rather than
|
|
inherited: the open state needs the same two animated properties with the delay gone.
|
|
"""
|
|
nav_css = _code(_css_section(_css(client), "mobile nav"))
|
|
open_state = _declarations(nav_css, "nav.tabs.is-open")
|
|
assert open_state.get("visibility") == "visible", "`.is-open` no longer reveals the menu"
|
|
assert open_state.get("opacity") == "1", "the open menu is still half-faded"
|
|
assert open_state.get("transform") == "none", \
|
|
"the open menu never came down the 8 px, so the slide has no end"
|
|
assert open_state.get("pointer-events") == "auto", \
|
|
"the open menu's five links would not take a tap"
|
|
assert open_state.get("transition") == (
|
|
f"opacity {NAV_SLIDE_MS}, transform {NAV_SLIDE_MS}, visibility 0s"), (
|
|
"the open state must animate the same two properties with the `visibility` delay "
|
|
"dropped — the same duration, in the other direction")
|
|
|
|
|
|
def test_the_menu_snaps_in_both_directions_under_reduced_motion(client) -> None:
|
|
"""One reduced-motion block, and it names **both** menu states.
|
|
|
|
The reference project's phase-46 comment is the reason: `.nav.tabs.is-open` out-specifies
|
|
`.nav.tabs`, so stilling only the closed state leaves the open transition running and the
|
|
menu still animates for someone who asked not to. The block is the file's only one, merged
|
|
with the pre-existing universal rule rather than a second block that would have to be kept
|
|
in step with it.
|
|
"""
|
|
code = _code(_css(client))
|
|
blocks = re.findall(r"@media\s*\(prefers-reduced-motion:\s*reduce\)", code)
|
|
assert len(blocks) == 1, (
|
|
f"dash.css carries {len(blocks)} reduced-motion blocks; the menu's rules belong in the "
|
|
"one universal rule, or the two drift")
|
|
block = re.search(r"@media\s*\(prefers-reduced-motion:\s*reduce\)\s*\{(.*?)\n\}", code, re.S)
|
|
assert block, "the reduced-motion block no longer has a body to read"
|
|
selector = re.search(r"(nav\.tabs[^{]*)\{", block.group(1))
|
|
assert selector, "the reduced-motion block stopped naming the menu at all"
|
|
named = {s.strip() for s in selector.group(1).split(",")}
|
|
assert named == {"nav.tabs", "nav.tabs.is-open"}, (
|
|
"the reduced-motion block must name both menu states: naming only the closed one "
|
|
f"leaves the open transition animating — got {sorted(named)}")
|
|
assert _declarations(block.group(1), "nav.tabs.is-open").get("transition") == "none", \
|
|
"the named open rule stopped stilling the transition"
|
|
|
|
|
|
# ---- the masthead with no polling control (phase 21 task 01) -------------- #
|
|
#
|
|
# `TODO.md:7` — "Completely remove the pause button". The control was four things: a masthead
|
|
# button on the four dashboards, a `Space` binding, the `setPaused`/`_paused` plumbing inside
|
|
# `Dash.poll()`, and a stylesheet comment naming it as a row-3 tenant. A browser is the only
|
|
# thing that can prove the operator no longer sees a button, but the served bytes are what ship
|
|
# it, so those are what fail a test when someone puts it back.
|
|
#
|
|
# The **hidden-tab** skip is deliberately *not* part of the removal and is pinned on the same
|
|
# page for that reason: a backgrounded tab asking for nothing is idle hygiene rather than a
|
|
# decision the operator made, and a poll loop "simplified" back to an unconditional `setTimeout`
|
|
# would be invisible until a laptop battery complained.
|
|
|
|
#: Every spelling the deleted control ever had, in markup, script or stylesheet.
|
|
PAUSE_ARTIFACTS = ('id="pause"', 'name="pause"', "⏸", "▶ Paused",
|
|
"bindPause", "setPaused", "_paused")
|
|
|
|
|
|
def test_no_page_serves_a_button_that_stops_the_polls(client, tmp_path: Path) -> None:
|
|
"""Eight pages, eleven assets, zero pause control.
|
|
|
|
`/login` needs its own anonymous `TestClient` for the reason every other login-page audit
|
|
gives: this `client` is signed in, and a signed-in `GET /login` is a `303` to the dashboard,
|
|
which would audit `/`'s bytes and call them `/login`. The other seven are read as themselves.
|
|
"""
|
|
bodies = {path: client.get(path).text for path in MARKUP_PAGES}
|
|
|
|
db = str(tmp_path / "no-pause-login.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
try:
|
|
with TestClient(app) as anon:
|
|
response = anon.get("/login")
|
|
assert response.status_code == 200, \
|
|
"a signed-out /login must be the form, not a bounce"
|
|
bodies["/login"] = response.text
|
|
finally:
|
|
store.close()
|
|
|
|
assert len(bodies) == 8, "the audit lost a page; every served page owes this check"
|
|
for path, html in bodies.items():
|
|
for artifact in PAUSE_ARTIFACTS:
|
|
assert artifact not in html, f"{path} serves the removed control again: {artifact!r}"
|
|
|
|
for asset in ASSETS:
|
|
body = client.get(asset).text
|
|
for artifact in PAUSE_ARTIFACTS:
|
|
assert artifact not in body, \
|
|
f"{asset} still carries the removed control: {artifact!r}"
|
|
|
|
|
|
def test_the_poll_loop_still_idles_on_a_hidden_tab_and_binds_no_key_for_the_old_control(
|
|
client,
|
|
) -> None:
|
|
"""What survives the deletion, pinned beside what must not come back.
|
|
|
|
The first block is the hidden-tab behaviour the phase keeps (`document.hidden` skips the run
|
|
and reschedules; `visibilitychange` fires one poll on return). The second is the `Space`
|
|
binding: no `event.code` check may live in the shared script, because a key that stops the
|
|
numbers with no control on screen to say so is the failure mode this task exists to retire.
|
|
The third is the half of the old listener that was never about pausing — `T` still toggles
|
|
the theme, on the same four pages it always did.
|
|
"""
|
|
app_js = client.get("/static/app.js").text
|
|
loop = app_js[app_js.index("function poll(fn, ms) {"):app_js.index(" // ---- chrome")]
|
|
assert "if (stopped || inFlight || document.hidden) { schedule(); return; }" in loop, \
|
|
"a hidden tab no longer skips its poll — the idle-hygiene guard is not the button"
|
|
assert "document.addEventListener('visibilitychange', () => { if (!document.hidden) tick(); })" in loop, \
|
|
"returning to a backgrounded tab no longer fires a poll — the page would sit on old numbers"
|
|
|
|
assert "event.code" not in app_js, \
|
|
"a key-code binding is back in the shared script — the only one it carried was the deleted control's"
|
|
assert "'Space'" not in app_js, "the deleted control's keyboard shortcut came back"
|
|
|
|
assert "event.key === 't' || event.key === 'T'" in app_js, \
|
|
"`T` stopped toggling the theme: the shortcut that shared the deleted binding survives"
|
|
for name in ("page-system.js", "page-disks.js", "page-llm.js", "page-ups.js"):
|
|
assert "Dash.bindShortcuts();" in client.get(f"/static/{name}").text, \
|
|
f"{name} no longer opts in to the theme shortcut it always had"
|
|
|
|
|
|
def test_the_header_status_dot_picks_up_the_disks_job(client) -> None:
|
|
"""`disks` gets its own dot instead of hiding inside `hosts`.
|
|
|
|
The same agent can answer the 10-second utilisation poll and have a wedged SMART pass;
|
|
one dot for both would have to be right about neither.
|
|
"""
|
|
app_js = client.get("/static/app.js").text
|
|
assert "{ key: 'disks', label: 'disks', href: '/disks' }" in app_js
|
|
assert "s.dashboard === job.key" in app_js, "the dot stopped filtering by dashboard"
|
|
|
|
|
|
def test_disks_page_draws_one_card_per_physical_drive_from_the_api(client) -> None:
|
|
"""The card is a drive, keyed by the agent's device name, patched in place.
|
|
|
|
A mount or a partition would need a filesystem field, and `/system` already answers "is
|
|
the filesystem full"; if one leaks back in here, the page stops being about the drive.
|
|
"""
|
|
html = client.get("/disks").text
|
|
js = _disks_js(client)
|
|
assert 'id="hosts"' in html and 'id="hosts-empty"' in html
|
|
assert "fetchJSON('/api/disks/snapshot')" in js
|
|
assert "row.path" in js and "/dev/" in js, "the card stopped naming the device it is about"
|
|
# identity + verdict lead the card: no hero row, no big single number
|
|
assert 'class="heroes"' not in html, "the disk card grew a hero row"
|
|
assert "Dash.hero(" not in js
|
|
for filesystem in ("mountpoint", "fstype", "free_bytes", "worst_mount"):
|
|
assert filesystem not in js, f"{filesystem} leaked into the drive page"
|
|
# built once per device and patched, so a 16-drive host does not rebuild 16 panels a poll
|
|
assert "seen.get(row.device)" in js and "renderDiskCard(panel, row)" in js
|
|
assert "pruneHosts(" in js, "a drive that left the host keeps its card forever"
|
|
|
|
|
|
def test_disk_cards_print_the_drive_numbers_and_colour_only_rated_ones(client) -> None:
|
|
"""Temp, power-on hours and wear, coloured by the API's verdicts and nothing else.
|
|
|
|
The odometer is the deliberate exception: `severity.THRESHOLDS` has no `disk.power_on_hours`
|
|
row, so the cell is written with a `null` verdict and cannot go green at 100 h or red at
|
|
60 000 h. A drive that publishes no wear (the recorded HDD shape) prints `n/a` because
|
|
`setTxt` is the only writer.
|
|
"""
|
|
js = _disks_js(client)
|
|
for field in ("smart.temp_c", "smart.power_on_hours", "smart.percentage_used",
|
|
"row.health"):
|
|
assert field in js, f"{field} is no longer on the card"
|
|
for verdict in ("severity.temp_c", "severity.percentage_used", "severity.health"):
|
|
assert verdict in js, f"the card stopped rendering {verdict}"
|
|
assert "severityFor" not in js, "the drive page wrote a threshold of its own"
|
|
body = js[js.index("function renderDiskCard("):js.index("/** Text that prints nothing")]
|
|
assert "severity.temp_c" in body and "severity.percentage_used" in body
|
|
assert re.search(r"setMetric\(card, 'hours',[^;]*null\)", body), (
|
|
"the odometer is being graded by something severity.py does not have")
|
|
# the drive's own word, and `unknown` for a drive that gave none — never a green pass
|
|
assert "'FAILED'" in body and "'unknown'" in body
|
|
|
|
|
|
def test_disk_alarm_chips_exist_only_while_a_counter_is_non_zero(client) -> None:
|
|
"""Zero and unknown both draw no chip; only a non-zero counter draws a red one.
|
|
|
|
The `/llm` preemption rule: a permanent "0 pending sectors" row trains the eye to ignore
|
|
it, and grading a counter green would call a quiet drive an achievement. `severity.py`
|
|
rates none of these, so the chip's presence is the entire signal here.
|
|
"""
|
|
js = _disks_js(client)
|
|
for counter in ("reallocated_sector_ct", "current_pending_sector_ct",
|
|
"offline_uncorrectable"):
|
|
assert counter in js, f"{counter} is no longer an alarm"
|
|
assert "value <= 0) continue" in js, "a zero now draws a chip"
|
|
assert "alarms.hidden = chips.length === 0" in js, "the empty chip row stays on screen"
|
|
assert 'class="chip fail"' in js
|
|
css = client.get("/static/dash.css").text
|
|
assert ".alarms {" in css and ".chip.fail" in css
|
|
|
|
|
|
def test_a_host_without_smartctl_is_dimmed_with_a_one_line_why_and_never_red(client) -> None:
|
|
"""Unconfigured is not broken (invariant 3), and the three answers stay three.
|
|
|
|
`smartctl_available` is `false` (install smartmontools — the agent said so, so the page
|
|
may say it), `null` (nobody told us either way, so name the fix without claiming to know
|
|
which half is missing), or the agent is not answering at all, which is a different
|
|
sentence again. None of the three reddens anything.
|
|
"""
|
|
js = _disks_js(client)
|
|
html = client.get("/disks").text
|
|
assert "smartctl not available on" in js
|
|
assert "disk group" in js, "the why stopped naming the grant that fixes it"
|
|
assert "smartctl_available === false" in js and "smartctl_available === true" in js
|
|
assert "host.unreachable" in js, "a dead agent must not be reported as a missing binary"
|
|
assert "smart_errors" in js, "the agent's own sentence is no longer printed"
|
|
assert "classList.toggle('stale'" in js
|
|
assert 'class="banner info why" hidden' in js, "the why-line has no home on the card"
|
|
assert "banner fail" not in js and html.count("banner") == 0, (
|
|
"a missing binary reddens the page")
|
|
|
|
|
|
def test_disks_charts_are_collapsed_lazy_and_cost_one_history_request_per_host(client) -> None:
|
|
"""`/system`'s chart discipline, reused: collapsed, painted only while open, one XHR.
|
|
|
|
The request still names every series the host could draw (temperature and wear per
|
|
drive), because `/api/disks/history` answers an unknown name as an empty series and a
|
|
panel should never have to discover series it did not ask for.
|
|
"""
|
|
js = _disks_js(client)
|
|
assert '<details class="charts-box"' in js
|
|
guard = js[js.index("function drawHostCharts("):js.index("async function renderHostCharts(")]
|
|
assert "if (!chartsOpen(card)) return;" in guard, (
|
|
"a collapsed stack is being built and drawn every poll")
|
|
history = js[js.index("async function renderHostCharts("):js.index("/**\n * Drop cards")]
|
|
assert history.count("fetchJSON(") == 1, "more than one history request for one host"
|
|
assert "/api/disks/history" in history
|
|
assert "streams.temp.concat(streams.wear)" in history, (
|
|
"the request no longer names every series the panel could draw")
|
|
prune = js[js.index("function pruneHosts("):js.index("// ---- fleet header")]
|
|
for entry in ("state.hosts.delete", "state.disks.delete", "state.charts.delete",
|
|
"state.history.delete", "state.payloads.delete"):
|
|
assert entry in prune, f"a pruned host keeps its {entry}"
|
|
|
|
|
|
def test_every_disks_chart_names_itself_for_assistive_tech(client) -> None:
|
|
"""An unlabelled plot is silence, and the legend is a sibling that labels nothing."""
|
|
js = _code(_disks_js(client))
|
|
labels = re.findall(r"ariaLabel: `([^`]+)`", js)
|
|
assert len(labels) == 2, "every drive chart must name itself (temperature, wear)"
|
|
assert all(label.startswith("${host.name}") for label in labels), (
|
|
"a chart that does not say which host it plots belongs on no page")
|
|
|
|
|
|
def test_the_disks_fleet_header_reads_the_api_sums_and_invents_none(client) -> None:
|
|
"""The header's arithmetic is the API's (`fleet_disk_summary`), never the page's.
|
|
|
|
Hidden below two hosts, because with one box in `HOST_TARGETS` these sums only duplicate
|
|
that host's own cards — `/llm`'s fleet-block rule. And no cell is coloured: a count of
|
|
drives is not a graded number (`severity.py` has no row for one).
|
|
"""
|
|
html = client.get("/disks").text
|
|
js = _disks_js(client)
|
|
for element in ("fleet", "fleet-dot", "fleet-count", "fleet-disks", "fleet-smart",
|
|
"fleet-failed", "fleet-alarms"):
|
|
assert f'id="{element}"' in html, f"the header lost #{element}"
|
|
fleet = js[js.index("function renderFleet("):js.index("// ---- snapshot")]
|
|
for path in ("fleet.hosts_live", "fleet.disks", "fleet.with_smart",
|
|
"fleet.health_failed", "fleet.alarm_disks"):
|
|
assert path in fleet, f"the header no longer reads {path}"
|
|
assert "reduce(" not in fleet and "+=" not in fleet, "the page is summing drives itself"
|
|
assert "severity" not in fleet, "a drive count grew a verdict"
|
|
assert "block.hidden = true" in fleet, "one host's sums duplicate its own cards"
|
|
|
|
|
|
def test_the_disk_panel_style_is_existing_vocabulary_with_no_new_palette(client) -> None:
|
|
"""`.disk` is the `.slot` shape, and the only colours it reaches for are the verdicts."""
|
|
css = _css(client)
|
|
block = _css_section(css, "drive panels")
|
|
for selector in (".disks {", ".disk {", ".disk-head", ".disk-dev", ".disk-model",
|
|
".disk .metrics", ".disk .alarms", ".disk .metric.warn .value",
|
|
".disk .metric.fail .value"):
|
|
assert selector in block, f"dash.css lost {selector}"
|
|
for token in ("var(--warn)", "var(--fail)", "var(--border)", "var(--bg-sunken)",
|
|
"var(--text)"):
|
|
assert token in block, f"the drive panel no longer uses {token}"
|
|
assert "--disk" not in css, "a drive-only colour token appeared"
|
|
assert ".stale" in css, "the dim that marks an unreadable host is gone"
|
|
|
|
|
|
def test_the_overview_prints_the_fourth_source_cadence(client) -> None:
|
|
"""`/` names every source's interval, so a drive card four minutes old reads as behaving."""
|
|
js = client.get("/static/page-index.js").text
|
|
assert "config.disks.interval_seconds" in js
|
|
|
|
|
|
@pytest.mark.parametrize("fixture", ["client", "data_client"])
|
|
def test_summary_carries_the_drive_block_for_programmatic_consumers(
|
|
request, fixture: str) -> None:
|
|
"""`disks` is additive: the same configured-ness, the same hosts, no key retired.
|
|
|
|
`client` has no targets, so it proves the block answers as `False`/`None`/`[]` instead of
|
|
disappearing; `data_client` proves it lists the host the fleet has.
|
|
"""
|
|
test_client = request.getfixturevalue(fixture)
|
|
body = test_client.get("/api/summary").json()
|
|
populated = fixture == "data_client"
|
|
assert body["disks"]["enabled"] is populated
|
|
assert (body["disks"]["fleet"] is not None) is populated
|
|
assert body["disks"]["hosts"] == ([] if not populated else [
|
|
{"name": "driveripper", "disks": None, "with_smart": None, "health_failed": None,
|
|
"alarm_disks": None, "smartctl_available": None, "failed_devices": [],
|
|
"unreachable": True}]), "a configured host with no payload is a row of nones"
|
|
assert [row["name"] for row in body["disks"]["hosts"]] == [
|
|
row["name"] for row in body["system"]["hosts"]]
|
|
|
|
|
|
def test_summary_serves_the_same_counts_the_disks_page_prints(disks_client) -> None:
|
|
"""One projection, two consumers: the summary's numbers *are* the snapshot's."""
|
|
body = disks_client.get("/api/summary").json()
|
|
snapshot = disks_client.get("/api/disks/snapshot").json()
|
|
disks = body["disks"]
|
|
assert disks["fleet"] == snapshot["fleet"], "the header would disagree with the page"
|
|
rows = {row["name"]: row for row in disks["hosts"]}
|
|
live = next(host for host in snapshot["hosts"] if host["name"] == "driveripper")
|
|
assert (rows["driveripper"]["disks"], rows["driveripper"]["with_smart"],
|
|
rows["driveripper"]["health_failed"], rows["driveripper"]["alarm_disks"]) == (
|
|
live["summary"]["disks"], live["summary"]["with_smart"],
|
|
live["summary"]["health_failed"], live["summary"]["alarm_disks"])
|
|
assert rows["driveripper"]["smartctl_available"] is True
|
|
assert rows["driveripper"]["unreachable"] is False
|
|
assert rows["deskwork"]["unreachable"] is True
|
|
assert rows["deskwork"]["disks"] is None, "unknown, never 0 (rule 2)"
|
|
assert sorted(state["key"] for state in disks["states"]) == ["disks:deskwork",
|
|
"disks:driveripper"]
|
|
|
|
|
|
def test_summary_names_the_drive_the_health_verdict_called_failed(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, disk_snapshot: Any) -> None:
|
|
"""Which drive the hardware itself called `FAILED` is part of the fleet answer.
|
|
|
|
A notifier reading `/api/summary` should not have to fetch a second endpoint to name the
|
|
device, and the count and the list have to agree or one of them is wrong.
|
|
"""
|
|
db = str(tmp_path / "summary-failed.sqlite3")
|
|
store = SampleStore(db)
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2,
|
|
host_targets=(Target("driveripper", DEAD_URL),)), store)
|
|
with TestClient(app) as test_client:
|
|
sign_in(test_client)
|
|
now = time.time()
|
|
snapshot = disk_snapshot("driveripper", mode="failed")
|
|
snapshot["ts"] = now
|
|
result = parse_disks_snapshot("driveripper", snapshot, DisksCollector(), now=now)
|
|
app.state.ctx.hub.jobs["disks:driveripper"].state.payload = result.payload
|
|
row = test_client.get("/api/summary").json()["disks"]["hosts"][0]
|
|
assert len(row["failed_devices"]) == 1
|
|
assert row["failed_devices"][0] == result.payload["disks"][0]["device"]
|
|
assert row["health_failed"] == result.payload["summary"]["health_failed"] == 1
|
|
store.close()
|
|
|
|
|
|
# ---- events --------------------------------------------------------------- #
|
|
|
|
def test_events_are_returned_in_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
db = str(tmp_path / "events.sqlite3")
|
|
store = SampleStore(db)
|
|
store.log_event(Event(ts=time.time(), source="host", host="deskwork", kind="down",
|
|
severity="error", message="deskwork is not answering"))
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
with TestClient(app) as client:
|
|
sign_in(client)
|
|
events = client.get("/api/events").json()["events"]
|
|
assert events[0]["host"] == "deskwork"
|
|
assert events[0]["severity"] == "error"
|
|
store.close()
|
|
|
|
|
|
def test_event_rows_say_their_subject_once(client) -> None:
|
|
"""`poller.py` messages name their own subject, and the row prints a host chip too.
|
|
|
|
"gufo gufo is answering again after 1 failure" reads as two problems — the same defect
|
|
`alerts._clean_error()` already fixes for the overview's alert list. `/ups` had the
|
|
mirror-image bug: it `replace()`d `host + ':'` *anywhere*, so "Quick battery test sent
|
|
to apc: OK" printed as "Quick battery test sent to OK". All three feeds now share
|
|
`Dash.eventText()`, which strips the wrapper and a **leading** name only.
|
|
"""
|
|
app_js = client.get("/static/app.js").text
|
|
helper = app_js[app_js.index("function eventText("):app_js.index("// ---- DOM helpers")]
|
|
assert "eventText" in app_js[app_js.index("return {"):], "eventText is not exported"
|
|
assert "^UpstreamError:\\s+" in helper, "the wrapper `alerts.py` drops is back"
|
|
assert "new RegExp" in helper and "stripped.trim() ? stripped : out" in helper, \
|
|
"the host strip is no longer anchored to the front, or can empty a message"
|
|
for page in ("/static/page-system.js", "/static/page-llm.js", "/static/page-ups.js"):
|
|
js = client.get(page).text
|
|
assert "eventText(event)" in js, f"{page} prints the raw message again"
|
|
assert "replace(event.host" not in client.get("/static/page-ups.js").text, \
|
|
"/ups is mangling the host name mid-message again"
|
|
|
|
|
|
# ---- the one write endpoint ------------------------------------------------ #
|
|
|
|
def test_command_endpoint_is_post_only(ups_client) -> None:
|
|
assert ups_client.post("/api/ups/command", json={}).status_code in (400, 403, 404)
|
|
assert ups_client.put("/api/ups/command", json={}).status_code == 405
|
|
assert ups_client.delete("/api/ups/command").status_code == 405
|
|
|
|
|
|
def test_commands_are_offered_when_actions_are_enabled(ups_client) -> None:
|
|
body = wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
assert body["simulated"] is True, "the simulator must label itself"
|
|
assert body["actions_enabled"] is True
|
|
ups = body["ups"][0]
|
|
enabled = [item for item in ups["commands"] if item["enabled"]]
|
|
assert enabled, "with NUT_ALLOW_ACTIONS=1 and a simulator, safe commands are sendable"
|
|
assert all(item["risk"] in ("safe", "caution", "dangerous") for item in ups["commands"])
|
|
|
|
|
|
def test_a_safe_command_actually_reaches_the_driver(ups_client) -> None:
|
|
body = wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.panel.start"})
|
|
assert response.status_code == 200, response.text
|
|
payload = response.json()
|
|
assert payload["ok"] is True and payload["simulated"] is True
|
|
events = ups_client.get("/api/events").json()["events"]
|
|
assert any("test.panel.start" in event["message"] or "Panel test" in event["message"]
|
|
for event in events), "every sent command must leave an audit trail"
|
|
|
|
|
|
def test_a_dangerous_command_needs_the_confirmation_word(ups_client) -> None:
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
deep = "test.battery.start.deep"
|
|
refusal = ups_client.post("/api/ups/command", json={"ups": name, "command": deep})
|
|
assert refusal.status_code == 422
|
|
assert "DANGER" in refusal.json()["detail"]
|
|
assert ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": deep, "confirm": "DANGER"}).status_code == 200
|
|
|
|
|
|
def test_a_command_the_driver_does_not_advertise_is_refused_before_it_is_sent(
|
|
ups_client) -> None:
|
|
"""Being in the risk catalog is not permission; `LIST CMD` from the driver is (invariant 1).
|
|
|
|
`shutdown.return` is a real NUT command the catalog knows and this UPS does not offer,
|
|
so the refusal must come from the advertisement check and not from the confirmation gate.
|
|
"""
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "shutdown.return"})
|
|
assert response.status_code == 400
|
|
assert "does not advertise" in response.json()["detail"]
|
|
|
|
|
|
def test_cutting_the_output_is_refused_while_the_ups_is_already_on_battery(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Invariant 1, the one that matters most: no power cut while the pack is carrying the load.
|
|
|
|
Driven through the simulator's own state machine — a quick battery test puts the pack on
|
|
battery — so the 409 comes from the same poll the buttons are drawn from, with a 0.5s
|
|
interval to keep the wait short. Nothing here can touch real hardware: `NUT_SIMULATE=1`.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True, nut_interval=0.5) as test_client:
|
|
name = wait_for_payload(test_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
start = test_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.battery.start.quick"})
|
|
assert start.status_code == 200
|
|
deadline = time.time() + 6.0
|
|
card: dict[str, Any] = {}
|
|
while time.time() < deadline:
|
|
card = next(item for item in test_client.get("/api/ups/snapshot").json()["ups"]
|
|
if item["name"] == name)
|
|
if card["on_battery"]:
|
|
break
|
|
time.sleep(0.05)
|
|
assert card.get("on_battery") is True, "the quick test must put the simulated pack on battery"
|
|
refusal = test_client.post("/api/ups/command",
|
|
json={"ups": name, "command": "load.off",
|
|
"confirm": "DANGER"})
|
|
assert refusal.status_code == 409
|
|
assert "on battery" in refusal.json()["detail"]
|
|
# and the same command is still offered once the pack is back on line power
|
|
assert not any(item["command"] == "load.off" and item["enabled"]
|
|
for item in card["commands"]), "the panel must not offer a blocked button"
|
|
|
|
|
|
class ScriptedNutClient:
|
|
"""`nut.NutClient` with the socket left out: the real code path, no battery attached.
|
|
|
|
`AGENTS.md` keeps live tests read-only, so this is where the one write endpoint's real
|
|
branch gets proven. The variable names are the ones `VAR_MAP` already pins from real
|
|
upsd captures; only the replies are scripted (a fake for behaviour, per the test policy).
|
|
"""
|
|
|
|
#: set per test: an exception instance to raise from `instcmd`, e.g. a refusal
|
|
refusal: Exception | None = None
|
|
sent: list[tuple[str, str]]
|
|
logged_in: list[str]
|
|
|
|
def __init__(self, host: str, port: int, *, timeout: float = 4.0,
|
|
username: str | None = None, password: str | None = None) -> None:
|
|
self.credentials = (username, password)
|
|
|
|
async def __aenter__(self) -> ScriptedNutClient:
|
|
return self
|
|
|
|
async def __aexit__(self, *_exc: object) -> None:
|
|
return None
|
|
|
|
async def list_ups(self) -> list[dict[str, str]]:
|
|
return [{"name": "apc", "description": "APC Back-UPS RS 1500MS"}]
|
|
|
|
async def list_vars(self, ups: str) -> dict[str, str]:
|
|
return {"ups.mfr": "APC", "ups.model": "Back-UPS RS 1500MS",
|
|
"ups.status": "OL", "battery.charge": "100", "ups.load": "18",
|
|
"battery.voltage": "13.6", "input.voltage": "230",
|
|
"battery.mfr.date": "2022-04", "ups.test.result": "Ok last test passed"}
|
|
|
|
async def list_cmds(self, ups: str) -> list[str]:
|
|
return ["test.panel.start", "beeper.mute", "load.off"]
|
|
|
|
async def list_rw_vars(self, ups: str) -> list[str]:
|
|
return []
|
|
|
|
async def login(self) -> None:
|
|
assert self.credentials[0], "a command send must authenticate as a named user"
|
|
ScriptedNutClient.logged_in.append(str(self.credentials[0]))
|
|
|
|
async def instcmd(self, ups: str, command: str) -> str:
|
|
if ScriptedNutClient.refusal is not None:
|
|
raise ScriptedNutClient.refusal
|
|
ScriptedNutClient.sent.append((ups, command))
|
|
return "OK Start Panel Test"
|
|
|
|
|
|
@pytest.fixture
|
|
def real_upsd_client(monkeypatch: pytest.MonkeyPatch) -> Iterator[type[ScriptedNutClient]]:
|
|
"""Swap the module-level `NutClient` so the collector and the send use the same stand-in."""
|
|
ScriptedNutClient.sent = []
|
|
ScriptedNutClient.logged_in = []
|
|
ScriptedNutClient.refusal = None
|
|
monkeypatch.setattr(nut, "NutClient", ScriptedNutClient)
|
|
yield ScriptedNutClient
|
|
|
|
|
|
@pytest.fixture
|
|
def nut_client_app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
|
real_upsd_client: type[ScriptedNutClient]) -> Iterator[TestClient]:
|
|
"""A non-simulated UPS app: real code paths, scripted upsd, 0.5s polls."""
|
|
with build_client(tmp_path, monkeypatch, nut_host="127.0.0.1", nut_user="dashboards",
|
|
nut_password="not-the-real-one", nut_allow_actions=True,
|
|
nut_interval=0.5) as test_client:
|
|
yield test_client
|
|
|
|
|
|
def test_a_real_upsd_send_authenticates_sends_and_audits(nut_client_app,
|
|
real_upsd_client) -> None:
|
|
body = wait_for_payload(nut_client_app, "/api/ups/snapshot")
|
|
assert body["simulated"] is False, "this test is about the branch that talks to upsd"
|
|
name = body["ups"][0]["name"]
|
|
response = nut_client_app.post("/api/ups/command",
|
|
json={"ups": name, "command": "test.panel.start"})
|
|
assert response.status_code == 200, response.text
|
|
assert response.json()["simulated"] is False
|
|
assert real_upsd_client.sent == [(name, "test.panel.start")]
|
|
assert real_upsd_client.logged_in, "instcmd requires a LOGIN first (upsd refuses otherwise)"
|
|
events = nut_client_app.get("/api/events").json()["events"]
|
|
assert any(event["kind"] == "action" and name in event["message"] for event in events)
|
|
|
|
|
|
def test_a_refusal_from_upsd_is_a_502_and_an_error_event(nut_client_app,
|
|
real_upsd_client) -> None:
|
|
real_upsd_client.refusal = nut.NutError("Access denied", code="ACCESS-DENIED")
|
|
name = wait_for_payload(nut_client_app, "/api/ups/snapshot")["ups"][0]["name"]
|
|
response = nut_client_app.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 502
|
|
detail = response.json()["detail"]
|
|
assert "ACCESS-DENIED" in detail or "Access denied" in detail, detail
|
|
assert real_upsd_client.sent == [], "a refused command was never accepted"
|
|
events = nut_client_app.get("/api/events").json()["events"]
|
|
failures = [event for event in events if event["kind"] == "action"
|
|
and event["severity"] == "error"]
|
|
assert failures, "an upsd refusal has to leave a trace, not just an HTTP status"
|
|
|
|
|
|
def test_unknown_commands_are_refused_even_with_actions_on(ups_client) -> None:
|
|
name = wait_for_payload(ups_client, "/api/ups/snapshot")["ups"][0]["name"]
|
|
for command in ("load.format", "rm -rf /", "INSTCMD x y"):
|
|
response = ups_client.post("/api/ups/command", json={"ups": name, "command": command})
|
|
assert response.status_code == 400, command
|
|
|
|
|
|
def test_commands_for_an_unknown_ups_are_404(ups_client) -> None:
|
|
wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
response = ups_client.post("/api/ups/command",
|
|
json={"ups": "not-real", "command": "beeper.mute"})
|
|
assert response.status_code == 404
|
|
|
|
|
|
def test_missing_fields_are_a_400(ups_client) -> None:
|
|
assert ups_client.post("/api/ups/command", json={"ups": ""}).status_code == 400
|
|
|
|
|
|
def test_actions_flag_off_blocks_every_command(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1") as client:
|
|
body = wait_for_payload(client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
assert body["actions_enabled"] is False
|
|
response = client.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 403
|
|
assert "NUT_ALLOW_ACTIONS" in response.json()["detail"]
|
|
|
|
|
|
def test_read_only_beats_the_actions_flag(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
with build_client(tmp_path, monkeypatch, simulate=True, nut_host="127.0.0.1",
|
|
nut_allow_actions=True, read_only=True) as client:
|
|
body = wait_for_payload(client, "/api/ups/snapshot")
|
|
name = body["ups"][0]["name"]
|
|
response = client.post("/api/ups/command",
|
|
json={"ups": name, "command": "beeper.mute"})
|
|
assert response.status_code == 403
|
|
commands = wait_for_payload(client, "/api/ups/commands")
|
|
assert commands["read_only"] is True
|
|
assert all(item["enabled"] is False for ups in commands["ups"]
|
|
for item in ups["commands"])
|
|
|
|
|
|
def test_ups_history_answers_for_simulated_ups(ups_client) -> None:
|
|
wait_for_payload(ups_client, "/api/ups/snapshot")
|
|
time.sleep(1.2)
|
|
body = ups_client.get("/api/ups/history", params={"metrics": "ups.charge_pct",
|
|
"range": "1h"}).json()
|
|
assert body["ups"], "the UPS page needs a name to chart before any history exists"
|
|
|
|
|
|
def response_ok(response) -> bool:
|
|
assert response.status_code == 200, response.text
|
|
return True
|
|
|
|
|
|
def test_samples_written_by_the_poller_are_queryable(tmp_path: Path,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The write path (poller -> store) and the read path (/api/history) must agree on names."""
|
|
db = str(tmp_path / "roundtrip.sqlite3")
|
|
store = SampleStore(db)
|
|
store.add_many([Sample("llm", "gufo", "llm.decode_tokens_per_second", 40.0, time.time() - 60),
|
|
Sample("llm", "gufo", "llm.decode_tokens_per_second", 45.0, time.time() - 30)])
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
with TestClient(app) as client:
|
|
sign_in(client)
|
|
metrics = client.get("/api/metrics", params={"source": "llm"}).json()["metrics"]
|
|
assert "llm.decode_tokens_per_second" in metrics
|
|
points = client.get("/api/history", params={"source": "llm", "host": "gufo",
|
|
"metric": "llm.decode_tokens_per_second",
|
|
"range": "15m"}).json()["points"]
|
|
assert [p["value"] for p in points] == [40.0, 45.0]
|
|
store.close()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# The certificate authority surface (phase 15 task 04)
|
|
#
|
|
# These routes mint, re-sign, and revoke the identities every agent in the fleet is dialled with,
|
|
# so the assertions are about **stored rows and wire effects** rather than about JSON being shaped
|
|
# nicely: a read that issued a certificate, a typo that wrote a row, a revocation that did not
|
|
# reach the collector, and a key that left `meta` are the four ways this surface can be wrong while
|
|
# every status code still reads 200.
|
|
#
|
|
# The fleet here is loopback ports nobody listens on. That is deliberate: nothing on this surface
|
|
# is allowed to need a live agent, so a refused connection is both the whole environment and the
|
|
# control that says "a dial was attempted, and it was not this".
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
#: Three configured hosts, none of them reachable, on the scheme a protected exporter is dialled
|
|
#: by. The URL host is also what a rotated certificate's SAN must carry, so these are the SANs the
|
|
#: assertions below read.
|
|
DEAD_TARGETS = (Target(name="deskwork", url="https://127.0.0.1:1"),
|
|
Target(name="driveripper", url="https://127.0.0.1:2"),
|
|
Target(name="gateway", url="https://127.0.0.1:3"))
|
|
|
|
#: The whole `GET /api/certs` document and one agent row, pinned as a table so a panel that
|
|
#: retires can never shrink the payload (`/api/disks`' idiom) and task 05's page has a contract.
|
|
CERT_DOCUMENT_KEYS = {"root", "client", "agents"}
|
|
CERT_TIMESERIAL_KEYS = {"serial", "not_after"}
|
|
CERT_AGENT_KEYS = {"name", "configured", "serial", "not_after", "status", "bundle_rev",
|
|
"deployed_bundle_rev", "san"}
|
|
|
|
|
|
def cert_document(client: TestClient) -> dict[str, Any]:
|
|
"""`GET /api/certs`, with the status asserted: a refusal here would pass a `.get()` chain."""
|
|
response = client.get("/api/certs")
|
|
assert response.status_code == 200, response.text
|
|
body = response.json()
|
|
assert set(body) == CERT_DOCUMENT_KEYS, "the CA document grew or lost a top-level key"
|
|
for block in ("root", "client"):
|
|
assert set(body[block]) == CERT_TIMESERIAL_KEYS, block
|
|
assert int(body[block]["serial"]) > 0, "a serial is the decimal text of a positive int"
|
|
assert body[block]["not_after"] > time.time(), "a stored CA may not already be expired"
|
|
return body
|
|
|
|
|
|
def cert_agent_rows(client: TestClient, name: str | None = None) -> list[dict[str, Any]]:
|
|
"""The agent rows, every one of them shaped, optionally just one host's."""
|
|
rows = cert_document(client)["agents"]
|
|
for row in rows:
|
|
assert set(row) == CERT_AGENT_KEYS, sorted(row)
|
|
if name is None:
|
|
return rows
|
|
return [row for row in rows if row["name"] == name]
|
|
|
|
|
|
def cert_context(client: TestClient) -> Any:
|
|
"""The app's `AppContext`.
|
|
|
|
`TestClient.app` is typed as the ASGI callable it wraps, so the one cast lives here instead of
|
|
one `# type: ignore` at every assertion that needs the store or a job.
|
|
"""
|
|
return client.app.state.ctx # type: ignore[attr-defined]
|
|
|
|
|
|
def cert_meta(store: SampleStore) -> dict[str, str]:
|
|
"""Every `certs.*` row, for the before/after comparisons that prove a route wrote nothing.
|
|
|
|
Coalesced to `""` because the comparison is "did any byte of these five rows move", and an
|
|
absent row is the empty answer to that question rather than a second kind of nothing.
|
|
"""
|
|
return {key: store.get_meta(key) or "" for key in certs.META_KEYS}
|
|
|
|
|
|
def revoked_serials(bundle: str) -> list[str]:
|
|
"""The denylist of a bundle, as the agent reads it: the `# REVOKED <serial>` lines."""
|
|
return [line.removeprefix(certs.REVOKED_LINE).strip()
|
|
for line in bundle.splitlines() if line.startswith(certs.REVOKED_LINE)]
|
|
|
|
|
|
#: One PEM block out of a bundle, by text — the same read task 02's agent does with stdlib.
|
|
_PEM_BLOCK = re.compile(r"-----BEGIN [A-Z0-9 ]+-----.*?-----END [A-Z0-9 ]+-----\n", re.DOTALL)
|
|
|
|
|
|
def bundle_certs(bundle: str) -> list[x509.Certificate]:
|
|
"""The two certificates of a bundle, in the order the format writes them.
|
|
|
|
`[0]` is the agent's own server certificate and `[1]` the root it chains to (`certs.py`'s
|
|
`_bundle_text` order: metadata lines, agent cert, agent key, root cert). Reading them out of the
|
|
served bytes rather than trusting the response's echo is what makes "did the rotation reach the
|
|
artifact?" a checked claim.
|
|
"""
|
|
blocks = [block for block in _PEM_BLOCK.findall(bundle) if "CERTIFICATE" in block]
|
|
assert len(blocks) == 2, f"a bundle is two certificates and a key, got {len(blocks)}"
|
|
return [x509.load_pem_x509_certificate(block.encode()) for block in blocks]
|
|
|
|
|
|
def bundle_of(client: TestClient, name: str) -> str:
|
|
"""The served bundle, asserted to still be a bundle (2 certs, 1 key) before it is compared."""
|
|
response = client.get(f"/api/certs/{name}/bundle")
|
|
assert response.status_code == 200, response.text
|
|
assert response.headers["content-type"].startswith("application/x-pem-file")
|
|
assert "no-store" in response.headers["cache-control"], "a private key may not be cached"
|
|
blocks = re.findall(r"-----BEGIN [A-Z0-9 ]+-----", response.text)
|
|
assert blocks.count("-----BEGIN CERTIFICATE-----") == 2, blocks
|
|
assert blocks.count("-----BEGIN PRIVATE KEY-----") == 1, blocks
|
|
return response.text
|
|
|
|
|
|
def stored_bundle(store: SampleStore, name: str) -> str:
|
|
"""The bundle exactly as the `certs.agents` row holds it."""
|
|
row: dict[str, Any] = json.loads(store.get_meta(certs.META_AGENTS) or "{}")
|
|
return str(row[name]["bundle"])
|
|
|
|
|
|
def cert_events(client: TestClient) -> list[dict[str, Any]]:
|
|
return client.get("/api/events", params={"source": "certs"}).json()["events"]
|
|
|
|
|
|
def cert_hosts(count: int) -> tuple[Target, ...]:
|
|
return DEAD_TARGETS[:count]
|
|
|
|
|
|
@pytest.mark.parametrize("hosts", [0, 1, 3], ids=["none", "one", "three"])
|
|
def test_the_cert_list_is_the_stored_table_for_zero_one_and_three_hosts(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, hosts: int) -> None:
|
|
"""The list works with no fleet at all, and says *unissued* rather than omitting a host.
|
|
|
|
The CA exists whether or not `HOST_TARGETS` does — a fresh database boots with a root and a
|
|
client certificate — so this is not the `/api/disks` "not configured" surface and does not 404.
|
|
A configured host with nothing issued still gets a row (invariant 3: a machine that is deployed
|
|
and not yet installed is not an absent machine), with `None` and not `0` where there is no
|
|
certificate (rule 2).
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(hosts),
|
|
host_interval=3600.0) as client:
|
|
rows = cert_agent_rows(client)
|
|
assert len(rows) == hosts, "one row per configured host, and no more"
|
|
assert [row["name"] for row in rows] == sorted(row["name"] for row in rows), (
|
|
"one order in the payload and one in the table")
|
|
for row in rows:
|
|
assert row["configured"] is True
|
|
assert row["status"] == "unissued", "nothing has been signed for it yet"
|
|
assert row["serial"] is None and row["bundle_rev"] is None, "no invented zeros"
|
|
assert row["deployed_bundle_rev"] is None, "nobody has answered, so nobody reported"
|
|
assert row["san"] == [], "no names on record, not a certificate with no names"
|
|
|
|
|
|
def test_the_cert_list_carries_the_rev_the_agent_reported(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""`deployed_bundle_rev` is what the host said about itself, not what the CA hopes.
|
|
|
|
Task 05's "stale bundle" sentence is a comparison of two numbers that come from two different
|
|
places: the row this dashboard issued and the `agent` block of the last `system` payload. Both
|
|
halves are pinned here, including the direction that matters — issued 2 against deployed 1 is
|
|
the operator's "install the new bundle" case, and it has to be visible rather than smoothed
|
|
over.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(1),
|
|
host_interval=3600.0) as client:
|
|
assert client.post("/api/certs/deskwork/rotate").status_code == 200
|
|
job = cert_context(client).hub.jobs["host:deskwork"]
|
|
assert cert_agent_rows(client, "deskwork")[0]["deployed_bundle_rev"] is None, (
|
|
"a host that has never answered reports nothing")
|
|
job.state.payload = {"name": "deskwork", "agent": {"tls": True, "bundle_rev": 1}}
|
|
assert cert_agent_rows(client, "deskwork")[0]["deployed_bundle_rev"] == 1
|
|
assert client.post("/api/certs/deskwork/rotate").json()["bundle_rev"] == 2
|
|
row = cert_agent_rows(client, "deskwork")[0]
|
|
assert (row["bundle_rev"], row["deployed_bundle_rev"]) == (2, 1), "the stale pair, both"
|
|
# A token-mode agent publishes no `agent.tls` block at all, and that is `n/a`, not rev 0.
|
|
job.state.payload = {"name": "deskwork", "agent": {"version": "1.0.0"}}
|
|
assert cert_agent_rows(client, "deskwork")[0]["deployed_bundle_rev"] is None
|
|
|
|
|
|
def test_an_entry_for_a_host_that_left_the_config_is_listed_and_not_writable(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A row issued for a machine that has gone is a key still sitting in the table.
|
|
|
|
Listing it is the honest answer (`/certs` is where you learn you still hold one); refusing to
|
|
rotate or revoke it is the same invariant from the other side — `HOST_TARGETS` is what this
|
|
dashboard dials, and a name it no longer dials is not a name to install a bundle on. The row is
|
|
written straight to the store because that is the only way this state arises: a host removed
|
|
from the config, never a route that created it.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(1),
|
|
host_interval=3600.0) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
certs.issue_agent_cert(store, certs.read_state(store), "old-host", ["10.9.8.7"])
|
|
|
|
rows = cert_agent_rows(client)
|
|
assert [row["name"] for row in rows] == ["deskwork", "old-host"]
|
|
ghost = rows[1]
|
|
assert ghost["configured"] is False, "listed, but this dashboard does not dial it"
|
|
assert ghost["status"] == certs.ACTIVE and ghost["bundle_rev"] == 1
|
|
assert ghost["deployed_bundle_rev"] is None, "no job means no report, not rev 0"
|
|
assert client.post("/api/certs/old-host/rotate").status_code == 404
|
|
assert client.post("/api/certs/old-host/revoke").status_code == 404
|
|
assert cert_events(client) == [], "a refusal is not an audit line"
|
|
|
|
|
|
def test_a_cert_route_for_an_unknown_name_dials_nothing_and_writes_nothing(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The invariant-3 gate on the per-host routes, taken before anything can have a side effect.
|
|
|
|
`not-installed` is refused in the first line of each route, so no client is built, no store row
|
|
is written, and no key is minted. All three are measured: the two network spies raise, `set_meta`
|
|
is watched for any `certs.*` write, and the five rows are compared byte for byte. A typo that
|
|
minted a certificate would be invisible on every status code and permanent in the table.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(1),
|
|
host_interval=3600.0) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
calls: list[str] = []
|
|
written: list[str] = []
|
|
|
|
async def spy_get(self: Any, url: Any, **kwargs: Any) -> Any:
|
|
calls.append(f"GET {url}")
|
|
raise AssertionError(f"an unconfigured cert name dialled {url}")
|
|
|
|
def spy_fetch(*args: Any, **kwargs: Any) -> Any:
|
|
calls.append("fetch_snapshot")
|
|
raise AssertionError("an unconfigured cert name fetched an agent snapshot")
|
|
|
|
real_set_meta = SampleStore.set_meta
|
|
|
|
def spy_set_meta(self: SampleStore, key: str, value: str) -> None:
|
|
written.append(key)
|
|
real_set_meta(self, key, value)
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "get", spy_get)
|
|
monkeypatch.setattr("dashboards.collectors.fetch_snapshot", spy_fetch)
|
|
monkeypatch.setattr(SampleStore, "set_meta", spy_set_meta)
|
|
before = cert_meta(store)
|
|
|
|
for response in (client.get("/api/certs/not-configured/bundle"),
|
|
client.post("/api/certs/not-configured/rotate"),
|
|
client.post("/api/certs/not-configured/revoke")):
|
|
assert response.status_code == 404, response.text
|
|
body = response.json()
|
|
assert body["configured"] is False
|
|
assert "HOST_TARGETS" in body["detail"], "the sentence names the fix"
|
|
|
|
assert calls == [], "the gate is in front of the network, not behind it"
|
|
assert [key for key in written if key.startswith("certs.")] == [], "a write happened"
|
|
assert cert_meta(store) == before, "the stored CA and agent table did not move"
|
|
assert cert_events(client) == [], "and no audit line was written for a name that is not here"
|
|
# …and the same route still works for the host that *is* configured, so the refusal above
|
|
# is the gate and not a broken router.
|
|
assert client.post("/api/certs/deskwork/rotate").status_code == 200
|
|
|
|
|
|
def test_rotating_a_host_mints_the_bundle_it_asks_for(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Rotate is also *issue*: first call is `bundle_rev` 1, and the SAN is the config's address.
|
|
|
|
The SAN assertion is the one a rollout fails on silently — a certificate that does not name the
|
|
address the dashboard dials is a handshake error and a red source, not a warning — so it is read
|
|
out of the served bundle's own certificate rather than out of the response's echo.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(1),
|
|
host_interval=3600.0) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
body = client.post("/api/certs/deskwork/rotate").json()
|
|
assert body["ok"] is True and body["status"] == certs.ACTIVE
|
|
assert body["bundle_rev"] == 1 and body["serial"] and body["not_after"] > time.time()
|
|
assert body["san"] == ["127.0.0.1", "deskwork"], "the dialled address and the name"
|
|
|
|
cert = bundle_certs(bundle_of(client, "deskwork"))[0]
|
|
names = cert.extensions.get_extension_for_class(
|
|
x509.SubjectAlternativeName).value.get_values_for_type(x509.IPAddress)
|
|
assert names == [ipaddress.ip_address("127.0.0.1")], "an IP SAN, or an IP dial fails"
|
|
assert cert.subject == x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "deskwork")])
|
|
|
|
events = cert_events(client)
|
|
assert len(events) == 1 and events[0]["kind"] == "cert-rotated"
|
|
assert events[0]["host"] == "deskwork" and events[0]["message"] == (
|
|
"agent certificate rotated for deskwork")
|
|
assert events[0]["data"] == {"bundle_rev": 1}
|
|
|
|
second = client.post("/api/certs/deskwork/rotate").json()
|
|
assert second["bundle_rev"] == 2 and second["serial"] != body["serial"]
|
|
# The new identity is not on the denylist: rotation replaces an identity, it does not strike
|
|
# one out, and a bundle that listed its own serial would refuse the client it ships.
|
|
bundle = bundle_of(client, "deskwork")
|
|
assert revoked_serials(bundle) == [], revoked_serials(bundle)
|
|
assert second["serial"] not in bundle.splitlines(), "no denylist line for the live serial"
|
|
assert stored_bundle(store, "deskwork") == bundle, "served bytes are the stored bytes"
|
|
|
|
|
|
def test_revoking_a_host_marks_the_table_and_the_next_poll_refuses_without_a_dial(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""The API half of the revocation chain: this route is what task 03's refusal reads.
|
|
|
|
The control is the sentence from *before* the click (`unreachable`, which is what a dial to a
|
|
dead port says), and after it the poll has to say `agent certificate revoked` with a spy on
|
|
`httpx.AsyncClient.get` proving no socket was opened. The stored denylist is the other half: an
|
|
agent holding a bundle that omits the serial keeps accepting the client it was given.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(1),
|
|
host_interval=0.05, disks_interval=0.05) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
wait_for_source_error(client, "host:deskwork", "unreachable")
|
|
serial = client.post("/api/certs/deskwork/rotate").json()["serial"]
|
|
|
|
body = client.post("/api/certs/deskwork/revoke").json()
|
|
assert body["ok"] is True and body["status"] == certs.REVOKED
|
|
assert body["serial"] == serial, "revocation marks the identity it has, it does not replace"
|
|
assert body["hint"] and "next poll" in body["hint"]
|
|
|
|
calls: list[str] = []
|
|
|
|
async def spy_get(self: Any, url: Any, **kwargs: Any) -> Any:
|
|
calls.append(str(url))
|
|
raise AssertionError(f"a revoked agent was dialled at {url}")
|
|
|
|
monkeypatch.setattr(httpx.AsyncClient, "get", spy_get)
|
|
for key in ("host:deskwork", "disks:deskwork"):
|
|
error = wait_for_source_error(client, key, "agent certificate revoked")
|
|
assert "unreachable" not in error
|
|
assert calls == []
|
|
|
|
assert revoked_serials(bundle_of(client, "deskwork")) == [serial]
|
|
events = cert_events(client)
|
|
assert [row["kind"] for row in events] == ["cert-revoked", "cert-rotated"]
|
|
assert events[0]["message"] == "agent certificate revoked for deskwork"
|
|
assert certs.agent_status(store, "deskwork") == certs.REVOKED, (
|
|
"the collector's own read agrees with the row the route wrote")
|
|
|
|
|
|
def test_root_rotation_reissues_every_host_and_answers_restart_required(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""One action re-arms the fleet — and says the part it cannot do from inside the process.
|
|
|
|
`restart_required` is not decoration: the CA and the client pair are startup material, so the
|
|
table is new immediately while the sockets keep the old identity. The page (task 05) prints
|
|
that sentence because the API answered it, and the alternative is an operator who rotated a
|
|
root and watches a fleet that still chains to the previous one.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(3),
|
|
host_interval=3600.0) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
for target in DEAD_TARGETS:
|
|
assert client.post(f"/api/certs/{target.name}/rotate").status_code == 200
|
|
before = cert_document(client)
|
|
rows_before = {row["name"]: row for row in before["agents"]}
|
|
root_pem_before = store.get_meta(certs.META_ROOT_CERT) or ""
|
|
key_before = store.get_meta(certs.META_ROOT_KEY) or ""
|
|
assert root_pem_before and key_before, "both halves of the CA exist before the rotation"
|
|
|
|
body = client.post("/api/certs/root/rotate").json()
|
|
assert body["ok"] is True and body["restart_required"] is True
|
|
assert body["agents"] == 3
|
|
assert body["root"]["serial"] != before["root"]["serial"], "a fresh root"
|
|
|
|
after = cert_document(client)
|
|
assert after["root"] == body["root"], "the list reports what the rotation wrote"
|
|
assert after["client"]["serial"] != before["client"]["serial"], "and a fresh client leaf"
|
|
for row in after["agents"]:
|
|
was = rows_before[row["name"]]
|
|
assert row["bundle_rev"] == was["bundle_rev"] + 1, "every host reissued, rev bumped"
|
|
assert row["serial"] != was["serial"] and row["status"] == certs.ACTIVE
|
|
bundle = bundle_of(client, row["name"])
|
|
leaf, root = bundle_certs(bundle)
|
|
assert leaf.serial_number == int(row["serial"]), "the served bundle is the listed one"
|
|
assert root.serial_number == int(after["root"]["serial"]), (
|
|
"and it chains to the authority the list now reports")
|
|
assert revoked_serials(bundle) == [], "the denylist resets with the generation"
|
|
assert store.get_meta(certs.META_ROOT_KEY) != key_before
|
|
assert store.get_meta(certs.META_ROOT_CERT) != root_pem_before
|
|
assert root_pem_before not in "".join(cert_meta(store).values()), (
|
|
"the old root is gone")
|
|
events = cert_events(client)
|
|
assert events[0]["kind"] == "cert-root-rotated"
|
|
assert events[0]["message"] == (
|
|
"root certificate rotated — every agent certificate reissued, restart required")
|
|
assert events[0]["data"] == {"agents": 3}
|
|
|
|
|
|
def test_the_bundle_route_names_a_file_and_changes_when_the_identity_does(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Byte-identical to the stored row, an `attachment`, and different bytes after a reissue.
|
|
|
|
This is the installer's input (task 06), so the claim is about bytes and not about a bundle
|
|
that merely re-verifies: `install-agent.sh` is handed the file and compares nothing else.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(1),
|
|
host_interval=3600.0) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
missing = client.get("/api/certs/deskwork/bundle")
|
|
assert missing.status_code == 404, "nothing issued yet, so nothing to download"
|
|
body = missing.json()
|
|
assert body["configured"] is True and body["issued"] is False, "not the same 404"
|
|
assert "rotate" in body["detail"], "and the answer names the way to get one"
|
|
|
|
first = client.post("/api/certs/deskwork/rotate").json()["serial"]
|
|
response = client.get("/api/certs/deskwork/bundle")
|
|
assert response.headers["content-disposition"] == (
|
|
'attachment; filename="agent-deskwork.bundle"')
|
|
assert response.text == stored_bundle(store, "deskwork")
|
|
|
|
client.post("/api/certs/deskwork/rotate")
|
|
second = client.post("/api/certs/deskwork/rotate").json()["serial"]
|
|
assert second != first
|
|
updated = client.get("/api/certs/deskwork/bundle")
|
|
assert updated.text != response.text, "a reissue changes the served bytes"
|
|
assert updated.text == stored_bundle(store, "deskwork")
|
|
|
|
|
|
def test_no_certificate_material_leaves_meta_through_the_cert_surface(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""invariant 6 on the API that owns the keys: serials and dates go out, PEMs do not.
|
|
|
|
Every write here is exercised first, so the grep is over a table that has been rotated and
|
|
revoked rather than a pristine one, and it covers both the payloads a browser reads and the
|
|
stored event rows those payloads are built from — an event row is kept forever, which is what
|
|
makes a PEM in a message permanent rather than transient.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(2),
|
|
host_interval=3600.0) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
client.post("/api/certs/deskwork/rotate")
|
|
client.post("/api/certs/driveripper/rotate")
|
|
client.post("/api/certs/driveripper/revoke")
|
|
client.post("/api/certs/root/rotate")
|
|
secrets = [value for value in cert_meta(store).values() if value]
|
|
assert any("PRIVATE KEY" in value for value in secrets), "the table really does hold keys"
|
|
|
|
for path in ("/api/certs", "/api/health", "/api/events", "/api/summary", "/api/config"):
|
|
text = client.get(path).text
|
|
for marker in ("BEGIN CERTIFICATE", "PRIVATE KEY", "BUNDLE-REV"):
|
|
assert marker not in text, f"{path} served {marker}"
|
|
assert client.get("/api/health").json()["db"], "health still answers, keys or no keys"
|
|
for event in store.recent_events(50, source="certs"):
|
|
assert "BEGIN" not in event.message and "PRIVATE KEY" not in event.message
|
|
assert not any(secret in json.dumps(event.as_json()) for secret in secrets)
|
|
|
|
|
|
def test_a_host_named_root_cannot_be_rotated_past_the_certificate_authority(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""`/api/certs/root/rotate` and `/api/certs/{name}/rotate` are one URL for a host called root.
|
|
|
|
The CA route is registered first, so it wins — which means the wrong button would rotate the
|
|
whole authority instead of one host's leaf. Refusing with the reason is the only honest answer:
|
|
the alternative is a fleet-wide outage behind a click that looked like routine maintenance.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch,
|
|
host_targets=(Target(name="root", url="https://127.0.0.1:1"),),
|
|
host_interval=3600.0) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
before = cert_document(client)["root"]["serial"]
|
|
refused = client.post("/api/certs/root/rotate")
|
|
assert refused.status_code == 409, refused.text
|
|
assert "rename" in refused.json()["detail"], "the sentence says what to do"
|
|
assert cert_document(client)["root"]["serial"] == before, "and the CA did not move"
|
|
assert cert_events(client) == []
|
|
assert cert_meta(store)[certs.META_AGENTS] == "{}", "nothing was issued for the name either"
|
|
|
|
|
|
def test_a_read_of_the_cert_surface_issues_nothing_and_writes_nothing(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A GET that mints a certificate is the surprise this route must never be.
|
|
|
|
`certs.ensure_certs` is the bootstrapping read and `certs.read_state` is the request-time one;
|
|
making `ensure_certs` explode is what proves the difference rather than asserting it in prose,
|
|
and the meta rows are compared byte for byte across three reads.
|
|
"""
|
|
with build_client(tmp_path, monkeypatch, host_targets=cert_hosts(1),
|
|
host_interval=3600.0) as client:
|
|
store: SampleStore = cert_context(client).store
|
|
before = cert_meta(store)
|
|
|
|
def explode(*args: Any, **kwargs: Any) -> Any:
|
|
raise AssertionError("a GET tried to bootstrap the certificate authority")
|
|
|
|
monkeypatch.setattr(certs, "ensure_certs", explode)
|
|
assert cert_agent_rows(client)
|
|
assert cert_document(client)["root"]["serial"] == str(
|
|
x509.load_pem_x509_certificate(
|
|
str(before[certs.META_ROOT_CERT]).encode()).serial_number)
|
|
# A bundle that has not been issued is still a 404 rather than a mint, on a read too.
|
|
assert client.get("/api/certs/deskwork/bundle").status_code == 404
|
|
assert cert_meta(store) == before, "three reads, no write"
|
|
assert cert_events(client) == []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# phase 14 task 04: the endpoint budgets
|
|
#
|
|
# `store_perf.BUDGETS` is the one table of ceilings (it is what the probe's report and the notes
|
|
# render, so a ceiling relaxed in one place is relaxed everywhere). The ceilings are 2-3x the
|
|
# measured "after" on the seeded production-shaped store, which makes them a regression gate
|
|
# rather than a stopwatch: the `/system` page's own chart request costs ~2 ms and still gets
|
|
# 100 ms, while putting a full-table scan back on a per-request path crosses by three orders of
|
|
# magnitude. Wall time lives behind `--store-perf` for that reason; what runs in every `uv run
|
|
# pytest` is the key-name pin below, because a budget that silently stops being priced is as
|
|
# useless as a budget that is crossed.
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
|
|
def test_every_http_budget_names_a_route_the_probe_prices(tmp_path: Path) -> None:
|
|
"""The stale-pointer check for the HTTP half, on a throwaway store.
|
|
|
|
`measure_http` is what the priced test runs, and it owns the request shapes; if a key there is
|
|
renamed, or a route stops being measured, the priced test would report the gap rather than the
|
|
cost — which is why this asserts the key set here, cheaply, on every run.
|
|
"""
|
|
report = store_perf.measure_budget_http(tmp_path / "budget-keys.sqlite3",
|
|
hosts=("driveripper",), metrics=("cpu.usage_pct",))
|
|
assert set(report["http"]) == set(store_perf.HTTP_MEASURED_KEYS), (
|
|
"the HTTP pass and the budget table no longer describe the same requests")
|
|
assert store_perf.missing_budget_keys(report, ("http",)) == []
|
|
# and the request the budgets describe is the one the task names: 3 hosts x 9 series
|
|
nine = store_perf.HISTORY_METRICS
|
|
assert len(nine) == 9, "the 90 d budget is priced against a 3x9 request, not some other shape"
|
|
|
|
|
|
def test_metrics_stops_scanning_for_series_names_once_the_hub_is_polling(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""The live shape behind one budgeted endpoint, guarded by statements rather than a clock.
|
|
|
|
Task 04's watch caught `/api/metrics` answering with its full `SELECT DISTINCT` scan on
|
|
practically every call of the running server (535 ms, measured): task 02's cache was invalidated
|
|
by every sample write, and the hub writes every 2-300 s, so the cache never lived long enough to
|
|
serve. The store-level half of this lives in `tests/test_store.py`; this is the half that says
|
|
the *endpoint* is what stops scanning, on the app the operator runs.
|
|
"""
|
|
db = str(tmp_path / "metrics-busy.sqlite3")
|
|
store = SampleStore(db)
|
|
store.add_many([Sample("host", "driveripper", "cpu.usage_pct", 1.0, time.time()),
|
|
Sample("host", "deskwork", "cpu.usage_pct", 1.0, time.time())])
|
|
app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
|
|
pbkdf2_iterations=FAST_PBKDF2), store)
|
|
try:
|
|
with TestClient(app) as http:
|
|
sign_in(http)
|
|
assert "cpu.usage_pct" in http.get("/api/metrics", params={"source": "host"}).json()["metrics"]
|
|
|
|
# a poll of series the store already has, then the same request with the log open
|
|
store.add_many([Sample("host", "driveripper", "cpu.usage_pct", 2.0, time.time()),
|
|
Sample("host", "deskwork", "cpu.usage_pct", 2.0, time.time())])
|
|
seen: list[str] = []
|
|
# the statement log is the test: `SampleStore._db` is the handle the request runs on
|
|
store._db.set_trace_callback(seen.append)
|
|
try:
|
|
assert "cpu.usage_pct" in http.get("/api/metrics",
|
|
params={"source": "host"}).json()["metrics"]
|
|
finally:
|
|
store._db.set_trace_callback(None)
|
|
assert [s for s in seen if "DISTINCT metric" in s] == [], (
|
|
f"`/api/metrics` went back to the table while nothing changed: {seen}")
|
|
|
|
# and a source that starts publishing something new is in the very next answer
|
|
store.add("host", "driveripper", "gpu.busy_pct@0", 1.0)
|
|
assert "gpu.busy_pct@0" in http.get("/api/metrics",
|
|
params={"source": "host"}).json()["metrics"]
|
|
finally:
|
|
store.close()
|
|
|
|
|
|
@pytest.mark.store_perf
|
|
def test_the_budgeted_http_paths_fit_their_pinned_margins(perf_store_path: Path) -> None:
|
|
"""`/api/health`, `/api/metrics`, `/api/system/history` (1 h and 90 d), `/api/summary`.
|
|
|
|
Priced through the real ASGI gate with a signed-in session, on the seeded 6.4 M-row store —
|
|
the same call the operator's browser makes, in-process so no network is measured. This is the
|
|
line `TODO.md:7` is about: `/api/health` used to answer in 1.27 s because it counted the whole
|
|
`sample` table on every poll, on every tab.
|
|
"""
|
|
report = store_perf.measure_budget_http(perf_store_path)
|
|
print()
|
|
print(store_perf.render_budgets_markdown(store_perf.budget_rows(report, ("http",))))
|
|
assert store_perf.budget_violations(report, ("http",)) == []
|