phase: 01_severity_language
**Gate defect found & fixed (the "does not lint" bug).** `.agents/validate.sh` — the phase gate — ran `python3 -m pytest` on `/usr/bin/python3` + `~/.local` site-packages (not the uv venv), ran `ruff check .` *only if* a ruff happened to be on PATH, and **never ran pyright or the coverage floor**. Rewrote it to the project's real gate: cd to repo root → `uv run pytest --cov --cov-fail-under=94` → `uv run ruff check src tests agent` → `uv run pyright` → `node --check web/static/*.js` → hygiene gates; a missing toolchain now fails loudly. Provoked it to confirm it bites (floor + a seeded F401 both fail).
**Verification (all green)**
- `bash .agents/validate.sh` → **exit 0**: 356 passed, 6 skipped · coverage **94.75%** (floor 94) · ruff `All checks passed!` · pyright `0 errors` · node syntax clean
- `uv run pytest --cov` → `severity.py` **100%**; `uv run pytest -m live --live` → **6 passed**; `uv pip list --outdated` → empty
- Criteria: `classify('cpu.usage_pct',…)` → `fail`/`ok`/`None` ✓ · `/api/ups/snapshot` carries `severity` beside `charge_pct`/`load_pct`/`battery_runtime` (apc now live-`warn` at 498 s — threshold working) ✓ · no `.sev-*`/synonym vocabulary (only a comment saying so) ✓ · pytest/coverage/lint ✓ · dev server restarted, `/api/health` ok 6/6 sources, all pages+assets 200 ✓
- Chromium against live fleet: `severityFrom` cell→`fail`, bare→`warn`, null→`''`, unrated→`''`; `hero` fail → `rgb(207,34,46)`, null → `n/a` with **no** verdict class, warn+bar 42%; all four pages render unchanged → `data/logs/phase01-final-{index,system,llm,ups}.png`
**Docs (full review, incl. pre-existing drift).** README: `/ups` row claimed "upsd on 3493" (fleet is 3051, stated elsewhere in the same file); config table omitted `DASH_HOST/PORT`, llm/ups `*_INTERVAL_SECONDS`, `*_TIMEOUT_SECONDS`, `GUFO/VLLM_API_KEY`, `POLL_MS_*`, `DEBUGPY` (+ dedup); severity bullet omitted `disks`/`gpus[]`/rated LLM cells and the ratio rescale; agent section listed only `/snapshot` though `/info`+`/health` exist. AGENTS.md: same 3493 claim; `install-agent.sh --token-file` (its flags are `--bind/--port/--token/--rotate-token/--status/--uninstall` — it hands *the agent* `--token-file`); gap 3 "Two commits" (now 6, still no `origin`); gap 1 given the sudoers install command + verified live gating reason; coverage line now 80% repo floor vs 94% gate floor; cyberpower `identity.model: GX1500U` vs `ups.description` quirk. `.env.dev.example`: `DEV_FORWARDS` remote port 3493→3051 and engine URLs that pointed at `127.0.0.1` with nothing forwarding them.
**Deviations.** No `ruff format` (30 files would reformat; the project gate is `ruff check` + pyright — reformatting completed phases is out of scope). **No live-infra changes**: enabling the battery-test buttons needs `deploy/nut-sudoers` installed on driveripper + a nut-server restart and a scoped `[dashboards]` `instcmds` user — prod write with a documented shutdown risk (AGENTS gap 1), so repo-side only. 36 pre-existing `ResourceWarning: unclosed database` from test fixtures are noise, not failures; left rather than churn ~15 fixtures. `severity.attach()` is unit-tested but has no production call site until phases 02–05.
**Next pending phase:** `02_overview_fleet_and_alerts` (`01_alerts_builder.md`).
This commit is contained in:
@@ -7,6 +7,8 @@ asset the network can fail to deliver.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
@@ -17,6 +19,9 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from dashboards.app import create_app
|
||||
from dashboards.config import Settings, Target
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.severity import classify
|
||||
from dashboards.sources.host import HostCollector, parse_host_snapshot
|
||||
from dashboards.store import Event, Sample, SampleStore
|
||||
|
||||
# refused immediately, and never dials out of the test environment
|
||||
@@ -63,6 +68,107 @@ def wait_for_payload(client: TestClient, path: str, *, timeout: float = 5.0) ->
|
||||
pytest.fail(f"{path} never became available")
|
||||
|
||||
|
||||
def host_payload(agent_snapshot: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Two scrapes of the real driveripper capture, nudged onto known sides of the table.
|
||||
|
||||
Driven from the fixture rather than hand-written so a schema drift breaks here first.
|
||||
"""
|
||||
snap = copy.deepcopy(agent_snapshot)
|
||||
snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.95 # mem.used_pct -> fail
|
||||
snap["cpu_temperature_celsius"] = 91.0 # cpu.temp_c -> fail
|
||||
for fs in snap["filesystems"]:
|
||||
fs["used_pct"] = 93.0 # disk -> fail
|
||||
collector = HostCollector(RateEstimator(span=60.0))
|
||||
parse_host_snapshot("driveripper", snap, collector, now=1_000.0)
|
||||
second = copy.deepcopy(snap)
|
||||
second["ts"] = 1_010.0
|
||||
modes = second["cpu"]["modes"]
|
||||
modes["total"] += 1_000.0
|
||||
modes["idle"] += 900.0 # 10 % busy over the window -> ok
|
||||
return parse_host_snapshot("driveripper", second, collector, now=1_010.0).payload
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def data_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
|
||||
agent_snapshot: dict[str, Any]) -> Iterator[TestClient]:
|
||||
"""Overview-shaped app: one host payload in the hub plus the simulated UPSes.
|
||||
|
||||
The host job points at a dead port, so a poll can never overwrite the injected payload
|
||||
(a failed poll keeps the last good one) — the numbers below are ours, not the fleet's.
|
||||
"""
|
||||
monkeypatch.setenv("NUT_SIMULATE", "1")
|
||||
db = str(tmp_path / "summary.sqlite3")
|
||||
store = SampleStore(db)
|
||||
app = create_app(Settings(db_path=db, nut_host="127.0.0.1",
|
||||
host_targets=(Target("driveripper", DEAD_URL),)), store)
|
||||
test_client = TestClient(app)
|
||||
test_client.__enter__()
|
||||
try:
|
||||
app.state.ctx.hub.jobs["host:driveripper"].state.payload = host_payload(agent_snapshot)
|
||||
wait_for_payload(test_client, "/api/ups/snapshot")
|
||||
yield test_client
|
||||
finally:
|
||||
test_client.__exit__(None, None, None)
|
||||
store.close()
|
||||
|
||||
|
||||
# ---- severity travels with the value --------------------------------------- #
|
||||
|
||||
def test_summary_carries_the_verdict_for_every_column_the_page_colours(data_client) -> None:
|
||||
body = data_client.get("/api/summary").json()
|
||||
row = body["system"]["hosts"][0]
|
||||
assert row["severity"] == {"cpu_usage_pct": "ok", "mem_used_pct": "fail",
|
||||
"disk_worst_used_pct": "fail", "temp_c": "fail"}
|
||||
# the fleet card is judged by the same table, so amber means the same thing at both sizes
|
||||
assert body["system"]["fleet"]["severity"] == {
|
||||
"cpu_usage_pct_avg": "ok", "memory_used_pct": "fail",
|
||||
"disk_worst_used_pct": "fail", "gpu_busy_pct_avg": "ok",
|
||||
}
|
||||
for ups in body["ups"]["ups"]:
|
||||
assert set(ups["severity"]) == {"charge_pct", "load_pct", "runtime_seconds"}
|
||||
assert ups["severity"]["charge_pct"] == classify("ups.charge_pct", ups["charge_pct"])
|
||||
assert ups["severity"]["charge_pct"] == "ok", "the simulator starts at float charge"
|
||||
|
||||
|
||||
def test_summary_shape_did_not_shrink(data_client) -> None:
|
||||
"""Additive-only anchor: /api/summary gained `severity` and lost nothing.
|
||||
|
||||
The overview, the live tests, and any external consumer read the flat keys; this test is
|
||||
the tripwire for a well-meaning refactor that "tidies" them away.
|
||||
"""
|
||||
body = data_client.get("/api/summary").json()
|
||||
fleet_keys = {"hosts_total", "hosts_live", "cpu_cores", "cpu_usage_pct_avg",
|
||||
"memory_total_bytes", "memory_used_bytes", "memory_used_pct",
|
||||
"disk_worst_used_pct", "load1_total", "net_rx_bytes_per_s",
|
||||
"net_tx_bytes_per_s", "gpu_count", "gpu_busy_pct_avg", "power_estimate_w"}
|
||||
host_keys = {"name", "os", "cpu_usage_pct", "load1", "cores", "mem_used_pct",
|
||||
"disk_worst_used_pct", "gpu_busy_pct", "temp_c", "uptime_seconds"}
|
||||
ups_keys = {"name", "overall", "status", "charge_pct", "load_pct", "runtime_seconds"}
|
||||
assert fleet_keys <= set(body["system"]["fleet"]), fleet_keys - set(body["system"]["fleet"])
|
||||
assert host_keys <= set(body["system"]["hosts"][0]), host_keys - set(body["system"]["hosts"][0])
|
||||
assert ups_keys <= set(body["ups"]["ups"][0]), ups_keys - set(body["ups"]["ups"][0])
|
||||
assert set(body) == {"ts", "system", "llm", "ups", "recent_events"}
|
||||
# ...and the new keys are there beside the old ones, not instead of them
|
||||
assert "severity" in body["system"]["fleet"]
|
||||
assert "severity" in body["system"]["hosts"][0]
|
||||
assert "severity" in body["ups"]["ups"][0]
|
||||
|
||||
|
||||
def test_snapshot_payloads_carry_their_own_severity_sections(data_client) -> None:
|
||||
"""/api/system/snapshot and /api/ups/snapshot ship the same verdicts the summary reuses."""
|
||||
system = data_client.get("/api/system/snapshot").json()
|
||||
host = system["hosts"][0]
|
||||
assert set(host["cpu"]["severity"]) == {"usage_pct", "temp_c"}
|
||||
assert set(host["memory"]["severity"]) == {"used_pct"}
|
||||
assert set(host["disks"]["severity"]) == {"worst_used_pct"}
|
||||
assert set(host["gpus"][0]["severity"]) == {"busy_pct", "mem_used_pct"}
|
||||
ups = data_client.get("/api/ups/snapshot").json()
|
||||
for card in ups["ups"]:
|
||||
assert set(card["severity"]) == {"charge_pct", "load_pct", "battery_runtime"}
|
||||
# numbers keep their own shape; severity is a sibling, never a replacement
|
||||
assert isinstance(card["numbers"], dict) and "charge_pct" in card["numbers"]
|
||||
|
||||
|
||||
# ---- unconfigured: say so, and do not dial -------------------------------- #
|
||||
|
||||
def test_config_reports_every_source_as_off(client) -> None:
|
||||
@@ -175,6 +281,104 @@ def test_static_assets_are_served(client) -> None:
|
||||
assert client.get(asset).status_code == 200, asset
|
||||
|
||||
|
||||
# ---- the severity language in the static assets --------------------------- #
|
||||
#
|
||||
# There is no JS test runner (a locked anchor: adding one would mean a build step).
|
||||
# What *can* be asserted end-to-end, from the served bytes, is the contract the pages
|
||||
# depend on: the hero style exists, its colour comes from the shared ok/warn/fail tokens,
|
||||
# no parallel `.sev-*` vocabulary appeared, and `Dash.hero()` reads the verdict instead of
|
||||
# comparing the number to a threshold in JavaScript.
|
||||
|
||||
ASSETS = ("/static/dash.css", "/static/app.js", "/static/charts.js",
|
||||
"/static/page-index.js", "/static/page-system.js", "/static/page-llm.js",
|
||||
"/static/page-ups.js")
|
||||
|
||||
|
||||
def _css(client: TestClient) -> str:
|
||||
return client.get("/static/dash.css").text
|
||||
|
||||
|
||||
def _hero_css(css: str) -> str:
|
||||
"""Just the `hero numbers` section of the stylesheet."""
|
||||
return css.split("/* ---- hero numbers", 1)[1].split("/* ---- bars", 1)[0]
|
||||
|
||||
|
||||
def test_css_ships_the_hero_number_style(client) -> None:
|
||||
css = _css(client)
|
||||
hero_block = _hero_css(css)
|
||||
for rule in (".hero {", ".hero .label", ".hero .value", ".hero .unit", ".hero .sub",
|
||||
".hero.ok .value", ".hero.warn .value", ".hero.fail .value",
|
||||
".hero-bar", ".hero-bar.ok .bar"):
|
||||
assert rule in hero_block, f"dash.css lost the {rule} rule"
|
||||
# a display size with fixed-width digits, so the number does not jitter between polls
|
||||
assert "2.4rem" in hero_block
|
||||
assert "tabular-nums" in hero_block
|
||||
|
||||
|
||||
def test_hero_colour_comes_from_the_shared_verdict_tokens(client) -> None:
|
||||
"""`.hero` reuses `--ok/--warn/--fail`; it does not invent a palette of its own."""
|
||||
hero_block = _hero_css(_css(client))
|
||||
for token in ("var(--ok)", "var(--warn)", "var(--fail)", "var(--text)",
|
||||
"var(--text-muted)", "var(--text-faint)"):
|
||||
assert token in hero_block, f"the hero style no longer uses {token}"
|
||||
|
||||
|
||||
def test_no_stylesheet_rule_uses_an_undeclared_token(client) -> None:
|
||||
"""A typo'd `var(--x)` renders unstyled and reads as a broken panel, not an error."""
|
||||
css = _css(client)
|
||||
root = re.search(r":root\s*\{(.*?)\n\}", css, re.S)
|
||||
assert root is not None
|
||||
declared = set(re.findall(r"(--[a-z0-9-]+)\s*:", root.group(1)))
|
||||
used = set(re.findall(r"var\(\s*(--[a-z0-9-]+)", css))
|
||||
assert not used - declared
|
||||
|
||||
|
||||
def _code(body: str) -> str:
|
||||
"""An asset with its comments stripped — prose may *name* a class the code must not define."""
|
||||
body = re.sub(r"/\*.*?\*/", " ", body, flags=re.S)
|
||||
body = re.sub(r"<!--.*?-->", " ", body, flags=re.S)
|
||||
return re.sub(r"^\s*//.*$", " ", body, flags=re.M)
|
||||
|
||||
|
||||
def test_no_parallel_severity_vocabulary_exists_anywhere(client) -> None:
|
||||
"""One word per colour: `ok|warn|fail`, as in severity.py and the existing components."""
|
||||
for asset in (*ASSETS, "/", "/system", "/llm", "/ups"):
|
||||
body = _code(client.get(asset).text)
|
||||
assert ".sev-" not in body, f"{asset} invented a .sev-* class"
|
||||
assert not re.search(r"classList\.(?:add|toggle)\(\s*[\"'](?:danger|good|warning)[\"']",
|
||||
body), f"{asset} invented a synonym for fail"
|
||||
|
||||
|
||||
def test_app_js_exposes_the_hero_helpers(client) -> None:
|
||||
js = client.get("/static/app.js").text
|
||||
assert "function hero(" in js
|
||||
assert "function severityFrom(" in js
|
||||
exports = js.rsplit("return {", 1)[1].split("}", 1)[0]
|
||||
for name in ("hero", "severityFrom", "severityFor"):
|
||||
assert re.search(rf"\b{name}\b", exports), f"Dash.{name} is not exported"
|
||||
|
||||
|
||||
def test_hero_reads_the_verdict_instead_of_a_threshold(client) -> None:
|
||||
"""`hero()` may only colour what the API rated — no thresholds re-derived in JS."""
|
||||
js = client.get("/static/app.js").text
|
||||
start = js.index("function hero(")
|
||||
body = js[start:js.index("\n function setDot", start)]
|
||||
assert "severityFrom(" in body
|
||||
assert "severityFor(" not in body
|
||||
# an unknown value gets n/a and no colour class at all
|
||||
assert "naText" in body
|
||||
assert re.search(r"raw != null", body)
|
||||
|
||||
|
||||
def test_severity_for_is_kept_as_the_documented_fallback(client) -> None:
|
||||
"""The old call sites still work until phases 02-05 convert each page."""
|
||||
js = client.get("/static/app.js").text
|
||||
assert "FALLBACK ONLY" in js
|
||||
for page in ("/static/page-index.js", "/static/page-system.js",
|
||||
"/static/page-llm.js", "/static/page-ups.js"):
|
||||
assert "severityFor" in client.get(page).text, f"{page} lost its fallback call site"
|
||||
|
||||
|
||||
def test_api_responses_are_not_cacheable(client) -> None:
|
||||
for path in ("/api/config", "/api/health", "/api/summary"):
|
||||
assert "no-store" in client.get(path).headers.get("cache-control", "")
|
||||
|
||||
@@ -13,6 +13,7 @@ from typing import Any
|
||||
import pytest
|
||||
|
||||
from dashboards.rates import RateEstimator
|
||||
from dashboards.severity import classify
|
||||
from dashboards.sources.host import HostCollector, fleet_summary, parse_host_snapshot
|
||||
|
||||
TICKS_PER_SECOND = 100.0
|
||||
@@ -234,6 +235,80 @@ def test_fleet_summary_worst_disk_and_memory_totals(agent_snapshot) -> None:
|
||||
base["memory"]["used_bytes"] / base["memory"]["total_bytes"] * 100)
|
||||
|
||||
|
||||
# ---- severity: the verdict travels with the value ------------------------- #
|
||||
|
||||
def test_payloads_carry_a_severity_beside_every_rated_value(agent_snapshot) -> None:
|
||||
"""The page paints a class; it must never hold its own opinion about where amber starts."""
|
||||
_first, payload = parse_twice(agent_snapshot)
|
||||
assert payload["cpu"]["severity"] == {
|
||||
"usage_pct": classify("cpu.usage_pct", payload["cpu"]["usage_pct"]),
|
||||
"temp_c": classify("cpu.temp_c", payload["cpu"]["temp_c"]),
|
||||
}
|
||||
assert payload["cpu"]["severity"]["usage_pct"] == "ok" # the window is 10 % busy
|
||||
assert payload["cpu"]["severity"]["temp_c"] == "ok" # 68 °C in the capture
|
||||
assert payload["memory"]["severity"]["used_pct"] == "ok" # 68 % of 128 GB
|
||||
assert payload["disks"]["severity"]["worst_used_pct"] == "ok" # 74.6 % worst mount
|
||||
gpu = payload["gpus"][0]
|
||||
assert gpu["severity"] == {
|
||||
"busy_pct": classify("gpu.busy_pct", gpu["busy_pct"]),
|
||||
"mem_used_pct": classify("gpu.mem_used_pct", gpu["mem_used_pct"]),
|
||||
}
|
||||
assert gpu["severity"]["busy_pct"] == "ok"
|
||||
|
||||
|
||||
def test_severity_trips_on_the_ratified_boundaries(agent_snapshot) -> None:
|
||||
"""Values pushed onto the table's edges in the real fixture, not on a hand-drawn snapshot."""
|
||||
snap = copy.deepcopy(agent_snapshot)
|
||||
snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.92 # mem.used_pct danger
|
||||
snap["cpu_temperature_celsius"] = 90.0 # cpu.temp_c danger
|
||||
for fs in snap["filesystems"]:
|
||||
fs["used_pct"] = 80.0 # disk.used_pct warn
|
||||
snap["gpus"][0]["busy_pct"] = 95.0 # gpu.busy_pct danger
|
||||
snap["gpus"][0]["gtt_used_bytes"] = 65e9 # whole-pool usage well past danger
|
||||
payload = parse_host_snapshot("driveripper", snap, make_collector(), now=1_000.0).payload
|
||||
assert payload["memory"]["severity"]["used_pct"] == "fail"
|
||||
assert payload["cpu"]["severity"]["temp_c"] == "fail"
|
||||
assert payload["disks"]["severity"]["worst_used_pct"] == "warn"
|
||||
assert payload["gpus"][0]["severity"]["busy_pct"] == "fail"
|
||||
assert payload["gpus"][0]["severity"]["mem_used_pct"] == "fail"
|
||||
|
||||
|
||||
def test_a_value_the_agent_did_not_report_gets_no_colour(agent_snapshot) -> None:
|
||||
"""Unknown is not ok: `severity: null` renders `n/a`, a green pass would be a lie."""
|
||||
snap = copy.deepcopy(agent_snapshot)
|
||||
snap["cpu_temperature_celsius"] = None
|
||||
snap["gpus"][0]["busy_pct"] = None
|
||||
snap["filesystems"] = []
|
||||
payload = parse_host_snapshot("driveripper", snap, make_collector(), now=1_000.0).payload
|
||||
assert payload["cpu"]["severity"]["temp_c"] is None
|
||||
assert payload["cpu"]["severity"]["usage_pct"] is None, "one scrape is not a rate"
|
||||
assert payload["gpus"][0]["severity"]["busy_pct"] is None
|
||||
assert payload["disks"]["severity"]["worst_used_pct"] is None
|
||||
# the numbers themselves are unchanged by the verdict sitting next to them
|
||||
assert payload["memory"]["used_pct"] == pytest.approx(
|
||||
snap["memory"]["used_bytes"] / snap["memory"]["total_bytes"] * 100)
|
||||
|
||||
|
||||
def test_attaching_severity_renamed_or_removed_nothing(agent_snapshot) -> None:
|
||||
"""Additive-only anchor, at the source: the flat fields the pages read are all still there."""
|
||||
payload = parse_host_snapshot("driveripper", agent_snapshot, make_collector(),
|
||||
now=1_000.0).payload
|
||||
expected = {
|
||||
"cpu": {"usage_pct", "modes_pct", "load1", "load5", "load15", "cores",
|
||||
"core_usage_pct", "freq_mhz", "freq_max_mhz", "context_switches_per_s",
|
||||
"forks_per_s", "procs_running", "procs_blocked", "temp_c"},
|
||||
"memory": {"total_bytes", "used_bytes", "available_bytes", "used_pct", "cached_bytes",
|
||||
"buffers_bytes", "swap_total_bytes", "swap_used_bytes", "swap_used_pct",
|
||||
"committed_bytes"},
|
||||
"disks": {"filesystems", "worst_used_pct", "worst_mount", "read_bytes_per_s",
|
||||
"write_bytes_per_s", "reads_per_s", "writes_per_s", "util_pct", "devices"},
|
||||
}
|
||||
for section, keys in expected.items():
|
||||
assert keys <= set(payload[section]), f"{section} lost {keys - set(payload[section])}"
|
||||
for gpu in payload["gpus"]:
|
||||
assert {"busy_pct", "mem_used_pct"} <= set(gpu)
|
||||
|
||||
|
||||
def test_fleet_summary_of_nothing_is_empty_not_zero() -> None:
|
||||
fleet = fleet_summary([])
|
||||
assert fleet["hosts_total"] == 0
|
||||
|
||||
@@ -100,6 +100,35 @@ def test_absent_gufo_metrics_are_none_with_no_source() -> None:
|
||||
assert metrics.get("spec_accept_rate", {}).get("value") is None
|
||||
|
||||
|
||||
# ---- severity: KV pressure arrives with a verdict ------------------------- #
|
||||
|
||||
def test_gufo_kv_cache_cell_keeps_provenance_and_gains_severity(metrics_text) -> None:
|
||||
metrics = parse_gufo_metrics(metrics_text("gufo"))
|
||||
cell = metrics["kv_cache_usage_ratio"]
|
||||
assert cell["value"] == 0.0
|
||||
assert cell["source"] == "llamacpp:kv_cache_usage_ratio"
|
||||
assert cell["derived"] is False, "the verdict is additive; the cell still says what it was"
|
||||
assert cell["severity"] == "ok"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ratio,expected", [
|
||||
("0.74", "ok"), ("0.75", "warn"), ("0.89", "warn"), ("0.9", "fail"), ("1.0", "fail"),
|
||||
])
|
||||
def test_kv_ratio_is_classified_on_a_percent_scale(metrics_text, ratio, expected) -> None:
|
||||
"""0.9 of a KV cache is 90 %, not 0.9 % — classifying the raw ratio is a permanent green."""
|
||||
text = re.sub(r"^(llamacpp:kv_cache_usage_ratio) \S+$", rf"\g<1> {ratio}",
|
||||
metrics_text("gufo"), count=1, flags=re.M)
|
||||
metrics = parse_gufo_metrics(text)
|
||||
assert metrics["kv_cache_usage_ratio"]["severity"] == expected
|
||||
assert metrics["kv_cache_usage_ratio"]["value"] == pytest.approx(float(ratio)), \
|
||||
"the scale is for the verdict only; the published value is untouched"
|
||||
|
||||
|
||||
def test_absent_kv_cell_gets_no_verdict() -> None:
|
||||
metrics = parse_gufo_metrics("# nothing here\n")
|
||||
assert metrics["kv_cache_usage_ratio"]["severity"] is None
|
||||
|
||||
|
||||
# ---- vLLM ----------------------------------------------------------------- #
|
||||
|
||||
def test_vllm_counters_and_gauges(metrics_text) -> None:
|
||||
@@ -112,6 +141,23 @@ def test_vllm_counters_and_gauges(metrics_text) -> None:
|
||||
assert metrics["num_preemptions_total"]["value"] is not None
|
||||
|
||||
|
||||
def test_vllm_kv_cache_cell_carries_a_severity(metrics_text) -> None:
|
||||
assert parse_vllm_metrics(metrics_text("vllm"))["gpu_cache_usage"]["severity"] == "ok"
|
||||
pressed = bump_metric(metrics_text("vllm"), "vllm:kv_cache_usage_perc", 0.95)
|
||||
cell = parse_vllm_metrics(pressed)["gpu_cache_usage"]
|
||||
assert cell["value"] == pytest.approx(0.95)
|
||||
assert cell["severity"] == "fail"
|
||||
assert cell["source"] == "vllm:kv_cache_usage_perc"
|
||||
|
||||
|
||||
def test_vllm_absent_kv_cell_gets_no_verdict(metrics_text) -> None:
|
||||
stripped = "\n".join(line for line in metrics_text("vllm").splitlines()
|
||||
if "kv_cache_usage" not in line and "gpu_cache_usage" not in line)
|
||||
metrics = parse_vllm_metrics(stripped)
|
||||
assert metrics["gpu_cache_usage"]["value"] is None
|
||||
assert metrics["gpu_cache_usage"]["severity"] is None
|
||||
|
||||
|
||||
def test_vllm_latency_histograms(metrics_text) -> None:
|
||||
metrics = parse_vllm_metrics(metrics_text("vllm"))
|
||||
ttft = metrics["time_to_first_token_seconds"]
|
||||
|
||||
@@ -435,6 +435,61 @@ def test_payload_carries_numbers_health_and_what_the_driver_offers() -> None:
|
||||
assert payload["extra"] or payload["status"] == "OL CHRG"
|
||||
|
||||
|
||||
def test_payload_severity_colours_the_hero_battery_numbers() -> None:
|
||||
"""charge / load / runtime carry the verdict so the page paints a class, not a threshold."""
|
||||
payload = nut.payload(reading(), now=1_767_225_600.0) # charge 100, runtime 3720 s, load 34 %
|
||||
assert payload["severity"] == {"charge_pct": "ok", "load_pct": "ok", "battery_runtime": "ok"}
|
||||
# the verdict sits beside the numbers; it does not replace or rename any of them
|
||||
assert payload["numbers"]["charge_pct"] == 100.0
|
||||
assert payload["numbers"]["battery_runtime"] == 3720.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("overrides,expected", [
|
||||
({"battery.charge": "15"}, {"charge_pct": "fail"}),
|
||||
({"battery.charge": "50"}, {"charge_pct": "warn"}), # reaching warn trips it
|
||||
({"battery.charge": "50.1"}, {"charge_pct": "ok"}),
|
||||
({"battery.runtime": "240"}, {"battery_runtime": "fail"}),
|
||||
({"battery.runtime": "600"}, {"battery_runtime": "warn"}),
|
||||
({"ups.load": "95"}, {"load_pct": "fail"}),
|
||||
({"ups.load": "70"}, {"load_pct": "warn"}),
|
||||
])
|
||||
def test_severity_trips_the_ratified_ups_thresholds(overrides, expected) -> None:
|
||||
payload = nut.payload(reading(**overrides), now=1_767_225_600.0)
|
||||
for key, verdict in expected.items():
|
||||
assert payload["severity"][key] == verdict, f"{key} with {overrides}"
|
||||
|
||||
|
||||
def test_severity_is_null_when_the_driver_publishes_nothing() -> None:
|
||||
"""Neither real UPS reports a battery temperature and either can drop `battery.runtime`;
|
||||
an absent number gets no colour, and the health check keeps saying `unknown`."""
|
||||
values = {key: value for key, value in VAR_ROWS.items()
|
||||
if key not in ("battery.runtime", "battery.temperature")}
|
||||
payload = nut.payload(nut.UpsReading(name="apc", vars=values, commands=[]),
|
||||
now=1_767_225_600.0)
|
||||
assert payload["numbers"]["battery_runtime"] is None
|
||||
assert payload["severity"]["battery_runtime"] is None
|
||||
assert payload["numbers"]["battery_temperature"] is None
|
||||
temp = next(c for c in payload["health"]["checks"] if c["name"] == "Battery temperature")
|
||||
assert temp["status"] == "unknown", "severity is additive: assess() keeps its own judgement"
|
||||
|
||||
|
||||
def test_severity_does_not_touch_health_checks_or_numbers() -> None:
|
||||
ups = reading()
|
||||
payload = nut.payload(ups, now=1_767_225_600.0)
|
||||
assert all(check["status"] in ("ok", "warn", "fail", "unknown")
|
||||
for check in payload["health"]["checks"])
|
||||
assert payload["health"] == nut.assess(ups, now=1_767_225_600.0), \
|
||||
"the condition verdict and the value verdict must not contaminate each other"
|
||||
|
||||
|
||||
async def test_simulated_ups_cards_carry_severity_too() -> None:
|
||||
"""The UPS page is developable without a battery, and still gets real verdicts."""
|
||||
for ups in await NutSimulator().read():
|
||||
card = nut.payload(ups, now=1_767_225_600.0)
|
||||
assert set(card["severity"]) == {"charge_pct", "load_pct", "battery_runtime"}
|
||||
assert card["severity"]["charge_pct"] == "ok", "the sim starts at float charge"
|
||||
|
||||
|
||||
def test_history_samples_only_include_real_numbers() -> None:
|
||||
values = {key: value for key, value in VAR_ROWS.items() if key != "battery.runtime"}
|
||||
samples = nut.history_samples(nut.UpsReading(name="cyberpower", vars=values, commands=[]))
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
"""The single threshold table and its classifier.
|
||||
|
||||
Colour is a judgement, and a judgement made in four places drifts into four meanings.
|
||||
These tests pin the boundaries (reaching a threshold trips it, in both directions), the
|
||||
"unknown is not ok" rule, and the shape of the two payload helpers that put `severity`
|
||||
next to a value.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboards import severity
|
||||
from dashboards.severity import FAIL, INVERTED, OK, THRESHOLDS, WARN, attach, classify, with_severity
|
||||
|
||||
# The ratified table (chat, 2026-10-16). Edit a number here and in THRESHOLDS together —
|
||||
# that coupling is the point: thresholds are a contract with the pages, not a tuning knob.
|
||||
RATIFIED = {
|
||||
"cpu.usage_pct": (75.0, 90.0),
|
||||
"mem.used_pct": (80.0, 92.0),
|
||||
"disk.used_pct": (80.0, 90.0),
|
||||
"cpu.temp_c": (75.0, 90.0),
|
||||
"gpu.busy_pct": (85.0, 95.0),
|
||||
"gpu.mem_used_pct": (85.0, 95.0),
|
||||
"llm.kv_cache_usage_pct": (75.0, 90.0),
|
||||
"ups.charge_pct": (50.0, 20.0),
|
||||
"ups.battery_runtime": (600.0, 300.0),
|
||||
"ups.load_pct": (70.0, 90.0),
|
||||
}
|
||||
|
||||
|
||||
def _samples(metric: str) -> list[tuple[float, str]]:
|
||||
"""(value, expected) walking across the boundaries of one metric, in its own direction."""
|
||||
warn, danger = THRESHOLDS[metric]
|
||||
middle = (warn + danger) / 2
|
||||
if metric in INVERTED: # lower is worse: reaching warn trips it, going below danger fails
|
||||
step = 0.1
|
||||
return [
|
||||
(warn + step, OK),
|
||||
(warn, WARN),
|
||||
(middle, WARN),
|
||||
(danger, FAIL),
|
||||
(danger - 1.0, FAIL),
|
||||
]
|
||||
return [
|
||||
(warn - 0.1, OK),
|
||||
(warn, WARN),
|
||||
(middle, WARN),
|
||||
(danger, FAIL),
|
||||
(danger + 1.0, FAIL),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metric", sorted(THRESHOLDS))
|
||||
def test_boundaries_hold_for_every_metric_in_the_table(metric: str) -> None:
|
||||
for value, expected in _samples(metric):
|
||||
assert classify(metric, value) == expected, f"{metric}={value} should be {expected}"
|
||||
|
||||
|
||||
def test_completion_criteria_examples() -> None:
|
||||
assert classify("cpu.usage_pct", 95) == "fail"
|
||||
assert classify("cpu.usage_pct", 60) == "ok"
|
||||
assert classify("cpu.usage_pct", None) is None
|
||||
assert classify("ups.charge_pct", 15) == "fail"
|
||||
|
||||
|
||||
def test_warn_edge_is_inclusive_and_just_below_is_still_good() -> None:
|
||||
assert classify("cpu.usage_pct", 74.9) == "ok"
|
||||
assert classify("cpu.usage_pct", 75) == "warn"
|
||||
assert classify("cpu.usage_pct", 75.0001) == "warn"
|
||||
assert classify("cpu.usage_pct", 89.9) == "warn"
|
||||
assert classify("cpu.usage_pct", 90) == "fail"
|
||||
|
||||
|
||||
def test_inverted_metrics_read_the_other_way() -> None:
|
||||
"""A flat battery and a flat runtime are the bad end, not the good one."""
|
||||
assert classify("ups.charge_pct", 100) == "ok"
|
||||
assert classify("ups.charge_pct", 50) == "warn"
|
||||
assert classify("ups.charge_pct", 19.9) == "fail"
|
||||
assert classify("ups.battery_runtime", 3600) == "ok"
|
||||
assert classify("ups.battery_runtime", 600) == "warn"
|
||||
assert classify("ups.battery_runtime", 120) == "fail"
|
||||
|
||||
|
||||
def test_a_metric_is_only_in_one_direction() -> None:
|
||||
"""Sanity: bounds are ordered for their direction, so an edit cannot invert a metric."""
|
||||
for metric, (warn, danger) in THRESHOLDS.items():
|
||||
assert math.isfinite(warn) and math.isfinite(danger), metric
|
||||
if metric in INVERTED:
|
||||
assert warn > danger, f"{metric} is inverted: warn must sit above danger"
|
||||
else:
|
||||
assert warn < danger, f"{metric} is not inverted: warn must sit below danger"
|
||||
|
||||
|
||||
def test_inverted_is_a_subset_of_the_table() -> None:
|
||||
assert set(THRESHOLDS) >= INVERTED, "an inverted metric with no thresholds has no colour"
|
||||
|
||||
|
||||
def test_the_ratified_numbers_are_the_ones_in_the_table() -> None:
|
||||
assert THRESHOLDS == RATIFIED
|
||||
|
||||
|
||||
def test_unknown_metric_is_none_rather_than_an_exception() -> None:
|
||||
assert classify("gpu.vram_used_pct", 99) is None
|
||||
assert classify("nope.nope", 0) is None
|
||||
assert classify("", 100) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metric", sorted(THRESHOLDS))
|
||||
def test_none_is_never_ok(metric: str) -> None:
|
||||
assert classify(metric, None) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metric", sorted(THRESHOLDS))
|
||||
@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
|
||||
def test_non_finite_values_are_treated_as_missing(metric: str, value: float) -> None:
|
||||
assert classify(metric, value) is None
|
||||
|
||||
|
||||
def test_per_device_streams_inherit_their_base_metric() -> None:
|
||||
assert classify("disk.used_pct@/srv", 95) == "fail"
|
||||
assert classify("disk.used_pct@/srv", 10) == "ok"
|
||||
assert classify("gpu.busy_pct@0", 84) == "ok"
|
||||
assert classify("gpu.busy_pct@0", 90) == "warn"
|
||||
assert classify("gpu.busy_pct@1", 95) == "fail"
|
||||
assert classify("cpu.usage_pct@core7", 80) == "warn"
|
||||
|
||||
|
||||
def test_vocabulary_is_exactly_ok_warn_fail_or_none() -> None:
|
||||
seen = {classify(metric, value) for metric in THRESHOLDS for value, _ in _samples(metric)}
|
||||
assert seen == {OK, WARN, FAIL}
|
||||
inverted = severity.INVERTED
|
||||
assert inverted == {"ups.charge_pct", "ups.battery_runtime"}
|
||||
|
||||
|
||||
def test_attach_pairs_a_plain_number_with_its_verdict() -> None:
|
||||
assert attach("cpu.usage_pct", 95) == {"value": 95.0, "severity": "fail"}
|
||||
assert attach("mem.used_pct", 10) == {"value": 10.0, "severity": "ok"}
|
||||
assert attach("ups.load_pct", 70) == {"value": 70.0, "severity": "warn"}
|
||||
|
||||
|
||||
def test_attach_keeps_the_value_when_it_has_no_opinion() -> None:
|
||||
assert attach("cpu.usage_pct", None) == {"value": None, "severity": None}
|
||||
assert attach("telemetry.mystery", 42) == {"value": 42.0, "severity": None}
|
||||
|
||||
|
||||
def test_attach_never_puts_a_non_finite_number_in_a_payload() -> None:
|
||||
for value in (float("nan"), float("inf"), float("-inf")):
|
||||
assert attach("cpu.usage_pct", value) == {"value": None, "severity": None}
|
||||
assert json.dumps(attach("cpu.usage_pct", float("nan"))) # would raise otherwise
|
||||
|
||||
|
||||
def test_with_severity_is_additive_on_a_llm_shaped_cell() -> None:
|
||||
cell = {"value": 82.0, "source": "llamacpp:kv_cache_usage_ratio", "derived": False}
|
||||
out = with_severity("llm.kv_cache_usage_pct", cell)
|
||||
assert out["source"] == "llamacpp:kv_cache_usage_ratio"
|
||||
assert out["derived"] is False
|
||||
assert out["value"] == 82.0
|
||||
assert out["severity"] == "warn"
|
||||
assert cell == {"value": 82.0, "source": "llamacpp:kv_cache_usage_ratio", "derived": False}, \
|
||||
"the caller's mapping must not be mutated"
|
||||
|
||||
|
||||
def test_with_severity_handles_missing_and_non_numeric_cells() -> None:
|
||||
# purely additive: a missing cell gains `severity` and nothing else
|
||||
assert with_severity("llm.kv_cache_usage_pct", None) == {"severity": None}
|
||||
assert with_severity("llm.kv_cache_usage_pct", {"value": None}) == {"value": None, "severity": None}
|
||||
assert with_severity("llm.decode_tps", {"value": "n/a", "source": None}) == {
|
||||
"value": "n/a", "source": None, "severity": None,
|
||||
}
|
||||
# a value the table has no thresholds for still keeps its other keys
|
||||
out = with_severity("llm.ttft_seconds", {"value": 1.2, "source": "computed"})
|
||||
assert out == {"value": 1.2, "source": "computed", "severity": None}
|
||||
Reference in New Issue
Block a user