ducoterra
51772ffef2
Fixture and live stream.conf are byte-identical (modulo header comment). Let me test the parser directly:
846 lines
44 KiB
Python
846 lines
44 KiB
Python
"""`alerts.build`: the whole "what is wrong right now" answer, with no I/O anywhere.
|
|
|
|
The builder is a pure projection of state the poller already keeps and verdicts
|
|
`severity.py` / `sources/nut.assess()` already reached, so every rule can be tested by
|
|
handing it a dict — no hub, no DB, no sockets, no clock it was not given. The cases below
|
|
are the ones that make the landing page trustworthy rather than merely busy:
|
|
|
|
* a source nobody configured is **never** an alert (`AGENTS.md` invariant 3);
|
|
* a source that is stale but still serving is amber, not red;
|
|
* a dead source is **one** line, not one per stale metric in its last good payload (and not one
|
|
more for the config warnings its last good payload still carries — phase 30 task 03);
|
|
* only `fail` and `warn` come through — `ok`, `unknown`, and a missing number are silence;
|
|
* the list dedupes per (source, metric), is bounded, and says how much it hid.
|
|
|
|
The one non-reachability rule for a source with no graded numbers is the gateway's config-integrity
|
|
row: a proxy config that does not connect to itself is a structural fact, so it gets exactly one
|
|
amber line naming how many there are and quoting the first, and it is suppressed whenever that
|
|
job's own reachability line is on screen.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import inspect
|
|
from typing import Any
|
|
|
|
from dashboards import alerts, severity
|
|
from dashboards.collectors import ADDON_ERROR_PREFIXES
|
|
from dashboards.config import Settings
|
|
from dashboards.sources import nut
|
|
from dashboards.sources.nut_sim import NutSimulator
|
|
|
|
SETTINGS = Settings() # defaults: no targets, intervals 10s system / 2s llm / 5s ups
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# state shapers
|
|
# --------------------------------------------------------------------------- #
|
|
def state(key: str = "host:driveripper", dashboard: str = "system",
|
|
label: str = "driveripper", *, ok: bool = True, configured: bool = True,
|
|
error: str | None = None, age: float | None = 0.0) -> dict[str, Any]:
|
|
"""A `SourceState.as_json()`-shaped health record — the only shape the builder reads."""
|
|
return {"key": key, "dashboard": dashboard, "label": label, "configured": configured,
|
|
"ok": ok, "error": error, "age_seconds": age, "attempts": 4, "successes": 3,
|
|
"consecutive_failures": 0 if ok else 1, "last_ok": None, "latency_ms": None}
|
|
|
|
|
|
def check(name: str, status: str, detail: str = "") -> dict[str, Any]:
|
|
return {"name": name, "status": status, "detail": detail, "metric": None, "value": None}
|
|
|
|
|
|
def ups_payload(*cards: dict[str, Any], simulated: bool = False) -> dict[str, Any]:
|
|
return {"ts": 1_000.0, "simulated": simulated, "ups": list(cards)}
|
|
|
|
|
|
def ups_card(name: str = "apc", *checks: dict[str, Any]) -> dict[str, Any]:
|
|
return {"name": name, "ts": 1_000.0, "health": {"overall": "healthy", "checks": list(checks)}}
|
|
|
|
|
|
def engine(name: str = "gufo", **cells: Any) -> dict[str, Any]:
|
|
return {"engine": name, "scraped_at": 1_000.0, "metrics": dict(cells)}
|
|
|
|
|
|
def ratio(value: float | None, verdict: str | None) -> dict[str, Any]:
|
|
"""A rated LLM cell, exactly as `sources/llm.py` publishes it (0-1 ratio + verdict)."""
|
|
return {"value": value, "source": "llamacpp:kv_cache_usage_ratio", "derived": False,
|
|
"severity": verdict}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# the all-clear, and the things that must not look like a problem
|
|
# --------------------------------------------------------------------------- #
|
|
def test_a_healthy_fleet_alerts_nothing(host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
assert alerts.build(SETTINGS, [state()], [payload], [], ups_payload(), [], now=2_000.0) == []
|
|
|
|
|
|
def test_no_inputs_at_all_is_all_clear_rather_than_an_error() -> None:
|
|
"""A fresh install with nothing wired up has nothing wrong with it."""
|
|
assert alerts.build(SETTINGS) == []
|
|
|
|
|
|
def test_an_unconfigured_source_never_alerts() -> None:
|
|
"""`AGENTS.md` invariant 3: not installed is not broken.
|
|
|
|
The health record for a disabled source says `ok: false` exactly like a dead one, so
|
|
`configured` is the only thing that tells those two apart — and the state map having
|
|
nothing for a source must not be read as a failure either.
|
|
"""
|
|
out = alerts.build(SETTINGS, [
|
|
state(ok=False, configured=False, error="Connection refused"),
|
|
state(key="nut", dashboard="ups", label="NUT server", ok=False, configured=False),
|
|
], [], [], ups_payload(), now=2_000.0)
|
|
assert out == []
|
|
|
|
|
|
def test_a_card_with_no_checks_and_a_source_with_no_state_say_nothing() -> None:
|
|
assert alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc"))) == []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# reachability
|
|
# --------------------------------------------------------------------------- #
|
|
def test_a_source_that_stopped_answering_is_a_fail_naming_it_and_linking_its_page() -> None:
|
|
out = alerts.build(SETTINGS, [state(ok=False, error="Connection refused")], now=2_000.0)
|
|
assert len(out) == 1
|
|
alert = out[0]
|
|
assert alert["severity"] == "fail"
|
|
assert alert["source"] == "driveripper"
|
|
assert alert["href"] == "/system"
|
|
assert alert["message"] == "driveripper: Connection refused"
|
|
assert alert["at"] == 2_000.0
|
|
|
|
|
|
def test_the_collectors_host_prefix_is_not_doubled() -> None:
|
|
"""Collectors already say `driveripper: …`; the alert starts with the name too."""
|
|
out = alerts.build(SETTINGS, [state(ok=False,
|
|
error="driveripper: http://h:9120/snapshot unreachable")])
|
|
assert out[0]["message"] == "driveripper: http://h:9120/snapshot unreachable"
|
|
|
|
|
|
def test_the_pollers_exception_prefix_is_not_printed_either() -> None:
|
|
"""The live shape is `poller`'s `UpstreamError: name: …`; one name, no class noise.
|
|
|
|
Verified against a real stopped agent: the health record reads
|
|
`UpstreamError: gateway: http://<host>:9120/snapshot unreachable (ConnectError)`,
|
|
and printing that verbatim put the host name on the page twice.
|
|
"""
|
|
out = alerts.build(SETTINGS, [state(key="host:gateway", label="gateway", ok=False,
|
|
error="UpstreamError: gateway: http://h:9120/snapshot "
|
|
"unreachable (ConnectError)")])
|
|
assert out[0]["message"] == ("gateway: http://h:9120/snapshot unreachable (ConnectError)")
|
|
|
|
|
|
def test_an_unexpected_exception_class_is_still_named() -> None:
|
|
"""Only `UpstreamError` is noise; a bug in our own code is worth reading about."""
|
|
out = alerts.build(SETTINGS, [state(ok=False, error="ValueError: bad snapshot")])
|
|
assert out[0]["message"] == "driveripper: ValueError: bad snapshot"
|
|
|
|
|
|
def test_a_failing_source_with_no_recorded_error_still_says_so() -> None:
|
|
out = alerts.build(SETTINGS, [state(key="llm:gufo", dashboard="llm", label="gufo",
|
|
ok=False, error=None)])
|
|
assert out[0]["message"] == "gufo: is not answering"
|
|
assert out[0]["href"] == "/llm"
|
|
|
|
|
|
def test_a_stale_but_serving_source_is_a_warn_because_the_page_still_shows_numbers() -> None:
|
|
"""Last good value on screen + no refresh for 3 intervals = amber, not `dead`."""
|
|
out = alerts.build(SETTINGS, [state(age=45.0)], now=2_000.0)
|
|
assert [item["severity"] for item in out] == ["warn"]
|
|
assert "45s ago" in out[0]["message"]
|
|
assert "last good value" in out[0]["message"]
|
|
assert out[0]["at"] == 2_000.0 - 45.0
|
|
# inside the grace period (3 x 10s) the same source is not news at all
|
|
assert alerts.build(SETTINGS, [state(age=25.0)]) == []
|
|
|
|
|
|
def test_staleness_is_measured_in_each_sources_own_poll_interval() -> None:
|
|
"""system polls every 10s and the UPS every 5s — one global number is wrong for one."""
|
|
llm = state(key="llm:gufo", dashboard="llm", label="gufo", age=5.0) # 3 x 2s = 6s
|
|
ups = state(key="nut", dashboard="ups", label="NUT server", age=20.0) # 3 x 5s = 15s
|
|
assert alerts.build(SETTINGS, [llm]) == []
|
|
out = alerts.build(SETTINGS, [ups])
|
|
assert out[0]["severity"] == "warn" and out[0]["href"] == "/ups"
|
|
|
|
|
|
def test_a_dashboard_the_builder_does_not_know_about_still_gets_a_verdict_and_a_home() -> None:
|
|
"""A new dashboard must not be silently un-alertable while its interval is unknown."""
|
|
unknown_down = {"key": "probe:0", "dashboard": "telemetry", "label": "probe",
|
|
"configured": True, "ok": False, "error": "no route"}
|
|
assert alerts.build(SETTINGS, [unknown_down])[0]["href"] == "/"
|
|
unknown_stale = {**unknown_down, "ok": True, "error": None, "age_seconds": 200.0}
|
|
out = alerts.build(SETTINGS, [unknown_stale])
|
|
assert "every 60s" in out[0]["message"]
|
|
|
|
|
|
def test_the_disk_job_is_stale_only_after_three_of_its_own_five_minute_polls() -> None:
|
|
"""SMART is a 300 s signal: judging it by the 10 s system interval would amber `/`
|
|
every five minutes for a source that is behaving exactly as configured."""
|
|
disks = Settings(disks_interval=300.0)
|
|
fresh = state(key="disks:driveripper", dashboard="disks", label="driveripper", age=600.0)
|
|
assert alerts.build(disks, [fresh]) == [], "600s is two polls, not an outage"
|
|
stale = {**fresh, "age_seconds": 1_000.0}
|
|
out = alerts.build(disks, [stale])
|
|
assert out[0]["severity"] == "warn"
|
|
assert "every 300s" in out[0]["message"], "the message must print the disks interval"
|
|
|
|
|
|
def test_a_disk_job_that_failed_because_its_agent_is_down_is_one_line_not_two() -> None:
|
|
"""`disks` reads the same agent snapshot `system` does.
|
|
|
|
A dead host would otherwise print the same sentence twice (`alerts` rule: one dead
|
|
source, one line), with only the internal id telling the two apart — which is a lie of
|
|
the same shape as a duplicated host name.
|
|
"""
|
|
out = alerts.build(SETTINGS, [
|
|
state(key="disks:driveripper", dashboard="disks", label="driveripper",
|
|
ok=False, error="UpstreamError: driveripper: http://h:9120/snapshot "
|
|
"unreachable (ConnectError)"),
|
|
state(ok=False, error="UpstreamError: driveripper: http://h:9120/snapshot "
|
|
"unreachable (ConnectError)"),
|
|
], now=2_000.0)
|
|
assert [item["source"] for item in out] == ["driveripper"]
|
|
assert len(out) == 1, "the echo of the same outage must not read as two problems"
|
|
assert out[0]["id"].startswith("driveripper:host:")
|
|
|
|
|
|
def test_a_disk_job_that_fails_while_its_host_answers_is_the_only_voice_it_has() -> None:
|
|
"""SMART wedged inside the agent is a `disks`-only fault, and must still be said."""
|
|
out = alerts.build(SETTINGS, [
|
|
state(key="disks:driveripper", dashboard="disks", label="driveripper",
|
|
ok=False, error="UpstreamError: driveripper: timed out"),
|
|
state(),
|
|
], now=2_000.0)
|
|
assert len(out) == 1
|
|
assert out[0]["id"].startswith("driveripper:disks:")
|
|
assert "timed out" in out[0]["message"]
|
|
|
|
|
|
def test_a_disk_only_alert_links_to_the_drive_page() -> None:
|
|
"""The overview's row is a link, so it has to point at the page that explains it.
|
|
|
|
A wedged SMART pass has no other voice (the host itself is answering), and `/system`
|
|
cannot tell the operator which drive stopped reporting — `/disks` can. Before that route
|
|
existed this link pointed at `/` on purpose; now it must not point anywhere else.
|
|
"""
|
|
out = alerts.build(SETTINGS, [
|
|
state(key="disks:driveripper", dashboard="disks", label="driveripper",
|
|
ok=False, error="UpstreamError: driveripper: timed out"),
|
|
state(),
|
|
], now=2_000.0)
|
|
assert len(out) == 1
|
|
assert out[0]["href"] == "/disks"
|
|
|
|
|
|
def test_an_unconfigured_disk_source_says_nothing_even_when_its_job_says_down() -> None:
|
|
"""No `HOST_TARGETS` means no `disks` job exists; a reported-but-unconfigured one must
|
|
still be silence (invariant 3).
|
|
|
|
The builder is fed job states rather than settings, so a hub that ever hands over a
|
|
`disks` record with `configured: false` — a config that arrives between two polls, a
|
|
future target list of its own — must not amber `/` over a page nobody installed.
|
|
"""
|
|
out = alerts.build(Settings(disks_interval=300.0), [
|
|
state(key="disks:driveripper", dashboard="disks", label="driveripper",
|
|
ok=False, configured=False, error="no hosts configured"),
|
|
], now=2_000.0)
|
|
assert out == []
|
|
|
|
|
|
def test_a_host_threshold_reads_the_system_job_not_the_disk_job(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
"""Two jobs share a host label, so the lookup has to be filtered by dashboard.
|
|
|
|
`disks` is registered after `system`, but the builder must not depend on that: if an
|
|
unfiltered lookup found the failing disk job first, the host would be called
|
|
unreachable and its real threshold breach would vanish.
|
|
"""
|
|
payload = host_scrape(agent_snapshot, cpu_busy=95.0) # cpu.usage_pct -> fail
|
|
out = alerts.build(SETTINGS, [
|
|
state(key="disks:driveripper", dashboard="disks", label="driveripper",
|
|
ok=False, error="timed out"),
|
|
state(),
|
|
], [payload], [], None, now=2_000.0)
|
|
ids = [item["id"] for item in out]
|
|
assert "driveripper:cpu.usage_pct" in ids, "the breach is real; the disk job is not it"
|
|
|
|
|
|
def test_a_failing_source_says_when_the_poller_first_noticed() -> None:
|
|
events = [{"ts": 1_500.0, "source": "system", "host": "driveripper", "kind": "unreachable",
|
|
"severity": "error", "message": "ConnectError", "data": None},
|
|
{"ts": 1_600.0, "source": "host", "host": "deskwork", "kind": "reboot",
|
|
"severity": "warn", "message": "deskwork rebooted", "data": None}]
|
|
out = alerts.build(SETTINGS, [state(ok=False, error="nope")], [], [], None, events,
|
|
now=2_000.0)
|
|
assert out[0]["at"] == 1_500.0, "the outage clock must not reset on every refresh"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# host thresholds, degraded agents
|
|
# --------------------------------------------------------------------------- #
|
|
def test_a_threshold_breach_names_the_host_the_metric_and_the_value(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
payload = host_scrape(agent_snapshot, cpu_busy=95.0)
|
|
out = alerts.build(SETTINGS, [state()], [payload], now=2_000.0)
|
|
assert out[0]["id"] == "driveripper:cpu.usage_pct"
|
|
assert out[0]["message"] == "driveripper: CPU 95% (fail ≥90)"
|
|
assert out[0]["href"] == "/system"
|
|
assert out[0]["at"] == 1_010.0, "`at` is the poll that measured it"
|
|
|
|
|
|
def test_every_rated_host_field_alerts_at_its_own_verdict(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
def nudge(snap: dict[str, Any]) -> None:
|
|
snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.85 # warn ≥80
|
|
snap["cpu_temperature_celsius"] = 80.0 # warn ≥75
|
|
for fs in snap["filesystems"]:
|
|
if fs["mountpoint"] == "/btrfs/pool1":
|
|
fs["used_pct"] = 91.0 # fail ≥90
|
|
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0, mutate=nudge)
|
|
out = {item["id"]: item for item in alerts.build(SETTINGS, [state()], [payload])}
|
|
assert out["driveripper:mem.used_pct"]["message"] == "driveripper: memory 85% (warn ≥80)"
|
|
assert out["driveripper:cpu.temp_c"]["message"] == (
|
|
"driveripper: CPU temperature 80°C (warn ≥75)")
|
|
assert out["driveripper:disk.used_pct"]["message"] == (
|
|
"driveripper: disk 91% on /btrfs/pool1 (fail ≥90)")
|
|
assert "driveripper:cpu.usage_pct" not in out, "10% busy is not a problem"
|
|
|
|
|
|
def test_a_number_the_upstream_does_not_publish_is_never_a_breach(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
"""No measurement, no verdict, no alert — the same rule `severity.classify` upholds."""
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0,
|
|
mutate=lambda snap: snap.pop("cpu_temperature_celsius"))
|
|
assert alerts.build(SETTINGS, [state()], [payload]) == []
|
|
|
|
|
|
def test_a_saturated_gpu_is_not_an_alert(host_scrape: Any,
|
|
agent_snapshot: dict[str, Any]) -> None:
|
|
"""Phase 06: these are LLM servers — a fully-worked GPU is the job, not a fault.
|
|
|
|
The host is otherwise all-clear, so a card at 100 % busy with both memory pools
|
|
filled must produce **no** alert at all: `severity.THRESHOLDS` has no `gpu.*` row,
|
|
the payload carries no GPU verdict, and this module owns no threshold of its own
|
|
from which to resurrect one.
|
|
"""
|
|
def fill_the_card(snap: dict[str, Any]) -> None:
|
|
gpu = snap["gpus"][0]
|
|
gpu["busy_pct"] = 100.0
|
|
for used, total in (("vram_used_bytes", "vram_total_bytes"),
|
|
("gtt_used_bytes", "gtt_total_bytes")):
|
|
if gpu.get(total) is not None:
|
|
gpu[used] = gpu[total]
|
|
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0, mutate=fill_the_card)
|
|
assert payload["gpus"][0]["busy_pct"] == 100.0, "the card really is saturated here"
|
|
assert alerts.build(SETTINGS, [state()], [payload]) == [], (
|
|
"a busy GPU produced an alert — GPU verdicts were removed in phase 06"
|
|
)
|
|
|
|
|
|
def test_a_degraded_agent_warns_and_names_the_sections_that_failed(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
payload["degraded"] = {"temperatures": "no hwmon sensors", "disk_io": ""}
|
|
out = alerts.build(SETTINGS, [state()], [payload])
|
|
assert len(out) == 1, "a partial snapshot is one warning, not one per number that is missing"
|
|
assert out[0]["id"] == "driveripper:agent.sections"
|
|
assert out[0]["severity"] == "warn"
|
|
assert "temperatures (no hwmon sensors)" in out[0]["message"]
|
|
assert "disk_io" in out[0]["message"]
|
|
|
|
|
|
def test_an_addon_absence_line_raises_no_row_and_a_section_that_failed_raises_one(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
"""Invariant 3's last link, `resolver:` included: filtered out means no row, unfiltered means one.
|
|
|
|
`collectors.ADDON_ERROR_PREFIXES` is what keeps an agent's add-on lines out of `degraded`
|
|
(`tests/test_collectors.py` pins that), and `degraded` is what this module turns into the
|
|
overview row pinned just above. So: a host whose agent said it runs no haproxy, no nginx, no
|
|
smartmontools and had **nothing to resolve** is all-clear on every panel — the phase-32 pair of
|
|
lines (`nothing to resolve`, `pass budget … exhausted`) included, because a truncated lookup is
|
|
one route without a count and not a broken box. And a line that *did* reach `degraded` — which
|
|
is what dropping a prefix from that list would do — does raise exactly one row, naming it.
|
|
"""
|
|
addon_lines = (
|
|
"resolver: nothing to resolve — haproxy not found and nginx not found; --no-resolve opts out",
|
|
"resolver: pass budget 3.0s exhausted, 41 of 58 targets looked up",
|
|
"haproxy: /etc/haproxy/haproxy.cfg not found — haproxy is not installed here",
|
|
"smart: smartctl 7.5 not found",
|
|
)
|
|
assert all(line.lower().startswith(ADDON_ERROR_PREFIXES) for line in addon_lines), (
|
|
"a prefix dropped from collectors' list would put this fleet's non-gateway hosts on warn")
|
|
|
|
clean = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
clean["agent"]["errors"] = list(addon_lines)
|
|
assert "degraded" not in clean, "the collector filtered them, so there is nothing to grade"
|
|
assert alerts.build(SETTINGS, [state()], [clean]) == []
|
|
|
|
for line in addon_lines:
|
|
broken = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
broken["degraded"] = [line]
|
|
out = alerts.build(SETTINGS, [state()], [broken])
|
|
assert [row["id"] for row in out] == ["driveripper:agent.sections"], line
|
|
|
|
|
|
def test_a_degraded_list_from_an_older_agent_is_readable_too(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
payload["degraded"] = ["network"]
|
|
out = alerts.build(SETTINGS, [state()], [payload])
|
|
assert out[0]["message"] == "driveripper: the agent could not read network"
|
|
|
|
|
|
def test_a_degraded_note_that_is_a_plain_string_is_still_reported(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
"""An agent (or a future collector) that reports one string instead of a map."""
|
|
payload = host_scrape(agent_snapshot, cpu_busy=10.0)
|
|
payload["degraded"] = "hwmon"
|
|
out = alerts.build(SETTINGS, [state()], [payload])
|
|
assert out[0]["message"] == "driveripper: the agent could not read hwmon"
|
|
|
|
|
|
def test_an_unreachable_host_is_one_alert_even_with_its_last_good_payload(
|
|
host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
|
|
"""The exact completion criterion for the phase: one line naming the dead host."""
|
|
def nudge(snap: dict[str, Any]) -> None:
|
|
snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.99
|
|
snap["cpu_temperature_celsius"] = 95.0
|
|
|
|
payload = host_scrape(agent_snapshot, cpu_busy=96.0, mutate=nudge)
|
|
out = alerts.build(SETTINGS, [state(ok=False, error="timed out")], [payload])
|
|
assert len(out) == 1
|
|
assert out[0]["severity"] == "fail" and "timed out" in out[0]["message"]
|
|
|
|
|
|
def test_a_stub_payload_for_a_host_that_never_answered_alerts_once() -> None:
|
|
"""/api/system/snapshot represents that host as {name, unreachable}; still one line."""
|
|
stub = {"name": "gateway", "ts": 1_000.0, "unreachable": True}
|
|
out = alerts.build(SETTINGS, [state(key="host:gateway", label="gateway", ok=False,
|
|
error="unreachable")], [stub])
|
|
assert [item["id"] for item in out] == ["gateway:host:gateway:down"]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# LLM engines
|
|
# --------------------------------------------------------------------------- #
|
|
def test_kv_pressure_comes_from_the_rated_cell_and_reads_as_a_percent() -> None:
|
|
"""The engine publishes 0-1; the operator reads percent, matching the verdict's scale."""
|
|
out = alerts.build(SETTINGS, [state(key="llm:gufo", dashboard="llm", label="gufo")], [],
|
|
[engine(kv_cache_usage_ratio=ratio(0.95, "fail"))])
|
|
assert out[0]["message"] == "gufo: KV cache 95% (fail ≥90)"
|
|
assert out[0]["href"] == "/llm"
|
|
assert out[0]["id"] == "gufo:llm.kv_cache_usage_pct@kv_cache_usage_ratio"
|
|
|
|
|
|
def test_vllm_publishes_the_same_verdict_under_its_own_series_name() -> None:
|
|
out = alerts.build(SETTINGS, [], [], [engine("vllm", gpu_cache_usage=ratio(0.8, "warn"))])
|
|
assert out[0]["message"] == "vllm: KV cache 80% (warn ≥75)"
|
|
|
|
|
|
def test_a_cool_or_absent_kv_cell_is_not_an_alert() -> None:
|
|
for cell in (ratio(0.10, "ok"), ratio(None, None), ratio(0.95, None), {}, "not a cell"):
|
|
out = alerts.build(SETTINGS, [], [], [engine(kv_cache_usage_ratio=cell)])
|
|
assert out == [], cell
|
|
|
|
|
|
def test_an_engine_that_is_not_answering_alerts_only_its_own_job() -> None:
|
|
dead = {"engine": "vllm", "metrics": {}, "sessions": {}, "unreachable": True}
|
|
out = alerts.build(SETTINGS, [state(key="llm:vllm", dashboard="llm", label="vLLM",
|
|
ok=False, error="502 Bad Gateway")], [], [dead])
|
|
assert [item["id"] for item in out] == ["vLLM:llm:vllm:down"]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# UPS
|
|
# --------------------------------------------------------------------------- #
|
|
def test_a_failing_ups_check_is_a_fail_carrying_the_check_and_its_detail() -> None:
|
|
payload = ups_payload(ups_card("apc",
|
|
check("Power path", "fail", "The UPS is on battery"),
|
|
check("Charge", "warn", "42% and discharging")))
|
|
out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server")],
|
|
[], [], payload)
|
|
by_id = {item["id"]: item for item in out}
|
|
assert by_id["apc:health:Power path"]["severity"] == "fail"
|
|
assert by_id["apc:health:Power path"]["message"] == "Power path: The UPS is on battery"
|
|
assert by_id["apc:health:Charge"]["severity"] == "warn"
|
|
assert all(item["href"] == "/ups" for item in out)
|
|
|
|
|
|
def test_the_landing_page_is_where_battery_age_belongs() -> None:
|
|
"""The task's own example: a sentence an operator can act on, not a score."""
|
|
detail = "5.3 years — past the 4-year replacement window"
|
|
out = alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc",
|
|
check("Battery age", "warn", detail))))
|
|
assert out[0]["message"] == f"Battery age: {detail}"
|
|
assert out[0]["source"] == "apc"
|
|
|
|
|
|
def test_an_unknown_check_is_silence_and_a_check_with_no_detail_is_still_readable() -> None:
|
|
payload = ups_payload(ups_card("apc", check("Battery temperature", "unknown",
|
|
"not reported by this driver"),
|
|
check("Alarms", "fail")))
|
|
out = alerts.build(SETTINGS, [], [], [], payload)
|
|
assert [item["message"] for item in out] == ["Alarms"]
|
|
|
|
|
|
def test_simulated_ups_data_is_announced_once_and_loudly() -> None:
|
|
"""With NUT_SIMULATE=1 and otherwise-healthy cards, that warning is the whole list."""
|
|
payload = ups_payload(ups_card("apc", check("Charge", "ok", "100%")),
|
|
ups_card("cyberpower", check("Charge", "ok", "100%")),
|
|
simulated=True)
|
|
out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server")],
|
|
[], [], payload)
|
|
assert len(out) == 1
|
|
assert out[0]["severity"] == "warn"
|
|
assert "NUT_SIMULATE=1" in out[0]["message"]
|
|
assert out[0]["href"] == "/ups"
|
|
|
|
|
|
async def test_the_simulator_on_its_own_produces_exactly_that_warning() -> None:
|
|
"""The task's acceptance line, through the real simulator and the real payload builder."""
|
|
readings = await NutSimulator().read()
|
|
payload = {"ts": 1_000.0, "simulated": True,
|
|
"ups": [nut.payload(reading) for reading in readings]}
|
|
out = alerts.build(SETTINGS,
|
|
[state(key="nut", dashboard="ups", label="NUT server")], [], [], payload)
|
|
assert [item["id"] for item in out] == ["ups:simulated"], out
|
|
|
|
|
|
def test_a_real_upsd_does_not_claim_to_be_simulated() -> None:
|
|
out = alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc")))
|
|
assert out == []
|
|
|
|
|
|
def test_a_dead_upsd_is_one_alert_and_not_its_stale_cards_too() -> None:
|
|
payload = ups_payload(ups_card("apc", check("Power path", "fail", "on battery")))
|
|
out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server",
|
|
ok=False, error="upsd closed the connection")],
|
|
[], [], payload)
|
|
assert [item["source"] for item in out] == ["NUT server"]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# the gateway page: reachability plus config integrity, and a dead agent is still one line
|
|
# --------------------------------------------------------------------------- #
|
|
def gateway_state(**kwargs: Any) -> dict[str, Any]:
|
|
"""The health record of the one `gateway` job (the `nut` shape: key `gateway`, label = host)."""
|
|
return state("gateway", "gateway", "gateway", **kwargs)
|
|
|
|
|
|
def gateway_payload(*messages: str, ts: float | None = 1_000.0) -> dict[str, Any]:
|
|
"""A gateway payload carrying exactly `messages` as its config warnings, in that order."""
|
|
return {"ts": ts, "host": "gateway", "supported": True,
|
|
"warnings": [{"id": f"synthetic:{index}", "kind": "synthetic",
|
|
"subject": str(index), "message": text}
|
|
for index, text in enumerate(messages)]}
|
|
|
|
|
|
def test_config_warnings_are_one_warn_row_naming_the_count_and_the_first_message() -> None:
|
|
"""`gateway:config`: the one row a proxy config that does not connect to itself earns.
|
|
|
|
Phase 30 task 03 changed the documented rule this file used to pin — a `gateway` alert used to
|
|
be reachability only. What it is now: one `warn` row, `href` `/gateway`, `at` the payload's own
|
|
timestamp (the config did not change between polls, so an alert whose clock reset every 30 s
|
|
would be unreadable), the count and the first message in the sentence.
|
|
"""
|
|
out = alerts.build(Settings(gateway_interval=30.0), [gateway_state()],
|
|
gateway_payload=gateway_payload(
|
|
'frontend haproxy-22 routes to backend "gone", which no backend '
|
|
"block declares",
|
|
'backend "old" has 2 servers and no frontend routes to it'),
|
|
now=2_000.0)
|
|
assert out == [{"id": "gateway:config", "severity": "warn", "source": "gateway",
|
|
"href": "/gateway", "at": 1_000.0,
|
|
"message": ("2 proxy config problems to address — "
|
|
'frontend haproxy-22 routes to backend "gone", which no '
|
|
"backend block declares")}]
|
|
|
|
|
|
def test_one_warning_and_seven_produce_exactly_one_row_either_way() -> None:
|
|
"""The row is aggregate on purpose: seven dangling routes are one config to fix.
|
|
|
|
Seven amber lines would push every other source off the overview panel to say one thing, and the
|
|
page beside them lists all seven — so the count is the number and the first message is the
|
|
sentence, once.
|
|
"""
|
|
settings = Settings(gateway_interval=30.0)
|
|
for count in (1, 2, 7):
|
|
out = alerts.build(settings, [gateway_state()],
|
|
gateway_payload=gateway_payload(*[f"problem {index}" for index
|
|
in range(count)]),
|
|
now=2_000.0)
|
|
assert len(out) == 1, f"{count} warnings produced {len(out)} rows"
|
|
assert out[0]["id"] == "gateway:config" and out[0]["severity"] == "warn"
|
|
assert out[0]["href"] == "/gateway" and out[0]["source"] == "gateway"
|
|
noun = "problem" if count == 1 else "problems"
|
|
assert out[0]["message"] == f"{count} proxy config {noun} to address — problem 0", (
|
|
"the count, the noun that matches it, and the first message verbatim")
|
|
|
|
|
|
def test_a_clean_ingress_warns_about_nothing_and_a_missing_payload_does_too() -> None:
|
|
"""Both directions of the row, because an absence-only assertion is satisfied by deleting it."""
|
|
settings = Settings(gateway_interval=30.0)
|
|
assert alerts.build(settings, [gateway_state()],
|
|
gateway_payload=gateway_payload(), now=2_000.0) == []
|
|
assert alerts.build(settings, [gateway_state()],
|
|
gateway_payload={"host": "gateway", "warnings": None},
|
|
now=2_000.0) == [], "a malformed list is not a fault either"
|
|
assert alerts.build(settings, [gateway_state()], now=2_000.0) == [], (
|
|
"no payload at all — unconfigured, or configured and never answered — says nothing")
|
|
|
|
|
|
def test_a_failing_gateway_job_keeps_its_reachability_row_and_gains_no_config_row() -> None:
|
|
"""A dead source is one line, and the payload it left behind is not a second one.
|
|
|
|
Both arms of the rule, in both of its shapes: `ok: false` (the job is down) and stale (`ok: true`
|
|
past three of its own polls). In each, the reachability row is what the operator sees, and the
|
|
last good payload's warnings are not re-listed beside it — the warnings will be on `/gateway`
|
|
when the source answers again, and a second line here would be the invariant the `disks` rule
|
|
already keeps.
|
|
"""
|
|
settings = Settings(gateway_interval=30.0)
|
|
broken = gateway_payload('backend "gone" declares no server — it routes nowhere')
|
|
|
|
down = alerts.build(settings, [gateway_state(ok=False, error="UpstreamError: gateway: timed out")],
|
|
gateway_payload=broken, now=2_000.0)
|
|
assert [row["id"] for row in down] == ["gateway:gateway:down"], down
|
|
assert "timed out" in down[0]["message"]
|
|
|
|
stale = alerts.build(settings, [gateway_state(age=100.0)], gateway_payload=broken,
|
|
now=2_000.0)
|
|
assert [row["id"] for row in stale] == ["gateway:gateway:stale"], stale
|
|
assert "every 30s" in stale[0]["message"]
|
|
|
|
# the control: the same payload with a job that IS answering does grow the config row
|
|
live = alerts.build(settings, [gateway_state(age=4.0)], gateway_payload=broken, now=2_000.0)
|
|
assert [row["id"] for row in live] == ["gateway:config"]
|
|
|
|
|
|
def test_an_unconfigured_gateway_with_a_warning_payload_still_says_nothing() -> None:
|
|
"""Invariant 3 with a payload attached: a source nobody installed has no config to be broken."""
|
|
out = alerts.build(Settings(gateway_interval=30.0),
|
|
[gateway_state(configured=False)],
|
|
gateway_payload=gateway_payload("anything"), now=2_000.0)
|
|
assert out == []
|
|
|
|
|
|
def test_a_gateway_config_row_survives_a_dead_host_beside_it_and_sorts_after_the_fail() -> None:
|
|
"""The config row is the gateway's own voice and the outage row is the host's — both stand.
|
|
|
|
The echo table suppresses a `gateway` job that failed *because* its agent did; a config warning
|
|
is not that, so a healthy host with a broken ingress gets exactly one overview row about the
|
|
ingress and nothing else, and a failing host plus a stale config list stays one row too.
|
|
"""
|
|
settings = Settings(gateway_interval=30.0)
|
|
out = alerts.build(settings, [gateway_state(), state("host:gateway", "system", "gateway")],
|
|
gateway_payload=gateway_payload("fix me"), now=2_000.0)
|
|
assert [row["id"] for row in out] == ["gateway:config"]
|
|
|
|
host_down = state("host:gateway", "system", "gateway", ok=False, error="unreachable")
|
|
out = alerts.build(settings, [gateway_state(ok=False), host_down],
|
|
gateway_payload=gateway_payload("fix me"), now=2_000.0)
|
|
assert [row["severity"] for row in out] == ["fail"], "the outage is the only line"
|
|
|
|
|
|
def test_a_config_warning_row_is_the_worst_thing_about_a_gateway_never_a_number() -> None:
|
|
"""The rule restated after it changed: reachability plus config integrity, no graded number.
|
|
|
|
Three ways, because deleting a rule satisfies an absence-only assertion:
|
|
|
|
* `severity.THRESHOLDS` has no `gateway.*` row, so there is no bound for this module to read
|
|
even if a threshold were added here (and a busy proxy is the job, the `net.*` case);
|
|
* the row's message is built from `warnings` and nothing else — a payload of nothing but counts
|
|
yields no row at all;
|
|
* `alerts.HREF` still carries `/gateway`, so both voices reach the page that explains them.
|
|
"""
|
|
assert not [name for name in severity.THRESHOLDS if name.startswith("gateway.")], (
|
|
"a connection count is not a graded number")
|
|
settings = Settings(gateway_interval=30.0)
|
|
counts = {"ts": 1_000.0, "host": "gateway", "listeners": [{"connections": 9_999}],
|
|
"connections": [{"connections": 9_999}], "sockets": {"established": 9_999}}
|
|
assert alerts.build(settings, [gateway_state()], gateway_payload=counts, now=2_000.0) == []
|
|
assert alerts.HREF["gateway"] == "/gateway"
|
|
assert alerts.AGENT_ECHO_DASHBOARDS == ("disks", "gateway"), (
|
|
"task 03 changed what a gateway may alert ON, not which lines it echoes")
|
|
# and a gateway with a payload whose rows are junk is silence, not a crash and not a claim
|
|
assert alerts.build(settings, [gateway_state()],
|
|
gateway_payload={"warnings": ["not a mapping", 7]},
|
|
now=2_000.0) == []
|
|
|
|
|
|
def test_a_stale_gateway_job_is_a_warn_measured_in_its_own_thirty_second_poll() -> None:
|
|
"""The pipes are drawn from a 30 s poll, so 60 s of silence is two healthy polls.
|
|
|
|
Judging this source by the 10 s system interval would amber the landing page every minute for
|
|
a job behaving exactly as configured — the same mistake the `disks` rule exists to avoid, one
|
|
cadence later.
|
|
"""
|
|
settings = Settings(gateway_interval=30.0)
|
|
assert alerts.build(settings, [gateway_state(age=60.0)]) == []
|
|
out = alerts.build(settings, [gateway_state(age=100.0)])
|
|
assert out[0]["severity"] == "warn"
|
|
assert out[0]["href"] == "/gateway", "the overview's row must link to the page it describes"
|
|
assert "every 30s" in out[0]["message"], "the message prints the gateway's own interval"
|
|
|
|
|
|
def test_a_gateway_job_that_fails_while_its_host_answers_is_the_only_voice_it_has() -> None:
|
|
"""A wedged proxy/socket section is a fault nothing else can report (the `disks` rule, once more)."""
|
|
out = alerts.build(SETTINGS, [
|
|
gateway_state(ok=False, error="UpstreamError: gateway: timed out"),
|
|
state("host:gateway", "system", "gateway"),
|
|
], now=2_000.0)
|
|
assert len(out) == 1
|
|
assert out[0]["source"] == "gateway"
|
|
assert out[0]["href"] == "/gateway"
|
|
assert "timed out" in out[0]["message"]
|
|
|
|
|
|
def test_a_gateway_job_that_failed_because_its_agent_is_down_is_one_line_not_two() -> None:
|
|
"""Three jobs dial one agent; the operator has one problem, so the overview says it once."""
|
|
down = {"ok": False,
|
|
"error": "UpstreamError: gateway: http://h:9120/snapshot unreachable (ConnectError)"}
|
|
out = alerts.build(SETTINGS, [
|
|
gateway_state(**down),
|
|
state("disks:gateway", "disks", "gateway", **down),
|
|
state("host:gateway", "system", "gateway", **down),
|
|
], now=2_000.0)
|
|
assert len(out) == 1, "the echo of one outage must not read as three problems"
|
|
assert out[0]["id"].startswith("gateway:host:"), out[0]["id"]
|
|
|
|
|
|
def test_an_unconfigured_gateway_source_says_nothing_even_when_its_job_says_down() -> None:
|
|
"""No `HOST_TARGETS` entry named `GATEWAY_HOST` means no job — and a reported one stays quiet."""
|
|
out = alerts.build(Settings(gateway_interval=30.0),
|
|
[gateway_state(ok=False, configured=False, error="not configured")],
|
|
now=2_000.0)
|
|
assert out == []
|
|
|
|
|
|
def test_a_gateway_payload_with_a_fabricated_verdict_alerts_nothing() -> None:
|
|
"""The no-verdict rule restated for the payload this task added a reader for.
|
|
|
|
`alerts.build` now *takes* a gateway payload (phase 30 task 03), so the old signature-only pin
|
|
is gone and what replaces it is stronger and narrower: the parameter is keyword-only with a
|
|
default (a positional gateway would be a rule this module grew by accident), it is the ONLY
|
|
gateway input, `severity.THRESHOLDS` still has no `gateway.*` row for it to consult, and a
|
|
payload with `severity` stamped into it stays silent — the row reads `warnings` and nothing
|
|
else. An answering gateway with a clean list is the all clear.
|
|
"""
|
|
signature = inspect.signature(alerts.build).parameters
|
|
gateway_argument = signature["gateway_payload"]
|
|
assert gateway_argument.kind is inspect.Parameter.KEYWORD_ONLY, (
|
|
"a positional gateway argument would let a caller pass a count where a payload belongs")
|
|
assert gateway_argument.default is None
|
|
assert not [name for name in signature
|
|
if name.startswith("gateway") and name != "gateway_payload"], (
|
|
"a second gateway input is a new rule in disguise")
|
|
assert not [name for name in severity.THRESHOLDS if name.startswith("gateway.")], (
|
|
"a connection count is not a graded number")
|
|
assert alerts.HREF["gateway"] == "/gateway"
|
|
assert alerts.AGENT_ECHO_DASHBOARDS == ("disks", "gateway")
|
|
# a fabricated verdict anywhere in the payload changes nothing: only `warnings` is read
|
|
rigged = gateway_payload()
|
|
rigged["severity"] = {"listeners": "fail", "connections": "fail"}
|
|
rigged["listeners"] = [{"connections": 100, "severity": "fail"}]
|
|
rigged["connections"] = [{"connections": 100, "severity": "fail"}]
|
|
settings = Settings(gateway_interval=30.0)
|
|
assert alerts.build(settings, [gateway_state()], gateway_payload=rigged, now=2_000.0) == []
|
|
assert alerts.build(settings, [gateway_state()], [], [], None, [], now=2_000.0) == []
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# configuration
|
|
# --------------------------------------------------------------------------- #
|
|
def test_a_bad_target_entry_is_a_warn_that_says_it_is_not_being_polled() -> None:
|
|
settings = Settings(host_target_problems=("nope is not name=url",
|
|
"deskwork: url must start with http://"))
|
|
out = alerts.build(settings)
|
|
assert len(out) == 2
|
|
assert all(item["severity"] == "warn" and item["source"] == "config" for item in out)
|
|
assert all(item["href"] == "/system" for item in out)
|
|
assert "HOST_TARGETS" in out[0]["message"] and "not being polled" in out[0]["message"]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# dedupe, bound, order, shape
|
|
# --------------------------------------------------------------------------- #
|
|
def test_one_alert_per_source_and_metric_with_the_worst_verdict_winning() -> None:
|
|
"""Two cards answering as the same name (a re-exported UPS) must not double the alarm."""
|
|
payload = ups_payload(ups_card("apc", check("Charge", "warn", "60%")),
|
|
ups_card("apc", check("Charge", "fail", "12%")))
|
|
out = alerts.build(SETTINGS, [], [], [], payload)
|
|
assert len(out) == 1
|
|
assert out[0]["severity"] == "fail"
|
|
assert out[0]["message"] == "Charge: 12%"
|
|
|
|
|
|
def test_the_list_is_bounded_and_says_how_many_it_hid() -> None:
|
|
cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery"))
|
|
for i in range(alerts.MAX_ALERTS + 5)]
|
|
out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards))
|
|
assert len(out) == alerts.MAX_ALERTS
|
|
marker = out[-1]
|
|
assert marker["id"] == "alerts:suppressed"
|
|
assert marker["severity"] == "fail", "hiding reds behind an amber marker is a lie"
|
|
hidden = len(cards) - (alerts.MAX_ALERTS - 1) # one slot is the marker itself
|
|
assert marker["message"] == f"{hidden} more not shown ({hidden} failing, 0 warning)"
|
|
|
|
|
|
def test_the_marker_is_the_worst_verdict_it_did_not_show() -> None:
|
|
cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery"))
|
|
for i in range(alerts.MAX_ALERTS)]
|
|
cards += [ups_card(f"warn{i:02d}", check("Charge", "warn", "40%")) for i in range(3)]
|
|
out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards))
|
|
assert len(out) == alerts.MAX_ALERTS
|
|
assert out[-1]["id"] == "alerts:suppressed"
|
|
assert out[-1]["severity"] == "fail", "a red is behind the marker, so the marker is red"
|
|
# and when only warnings are hidden, the marker says amber — it describes what it hid
|
|
mild = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery"))
|
|
for i in range(3)]
|
|
mild += [ups_card(f"warn{i:02d}", check("Charge", "warn", "40%"))
|
|
for i in range(alerts.MAX_ALERTS)]
|
|
out = alerts.build(SETTINGS, [], [], [], ups_payload(*mild))
|
|
assert out[-1]["severity"] == "warn"
|
|
|
|
|
|
def test_exactly_the_cap_adds_no_marker() -> None:
|
|
cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery"))
|
|
for i in range(alerts.MAX_ALERTS)]
|
|
out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards))
|
|
assert len(out) == alerts.MAX_ALERTS
|
|
assert all(item["id"] != "alerts:suppressed" for item in out)
|
|
|
|
|
|
def test_fails_sort_before_warns_and_then_by_source() -> None:
|
|
payload = ups_payload(ups_card("zeta", check("Power path", "fail", "on battery")),
|
|
ups_card("alpha", check("Charge", "warn", "40%")),
|
|
ups_card("beta", check("Load", "warn", "82%")))
|
|
out = alerts.build(SETTINGS, [], [], [], payload)
|
|
assert [(item["severity"], item["source"]) for item in out] == [
|
|
("fail", "zeta"), ("warn", "alpha"), ("warn", "beta")]
|
|
|
|
|
|
def test_the_trip_note_quotes_the_threshold_table_in_the_right_direction() -> None:
|
|
"""`_trip` reads `severity.THRESHOLDS` back instead of carrying its own numbers.
|
|
|
|
Reached through `build()` only by metrics that are not inverted today, so this checks
|
|
the helper directly: a low-is-worse metric must print `≤` (an `alert` for a battery is
|
|
exactly where a wrong sign would be published), and a metric with no threshold gets no
|
|
note rather than a made-up one.
|
|
"""
|
|
assert alerts._trip("cpu.usage_pct", "fail") == " (fail ≥90)"
|
|
assert alerts._trip("ups.battery_runtime", "warn") == " (warn ≤600)"
|
|
assert alerts._trip("ups.charge_pct", "fail") == " (fail ≤20)"
|
|
assert alerts._trip("cpu.usage_pct", "ok") == ""
|
|
assert alerts._trip("mystery.metric", "fail") == ""
|
|
|
|
|
|
def test_the_alert_shape_is_the_contract_the_pages_render() -> None:
|
|
out = alerts.build(SETTINGS, [state(ok=False, error="nope")], now=2_000.0)
|
|
assert out[0] == {"id": "driveripper:host:driveripper:down", "severity": "fail",
|
|
"source": "driveripper", "message": "driveripper: nope",
|
|
"href": "/system", "at": 2_000.0}
|