"""`alerts.build`: the whole "what is wrong right now" answer, with no I/O anywhere. The builder is a pure projection of state the poller already keeps and verdicts `severity.py` / `sources/nut.assess()` already reached, so every rule can be tested by handing it a dict — no hub, no DB, no sockets, no clock it was not given. The cases below are the ones that make the landing page trustworthy rather than merely busy: * a source nobody configured is **never** an alert (`AGENTS.md` invariant 3); * a source that is stale but still serving is amber, not red; * a dead source is **one** line, not one per stale metric in its last good payload (and not one more for the config warnings its last good payload still carries — phase 30 task 03); * only `fail` and `warn` come through — `ok`, `unknown`, and a missing number are silence; * the list dedupes per (source, metric), is bounded, and says how much it hid. The one non-reachability rule for a source with no graded numbers is the gateway's config-integrity row: a proxy config that does not connect to itself is a structural fact, so it gets exactly one amber line naming how many there are and quoting the first, and it is suppressed whenever that job's own reachability line is on screen. """ from __future__ import annotations import inspect from typing import Any from dashboards import alerts, severity from dashboards.config import Settings from dashboards.sources import nut from dashboards.sources.nut_sim import NutSimulator SETTINGS = Settings() # defaults: no targets, intervals 10s system / 2s llm / 5s ups # --------------------------------------------------------------------------- # # state shapers # --------------------------------------------------------------------------- # def state(key: str = "host:driveripper", dashboard: str = "system", label: str = "driveripper", *, ok: bool = True, configured: bool = True, error: str | None = None, age: float | None = 0.0) -> dict[str, Any]: """A `SourceState.as_json()`-shaped health record — the only shape the builder reads.""" return {"key": key, "dashboard": dashboard, "label": label, "configured": configured, "ok": ok, "error": error, "age_seconds": age, "attempts": 4, "successes": 3, "consecutive_failures": 0 if ok else 1, "last_ok": None, "latency_ms": None} def check(name: str, status: str, detail: str = "") -> dict[str, Any]: return {"name": name, "status": status, "detail": detail, "metric": None, "value": None} def ups_payload(*cards: dict[str, Any], simulated: bool = False) -> dict[str, Any]: return {"ts": 1_000.0, "simulated": simulated, "ups": list(cards)} def ups_card(name: str = "apc", *checks: dict[str, Any]) -> dict[str, Any]: return {"name": name, "ts": 1_000.0, "health": {"overall": "healthy", "checks": list(checks)}} def engine(name: str = "gufo", **cells: Any) -> dict[str, Any]: return {"engine": name, "scraped_at": 1_000.0, "metrics": dict(cells)} def ratio(value: float | None, verdict: str | None) -> dict[str, Any]: """A rated LLM cell, exactly as `sources/llm.py` publishes it (0-1 ratio + verdict).""" return {"value": value, "source": "llamacpp:kv_cache_usage_ratio", "derived": False, "severity": verdict} # --------------------------------------------------------------------------- # # the all-clear, and the things that must not look like a problem # --------------------------------------------------------------------------- # def test_a_healthy_fleet_alerts_nothing(host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: payload = host_scrape(agent_snapshot, cpu_busy=10.0) assert alerts.build(SETTINGS, [state()], [payload], [], ups_payload(), [], now=2_000.0) == [] def test_no_inputs_at_all_is_all_clear_rather_than_an_error() -> None: """A fresh install with nothing wired up has nothing wrong with it.""" assert alerts.build(SETTINGS) == [] def test_an_unconfigured_source_never_alerts() -> None: """`AGENTS.md` invariant 3: not installed is not broken. The health record for a disabled source says `ok: false` exactly like a dead one, so `configured` is the only thing that tells those two apart — and the state map having nothing for a source must not be read as a failure either. """ out = alerts.build(SETTINGS, [ state(ok=False, configured=False, error="Connection refused"), state(key="nut", dashboard="ups", label="NUT server", ok=False, configured=False), ], [], [], ups_payload(), now=2_000.0) assert out == [] def test_a_card_with_no_checks_and_a_source_with_no_state_say_nothing() -> None: assert alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc"))) == [] # --------------------------------------------------------------------------- # # reachability # --------------------------------------------------------------------------- # def test_a_source_that_stopped_answering_is_a_fail_naming_it_and_linking_its_page() -> None: out = alerts.build(SETTINGS, [state(ok=False, error="Connection refused")], now=2_000.0) assert len(out) == 1 alert = out[0] assert alert["severity"] == "fail" assert alert["source"] == "driveripper" assert alert["href"] == "/system" assert alert["message"] == "driveripper: Connection refused" assert alert["at"] == 2_000.0 def test_the_collectors_host_prefix_is_not_doubled() -> None: """Collectors already say `driveripper: …`; the alert starts with the name too.""" out = alerts.build(SETTINGS, [state(ok=False, error="driveripper: http://h:9120/snapshot unreachable")]) assert out[0]["message"] == "driveripper: http://h:9120/snapshot unreachable" def test_the_pollers_exception_prefix_is_not_printed_either() -> None: """The live shape is `poller`'s `UpstreamError: name: …`; one name, no class noise. Verified against a real stopped agent: the health record reads `UpstreamError: gateway: http://:9120/snapshot unreachable (ConnectError)`, and printing that verbatim put the host name on the page twice. """ out = alerts.build(SETTINGS, [state(key="host:gateway", label="gateway", ok=False, error="UpstreamError: gateway: http://h:9120/snapshot " "unreachable (ConnectError)")]) assert out[0]["message"] == ("gateway: http://h:9120/snapshot unreachable (ConnectError)") def test_an_unexpected_exception_class_is_still_named() -> None: """Only `UpstreamError` is noise; a bug in our own code is worth reading about.""" out = alerts.build(SETTINGS, [state(ok=False, error="ValueError: bad snapshot")]) assert out[0]["message"] == "driveripper: ValueError: bad snapshot" def test_a_failing_source_with_no_recorded_error_still_says_so() -> None: out = alerts.build(SETTINGS, [state(key="llm:gufo", dashboard="llm", label="gufo", ok=False, error=None)]) assert out[0]["message"] == "gufo: is not answering" assert out[0]["href"] == "/llm" def test_a_stale_but_serving_source_is_a_warn_because_the_page_still_shows_numbers() -> None: """Last good value on screen + no refresh for 3 intervals = amber, not `dead`.""" out = alerts.build(SETTINGS, [state(age=45.0)], now=2_000.0) assert [item["severity"] for item in out] == ["warn"] assert "45s ago" in out[0]["message"] assert "last good value" in out[0]["message"] assert out[0]["at"] == 2_000.0 - 45.0 # inside the grace period (3 x 10s) the same source is not news at all assert alerts.build(SETTINGS, [state(age=25.0)]) == [] def test_staleness_is_measured_in_each_sources_own_poll_interval() -> None: """system polls every 10s and the UPS every 5s — one global number is wrong for one.""" llm = state(key="llm:gufo", dashboard="llm", label="gufo", age=5.0) # 3 x 2s = 6s ups = state(key="nut", dashboard="ups", label="NUT server", age=20.0) # 3 x 5s = 15s assert alerts.build(SETTINGS, [llm]) == [] out = alerts.build(SETTINGS, [ups]) assert out[0]["severity"] == "warn" and out[0]["href"] == "/ups" def test_a_dashboard_the_builder_does_not_know_about_still_gets_a_verdict_and_a_home() -> None: """A new dashboard must not be silently un-alertable while its interval is unknown.""" unknown_down = {"key": "probe:0", "dashboard": "telemetry", "label": "probe", "configured": True, "ok": False, "error": "no route"} assert alerts.build(SETTINGS, [unknown_down])[0]["href"] == "/" unknown_stale = {**unknown_down, "ok": True, "error": None, "age_seconds": 200.0} out = alerts.build(SETTINGS, [unknown_stale]) assert "every 60s" in out[0]["message"] def test_the_disk_job_is_stale_only_after_three_of_its_own_five_minute_polls() -> None: """SMART is a 300 s signal: judging it by the 10 s system interval would amber `/` every five minutes for a source that is behaving exactly as configured.""" disks = Settings(disks_interval=300.0) fresh = state(key="disks:driveripper", dashboard="disks", label="driveripper", age=600.0) assert alerts.build(disks, [fresh]) == [], "600s is two polls, not an outage" stale = {**fresh, "age_seconds": 1_000.0} out = alerts.build(disks, [stale]) assert out[0]["severity"] == "warn" assert "every 300s" in out[0]["message"], "the message must print the disks interval" def test_a_disk_job_that_failed_because_its_agent_is_down_is_one_line_not_two() -> None: """`disks` reads the same agent snapshot `system` does. A dead host would otherwise print the same sentence twice (`alerts` rule: one dead source, one line), with only the internal id telling the two apart — which is a lie of the same shape as a duplicated host name. """ out = alerts.build(SETTINGS, [ state(key="disks:driveripper", dashboard="disks", label="driveripper", ok=False, error="UpstreamError: driveripper: http://h:9120/snapshot " "unreachable (ConnectError)"), state(ok=False, error="UpstreamError: driveripper: http://h:9120/snapshot " "unreachable (ConnectError)"), ], now=2_000.0) assert [item["source"] for item in out] == ["driveripper"] assert len(out) == 1, "the echo of the same outage must not read as two problems" assert out[0]["id"].startswith("driveripper:host:") def test_a_disk_job_that_fails_while_its_host_answers_is_the_only_voice_it_has() -> None: """SMART wedged inside the agent is a `disks`-only fault, and must still be said.""" out = alerts.build(SETTINGS, [ state(key="disks:driveripper", dashboard="disks", label="driveripper", ok=False, error="UpstreamError: driveripper: timed out"), state(), ], now=2_000.0) assert len(out) == 1 assert out[0]["id"].startswith("driveripper:disks:") assert "timed out" in out[0]["message"] def test_a_disk_only_alert_links_to_the_drive_page() -> None: """The overview's row is a link, so it has to point at the page that explains it. A wedged SMART pass has no other voice (the host itself is answering), and `/system` cannot tell the operator which drive stopped reporting — `/disks` can. Before that route existed this link pointed at `/` on purpose; now it must not point anywhere else. """ out = alerts.build(SETTINGS, [ state(key="disks:driveripper", dashboard="disks", label="driveripper", ok=False, error="UpstreamError: driveripper: timed out"), state(), ], now=2_000.0) assert len(out) == 1 assert out[0]["href"] == "/disks" def test_an_unconfigured_disk_source_says_nothing_even_when_its_job_says_down() -> None: """No `HOST_TARGETS` means no `disks` job exists; a reported-but-unconfigured one must still be silence (invariant 3). The builder is fed job states rather than settings, so a hub that ever hands over a `disks` record with `configured: false` — a config that arrives between two polls, a future target list of its own — must not amber `/` over a page nobody installed. """ out = alerts.build(Settings(disks_interval=300.0), [ state(key="disks:driveripper", dashboard="disks", label="driveripper", ok=False, configured=False, error="no hosts configured"), ], now=2_000.0) assert out == [] def test_a_host_threshold_reads_the_system_job_not_the_disk_job( host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: """Two jobs share a host label, so the lookup has to be filtered by dashboard. `disks` is registered after `system`, but the builder must not depend on that: if an unfiltered lookup found the failing disk job first, the host would be called unreachable and its real threshold breach would vanish. """ payload = host_scrape(agent_snapshot, cpu_busy=95.0) # cpu.usage_pct -> fail out = alerts.build(SETTINGS, [ state(key="disks:driveripper", dashboard="disks", label="driveripper", ok=False, error="timed out"), state(), ], [payload], [], None, now=2_000.0) ids = [item["id"] for item in out] assert "driveripper:cpu.usage_pct" in ids, "the breach is real; the disk job is not it" def test_a_failing_source_says_when_the_poller_first_noticed() -> None: events = [{"ts": 1_500.0, "source": "system", "host": "driveripper", "kind": "unreachable", "severity": "error", "message": "ConnectError", "data": None}, {"ts": 1_600.0, "source": "host", "host": "deskwork", "kind": "reboot", "severity": "warn", "message": "deskwork rebooted", "data": None}] out = alerts.build(SETTINGS, [state(ok=False, error="nope")], [], [], None, events, now=2_000.0) assert out[0]["at"] == 1_500.0, "the outage clock must not reset on every refresh" # --------------------------------------------------------------------------- # # host thresholds, degraded agents # --------------------------------------------------------------------------- # def test_a_threshold_breach_names_the_host_the_metric_and_the_value( host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: payload = host_scrape(agent_snapshot, cpu_busy=95.0) out = alerts.build(SETTINGS, [state()], [payload], now=2_000.0) assert out[0]["id"] == "driveripper:cpu.usage_pct" assert out[0]["message"] == "driveripper: CPU 95% (fail ≥90)" assert out[0]["href"] == "/system" assert out[0]["at"] == 1_010.0, "`at` is the poll that measured it" def test_every_rated_host_field_alerts_at_its_own_verdict( host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: def nudge(snap: dict[str, Any]) -> None: snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.85 # warn ≥80 snap["cpu_temperature_celsius"] = 80.0 # warn ≥75 for fs in snap["filesystems"]: if fs["mountpoint"] == "/btrfs/pool1": fs["used_pct"] = 91.0 # fail ≥90 payload = host_scrape(agent_snapshot, cpu_busy=10.0, mutate=nudge) out = {item["id"]: item for item in alerts.build(SETTINGS, [state()], [payload])} assert out["driveripper:mem.used_pct"]["message"] == "driveripper: memory 85% (warn ≥80)" assert out["driveripper:cpu.temp_c"]["message"] == ( "driveripper: CPU temperature 80°C (warn ≥75)") assert out["driveripper:disk.used_pct"]["message"] == ( "driveripper: disk 91% on /btrfs/pool1 (fail ≥90)") assert "driveripper:cpu.usage_pct" not in out, "10% busy is not a problem" def test_a_number_the_upstream_does_not_publish_is_never_a_breach( host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: """No measurement, no verdict, no alert — the same rule `severity.classify` upholds.""" payload = host_scrape(agent_snapshot, cpu_busy=10.0, mutate=lambda snap: snap.pop("cpu_temperature_celsius")) assert alerts.build(SETTINGS, [state()], [payload]) == [] def test_a_saturated_gpu_is_not_an_alert(host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: """Phase 06: these are LLM servers — a fully-worked GPU is the job, not a fault. The host is otherwise all-clear, so a card at 100 % busy with both memory pools filled must produce **no** alert at all: `severity.THRESHOLDS` has no `gpu.*` row, the payload carries no GPU verdict, and this module owns no threshold of its own from which to resurrect one. """ def fill_the_card(snap: dict[str, Any]) -> None: gpu = snap["gpus"][0] gpu["busy_pct"] = 100.0 for used, total in (("vram_used_bytes", "vram_total_bytes"), ("gtt_used_bytes", "gtt_total_bytes")): if gpu.get(total) is not None: gpu[used] = gpu[total] payload = host_scrape(agent_snapshot, cpu_busy=10.0, mutate=fill_the_card) assert payload["gpus"][0]["busy_pct"] == 100.0, "the card really is saturated here" assert alerts.build(SETTINGS, [state()], [payload]) == [], ( "a busy GPU produced an alert — GPU verdicts were removed in phase 06" ) def test_a_degraded_agent_warns_and_names_the_sections_that_failed( host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: payload = host_scrape(agent_snapshot, cpu_busy=10.0) payload["degraded"] = {"temperatures": "no hwmon sensors", "disk_io": ""} out = alerts.build(SETTINGS, [state()], [payload]) assert len(out) == 1, "a partial snapshot is one warning, not one per number that is missing" assert out[0]["id"] == "driveripper:agent.sections" assert out[0]["severity"] == "warn" assert "temperatures (no hwmon sensors)" in out[0]["message"] assert "disk_io" in out[0]["message"] def test_a_degraded_list_from_an_older_agent_is_readable_too( host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: payload = host_scrape(agent_snapshot, cpu_busy=10.0) payload["degraded"] = ["network"] out = alerts.build(SETTINGS, [state()], [payload]) assert out[0]["message"] == "driveripper: the agent could not read network" def test_a_degraded_note_that_is_a_plain_string_is_still_reported( host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: """An agent (or a future collector) that reports one string instead of a map.""" payload = host_scrape(agent_snapshot, cpu_busy=10.0) payload["degraded"] = "hwmon" out = alerts.build(SETTINGS, [state()], [payload]) assert out[0]["message"] == "driveripper: the agent could not read hwmon" def test_an_unreachable_host_is_one_alert_even_with_its_last_good_payload( host_scrape: Any, agent_snapshot: dict[str, Any]) -> None: """The exact completion criterion for the phase: one line naming the dead host.""" def nudge(snap: dict[str, Any]) -> None: snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.99 snap["cpu_temperature_celsius"] = 95.0 payload = host_scrape(agent_snapshot, cpu_busy=96.0, mutate=nudge) out = alerts.build(SETTINGS, [state(ok=False, error="timed out")], [payload]) assert len(out) == 1 assert out[0]["severity"] == "fail" and "timed out" in out[0]["message"] def test_a_stub_payload_for_a_host_that_never_answered_alerts_once() -> None: """/api/system/snapshot represents that host as {name, unreachable}; still one line.""" stub = {"name": "gateway", "ts": 1_000.0, "unreachable": True} out = alerts.build(SETTINGS, [state(key="host:gateway", label="gateway", ok=False, error="unreachable")], [stub]) assert [item["id"] for item in out] == ["gateway:host:gateway:down"] # --------------------------------------------------------------------------- # # LLM engines # --------------------------------------------------------------------------- # def test_kv_pressure_comes_from_the_rated_cell_and_reads_as_a_percent() -> None: """The engine publishes 0-1; the operator reads percent, matching the verdict's scale.""" out = alerts.build(SETTINGS, [state(key="llm:gufo", dashboard="llm", label="gufo")], [], [engine(kv_cache_usage_ratio=ratio(0.95, "fail"))]) assert out[0]["message"] == "gufo: KV cache 95% (fail ≥90)" assert out[0]["href"] == "/llm" assert out[0]["id"] == "gufo:llm.kv_cache_usage_pct@kv_cache_usage_ratio" def test_vllm_publishes_the_same_verdict_under_its_own_series_name() -> None: out = alerts.build(SETTINGS, [], [], [engine("vllm", gpu_cache_usage=ratio(0.8, "warn"))]) assert out[0]["message"] == "vllm: KV cache 80% (warn ≥75)" def test_a_cool_or_absent_kv_cell_is_not_an_alert() -> None: for cell in (ratio(0.10, "ok"), ratio(None, None), ratio(0.95, None), {}, "not a cell"): out = alerts.build(SETTINGS, [], [], [engine(kv_cache_usage_ratio=cell)]) assert out == [], cell def test_an_engine_that_is_not_answering_alerts_only_its_own_job() -> None: dead = {"engine": "vllm", "metrics": {}, "sessions": {}, "unreachable": True} out = alerts.build(SETTINGS, [state(key="llm:vllm", dashboard="llm", label="vLLM", ok=False, error="502 Bad Gateway")], [], [dead]) assert [item["id"] for item in out] == ["vLLM:llm:vllm:down"] # --------------------------------------------------------------------------- # # UPS # --------------------------------------------------------------------------- # def test_a_failing_ups_check_is_a_fail_carrying_the_check_and_its_detail() -> None: payload = ups_payload(ups_card("apc", check("Power path", "fail", "The UPS is on battery"), check("Charge", "warn", "42% and discharging"))) out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server")], [], [], payload) by_id = {item["id"]: item for item in out} assert by_id["apc:health:Power path"]["severity"] == "fail" assert by_id["apc:health:Power path"]["message"] == "Power path: The UPS is on battery" assert by_id["apc:health:Charge"]["severity"] == "warn" assert all(item["href"] == "/ups" for item in out) def test_the_landing_page_is_where_battery_age_belongs() -> None: """The task's own example: a sentence an operator can act on, not a score.""" detail = "5.3 years — past the 4-year replacement window" out = alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc", check("Battery age", "warn", detail)))) assert out[0]["message"] == f"Battery age: {detail}" assert out[0]["source"] == "apc" def test_an_unknown_check_is_silence_and_a_check_with_no_detail_is_still_readable() -> None: payload = ups_payload(ups_card("apc", check("Battery temperature", "unknown", "not reported by this driver"), check("Alarms", "fail"))) out = alerts.build(SETTINGS, [], [], [], payload) assert [item["message"] for item in out] == ["Alarms"] def test_simulated_ups_data_is_announced_once_and_loudly() -> None: """With NUT_SIMULATE=1 and otherwise-healthy cards, that warning is the whole list.""" payload = ups_payload(ups_card("apc", check("Charge", "ok", "100%")), ups_card("cyberpower", check("Charge", "ok", "100%")), simulated=True) out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server")], [], [], payload) assert len(out) == 1 assert out[0]["severity"] == "warn" assert "NUT_SIMULATE=1" in out[0]["message"] assert out[0]["href"] == "/ups" async def test_the_simulator_on_its_own_produces_exactly_that_warning() -> None: """The task's acceptance line, through the real simulator and the real payload builder.""" readings = await NutSimulator().read() payload = {"ts": 1_000.0, "simulated": True, "ups": [nut.payload(reading) for reading in readings]} out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server")], [], [], payload) assert [item["id"] for item in out] == ["ups:simulated"], out def test_a_real_upsd_does_not_claim_to_be_simulated() -> None: out = alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc"))) assert out == [] def test_a_dead_upsd_is_one_alert_and_not_its_stale_cards_too() -> None: payload = ups_payload(ups_card("apc", check("Power path", "fail", "on battery"))) out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server", ok=False, error="upsd closed the connection")], [], [], payload) assert [item["source"] for item in out] == ["NUT server"] # --------------------------------------------------------------------------- # # the gateway page: reachability plus config integrity, and a dead agent is still one line # --------------------------------------------------------------------------- # def gateway_state(**kwargs: Any) -> dict[str, Any]: """The health record of the one `gateway` job (the `nut` shape: key `gateway`, label = host).""" return state("gateway", "gateway", "gateway", **kwargs) def gateway_payload(*messages: str, ts: float | None = 1_000.0) -> dict[str, Any]: """A gateway payload carrying exactly `messages` as its config warnings, in that order.""" return {"ts": ts, "host": "gateway", "supported": True, "warnings": [{"id": f"synthetic:{index}", "kind": "synthetic", "subject": str(index), "message": text} for index, text in enumerate(messages)]} def test_config_warnings_are_one_warn_row_naming_the_count_and_the_first_message() -> None: """`gateway:config`: the one row a proxy config that does not connect to itself earns. Phase 30 task 03 changed the documented rule this file used to pin — a `gateway` alert used to be reachability only. What it is now: one `warn` row, `href` `/gateway`, `at` the payload's own timestamp (the config did not change between polls, so an alert whose clock reset every 30 s would be unreadable), the count and the first message in the sentence. """ out = alerts.build(Settings(gateway_interval=30.0), [gateway_state()], gateway_payload=gateway_payload( 'frontend haproxy-22 routes to backend "gone", which no backend ' "block declares", 'backend "old" has 2 servers and no frontend routes to it'), now=2_000.0) assert out == [{"id": "gateway:config", "severity": "warn", "source": "gateway", "href": "/gateway", "at": 1_000.0, "message": ("2 proxy config problems to address — " 'frontend haproxy-22 routes to backend "gone", which no ' "backend block declares")}] def test_one_warning_and_seven_produce_exactly_one_row_either_way() -> None: """The row is aggregate on purpose: seven dangling routes are one config to fix. Seven amber lines would push every other source off the overview panel to say one thing, and the page beside them lists all seven — so the count is the number and the first message is the sentence, once. """ settings = Settings(gateway_interval=30.0) for count in (1, 2, 7): out = alerts.build(settings, [gateway_state()], gateway_payload=gateway_payload(*[f"problem {index}" for index in range(count)]), now=2_000.0) assert len(out) == 1, f"{count} warnings produced {len(out)} rows" assert out[0]["id"] == "gateway:config" and out[0]["severity"] == "warn" assert out[0]["href"] == "/gateway" and out[0]["source"] == "gateway" noun = "problem" if count == 1 else "problems" assert out[0]["message"] == f"{count} proxy config {noun} to address — problem 0", ( "the count, the noun that matches it, and the first message verbatim") def test_a_clean_ingress_warns_about_nothing_and_a_missing_payload_does_too() -> None: """Both directions of the row, because an absence-only assertion is satisfied by deleting it.""" settings = Settings(gateway_interval=30.0) assert alerts.build(settings, [gateway_state()], gateway_payload=gateway_payload(), now=2_000.0) == [] assert alerts.build(settings, [gateway_state()], gateway_payload={"host": "gateway", "warnings": None}, now=2_000.0) == [], "a malformed list is not a fault either" assert alerts.build(settings, [gateway_state()], now=2_000.0) == [], ( "no payload at all — unconfigured, or configured and never answered — says nothing") def test_a_failing_gateway_job_keeps_its_reachability_row_and_gains_no_config_row() -> None: """A dead source is one line, and the payload it left behind is not a second one. Both arms of the rule, in both of its shapes: `ok: false` (the job is down) and stale (`ok: true` past three of its own polls). In each, the reachability row is what the operator sees, and the last good payload's warnings are not re-listed beside it — the warnings will be on `/gateway` when the source answers again, and a second line here would be the invariant the `disks` rule already keeps. """ settings = Settings(gateway_interval=30.0) broken = gateway_payload('backend "gone" declares no server — it routes nowhere') down = alerts.build(settings, [gateway_state(ok=False, error="UpstreamError: gateway: timed out")], gateway_payload=broken, now=2_000.0) assert [row["id"] for row in down] == ["gateway:gateway:down"], down assert "timed out" in down[0]["message"] stale = alerts.build(settings, [gateway_state(age=100.0)], gateway_payload=broken, now=2_000.0) assert [row["id"] for row in stale] == ["gateway:gateway:stale"], stale assert "every 30s" in stale[0]["message"] # the control: the same payload with a job that IS answering does grow the config row live = alerts.build(settings, [gateway_state(age=4.0)], gateway_payload=broken, now=2_000.0) assert [row["id"] for row in live] == ["gateway:config"] def test_an_unconfigured_gateway_with_a_warning_payload_still_says_nothing() -> None: """Invariant 3 with a payload attached: a source nobody installed has no config to be broken.""" out = alerts.build(Settings(gateway_interval=30.0), [gateway_state(configured=False)], gateway_payload=gateway_payload("anything"), now=2_000.0) assert out == [] def test_a_gateway_config_row_survives_a_dead_host_beside_it_and_sorts_after_the_fail() -> None: """The config row is the gateway's own voice and the outage row is the host's — both stand. The echo table suppresses a `gateway` job that failed *because* its agent did; a config warning is not that, so a healthy host with a broken ingress gets exactly one overview row about the ingress and nothing else, and a failing host plus a stale config list stays one row too. """ settings = Settings(gateway_interval=30.0) out = alerts.build(settings, [gateway_state(), state("host:gateway", "system", "gateway")], gateway_payload=gateway_payload("fix me"), now=2_000.0) assert [row["id"] for row in out] == ["gateway:config"] host_down = state("host:gateway", "system", "gateway", ok=False, error="unreachable") out = alerts.build(settings, [gateway_state(ok=False), host_down], gateway_payload=gateway_payload("fix me"), now=2_000.0) assert [row["severity"] for row in out] == ["fail"], "the outage is the only line" def test_a_config_warning_row_is_the_worst_thing_about_a_gateway_never_a_number() -> None: """The rule restated after it changed: reachability plus config integrity, no graded number. Three ways, because deleting a rule satisfies an absence-only assertion: * `severity.THRESHOLDS` has no `gateway.*` row, so there is no bound for this module to read even if a threshold were added here (and a busy proxy is the job, the `net.*` case); * the row's message is built from `warnings` and nothing else — a payload of nothing but counts yields no row at all; * `alerts.HREF` still carries `/gateway`, so both voices reach the page that explains them. """ assert not [name for name in severity.THRESHOLDS if name.startswith("gateway.")], ( "a connection count is not a graded number") settings = Settings(gateway_interval=30.0) counts = {"ts": 1_000.0, "host": "gateway", "listeners": [{"connections": 9_999}], "connections": [{"connections": 9_999}], "sockets": {"established": 9_999}} assert alerts.build(settings, [gateway_state()], gateway_payload=counts, now=2_000.0) == [] assert alerts.HREF["gateway"] == "/gateway" assert alerts.AGENT_ECHO_DASHBOARDS == ("disks", "gateway"), ( "task 03 changed what a gateway may alert ON, not which lines it echoes") # and a gateway with a payload whose rows are junk is silence, not a crash and not a claim assert alerts.build(settings, [gateway_state()], gateway_payload={"warnings": ["not a mapping", 7]}, now=2_000.0) == [] def test_a_stale_gateway_job_is_a_warn_measured_in_its_own_thirty_second_poll() -> None: """The pipes are drawn from a 30 s poll, so 60 s of silence is two healthy polls. Judging this source by the 10 s system interval would amber the landing page every minute for a job behaving exactly as configured — the same mistake the `disks` rule exists to avoid, one cadence later. """ settings = Settings(gateway_interval=30.0) assert alerts.build(settings, [gateway_state(age=60.0)]) == [] out = alerts.build(settings, [gateway_state(age=100.0)]) assert out[0]["severity"] == "warn" assert out[0]["href"] == "/gateway", "the overview's row must link to the page it describes" assert "every 30s" in out[0]["message"], "the message prints the gateway's own interval" def test_a_gateway_job_that_fails_while_its_host_answers_is_the_only_voice_it_has() -> None: """A wedged proxy/socket section is a fault nothing else can report (the `disks` rule, once more).""" out = alerts.build(SETTINGS, [ gateway_state(ok=False, error="UpstreamError: gateway: timed out"), state("host:gateway", "system", "gateway"), ], now=2_000.0) assert len(out) == 1 assert out[0]["source"] == "gateway" assert out[0]["href"] == "/gateway" assert "timed out" in out[0]["message"] def test_a_gateway_job_that_failed_because_its_agent_is_down_is_one_line_not_two() -> None: """Three jobs dial one agent; the operator has one problem, so the overview says it once.""" down = {"ok": False, "error": "UpstreamError: gateway: http://h:9120/snapshot unreachable (ConnectError)"} out = alerts.build(SETTINGS, [ gateway_state(**down), state("disks:gateway", "disks", "gateway", **down), state("host:gateway", "system", "gateway", **down), ], now=2_000.0) assert len(out) == 1, "the echo of one outage must not read as three problems" assert out[0]["id"].startswith("gateway:host:"), out[0]["id"] def test_an_unconfigured_gateway_source_says_nothing_even_when_its_job_says_down() -> None: """No `HOST_TARGETS` entry named `GATEWAY_HOST` means no job — and a reported one stays quiet.""" out = alerts.build(Settings(gateway_interval=30.0), [gateway_state(ok=False, configured=False, error="not configured")], now=2_000.0) assert out == [] def test_a_gateway_payload_with_a_fabricated_verdict_alerts_nothing() -> None: """The no-verdict rule restated for the payload this task added a reader for. `alerts.build` now *takes* a gateway payload (phase 30 task 03), so the old signature-only pin is gone and what replaces it is stronger and narrower: the parameter is keyword-only with a default (a positional gateway would be a rule this module grew by accident), it is the ONLY gateway input, `severity.THRESHOLDS` still has no `gateway.*` row for it to consult, and a payload with `severity` stamped into it stays silent — the row reads `warnings` and nothing else. An answering gateway with a clean list is the all clear. """ signature = inspect.signature(alerts.build).parameters gateway_argument = signature["gateway_payload"] assert gateway_argument.kind is inspect.Parameter.KEYWORD_ONLY, ( "a positional gateway argument would let a caller pass a count where a payload belongs") assert gateway_argument.default is None assert not [name for name in signature if name.startswith("gateway") and name != "gateway_payload"], ( "a second gateway input is a new rule in disguise") assert not [name for name in severity.THRESHOLDS if name.startswith("gateway.")], ( "a connection count is not a graded number") assert alerts.HREF["gateway"] == "/gateway" assert alerts.AGENT_ECHO_DASHBOARDS == ("disks", "gateway") # a fabricated verdict anywhere in the payload changes nothing: only `warnings` is read rigged = gateway_payload() rigged["severity"] = {"listeners": "fail", "connections": "fail"} rigged["listeners"] = [{"connections": 100, "severity": "fail"}] rigged["connections"] = [{"connections": 100, "severity": "fail"}] settings = Settings(gateway_interval=30.0) assert alerts.build(settings, [gateway_state()], gateway_payload=rigged, now=2_000.0) == [] assert alerts.build(settings, [gateway_state()], [], [], None, [], now=2_000.0) == [] # --------------------------------------------------------------------------- # # configuration # --------------------------------------------------------------------------- # def test_a_bad_target_entry_is_a_warn_that_says_it_is_not_being_polled() -> None: settings = Settings(host_target_problems=("nope is not name=url", "deskwork: url must start with http://")) out = alerts.build(settings) assert len(out) == 2 assert all(item["severity"] == "warn" and item["source"] == "config" for item in out) assert all(item["href"] == "/system" for item in out) assert "HOST_TARGETS" in out[0]["message"] and "not being polled" in out[0]["message"] # --------------------------------------------------------------------------- # # dedupe, bound, order, shape # --------------------------------------------------------------------------- # def test_one_alert_per_source_and_metric_with_the_worst_verdict_winning() -> None: """Two cards answering as the same name (a re-exported UPS) must not double the alarm.""" payload = ups_payload(ups_card("apc", check("Charge", "warn", "60%")), ups_card("apc", check("Charge", "fail", "12%"))) out = alerts.build(SETTINGS, [], [], [], payload) assert len(out) == 1 assert out[0]["severity"] == "fail" assert out[0]["message"] == "Charge: 12%" def test_the_list_is_bounded_and_says_how_many_it_hid() -> None: cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery")) for i in range(alerts.MAX_ALERTS + 5)] out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards)) assert len(out) == alerts.MAX_ALERTS marker = out[-1] assert marker["id"] == "alerts:suppressed" assert marker["severity"] == "fail", "hiding reds behind an amber marker is a lie" hidden = len(cards) - (alerts.MAX_ALERTS - 1) # one slot is the marker itself assert marker["message"] == f"{hidden} more not shown ({hidden} failing, 0 warning)" def test_the_marker_is_the_worst_verdict_it_did_not_show() -> None: cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery")) for i in range(alerts.MAX_ALERTS)] cards += [ups_card(f"warn{i:02d}", check("Charge", "warn", "40%")) for i in range(3)] out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards)) assert len(out) == alerts.MAX_ALERTS assert out[-1]["id"] == "alerts:suppressed" assert out[-1]["severity"] == "fail", "a red is behind the marker, so the marker is red" # and when only warnings are hidden, the marker says amber — it describes what it hid mild = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery")) for i in range(3)] mild += [ups_card(f"warn{i:02d}", check("Charge", "warn", "40%")) for i in range(alerts.MAX_ALERTS)] out = alerts.build(SETTINGS, [], [], [], ups_payload(*mild)) assert out[-1]["severity"] == "warn" def test_exactly_the_cap_adds_no_marker() -> None: cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery")) for i in range(alerts.MAX_ALERTS)] out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards)) assert len(out) == alerts.MAX_ALERTS assert all(item["id"] != "alerts:suppressed" for item in out) def test_fails_sort_before_warns_and_then_by_source() -> None: payload = ups_payload(ups_card("zeta", check("Power path", "fail", "on battery")), ups_card("alpha", check("Charge", "warn", "40%")), ups_card("beta", check("Load", "warn", "82%"))) out = alerts.build(SETTINGS, [], [], [], payload) assert [(item["severity"], item["source"]) for item in out] == [ ("fail", "zeta"), ("warn", "alpha"), ("warn", "beta")] def test_the_trip_note_quotes_the_threshold_table_in_the_right_direction() -> None: """`_trip` reads `severity.THRESHOLDS` back instead of carrying its own numbers. Reached through `build()` only by metrics that are not inverted today, so this checks the helper directly: a low-is-worse metric must print `≤` (an `alert` for a battery is exactly where a wrong sign would be published), and a metric with no threshold gets no note rather than a made-up one. """ assert alerts._trip("cpu.usage_pct", "fail") == " (fail ≥90)" assert alerts._trip("ups.battery_runtime", "warn") == " (warn ≤600)" assert alerts._trip("ups.charge_pct", "fail") == " (fail ≤20)" assert alerts._trip("cpu.usage_pct", "ok") == "" assert alerts._trip("mystery.metric", "fail") == "" def test_the_alert_shape_is_the_contract_the_pages_render() -> None: out = alerts.build(SETTINGS, [state(ok=False, error="nope")], now=2_000.0) assert out[0] == {"id": "driveripper:host:driveripper:down", "severity": "fail", "source": "driveripper", "message": "driveripper: nope", "href": "/system", "at": 2_000.0}