dashboards/tests/test_alerts.py

"""`alerts.build`: the whole "what is wrong right now" answer, with no I/O anywhere.

The builder is a pure projection of state the poller already keeps and verdicts
`severity.py` / `sources/nut.assess()` already reached, so every rule can be tested by
handing it a dict — no hub, no DB, no sockets, no clock it was not given. The cases below
are the ones that make the landing page trustworthy rather than merely busy:

* a source nobody configured is **never** an alert (`AGENTS.md` invariant 3);
* a source that is stale but still serving is amber, not red;
* a dead source is **one** line, not one per stale metric in its last good payload;
* only `fail` and `warn` come through — `ok`, `unknown`, and a missing number are silence;
* the list dedupes per (source, metric), is bounded, and says how much it hid.
"""

from __future__ import annotations

from typing import Any

from dashboards import alerts
from dashboards.config import Settings
from dashboards.sources import nut
from dashboards.sources.nut_sim import NutSimulator

SETTINGS = Settings()  # defaults: no targets, intervals 10s system / 2s llm / 5s ups


# --------------------------------------------------------------------------- #
# state shapers
# --------------------------------------------------------------------------- #
def state(key: str = "host:driveripper", dashboard: str = "system",
          label: str = "driveripper", *, ok: bool = True, configured: bool = True,
          error: str | None = None, age: float | None = 0.0) -> dict[str, Any]:
    """A `SourceState.as_json()`-shaped health record — the only shape the builder reads."""
    return {"key": key, "dashboard": dashboard, "label": label, "configured": configured,
            "ok": ok, "error": error, "age_seconds": age, "attempts": 4, "successes": 3,
            "consecutive_failures": 0 if ok else 1, "last_ok": None, "latency_ms": None}


def check(name: str, status: str, detail: str = "") -> dict[str, Any]:
    return {"name": name, "status": status, "detail": detail, "metric": None, "value": None}


def ups_payload(*cards: dict[str, Any], simulated: bool = False) -> dict[str, Any]:
    return {"ts": 1_000.0, "simulated": simulated, "ups": list(cards)}


def ups_card(name: str = "apc", *checks: dict[str, Any]) -> dict[str, Any]:
    return {"name": name, "ts": 1_000.0, "health": {"overall": "healthy", "checks": list(checks)}}


def engine(name: str = "gufo", **cells: Any) -> dict[str, Any]:
    return {"engine": name, "scraped_at": 1_000.0, "metrics": dict(cells)}


def ratio(value: float | None, verdict: str | None) -> dict[str, Any]:
    """A rated LLM cell, exactly as `sources/llm.py` publishes it (0-1 ratio + verdict)."""
    return {"value": value, "source": "llamacpp:kv_cache_usage_ratio", "derived": False,
            "severity": verdict}


# --------------------------------------------------------------------------- #
# the all-clear, and the things that must not look like a problem
# --------------------------------------------------------------------------- #
def test_a_healthy_fleet_alerts_nothing(host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
    payload = host_scrape(agent_snapshot, cpu_busy=10.0)
    assert alerts.build(SETTINGS, [state()], [payload], [], ups_payload(), [], now=2_000.0) == []


def test_no_inputs_at_all_is_all_clear_rather_than_an_error() -> None:
    """A fresh install with nothing wired up has nothing wrong with it."""
    assert alerts.build(SETTINGS) == []


def test_an_unconfigured_source_never_alerts() -> None:
    """`AGENTS.md` invariant 3: not installed is not broken.

    The health record for a disabled source says `ok: false` exactly like a dead one, so
    `configured` is the only thing that tells those two apart — and the state map having
    nothing for a source must not be read as a failure either.
    """
    out = alerts.build(SETTINGS, [
        state(ok=False, configured=False, error="Connection refused"),
        state(key="nut", dashboard="ups", label="NUT server", ok=False, configured=False),
    ], [], [], ups_payload(), now=2_000.0)
    assert out == []


def test_a_card_with_no_checks_and_a_source_with_no_state_say_nothing() -> None:
    assert alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc"))) == []


# --------------------------------------------------------------------------- #
# reachability
# --------------------------------------------------------------------------- #
def test_a_source_that_stopped_answering_is_a_fail_naming_it_and_linking_its_page() -> None:
    out = alerts.build(SETTINGS, [state(ok=False, error="Connection refused")], now=2_000.0)
    assert len(out) == 1
    alert = out[0]
    assert alert["severity"] == "fail"
    assert alert["source"] == "driveripper"
    assert alert["href"] == "/system"
    assert alert["message"] == "driveripper: Connection refused"
    assert alert["at"] == 2_000.0


def test_the_collectors_host_prefix_is_not_doubled() -> None:
    """Collectors already say `driveripper: …`; the alert starts with the name too."""
    out = alerts.build(SETTINGS, [state(ok=False,
                                        error="driveripper: http://h:9121/snapshot unreachable")])
    assert out[0]["message"] == "driveripper: http://h:9121/snapshot unreachable"


def test_the_pollers_exception_prefix_is_not_printed_either() -> None:
    """The live shape is `poller`'s `UpstreamError: name: …`; one name, no class noise.

    Verified against a real stopped agent tunnel: the health record reads
    `UpstreamError: gateway: http://127.0.0.1:9123/snapshot unreachable (ConnectError)`,
    and printing that verbatim put the host name on the page twice.
    """
    out = alerts.build(SETTINGS, [state(key="host:gateway", label="gateway", ok=False,
                                        error="UpstreamError: gateway: http://h:9123/snapshot "
                                              "unreachable (ConnectError)")])
    assert out[0]["message"] == ("gateway: http://h:9123/snapshot unreachable (ConnectError)")


def test_an_unexpected_exception_class_is_still_named() -> None:
    """Only `UpstreamError` is noise; a bug in our own code is worth reading about."""
    out = alerts.build(SETTINGS, [state(ok=False, error="ValueError: bad snapshot")])
    assert out[0]["message"] == "driveripper: ValueError: bad snapshot"


def test_a_failing_source_with_no_recorded_error_still_says_so() -> None:
    out = alerts.build(SETTINGS, [state(key="llm:gufo", dashboard="llm", label="gufo",
                                        ok=False, error=None)])
    assert out[0]["message"] == "gufo: is not answering"
    assert out[0]["href"] == "/llm"


def test_a_stale_but_serving_source_is_a_warn_because_the_page_still_shows_numbers() -> None:
    """Last good value on screen + no refresh for 3 intervals = amber, not `dead`."""
    out = alerts.build(SETTINGS, [state(age=45.0)], now=2_000.0)
    assert [item["severity"] for item in out] == ["warn"]
    assert "45s ago" in out[0]["message"]
    assert "last good value" in out[0]["message"]
    assert out[0]["at"] == 2_000.0 - 45.0
    # inside the grace period (3 x 10s) the same source is not news at all
    assert alerts.build(SETTINGS, [state(age=25.0)]) == []


def test_staleness_is_measured_in_each_sources_own_poll_interval() -> None:
    """system polls every 10s and the UPS every 5s — one global number is wrong for one."""
    llm = state(key="llm:gufo", dashboard="llm", label="gufo", age=5.0)      # 3 x 2s = 6s
    ups = state(key="nut", dashboard="ups", label="NUT server", age=20.0)    # 3 x 5s = 15s
    assert alerts.build(SETTINGS, [llm]) == []
    out = alerts.build(SETTINGS, [ups])
    assert out[0]["severity"] == "warn" and out[0]["href"] == "/ups"


def test_a_dashboard_the_builder_does_not_know_about_still_gets_a_verdict_and_a_home() -> None:
    """A new dashboard must not be silently un-alertable while its interval is unknown."""
    unknown_down = {"key": "probe:0", "dashboard": "telemetry", "label": "probe",
                    "configured": True, "ok": False, "error": "no route"}
    assert alerts.build(SETTINGS, [unknown_down])[0]["href"] == "/"
    unknown_stale = {**unknown_down, "ok": True, "error": None, "age_seconds": 200.0}
    out = alerts.build(SETTINGS, [unknown_stale])
    assert "every 60s" in out[0]["message"]


def test_a_failing_source_says_when_the_poller_first_noticed() -> None:
    events = [{"ts": 1_500.0, "source": "system", "host": "driveripper", "kind": "unreachable",
               "severity": "error", "message": "ConnectError", "data": None},
              {"ts": 1_600.0, "source": "host", "host": "deskwork", "kind": "reboot",
               "severity": "warn", "message": "deskwork rebooted", "data": None}]
    out = alerts.build(SETTINGS, [state(ok=False, error="nope")], [], [], None, events,
                       now=2_000.0)
    assert out[0]["at"] == 1_500.0, "the outage clock must not reset on every refresh"


# --------------------------------------------------------------------------- #
# host thresholds, degraded agents
# --------------------------------------------------------------------------- #
def test_a_threshold_breach_names_the_host_the_metric_and_the_value(
        host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
    payload = host_scrape(agent_snapshot, cpu_busy=95.0)
    out = alerts.build(SETTINGS, [state()], [payload], now=2_000.0)
    assert out[0]["id"] == "driveripper:cpu.usage_pct"
    assert out[0]["message"] == "driveripper: CPU 95% (fail ≥90)"
    assert out[0]["href"] == "/system"
    assert out[0]["at"] == 1_010.0, "`at` is the poll that measured it"


def test_every_rated_host_field_alerts_at_its_own_verdict(
        host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
    def nudge(snap: dict[str, Any]) -> None:
        snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.85   # warn ≥80
        snap["cpu_temperature_celsius"] = 80.0                                # warn ≥75
        for fs in snap["filesystems"]:
            if fs["mountpoint"] == "/btrfs/pool1":
                fs["used_pct"] = 91.0                                         # fail ≥90

    payload = host_scrape(agent_snapshot, cpu_busy=10.0, mutate=nudge)
    out = {item["id"]: item for item in alerts.build(SETTINGS, [state()], [payload])}
    assert out["driveripper:mem.used_pct"]["message"] == "driveripper: memory 85% (warn ≥80)"
    assert out["driveripper:cpu.temp_c"]["message"] == (
        "driveripper: CPU temperature 80°C (warn ≥75)")
    assert out["driveripper:disk.used_pct"]["message"] == (
        "driveripper: disk 91% on /btrfs/pool1 (fail ≥90)")
    assert "driveripper:cpu.usage_pct" not in out, "10% busy is not a problem"


def test_a_number_the_upstream_does_not_publish_is_never_a_breach(
        host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
    """No measurement, no verdict, no alert — the same rule `severity.classify` upholds."""
    payload = host_scrape(agent_snapshot, cpu_busy=10.0,
                          mutate=lambda snap: snap.pop("cpu_temperature_celsius"))
    assert alerts.build(SETTINGS, [state()], [payload]) == []


def test_a_saturated_gpu_is_not_an_alert(host_scrape: Any,
                                         agent_snapshot: dict[str, Any]) -> None:
    """Phase 06: these are LLM servers — a fully-worked GPU is the job, not a fault.

    The host is otherwise all-clear, so a card at 100 % busy with both memory pools
    filled must produce **no** alert at all: `severity.THRESHOLDS` has no `gpu.*` row,
    the payload carries no GPU verdict, and this module owns no threshold of its own
    from which to resurrect one.
    """
    def fill_the_card(snap: dict[str, Any]) -> None:
        gpu = snap["gpus"][0]
        gpu["busy_pct"] = 100.0
        for used, total in (("vram_used_bytes", "vram_total_bytes"),
                            ("gtt_used_bytes", "gtt_total_bytes")):
            if gpu.get(total) is not None:
                gpu[used] = gpu[total]

    payload = host_scrape(agent_snapshot, cpu_busy=10.0, mutate=fill_the_card)
    assert payload["gpus"][0]["busy_pct"] == 100.0, "the card really is saturated here"
    assert alerts.build(SETTINGS, [state()], [payload]) == [], (
        "a busy GPU produced an alert — GPU verdicts were removed in phase 06"
    )


def test_a_degraded_agent_warns_and_names_the_sections_that_failed(
        host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
    payload = host_scrape(agent_snapshot, cpu_busy=10.0)
    payload["degraded"] = {"temperatures": "no hwmon sensors", "disk_io": ""}
    out = alerts.build(SETTINGS, [state()], [payload])
    assert len(out) == 1, "a partial snapshot is one warning, not one per number that is missing"
    assert out[0]["id"] == "driveripper:agent.sections"
    assert out[0]["severity"] == "warn"
    assert "temperatures (no hwmon sensors)" in out[0]["message"]
    assert "disk_io" in out[0]["message"]


def test_a_degraded_list_from_an_older_agent_is_readable_too(
        host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
    payload = host_scrape(agent_snapshot, cpu_busy=10.0)
    payload["degraded"] = ["network"]
    out = alerts.build(SETTINGS, [state()], [payload])
    assert out[0]["message"] == "driveripper: the agent could not read network"


def test_a_degraded_note_that_is_a_plain_string_is_still_reported(
        host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
    """An agent (or a future collector) that reports one string instead of a map."""
    payload = host_scrape(agent_snapshot, cpu_busy=10.0)
    payload["degraded"] = "hwmon"
    out = alerts.build(SETTINGS, [state()], [payload])
    assert out[0]["message"] == "driveripper: the agent could not read hwmon"


def test_an_unreachable_host_is_one_alert_even_with_its_last_good_payload(
        host_scrape: Any, agent_snapshot: dict[str, Any]) -> None:
    """The exact completion criterion for the phase: one line naming the dead host."""
    def nudge(snap: dict[str, Any]) -> None:
        snap["memory"]["used_bytes"] = snap["memory"]["total_bytes"] * 0.99
        snap["cpu_temperature_celsius"] = 95.0

    payload = host_scrape(agent_snapshot, cpu_busy=96.0, mutate=nudge)
    out = alerts.build(SETTINGS, [state(ok=False, error="timed out")], [payload])
    assert len(out) == 1
    assert out[0]["severity"] == "fail" and "timed out" in out[0]["message"]


def test_a_stub_payload_for_a_host_that_never_answered_alerts_once() -> None:
    """/api/system/snapshot represents that host as {name, unreachable}; still one line."""
    stub = {"name": "gateway", "ts": 1_000.0, "unreachable": True}
    out = alerts.build(SETTINGS, [state(key="host:gateway", label="gateway", ok=False,
                                        error="unreachable")], [stub])
    assert [item["id"] for item in out] == ["gateway:host:gateway:down"]


# --------------------------------------------------------------------------- #
# LLM engines
# --------------------------------------------------------------------------- #
def test_kv_pressure_comes_from_the_rated_cell_and_reads_as_a_percent() -> None:
    """The engine publishes 0-1; the operator reads percent, matching the verdict's scale."""
    out = alerts.build(SETTINGS, [state(key="llm:gufo", dashboard="llm", label="gufo")], [],
                       [engine(kv_cache_usage_ratio=ratio(0.95, "fail"))])
    assert out[0]["message"] == "gufo: KV cache 95% (fail ≥90)"
    assert out[0]["href"] == "/llm"
    assert out[0]["id"] == "gufo:llm.kv_cache_usage_pct@kv_cache_usage_ratio"


def test_vllm_publishes_the_same_verdict_under_its_own_series_name() -> None:
    out = alerts.build(SETTINGS, [], [], [engine("vllm", gpu_cache_usage=ratio(0.8, "warn"))])
    assert out[0]["message"] == "vllm: KV cache 80% (warn ≥75)"


def test_a_cool_or_absent_kv_cell_is_not_an_alert() -> None:
    for cell in (ratio(0.10, "ok"), ratio(None, None), ratio(0.95, None), {}, "not a cell"):
        out = alerts.build(SETTINGS, [], [], [engine(kv_cache_usage_ratio=cell)])
        assert out == [], cell


def test_an_engine_that_is_not_answering_alerts_only_its_own_job() -> None:
    dead = {"engine": "vllm", "metrics": {}, "sessions": {}, "unreachable": True}
    out = alerts.build(SETTINGS, [state(key="llm:vllm", dashboard="llm", label="vLLM",
                                        ok=False, error="502 Bad Gateway")], [], [dead])
    assert [item["id"] for item in out] == ["vLLM:llm:vllm:down"]


# --------------------------------------------------------------------------- #
# UPS
# --------------------------------------------------------------------------- #
def test_a_failing_ups_check_is_a_fail_carrying_the_check_and_its_detail() -> None:
    payload = ups_payload(ups_card("apc",
                                   check("Power path", "fail", "The UPS is on battery"),
                                   check("Charge", "warn", "42% and discharging")))
    out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server")],
                       [], [], payload)
    by_id = {item["id"]: item for item in out}
    assert by_id["apc:health:Power path"]["severity"] == "fail"
    assert by_id["apc:health:Power path"]["message"] == "Power path: The UPS is on battery"
    assert by_id["apc:health:Charge"]["severity"] == "warn"
    assert all(item["href"] == "/ups" for item in out)


def test_the_landing_page_is_where_battery_age_belongs() -> None:
    """The task's own example: a sentence an operator can act on, not a score."""
    detail = "5.3 years — past the 4-year replacement window"
    out = alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc",
               check("Battery age", "warn", detail))))
    assert out[0]["message"] == f"Battery age: {detail}"
    assert out[0]["source"] == "apc"


def test_an_unknown_check_is_silence_and_a_check_with_no_detail_is_still_readable() -> None:
    payload = ups_payload(ups_card("apc", check("Battery temperature", "unknown",
                                                "not reported by this driver"),
                                   check("Alarms", "fail")))
    out = alerts.build(SETTINGS, [], [], [], payload)
    assert [item["message"] for item in out] == ["Alarms"]


def test_simulated_ups_data_is_announced_once_and_loudly() -> None:
    """With NUT_SIMULATE=1 and otherwise-healthy cards, that warning is the whole list."""
    payload = ups_payload(ups_card("apc", check("Charge", "ok", "100%")),
                          ups_card("cyberpower", check("Charge", "ok", "100%")),
                          simulated=True)
    out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server")],
                       [], [], payload)
    assert len(out) == 1
    assert out[0]["severity"] == "warn"
    assert "NUT_SIMULATE=1" in out[0]["message"]
    assert out[0]["href"] == "/ups"


async def test_the_simulator_on_its_own_produces_exactly_that_warning() -> None:
    """The task's acceptance line, through the real simulator and the real payload builder."""
    readings = await NutSimulator().read()
    payload = {"ts": 1_000.0, "simulated": True,
               "ups": [nut.payload(reading) for reading in readings]}
    out = alerts.build(SETTINGS,
                       [state(key="nut", dashboard="ups", label="NUT server")], [], [], payload)
    assert [item["id"] for item in out] == ["ups:simulated"], out


def test_a_real_upsd_does_not_claim_to_be_simulated() -> None:
    out = alerts.build(SETTINGS, [], [], [], ups_payload(ups_card("apc")))
    assert out == []


def test_a_dead_upsd_is_one_alert_and_not_its_stale_cards_too() -> None:
    payload = ups_payload(ups_card("apc", check("Power path", "fail", "on battery")))
    out = alerts.build(SETTINGS, [state(key="nut", dashboard="ups", label="NUT server",
                                        ok=False, error="upsd closed the connection")],
                       [], [], payload)
    assert [item["source"] for item in out] == ["NUT server"]


# --------------------------------------------------------------------------- #
# configuration
# --------------------------------------------------------------------------- #
def test_a_bad_target_entry_is_a_warn_that_says_it_is_not_being_polled() -> None:
    settings = Settings(host_target_problems=("nope is not name=url",
                                              "deskwork: url must start with http://"))
    out = alerts.build(settings)
    assert len(out) == 2
    assert all(item["severity"] == "warn" and item["source"] == "config" for item in out)
    assert all(item["href"] == "/system" for item in out)
    assert "HOST_TARGETS" in out[0]["message"] and "not being polled" in out[0]["message"]


# --------------------------------------------------------------------------- #
# dedupe, bound, order, shape
# --------------------------------------------------------------------------- #
def test_one_alert_per_source_and_metric_with_the_worst_verdict_winning() -> None:
    """Two cards answering as the same name (a re-exported UPS) must not double the alarm."""
    payload = ups_payload(ups_card("apc", check("Charge", "warn", "60%")),
                          ups_card("apc", check("Charge", "fail", "12%")))
    out = alerts.build(SETTINGS, [], [], [], payload)
    assert len(out) == 1
    assert out[0]["severity"] == "fail"
    assert out[0]["message"] == "Charge: 12%"


def test_the_list_is_bounded_and_says_how_many_it_hid() -> None:
    cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery"))
             for i in range(alerts.MAX_ALERTS + 5)]
    out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards))
    assert len(out) == alerts.MAX_ALERTS
    marker = out[-1]
    assert marker["id"] == "alerts:suppressed"
    assert marker["severity"] == "fail", "hiding reds behind an amber marker is a lie"
    hidden = len(cards) - (alerts.MAX_ALERTS - 1)  # one slot is the marker itself
    assert marker["message"] == f"{hidden} more not shown ({hidden} failing, 0 warning)"


def test_the_marker_is_the_worst_verdict_it_did_not_show() -> None:
    cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery"))
             for i in range(alerts.MAX_ALERTS)]
    cards += [ups_card(f"warn{i:02d}", check("Charge", "warn", "40%")) for i in range(3)]
    out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards))
    assert len(out) == alerts.MAX_ALERTS
    assert out[-1]["id"] == "alerts:suppressed"
    assert out[-1]["severity"] == "fail", "a red is behind the marker, so the marker is red"
    # and when only warnings are hidden, the marker says amber — it describes what it hid
    mild = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery"))
            for i in range(3)]
    mild += [ups_card(f"warn{i:02d}", check("Charge", "warn", "40%"))
             for i in range(alerts.MAX_ALERTS)]
    out = alerts.build(SETTINGS, [], [], [], ups_payload(*mild))
    assert out[-1]["severity"] == "warn"


def test_exactly_the_cap_adds_no_marker() -> None:
    cards = [ups_card(f"ups{i:02d}", check("Power path", "fail", "on battery"))
             for i in range(alerts.MAX_ALERTS)]
    out = alerts.build(SETTINGS, [], [], [], ups_payload(*cards))
    assert len(out) == alerts.MAX_ALERTS
    assert all(item["id"] != "alerts:suppressed" for item in out)


def test_fails_sort_before_warns_and_then_by_source() -> None:
    payload = ups_payload(ups_card("zeta", check("Power path", "fail", "on battery")),
                          ups_card("alpha", check("Charge", "warn", "40%")),
                          ups_card("beta", check("Load", "warn", "82%")))
    out = alerts.build(SETTINGS, [], [], [], payload)
    assert [(item["severity"], item["source"]) for item in out] == [
        ("fail", "zeta"), ("warn", "alpha"), ("warn", "beta")]


def test_the_trip_note_quotes_the_threshold_table_in_the_right_direction() -> None:
    """`_trip` reads `severity.THRESHOLDS` back instead of carrying its own numbers.

    Reached through `build()` only by metrics that are not inverted today, so this checks
    the helper directly: a low-is-worse metric must print `≤` (an `alert` for a battery is
    exactly where a wrong sign would be published), and a metric with no threshold gets no
    note rather than a made-up one.
    """
    assert alerts._trip("cpu.usage_pct", "fail") == " (fail ≥90)"
    assert alerts._trip("ups.battery_runtime", "warn") == " (warn ≤600)"
    assert alerts._trip("ups.charge_pct", "fail") == " (fail ≤20)"
    assert alerts._trip("cpu.usage_pct", "ok") == ""
    assert alerts._trip("mystery.metric", "fail") == ""


def test_the_alert_shape_is_the_contract_the_pages_render() -> None:
    out = alerts.build(SETTINGS, [state(ok=False, error="nope")], now=2_000.0)
    assert out[0] == {"id": "driveripper:host:driveripper:down", "severity": "fail",
                      "source": "driveripper", "message": "driveripper: nope",
                      "href": "/system", "at": 2_000.0}