dashboards/tests/test_collectors.py

"""Collectors: the fetch half of every poll, exercised with a fake transport.

`httpx.MockTransport` means these run at the speed of the parser and still cover what the
live hosts do badly: a 401 from a rotated agent token, a half-degraded agent snapshot, a
gufo server that only answers `/v1/slots`, an engine that counts nothing yet.

NUT is covered here for the *monitor* bookkeeping (payload shape, command gating, status
transitions); the socket protocol itself is in test_nut.py.
"""
from __future__ import annotations

import json
import threading
from http.server import ThreadingHTTPServer
from pathlib import Path
from types import SimpleNamespace
from typing import Any, ClassVar

import httpx
import pytest

from dashboards import certs
from dashboards.collectors import (
    AgentTls,
    NutMonitor,
    UpstreamError,
    agent_tls_map,
    disks_jobs,
    fetch_disk_snapshot,
    fetch_engine,
    fetch_host_snapshot,
    fetch_snapshot,
    host_jobs,
)
from dashboards.config import Target
from dashboards.poller import Result
from dashboards.rates import RateEstimator
from dashboards.sources import nut
from dashboards.sources.disks import DisksCollector
from dashboards.sources.host import HostCollector

FIXTURES = Path(__file__).parent / "fixtures"


def client_for(handler: Any) -> httpx.AsyncClient:
    return httpx.AsyncClient(transport=httpx.MockTransport(handler),
                             base_url="http://agent.test")


# --------------------------------------------------------------------------- #
# host agent
# --------------------------------------------------------------------------- #
def agent_body() -> dict[str, Any]:
    return json.loads((FIXTURES / "agent_driveripper.json").read_text())


def advance_cpu(body: dict[str, Any], *, total: float, idle: float) -> None:
    """Move the jiffy counters the way a real /proc/stat does between polls."""
    modes = body["cpu"]["modes"]
    for mode, value in modes.items():
        modes[mode] = value + (total if mode == "total" else idle if mode == "idle" else 0)


async def test_a_real_agent_snapshot_parses() -> None:
    body = agent_body()
    calls: list[str] = []

    def handler(request: httpx.Request) -> httpx.Response:
        assert request.url.path == "/snapshot"
        calls.append(request.url.path)
        return httpx.Response(200, json=body)

    target = Target("driveripper", "http://agent.test")
    # min_window=0: these fetches land microseconds apart, far under any real poll
    # cadence; this test pins the Δjiffies window semantics, not the phase-13 floor
    # (tests/test_rates.py pins the floor at a real cadence).
    collector = HostCollector(RateEstimator(min_window=0.0))
    async with client_for(handler) as client:
        first = (await fetch_host_snapshot(client, target, collector)).payload
        # CPU usage is Δjiffies/Δt: one scrape has no window, so it is n/a and not 0 %
        assert first["cpu"]["usage_pct"] is None
        # the counters have to actually move — total 0 delta means "cannot divide", not idle
        same = (await fetch_host_snapshot(client, target, collector)).payload
        assert same["cpu"]["usage_pct"] is None, "no counter movement must read as n/a"
        advance_cpu(body, total=1000, idle=700)
        result = await fetch_host_snapshot(client, target, collector)
    payload = result.payload
    assert payload["cpu"]["usage_pct"] == pytest.approx(30.0, abs=1.0)
    assert payload["name"] == "driveripper"
    assert payload["cpu"]["usage_pct"] is not None
    assert payload["memory"]["total_bytes"] > 0
    # real captures have real GPUs; the legend wants the short name, the tooltip the full one
    assert payload["gpus"], "driveripper's capture must yield at least one GPU"
    assert payload["gpus"][0]["model"] and "model_full" in payload["gpus"][0]
    assert any(s.metric == "cpu.usage_pct" for s in result.samples)


async def test_target_headers_are_sent_to_the_agent() -> None:
    seen: dict[str, str] = {}

    def handler(request: httpx.Request) -> httpx.Response:
        seen["authorization"] = request.headers.get("authorization", "")
        return httpx.Response(200, json=agent_body())

    target = Target("gateway", "http://agent.test", headers={"Authorization": "Bearer tok"})
    async with client_for(handler) as client:
        await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
    assert seen["authorization"] == "Bearer tok"


@pytest.mark.parametrize("status,fragment", [
    (401, "token"),
    (403, "HTTP 403"),
    (500, "HTTP 500"),
])
async def test_agent_rejections_say_which_one_it_was(status, fragment) -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(status, text="nope")

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError) as excinfo:
            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                      HostCollector(RateEstimator()))
    assert fragment in excinfo.value.message
    assert excinfo.value.status == 502


async def test_a_refused_connection_is_an_upstream_error_not_a_traceback() -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        raise httpx.ConnectError("connection refused")

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError, match="unreachable"):
            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                      HostCollector(RateEstimator()))


@pytest.mark.parametrize("failure", ["refused", "timeout", "http-500"])
async def test_a_failing_host_never_echoes_its_token_in_the_message(failure: str) -> None:
    """An `UpstreamError` message is not just a log line: the poller keeps it in the health
    record (`/api/health`) and writes it to the `event` table (`/api/events`), both of which
    the browser reads — so the URL it names has to be the redacted one, on every failure path
    (AGENTS.md invariant 6). `parse_targets` normally moves a userinfo token into a header, so
    this is the backstop that keeps the guarantee true for any `Target` that still carries one.
    """
    def handler(request: httpx.Request) -> httpx.Response:
        if failure == "refused":
            raise httpx.ConnectError("connection refused")
        if failure == "timeout":
            raise httpx.ReadTimeout("too slow")
        return httpx.Response(500, text="boom")

    target = Target("gw", "http://SEKRIT-AGENT-TOKEN@agent.test")
    async with client_for(handler) as client:
        with pytest.raises(UpstreamError) as excinfo:
            await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
    message = excinfo.value.message
    assert "SEKRIT-AGENT-TOKEN" not in message, f"the token leaked through {failure}"
    # …and the message still names the machine it failed to reach
    assert "http://agent.test/snapshot" in message


async def test_a_failing_engine_never_echoes_credentials_in_its_url() -> None:
    """Same rule for the engines: the URL in the message is the redacted one."""
    def handler(request: httpx.Request) -> httpx.Response:
        raise httpx.ConnectError("connection refused")

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError, match="unreachable") as excinfo:
            await fetch_engine(client, "gufo", "http://SEKRIT-ENGINE-TOKEN@agent.test", "",
                               RateEstimator())
    assert "SEKRIT-ENGINE-TOKEN" not in excinfo.value.message
    assert "http://agent.test/metrics" in excinfo.value.message


async def test_a_slow_agent_is_a_gateway_timeout() -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        raise httpx.ReadTimeout("too slow")

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError) as excinfo:
            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                      HostCollector(RateEstimator()))
    assert excinfo.value.status == 504


async def test_html_instead_of_json_is_reported_as_such() -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, text="<html><body>502 Bad Gateway</body></html>")

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError, match="non-JSON"):
            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                      HostCollector(RateEstimator()))


async def test_an_agent_answering_not_ok_is_not_silently_empty() -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"})

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError, match="procfs unreadable"):
            await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                      HostCollector(RateEstimator()))


async def test_a_degraded_agent_still_renders_and_says_what_broke() -> None:
    body = agent_body()
    body["errors"] = {"temperatures": "no hwmon sensors"}

    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json=body)

    async with client_for(handler) as client:
        result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                           HostCollector(RateEstimator()))
    assert result.payload["degraded"] == {"temperatures": "no hwmon sensors"}
    assert result.payload["memory"]["total_bytes"] > 0


async def test_smart_addon_errors_do_not_degrade_the_system_source() -> None:
    """A host without smartmontools (or without device read access) is unconfigured, not
    broken (invariant 3): its `smart:` lines must never hold the system page amber. They
    stay in `agent.errors`, where the /disks page explains them in its own panel.
    """
    body = agent_body()
    body["errors"] = ["smart: sdb: Smartctl open device: /dev/sdb failed: Permission denied"]

    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json=body)

    async with client_for(handler) as client:
        result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                           HostCollector(RateEstimator()))
    assert "degraded" not in result.payload
    assert any("smart:" in str(e) for e in result.payload["agent"]["errors"])


async def test_real_degradations_still_show_when_smart_noise_is_alongside() -> None:
    body = agent_body()
    body["errors"] = ["smart: smartctl not found", "temperatures: no hwmon sensors"]

    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json=body)

    async with client_for(handler) as client:
        result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                           HostCollector(RateEstimator()))
    assert result.payload["degraded"] == ["temperatures: no hwmon sensors"]


async def test_a_host_without_a_proxy_config_does_not_degrade_the_system_source() -> None:
    """The proxy add-on's absence lines, same rule and same reason as `smart:` above: two of this
    fleet's three hosts run no haproxy and no nginx, so an agent upgrade must not put a
    `degraded` payload (and therefore an overview alert, `alerts.py` reads this key) on a box that
    never had either. The lines survive in `agent.errors` for the page that owns them — phase 29's
    gateway source reads them the way `/disks` reads `smart_errors`.

    `collectors.ADDON_ERROR_PREFIXES` is the one list of these prefixes; this asserts every
    name by behaviour, so dropping any of them fails here, and `tests/test_agent.py` pins both
    the wording of the lines the agent actually writes and the equality of the two lists.
    """
    missing_haproxy = ("haproxy: /etc/haproxy/haproxy.cfg not found — haproxy is not installed "
                       "here, or its config lives elsewhere (--haproxy-config); --no-proxy "
                       "opts out")
    missing_nginx = ("nginx: /etc/nginx/nginx.conf not found — nginx is not installed here, or "
                     "its config lives elsewhere (--nginx-config); --no-proxy opts out")
    no_sockets = ("sockets: /proc/net/tcp not found, /proc/net/tcp6 not found — this host "
                  "publishes no kernel socket table; --no-proxy opts out")
    body = agent_body()
    body["errors"] = [missing_haproxy, missing_nginx, no_sockets,
                      "smart: smartctl 7.5 not found"]

    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json=body)

    async with client_for(handler) as client:
        result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                           HostCollector(RateEstimator()))
    assert "degraded" not in result.payload, "unconfigured is not broken (invariant 3)"
    for prefix in ("haproxy:", "nginx:", "sockets:"):
        assert any(prefix in str(e) for e in result.payload["agent"]["errors"]), \
            "the sentence survives for the page that explains it"


async def test_a_real_degradation_shows_alongside_the_proxy_addon_noise() -> None:
    """Filtering the add-ons must not swallow a genuine half-snapshot beside them."""
    body = agent_body()
    body["errors"] = ["nginx: /etc/nginx/nginx.conf is there but could not be read: boom",
                      "temperatures: no hwmon sensors"]

    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json=body)

    async with client_for(handler) as client:
        result = await fetch_host_snapshot(client, Target("gw", "http://agent.test"),
                                           HostCollector(RateEstimator()))
    assert result.payload["degraded"] == ["temperatures: no hwmon sensors"]


# --------------------------------------------------------------------------- #
# physical disks (the /disks source's fetch half)
# --------------------------------------------------------------------------- #
async def test_the_disk_fetch_reads_the_same_agent_snapshot_as_the_system_one(
        disk_snapshot: Any) -> None:
    body = disk_snapshot(mode="healthy")
    seen: list[str] = []

    def handler(request: httpx.Request) -> httpx.Response:
        assert request.url.path == "/snapshot", "disks has no endpoint of its own"
        seen.append(request.url.path)
        return httpx.Response(200, json=body)

    target = Target("driveripper", "http://agent.test")
    async with client_for(handler) as client:
        result = await fetch_disk_snapshot(client, target, DisksCollector(), timeout=30)
    assert seen == ["/snapshot"]
    assert result.payload["name"] == "driveripper"
    assert result.payload["disks"], "the physical rows come with the snapshot"
    assert any(sample.metric.startswith("disk.temp_c@") for sample in result.samples)


async def test_the_disk_fetch_reports_the_agent_failure_like_the_system_one(
        monkeypatch: pytest.MonkeyPatch) -> None:
    """One dead host must say the same sentence on both pages (shared `fetch_snapshot`)."""
    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(500, text="boom")

    target = Target("gw", "http://agent.test")
    async with client_for(handler) as client:
        with pytest.raises(UpstreamError) as system_error:
            await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
        with pytest.raises(UpstreamError) as disk_error:
            await fetch_disk_snapshot(client, target, DisksCollector())
    assert disk_error.value.message == system_error.value.message


async def test_an_agent_that_is_not_ok_fails_the_disk_poll_too() -> None:
    """A degraded agent that says `ok: false` has no trustworthy disk rows to hand over."""
    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json={"ok": False, "detail": "procfs unreadable"})

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError, match="procfs unreadable"):
            await fetch_disk_snapshot(client, Target("gw", "http://agent.test"),
                                      DisksCollector())


class TimeoutRecordingClient(httpx.AsyncClient):
    """An `AsyncClient` that remembers the `timeout=` each request was handed.

    A `MockTransport` replaces httpcore's connection pool, so it does not enforce read
    timeouts and a slow-handler test would prove nothing. The argument is what matters
    anyway: `timeout=None` means *no timeout at all* to httpx, which would hang a poll
    loop forever, and the client default means "the system poll's 4 s".
    """

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        super().__init__(*args, **kwargs)
        self.timeouts: list[Any] = []

    async def get(self, url: Any, **kwargs: Any) -> httpx.Response:  # type: ignore[override]
        self.timeouts.append(kwargs.get("timeout", "<unset>"))
        return await super().get(url, **kwargs)


async def test_the_slow_disk_poll_carries_its_own_timeout_and_never_disables_one() -> None:
    """`HOST_TIMEOUT_SECONDS` (4 s) stays `/system`'s budget; `disks` brings its own 30 s.

    The SMART pass runs inside the agent's snapshot handler, so the two dashboards cannot
    share one budget: too short and `/disks` reads red on every SMART poll, too long and
    `/system` stops noticing a dead host. And the default must stay httpx's sentinel —
    passing a literal `None` silently turns timeout protection off altogether.
    """
    body = {"ok": True, "ts": 1.0, "disk_io": {"devices": []}, "errors": []}

    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json=body)

    client = TimeoutRecordingClient(transport=httpx.MockTransport(handler),
                                    timeout=4.0, base_url="http://agent.test")
    target = Target("gw", "http://agent.test")
    async with client:
        await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))
        assert client.timeouts == [httpx.USE_CLIENT_DEFAULT]
        await fetch_disk_snapshot(client, target, DisksCollector(), timeout=30)
        assert client.timeouts[-1] == 30, "the disk poll waits for the SMART pass"
        await fetch_snapshot(client, target)
        assert client.timeouts[-1] is httpx.USE_CLIENT_DEFAULT
        assert all(timeout is not None for timeout in client.timeouts), (
            "`timeout=None` disables httpx's timeouts entirely — translate it, never pass it")


async def test_smart_addon_errors_reach_the_disk_payload_verbatim(
        disk_snapshot: Any) -> None:
    """`collectors` keeps the `smart:` lines out of `/system`'s `degraded`; `/disks` is
    where they earn their keep — they are the page's reason for an empty card."""
    body = disk_snapshot(mode="denied")

    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json=body)

    async with client_for(handler) as client:
        disks = await fetch_disk_snapshot(client, Target("driveripper", "http://agent.test"),
                                          DisksCollector())
        system = await fetch_host_snapshot(client, Target("driveripper", "http://agent.test"),
                                           HostCollector(RateEstimator()))
    assert disks.payload["smart_errors"], "the /disks page gets the agent's own sentence"
    assert "degraded" not in system.payload, "and /system stays unbothered by it"


# --------------------------------------------------------------------------- #
# TLS: the dashboard dials a protected exporter with its own client certificate
# --------------------------------------------------------------------------- #
#
# The certificates come from `dashboards.certs` over a dict store and the server is the real
# `agent/host_agent.py` in the TLS mode task 02 added, so this proves the dial itself: the
# client certificate is presented, the agent is verified against the root this dashboard minted,
# and a host the operator revoked is refused before a socket opens. `httpx.MockTransport` is
# kept for the sentences (a 403 body is the agent's words, not a handshake).


class DictMeta:
    """The three-method `certs.MetaStore` slice, backed by a dict (the `test_certs.py` idiom)."""

    def __init__(self) -> None:
        self.rows: dict[str, str] = {}

    def get_meta(self, key: str) -> str | None:
        return self.rows.get(key)

    def set_meta(self, key: str, value: str) -> None:
        self.rows[key] = value

    def delete_meta(self, key: str) -> int:
        return 1 if self.rows.pop(key, None) is not None else 0


@pytest.fixture
def tls_fleet(tmp_path: Path, host_agent_module: Any) -> Any:
    """One dashboard CA + client pair, one issued agent bundle, and the agent serving it.

    Everything production has: the material written to a private directory (`httpx` takes
    paths), the bundle installed where `install-agent.sh` will put it, and the `AgentTls` the
    jobs get — built by `agent_tls_map`, the same factory `app.build_hub` is handed, so the
    tests dial with the production wiring and not a hand-built guess of it.
    """
    store = DictMeta()
    state = certs.ensure_certs(store)
    state = certs.issue_agent_cert(store, state, "localhost", ["127.0.0.1", "localhost"])
    entry = state.agent("localhost")
    assert entry is not None
    material = certs.write_material(state, tmp_path)
    bundle_path = tmp_path / "dashboards-agent.certs"
    bundle_path.write_text(entry.bundle, encoding="ascii")
    bundle, error = host_agent_module.TlsBundle.load(str(bundle_path))
    assert error is None and bundle is not None
    host_agent_module.Handler.collector = host_agent_module.Collector("/", smart=False)
    host_agent_module.Handler.token = None
    host_agent_module.Handler.bundle = bundle
    httpd = ThreadingHTTPServer(("127.0.0.1", 0), host_agent_module.Handler)
    httpd.socket = bundle.context.wrap_socket(httpd.socket, server_side=True)
    thread = threading.Thread(target=httpd.serve_forever, daemon=True)
    thread.start()
    base = f"https://127.0.0.1:{httpd.server_address[1]}"
    target = Target("localhost", base)
    try:
        yield SimpleNamespace(
            store=store,
            state=state,
            material=material,
            bundle=bundle,
            bundle_rev=entry.bundle_rev,
            base=base,
            target=target,
            tls=agent_tls_map(material, store, [target])["localhost"],
        )
    finally:
        httpd.shutdown()
        httpd.server_close()
        # `Handler` is process-wide state: a later test must not inherit this bundle.
        host_agent_module.Handler.bundle = None
        host_agent_module.Handler.token = None
        certs.remove_material(material)


async def test_an_https_agent_answers_the_dashboard_that_presents_its_certificate(
        tls_fleet: SimpleNamespace) -> None:
    """The whole point of the phase, collector side: our cert opens the agent.

    The two fields the agent adds in TLS mode come back too, because task 05's page compares the
    revision `/certs` issued with the revision the host is actually running.
    """
    async with httpx.AsyncClient(timeout=15.0) as client:
        body = await fetch_snapshot(client, tls_fleet.target, tls=tls_fleet.tls)
        assert body["ok"] is True
        assert body["tls"] is True
        assert body["bundle_rev"] == tls_fleet.bundle_rev
        result = await fetch_host_snapshot(client, tls_fleet.target,
                                           HostCollector(RateEstimator()), tls=tls_fleet.tls)
    assert result.payload["agent"]["tls"] is True
    assert result.payload["agent"]["bundle_rev"] == tls_fleet.bundle_rev
    assert result.payload["memory"]["total_bytes"] > 0, "the snapshot is a real one"


async def test_an_agent_signed_by_another_authority_is_not_trusted(
        tls_fleet: SimpleNamespace, tmp_path: Path) -> None:
    """A root that is not ours verifies nothing here, so the dial fails instead of accepting.

    Trusting any certificate would make the whole phase a no-op with extra steps, and the
    failure has to reach the poller as an `UpstreamError` naming the host rather than as a
    traceback out of `ssl`.
    """
    stranger_base = tmp_path / "stranger"
    stranger_base.mkdir()
    stranger = certs.write_material(certs.ensure_certs(DictMeta()), stranger_base)
    foreign = agent_tls_map(stranger, tls_fleet.store, [tls_fleet.target])["localhost"]
    async with httpx.AsyncClient(timeout=15.0) as client:
        with pytest.raises(UpstreamError) as caught:
            await fetch_snapshot(client, tls_fleet.target, tls=foreign)
    assert "unreachable" in caught.value.message
    assert tls_fleet.base in caught.value.message
    certs.remove_material(stranger)


async def test_dashboard_material_that_cannot_be_read_fails_the_poll(
        tls_fleet: SimpleNamespace, tmp_path: Path) -> None:
    """A missing client-cert file is a red source with a sentence, not a dead poll loop.

    `httpx` loads the CA and the client chain when the client is *built*, so this arrives as a
    `FileNotFoundError` from inside the fetch. A TLS dial that can only fail with a non-HTTP
    exception is the invariant-4 failure mode: the task would die and the source would look
    healthy while its charts stopped.
    """
    broken = AgentTls(verify=str(tls_fleet.material.ca),
                      cert=(str(tmp_path / "gone-cert.pem"), str(tmp_path / "gone-key.pem")),
                      status=lambda: certs.ACTIVE)
    async with httpx.AsyncClient(timeout=15.0) as client:
        with pytest.raises(UpstreamError, match="unreachable") as caught:
            await fetch_snapshot(client, tls_fleet.target, tls=broken)
    assert "FileNotFoundError" in caught.value.message
    assert tls_fleet.base in caught.value.message, "the failure still names the host's URL"


async def test_an_https_agent_is_not_readable_without_the_material(tls_fleet: SimpleNamespace) -> None:
    """A shared client with no client certificate gets nowhere — "only this dashboard can read
    an agent" has to hold for the collector's own client too, not just for a stranger's.
    """
    async with httpx.AsyncClient(timeout=15.0) as client:
        with pytest.raises(UpstreamError) as caught:
            await fetch_snapshot(client, tls_fleet.target)
    assert "unreachable" in caught.value.message


async def test_a_revoked_agent_is_refused_before_anything_is_dialled(
        tls_fleet: SimpleNamespace, monkeypatch: pytest.MonkeyPatch) -> None:
    """The revocation gate is in front of the network, not behind it.

    The spy replaces `httpx.AsyncClient.get` on the class, so it catches the shared client and
    the per-dial TLS client alike: what is being proved is that no request is built at all, not
    that one particular client was unused. A dial-then-refuse would still put a socket on a host
    the operator just struck out.
    """
    calls: list[str] = []

    async def spy_get(self: Any, url: Any, **kwargs: Any) -> Any:
        calls.append(str(url))
        raise AssertionError(f"a revoked agent was dialled at {url}")

    monkeypatch.setattr(httpx.AsyncClient, "get", spy_get)
    certs.revoke(tls_fleet.store, tls_fleet.state, "localhost")
    async with httpx.AsyncClient(timeout=15.0) as client:
        with pytest.raises(UpstreamError, match="agent certificate revoked") as caught:
            await fetch_snapshot(client, tls_fleet.target, tls=tls_fleet.tls)
        with pytest.raises(UpstreamError, match="agent certificate revoked"):
            await fetch_disk_snapshot(client, tls_fleet.target, DisksCollector(),
                                      tls=tls_fleet.tls)
    assert calls == [], "a revoked host must not be dialled on either cadence"
    assert "revoked" in caught.value.message


async def test_the_revocation_answer_is_a_live_read_of_the_store(tls_fleet: SimpleNamespace) -> None:
    """`status` is a callable because the operator clicks revoke while this process is running.

    A status string read at startup would keep dialling a struck-out host until the next
    restart, and a dashboard that needs a restart to enforce a security decision does not
    enforce it.
    """
    tls = tls_fleet.tls
    assert tls.status() == certs.ACTIVE
    assert tls.revoked is False
    certs.revoke(tls_fleet.store, tls_fleet.state, "localhost")
    assert tls.status() == certs.REVOKED
    assert tls.revoked is True, "the same object, with no rebuild and no restart"


def test_the_revocation_is_per_host_and_the_material_is_shared(tls_fleet: SimpleNamespace) -> None:
    """One identity for every dial, one revocation answer per name.

    `certs.py` gives the dashboard a single client certificate, so `verify`/`cert` are the same
    paths for every host; what differs is whose identity the operator struck out. A name with no
    entry is active rather than refused — never-issued is not the same fact as revoked, and
    answering revoked for it would take a working host red with no operator action behind it
    (invariant 3).
    """
    targets = [tls_fleet.target, Target("deskwork", "https://10.1.0.51:9120")]
    mapping = agent_tls_map(tls_fleet.material, tls_fleet.store, targets)
    assert mapping["localhost"].verify == str(tls_fleet.material.ca)
    assert mapping["localhost"].cert == mapping["deskwork"].cert == (
        str(tls_fleet.material.client_cert), str(tls_fleet.material.client_key))
    certs.revoke(tls_fleet.store, tls_fleet.state, "localhost")
    assert mapping["localhost"].status() == certs.REVOKED
    assert mapping["deskwork"].status() == certs.ACTIVE
    assert agent_tls_map(None, tls_fleet.store, targets) == {}, (
        "no material means no TLS dial, exactly the pre-phase-15 path")


def scripted_refusal(body: Any, *, status: int = 403) -> AgentTls:
    """An agent that answers every `/snapshot` with a refusal, in whatever shape it sent the body."""
    def handler(request: httpx.Request) -> httpx.Response:
        if isinstance(body, dict):
            return httpx.Response(status, json=body)
        return httpx.Response(status, text=body)

    return AgentTls(verify="/unused/root.pem", cert=("/unused/cert.pem", "/unused/key.pem"),
                    status=lambda: certs.ACTIVE, transport=httpx.MockTransport(handler))


@pytest.mark.parametrize("body,expected", [
    ({"ok": False, "detail": "client certificate required"}, "client certificate required"),
    ({"ok": False, "detail": "client certificate revoked"}, "client certificate revoked"),
    ("<html><body>403 Forbidden</body></html>", "403 Forbidden"),
    ("", "no reason given"),
])
async def test_a_refusal_from_a_protected_agent_quotes_its_own_words(
        body: Any, expected: str) -> None:
    """The 403 sentence names the refusal the *agent* made, redacted and never blank.

    An `UpstreamError` string goes into the health record and the event table, so it has to say
    which of the agent's two refusals this was (a required certificate and a revoked one are two
    different operator actions) while quoting nothing but the URL it was dialling
    (AGENTS.md invariant 6). The `verify`/`cert` paths here do not exist: a scripted refusal
    proves the wording, and the tests above prove the handshake.
    """
    tls = scripted_refusal(body)
    target = Target("gw", "https://SEKRIT-AGENT-TOKEN@agent.test")
    async with httpx.AsyncClient(timeout=15.0) as client:
        with pytest.raises(UpstreamError) as caught:
            await fetch_snapshot(client, target, tls=tls)
    message = caught.value.message
    assert "agent refused the client certificate" in message
    assert expected in message
    assert "SEKRIT-AGENT-TOKEN" not in message, "the token leaked into a browser-readable string"
    assert "https://agent.test/snapshot" in message, "and it still says where"


@pytest.mark.parametrize("status", [401, 403, 500])
async def test_a_refusal_from_an_unprotected_agent_keeps_the_status_wording(status: int) -> None:
    """The certificate sentence is only for a dial where a certificate was presented.

    A token-mode host answers 401 for a bad token and would answer 403 for something that has
    nothing to do with certificates; calling that "refused the client certificate" would send
    the operator to `/certs` for a token problem.
    """
    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(status, text="nope")

    target = Target("gw", "http://agent.test")
    async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
        with pytest.raises(UpstreamError) as caught:
            await fetch_snapshot(client, target,
                                 tls=AgentTls(verify="/unused/root.pem",
                                              cert=("/unused/c.pem", "/unused/k.pem"),
                                              status=lambda: certs.ACTIVE))
    if status == 401:
        assert "token" in caught.value.message
    else:
        assert f"HTTP {status}" in caught.value.message
    assert "client certificate" not in caught.value.message


async def test_a_token_mode_host_still_uses_the_shared_client(tls_fleet: SimpleNamespace) -> None:
    """An `http://` entry is dialled exactly as it was before phase 15.

    The material exists in this deployment (one CA, one client pair, every host), so the scheme
    is the only thing that may decide the path — a rollout is per host, and the plain hosts
    cannot be allowed to pick up a client certificate they never asked for.
    """
    seen: list[str] = []

    def handler(request: httpx.Request) -> httpx.Response:
        seen.append(str(request.url))
        return httpx.Response(200, json=agent_body())

    plain = Target("localhost", "http://agent.test")
    async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
        body = await fetch_snapshot(client, plain, tls=tls_fleet.tls)
    assert body["ok"] is True
    assert seen == ["http://agent.test/snapshot"], "the shared client, the shared transport"


async def test_a_revoked_agent_fails_both_jobs_with_one_sentence(store: Any,
                                                                 tls_fleet: SimpleNamespace,
                                                                 monkeypatch: pytest.MonkeyPatch) -> None:
    """The phase's criterion, at the level the health record is written: `host:` and `disks:`.

    The two jobs dial one agent on two cadences, so a revocation has to fail both with the same
    sentence (one dead source must not blink the other, and both must stop talking to it).
    """
    calls: list[str] = []

    async def spy_get(self: Any, url: Any, **kwargs: Any) -> Any:
        calls.append(str(url))
        raise AssertionError(f"a revoked agent was dialled at {url}")

    monkeypatch.setattr(httpx.AsyncClient, "get", spy_get)
    certs.revoke(tls_fleet.store, tls_fleet.state, "localhost")
    async with httpx.AsyncClient(timeout=15.0) as client:
        jobs = host_jobs([tls_fleet.target], client, HostCollector(RateEstimator()), 10.0,
                         tls={"localhost": tls_fleet.tls})
        jobs += disks_jobs([tls_fleet.target], client, DisksCollector(), 300.0,
                           tls={"localhost": tls_fleet.tls})
        assert [job.key for job in jobs] == ["host:localhost", "disks:localhost"]
        for job in jobs:
            assert await job.run_once(store) is False
    assert [job.state.error for job in jobs] == [
        "UpstreamError: localhost: agent certificate revoked"] * 2
    assert calls == []


async def test_host_and_disk_jobs_hand_each_host_its_own_material(
        monkeypatch: pytest.MonkeyPatch) -> None:
    """The plumbing, pinned: a host in the map gets its entry, a host outside it gets `None`.

    Two hosts and one wrong-key miss is the minimum that shows the lookup is by name rather than
    "whatever was built first", which is how a revoked host would keep being dialled with
    another host's revocation answer.
    """
    seen: dict[str, Any] = {}

    def record(name: str) -> Any:
        async def scrape(client: Any, target: Target, *_args: Any, **kwargs: Any) -> Result:
            seen[f"{name}:{target.name}"] = kwargs.get("tls")
            return Result(payload={}, samples=[], events=[])

        return scrape

    fetch_host = record("system")
    fetch_disk = record("disks")
    monkeypatch.setattr("dashboards.collectors.fetch_host_snapshot", fetch_host)
    monkeypatch.setattr("dashboards.collectors.fetch_disk_snapshot", fetch_disk)
    entry = AgentTls(verify="/ca.pem", cert=("/c.pem", "/k.pem"),
                     status=lambda: certs.ACTIVE)
    targets = [Target("gw", "https://gw.test"), Target("desk", "http://desk.test")]
    async with httpx.AsyncClient() as client:
        for job in (host_jobs(targets, client, HostCollector(RateEstimator()), 10.0,
                              tls={"gw": entry})
                    + disks_jobs(targets, client, DisksCollector(), 300.0, tls={"gw": entry})):
            await job.fn()
    assert seen == {"system:gw": entry, "system:desk": None,
                    "disks:gw": entry, "disks:desk": None}


async def test_the_tls_self_report_reaches_both_payloads(disk_snapshot: Any) -> None:
    """`tls` + `bundle_rev` travel into `/api/system/snapshot` and `/api/disks/snapshot`.

    Task 05's page has to answer "is the installed bundle the one I just issued?", and both
    dashboards read the same snapshot, so both carry the deployed revision under the same
    `agent` key. Nothing grades it: a revision is an identity counter, not a threshold.
    """
    def stamped(snapshot: dict[str, Any]) -> dict[str, Any]:
        """What a TLS-mode agent's `/snapshot` looks like on the wire."""
        snapshot["tls"] = True
        snapshot["bundle_rev"] = 3
        return snapshot

    target = Target("driveripper", "http://agent.test")
    async with client_for(lambda request: httpx.Response(200,
                                                        json=stamped(agent_body()))) as client:
        system = (await fetch_host_snapshot(client, target,
                                           HostCollector(RateEstimator()))).payload
    assert system["agent"]["tls"] is True
    assert system["agent"]["bundle_rev"] == 3

    # the `/disks` payload is built from a capture that also carries SMART blocks, so it gets
    # its own stamped snapshot rather than sharing the one above
    with_smart = disk_snapshot(mutate=stamped)
    async with client_for(lambda request: httpx.Response(200, json=with_smart)) as client:
        disks = (await fetch_disk_snapshot(client, target, DisksCollector())).payload
    assert disks["agent"] == {"tls": True, "bundle_rev": 3}


async def test_a_token_mode_snapshot_gains_no_tls_fields() -> None:
    """The captures are token-mode agents, and their payloads must stay byte-identical.

    `false` and `0` would be numbers the agent never published (rule 2), and an empty block
    would be a container that means nothing.
    """
    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(200, json=agent_body())

    target = Target("driveripper", "http://agent.test")
    async with client_for(handler) as client:
        system = (await fetch_host_snapshot(client, target, HostCollector(RateEstimator()))).payload
    assert "tls" not in system["agent"] and "bundle_rev" not in system["agent"]
    assert system["agent"]["version"], "the block is still the block it was"


# --------------------------------------------------------------------------- #
# LLM engines
# --------------------------------------------------------------------------- #
GUFO = (FIXTURES / "metrics_gufo.txt").read_text()
VLLM = (FIXTURES / "metrics_vllm.txt").read_text()


def bump(text: str, series: str, amount: float) -> str:
    lines = []
    for line in text.splitlines():
        if line.startswith(series + " "):
            head, _, value = line.rpartition(" ")
            line = f"{head} {float(value) + amount}"
        lines.append(line)
    return "\n".join(lines) + "\n"


async def test_first_gufo_poll_has_no_measured_rate_yet() -> None:
    """One scrape cannot produce Δ/Δt. The cell is n/a, not a stale single-request gauge."""
    def handler(request: httpx.Request) -> httpx.Response:
        if request.url.path == "/metrics":
            return httpx.Response(200, text=GUFO)
        return httpx.Response(404, text="nope")

    async with client_for(handler) as client:
        result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(),
                                    with_slots=True)
    decode = result.payload["metrics"]["decode_tokens_per_second"]
    assert decode["value"] is None
    assert decode["source"] is None
    assert result.payload["sessions"]["active"] == 0.0
    assert "no /slots answer" in result.payload["sessions"]["detail"]


async def test_second_gufo_poll_measures_throughput_over_a_window() -> None:
    state = {"text": GUFO}

    def handler(request: httpx.Request) -> httpx.Response:
        if request.url.path == "/metrics":
            return httpx.Response(200, text=state["text"])
        if request.url.path == "/slots":
            return httpx.Response(200, json=[{"index": 0, "n_ctx": 8192, "predicted": 12}])
        return httpx.Response(404)

    # min_window=0: the two fetches are microseconds apart (no real loop polls this
    # fast); this test pins that the measured cell divides by the window it labels.
    rates = RateEstimator(span=10.0, min_window=0.0)
    async with client_for(handler) as client:
        await fetch_engine(client, "gufo", "http://engine.test", "", rates)
        state["text"] = bump(GUFO, "llamacpp:tokens_predicted_total", 40.0)
        result = await fetch_engine(client, "gufo", "http://engine.test", "", rates,
                                    with_slots=True)
    decode = result.payload["metrics"]["decode_tokens_per_second"]
    assert decode["source"].startswith("computed:Δllamacpp:")
    assert decode["derived"] is True
    assert decode["window"] > 0
    assert decode["value"] == pytest.approx(40.0 / decode["window"], rel=0.05)
    assert result.payload["sessions"]["source"] == "gufo /slots"
    assert result.payload["sessions"]["list"][0]["index"] == 0


async def test_gufo_slots_alias_is_used_when_plain_slots_is_missing() -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        if request.url.path == "/metrics":
            return httpx.Response(200, text=GUFO)
        if request.url.path == "/v1/slots":
            return httpx.Response(200, json={"slots": [{"index": 1}]})
        return httpx.Response(404)

    async with client_for(handler) as client:
        result = await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator(),
                                    with_slots=True)
    assert result.payload["sessions"]["list"] == [{"index": 1}]


async def test_vllm_metrics_parse_and_carry_the_api_key() -> None:
    seen: list[str] = []

    def handler(request: httpx.Request) -> httpx.Response:
        seen.append(request.headers.get("authorization", ""))
        return httpx.Response(200, text=VLLM)

    async with client_for(handler) as client:
        result = await fetch_engine(client, "vllm", "http://engine.test", "SECRET",
                                    RateEstimator())
    assert seen == ["Bearer SECRET"]
    metrics = result.payload["metrics"]
    assert metrics["num_requests_running"]["value"] is not None
    # vLLM publishes no per-session list: the page is told, rather than shown an empty table
    assert "counts, not sessions" in result.payload["sessions"]["detail"]
    assert any(s.metric.startswith("llm.") for s in result.samples)


@pytest.mark.parametrize("status", [404, 500, 503])
async def test_engine_http_errors_name_the_status(status) -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        return httpx.Response(status, text="")

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError, match=f"HTTP {status}"):
            await fetch_engine(client, "vllm", "http://engine.test", "", RateEstimator())


async def test_engine_timeout_is_504() -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        raise httpx.ReadTimeout("engine busy")

    async with client_for(handler) as client:
        with pytest.raises(UpstreamError) as excinfo:
            await fetch_engine(client, "gufo", "http://engine.test", "", RateEstimator())
    assert excinfo.value.status == 504


# --------------------------------------------------------------------------- #
# NUT monitor
# --------------------------------------------------------------------------- #
class FakeSimulator:
    """Stands in for NutSimulator: hands back readings we control."""

    def __init__(self, readings: list[nut.UpsReading]) -> None:
        self.readings = readings

    async def read(self) -> list[nut.UpsReading]:
        return list(self.readings)


def reading(status: str, **vars: str) -> nut.UpsReading:
    item = nut.UpsReading(name="cyberpower", description="CP1350PFCLCD", ts=1_700_000_000.0)
    item.vars = {"ups.status": status, "battery.charge": "92", "ups.load": "35",
                 "battery.runtime": "1800", **vars}
    item.commands = ["beeper.mute", "test.panel.start", "test.battery.start.deep"]
    return item


class SettingsStub:
    nut_host, nut_port, nut_timeout = "ups.test", 3493, 5.0
    nut_user = nut_password = ""
    nut_interval = 5.0
    nut_enabled = True
    actions_allowed = True
    read_only = False


async def test_monitor_payload_labels_the_simulator() -> None:
    monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
    result = await monitor.collect()
    payload = result.payload
    assert payload["simulated"] is True
    assert payload["server"] is None, "a simulator must not claim a server address"
    assert payload["actions_enabled"] is True
    card = payload["ups"][0]
    assert card["name"] == "cyberpower"
    assert card["status"] == "OL"
    assert card["numbers"]["charge_pct"] == 92.0
    # a driver that reports no battery date, temperature, or self-test result is `unknown`,
    # never a silent "healthy" — and never a failure either
    assert card["health"]["overall"] == "unknown", card["health"]
    assert card["health"]["counts"]["fail"] == 0
    assert card["health"]["counts"]["warn"] == 0
    # commands are offered with their risk, and the dangerous one is present but marked
    risks = {item["command"]: item for item in card["commands"]}
    assert risks["beeper.mute"]["enabled"] is True
    assert risks["test.battery.start.deep"]["risk"] == "dangerous"


async def test_a_fully_report_ups_is_called_healthy() -> None:
    full = reading("OL", **{"battery.temperature": "27.4", "battery.mfr.date": "2024-06-01",
                            "battery.test.result": "OK", "input.voltage": "230"})
    full.vars["battery.charge"] = "100"
    card = (await NutMonitor(SettingsStub(), simulator=FakeSimulator([full])).collect()).payload["ups"][0]
    assert card["health"]["overall"] == "healthy", card["health"]
    assert card["health"]["counts"]["unknown"] == 0


async def test_monitor_sends_history_samples_for_every_ups_metric() -> None:
    monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OB")]))
    result = await monitor.collect()
    names = {sample.metric for sample in result.samples}
    assert "ups.charge_pct" in names
    assert "ups.load_pct" in names
    assert all(sample.source == "ups" for sample in result.samples)


async def test_status_changes_become_one_event_each() -> None:
    monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
    assert (await monitor.collect()).events == []
    monitor.simulator = FakeSimulator([reading("OL")])
    assert (await monitor.collect()).events == [], "an unchanged status is not news"
    monitor.simulator = FakeSimulator([reading("OB")])
    events = (await monitor.collect()).events
    assert len(events) == 1
    assert events[0].kind == "status"
    assert "OL → OB" in events[0].message
    assert events[0].severity == "warn", "on battery is a warning, not a failure"
    monitor.simulator = FakeSimulator([reading("OL FUB")])
    events = (await monitor.collect()).events
    assert events[0].severity == "error", "FUB (the UPS decided to power off) is a failure"


async def test_a_self_test_result_is_reported_when_it_changes() -> None:
    monitor = NutMonitor(SettingsStub(), simulator=FakeSimulator([reading("OL")]))
    await monitor.collect()
    # the variable is `ups.test.result` (drivers also use battery.test.result) — see VAR_MAP
    monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "INPROGRESS"})])
    assert (await monitor.collect()).events == [], "first sighting is not news"
    monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "FAIL"})])
    events = (await monitor.collect()).events
    assert [event.kind for event in events] == ["test"]
    assert events[0].severity == "warn"
    monitor.simulator = FakeSimulator([reading("OL", **{"ups.test.result": "OK"})])
    assert (await monitor.collect()).events[0].severity == "info"
    # the word this fleet's CyberPower actually answers (phase 12 capture): the feed and
    # the health check grade one outcome, so a passed test is never an amber line
    monitor.simulator = FakeSimulator(
        [reading("OL", **{"ups.test.result": "Done and passed"})])
    assert (await monitor.collect()).events[0].severity == "info"


async def test_gating_is_applied_where_the_commands_are_built() -> None:
    class ReadOnly(SettingsStub):
        actions_allowed = False

    monitor = NutMonitor(ReadOnly(), simulator=FakeSimulator([reading("OL")]))
    card = (await monitor.collect()).payload["ups"][0]
    assert all(item["enabled"] is False for item in card["commands"])
    assert all(item["reason"] for item in card["commands"]), "every refusal must explain itself"
    assert set(card["advertised_commands"]) == {"beeper.mute", "test.panel.start",
                                                "test.battery.start.deep"}, (
        "the driver still advertises them — the refusal is ours, and must say so")


class FakeNutClient:
    """The real upsd branch without a socket: vars work, the optional lists do not."""

    instances: ClassVar[list[FakeNutClient]] = []

    def __init__(self, host: str, port: int, **kwargs: Any) -> None:
        self.host, self.port, self.kwargs = host, port, kwargs
        self.entered = False
        FakeNutClient.instances.append(self)

    async def __aenter__(self) -> FakeNutClient:
        self.entered = True
        return self

    async def __aexit__(self, *exc: Any) -> None:
        self.entered = False

    async def list_ups(self) -> list[dict[str, str]]:
        return [{"name": "cyberpower", "description": "CP1350PFCLCD"}]

    async def list_vars(self, ups: str) -> dict[str, str]:
        return {"ups.status": "OL", "battery.charge": "100", "battery.temperature": "301.1"}

    async def list_cmds(self, ups: str) -> list[str]:
        raise nut.NutError("this driver exports no commands")

    async def list_rw_vars(self, ups: str) -> list[str]:
        raise nut.NutError("LIST RW denied")


async def test_real_upsd_branch_survives_drivers_that_refuse_the_optional_lists(
        monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(nut, "NutClient", FakeNutClient)
    monitor = NutMonitor(SettingsStub())
    result = await monitor.collect()
    client = FakeNutClient.instances[-1]
    assert client.entered is False, "the connection must not outlive one poll"
    assert client.kwargs["timeout"] == 5.0
    card = result.payload["ups"][0]
    assert card["commands"] == [] and card["advertised_commands"] == []
    assert card["numbers"]["charge_pct"] == 100.0
    # a driver reporting Kelvin is normalised, not charted as 301 °C
    assert card["numbers"]["battery_temperature"] == pytest.approx(301.1 - 273.15, abs=0.01)
    samples = {sample.metric: sample.value for sample in result.samples}
    assert samples["ups.battery_temperature_c"] == pytest.approx(27.95, abs=0.01)


async def test_a_real_server_names_itself_in_the_payload(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(nut, "NutClient", FakeNutClient)
    payload = (await NutMonitor(SettingsStub()).collect()).payload
    assert payload["simulated"] is False
    assert payload["server"] == "ups.test:3493"