dashboards/tests/test_system_contract.py

"""The payload contract behind the `/system` hero cards (phase 03, task 01).

Phase 03 turns `/system` into one panel per host with six big colour-coded numbers. Before
any of it is rendered, the data has to be proven present — otherwise a card that shows `n/a`
gets "fixed" by inventing a field, and the field it invents is a number the agent never
reported. This file drives the verbatim captures in `tests/fixtures/` through
`parse_host_snapshot()` (and through a real app), so a rename breaks here rather than in a
browser at 2am (`AGENTS.md` invariant 7).

The path table itself lives in `tests/card_contract.py` so the live suite asserts the same
list against real agents (`tests/test_live_fleet.py::test_every_host_serves_the_hero_card_contract`)
and the two cannot drift apart.

What is **deliberately not on the card** — a legibility decision, not a missing feature:
per-core usage (still stored per logical CPU as `cpu.usage_pct@coreN`, see
`test_visuals_the_page_drops_are_still_stored`), swap, the process table, the raw sensor
list, and per-interface rows. Removing a plot is not removing data (PLAN.md *Dropped
visuals*), so the decision stays reversible.

Two choices are recorded here because the task asked for them in writing:

* **CPU temp comes from `cpu.temp_c`**, not `temperatures.cpu_c`. They are the same
  measurement (the agent's own `cpu_temperature_celsius`) on every capture, `cpu.temp_c` is
  what `/api/summary` flattens into its fleet row, and a test below keeps them equal.
* **The disk verdict lives at `disks.severity.worst_used_pct`** (phase 01 attaches severity
  as a sibling dict per section, so there is no nested `disks.worst` object to hang it on).
"""
from __future__ import annotations

import copy
import math
import time
from collections.abc import Iterator
from typing import Any

import pytest
from fastapi.testclient import TestClient

from auth_client import sign_in
from auth_contract import ADMIN_PASSWORD, FAST_PBKDF2
from card_contract import GPU_METRICS, HOST_SHAPES, assert_card_contract, dig
from dashboards.app import create_app
from dashboards.config import Settings, Target
from dashboards.rates import RateEstimator
from dashboards.severity import classify
from dashboards.sources.host import HostCollector, parse_host_snapshot
from dashboards.store import SampleStore

#: One scrape is not a rate, so the API-level app points at a port that refuses instantly.
DEAD_URL = "http://127.0.0.1:1"


def make_collector() -> HostCollector:
    return HostCollector(RateEstimator(span=60.0))


def _bump_modes(modes: dict[str, Any], delta: float) -> None:
    """Move a /proc/stat mode set so the window reads `delta` ticks, 10 % of them busy."""
    if not isinstance(modes.get("total"), (int, float)):
        return
    modes["total"] += delta
    for idle_mode, share in (("idle", 0.8), ("iowait", 0.1)):
        if isinstance(modes.get(idle_mode), (int, float)):
            modes[idle_mode] += delta * share


def _advance(snapshot: dict[str, Any], dt: float = 10.0) -> dict[str, Any]:
    """The same scrape `dt` later with counters moved — the second point every rate needs."""
    moved = copy.deepcopy(snapshot)
    moved["ts"] = float(snapshot["ts"]) + dt
    _bump_modes(moved["cpu"]["modes"], 1_000.0)
    for core in moved["cpu"].get("cores") or []:
        _bump_modes(core.get("modes") or {}, 1_000.0)
    for iface in (moved.get("network") or {}).get("interfaces") or []:
        for key, value in list(iface.items()):
            if key.endswith("_total") and isinstance(value, (int, float)):
                iface[key] = value + 50_000.0
    return moved


@pytest.fixture
def payloads(agent_snapshots: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:
    """Each capture parsed once, first scrape — the shape the cards must tolerate."""
    out: dict[str, dict[str, Any]] = {}
    for name, snapshot in agent_snapshots.items():
        out[name] = parse_host_snapshot(name, snapshot, make_collector(), now=1_000.0).payload
    return out


@pytest.fixture
def api_client(tmp_path: Any, agent_snapshot: dict[str, Any],
               host_scrape: Any) -> Iterator[TestClient]:
    """A running app serving one host payload, so the contract is checked over HTTP too."""
    db = str(tmp_path / "contract.sqlite3")
    store = SampleStore(db)
    app = create_app(Settings(db_path=db, admin_password=ADMIN_PASSWORD,
                              pbkdf2_iterations=FAST_PBKDF2,
                              host_targets=(Target("driveripper", DEAD_URL),)), store)
    client = TestClient(app)
    client.__enter__()
    try:
        # phase 11: the snapshot is behind the session, and `TestClient` follows the 303 to
        # /login — a contract checked against the login page would pass on every path check
        # below it, which is worse than the test failing.
        sign_in(client)
        # the job's port refuses, so a poll can never overwrite what is injected here
        app.state.ctx.hub.jobs["host:driveripper"].state.payload = host_scrape(
            agent_snapshot, cpu_busy=10.0)
        yield client
    finally:
        client.__exit__(None, None, None)
        store.close()


# --------------------------------------------------------------------------- #
# Every card metric has a path, on every host shape
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("name", HOST_SHAPES)
def test_every_card_metric_has_a_payload_path_on_every_host(
        payloads: dict[str, dict[str, Any]], name: str) -> None:
    assert_card_contract(payloads[name])


def test_the_contract_covers_every_captured_host(agent_snapshots: dict[str, dict[str, Any]],
                                                 payloads: dict[str, dict[str, Any]]) -> None:
    """A fourth capture must join `HOST_SHAPES`, or its host shape goes uncontracted."""
    assert set(HOST_SHAPES) == set(payloads) == set(agent_snapshots)


def test_the_discrete_gpu_capture_supplies_real_numbers(
        payloads: dict[str, dict[str, Any]], agent_snapshots: dict[str, dict[str, Any]]) -> None:
    """Not just present: correct, on the host that has all six metrics."""
    payload = payloads["driveripper"]
    snapshot = agent_snapshots["driveripper"]

    assert payload["cpu"]["temp_c"] == snapshot["cpu_temperature_celsius"]
    # the two places the CPU temperature is exposed are the same measurement, always
    assert payload["temperatures"]["cpu_c"] == payload["cpu"]["temp_c"]

    memory = snapshot["memory"]
    assert payload["memory"]["used_pct"] == pytest.approx(
        memory["used_bytes"] / memory["total_bytes"] * 100.0)

    worst = max(snapshot["filesystems"], key=lambda fs: fs["used_pct"])
    assert payload["disks"]["worst_mount"] == worst["mountpoint"]
    assert payload["disks"]["worst_used_pct"] == pytest.approx(worst["used_pct"])

    gpu = payload["gpus"][0]
    assert gpu["model"]
    assert gpu["busy_pct"] == pytest.approx(snapshot["gpus"][0]["busy_pct"])
    assert gpu["vram_used_pct"] == pytest.approx(
        gpu["vram_used_bytes"] / gpu["vram_total_bytes"] * 100.0)
    # the hero's footnote is the agent's own max over this card's hwmon labels (phase 06),
    # so the card and the `gpu.temp_c@N` history series can never disagree
    assert gpu["temp_c"] == max(snapshot["gpus"][0]["temps"].values())

    # one scrape is not a rate: usage and throughput are honestly n/a until the second
    assert payload["cpu"]["usage_pct"] is None
    assert payload["network"]["rx_bytes_per_s"] is None


def test_measured_values_appear_on_the_second_scrape(host_scrape: Any,
                                                     agent_snapshot: dict[str, Any]) -> None:
    """The payload the API actually serves (after two scrapes) has numbers in all six places."""
    payload = host_scrape(agent_snapshot, cpu_busy=33.0)
    assert payload["cpu"]["usage_pct"] == pytest.approx(33.0)
    assert math.isfinite(payload["network"]["rx_bytes_per_s"])
    assert math.isfinite(payload["network"]["tx_bytes_per_s"])
    assert payload["memory"]["used_pct"] is not None
    assert payload["disks"]["worst_used_pct"] is not None
    assert payload["cpu"]["temp_c"] is not None
    gpu = payload["gpus"][0]
    assert gpu["busy_pct"] is not None
    assert gpu["vram_used_pct"] is not None
    assert gpu["mem_used_pct"] is not None


def test_each_card_number_travels_with_its_own_verdict(host_scrape: Any,
                                                       agent_snapshot: dict[str, Any]) -> None:
    """The card colours a verdict from the payload; it never holds a threshold (phase 01)."""
    payload = host_scrape(agent_snapshot, cpu_busy=33.0)
    assert payload["cpu"]["severity"] == {
        "usage_pct": classify("cpu.usage_pct", payload["cpu"]["usage_pct"]),
        "temp_c": classify("cpu.temp_c", payload["cpu"]["temp_c"]),
    }
    assert payload["memory"]["severity"]["used_pct"] == classify(
        "mem.used_pct", payload["memory"]["used_pct"])
    assert payload["disks"]["severity"]["worst_used_pct"] == classify(
        "disk.used_pct", payload["disks"]["worst_used_pct"])


def test_the_gpu_card_carries_no_verdict_at_all(host_scrape: Any,
                                                agent_snapshot: dict[str, Any]) -> None:
    """Phase 06: a fully-worked GPU is the job on an LLM server, not a fault.

    The card prints busy % / VRAM with no colour class at any level, so the payload
    carries no `severity` dict to colour from — the absence is the contract (pushed past
    every old bound in the fixture, so a resurrected threshold row would trip and fail).
    """
    def work_the_card(snap: dict[str, Any]) -> None:
        gpu = snap["gpus"][0]
        gpu["busy_pct"] = 100.0
        for used, total in (("vram_used_bytes", "vram_total_bytes"),
                            ("gtt_used_bytes", "gtt_total_bytes")):
            if gpu.get(total) is not None:
                gpu[used] = gpu[total]

    payload = host_scrape(agent_snapshot, cpu_busy=33.0, mutate=work_the_card)
    gpu = payload["gpus"][0]
    assert gpu["busy_pct"] == 100.0, "the numbers themselves stay on the card"
    assert gpu["mem_used_pct"] > 90.0, "the whole pool is worked too"
    assert "severity" not in gpu, (
        "a GPU verdict came back — classify('gpu.busy_pct', …) must be None and the row "
        "must not carry a dict for the page to colour from"
    )
    assert classify("gpu.busy_pct", gpu["busy_pct"]) is None
    assert classify("gpu.mem_used_pct", gpu["mem_used_pct"]) is None


# --------------------------------------------------------------------------- #
# The two host shapes that are not "a box with a discrete GPU"
# --------------------------------------------------------------------------- #
def test_the_box_with_no_gpu_reports_an_empty_list_not_a_missing_key(
        payloads: dict[str, dict[str, Any]]) -> None:
    """gateway has no GPU: `gpus == []`, never a missing key and never a card at 0 % busy."""
    payload = payloads["gateway"]
    assert "gpus" in payload, "a host with no GPU still has the key; the page says 'no GPU'"
    assert payload["gpus"] == []
    # the other five numbers keep their paths, with the honest None where nothing is measured
    assert_card_contract(payload)
    assert payload["cpu"]["temp_c"] is None, "this agent reports no CPU sensor"
    assert payload["cpu"]["severity"]["temp_c"] is None, "unknown is not ok"


def test_the_apu_reports_vram_and_the_whole_addressable_pool(
        payloads: dict[str, dict[str, Any]]) -> None:
    """deskwork (Strix Halo) is why `mem_used_pct` exists: 3 % VRAM and 78 % memory, both true."""
    gpu = payloads["deskwork"]["gpus"][0]
    assert gpu["vram_total_bytes"] < gpu["mem_total_bytes"]
    assert gpu["vram_used_pct"] < gpu["mem_used_pct"]
    assert gpu["model"], "the card needs a name to put beside the numbers"


def test_a_gpu_with_no_temperature_input_reports_none_not_zero(agent_snapshot: dict[str, Any],
                                                              ) -> None:
    """Phase 06 put `gpus[].temp_c` on the hero footnote, so its absence must stay honest.

    A driver exposing no hwmon temperature has to come back as `None` (the page prints no
    temperature at all) rather than `0 °C`, the same rule
    `tests/test_host_source.py::test_missing_sensors_stay_missing` holds for the flat sensor
    list — and no `gpu.temp_c@N` sample may be written for a number nobody measured.
    """
    snapshot = copy.deepcopy(agent_snapshot)
    snapshot["gpus"][0]["temps"] = {}
    result = parse_host_snapshot("driveripper", snapshot, make_collector(), now=1_000.0)
    gpu = result.payload["gpus"][0]
    assert gpu["temp_c"] is None, "an unreadable sensor is not a cold card"
    assert gpu["temps"] == {}
    assert_card_contract(result.payload)
    assert not [s for s in result.samples if s.metric.startswith("gpu.temp_c@")], (
        "a None temperature must not reach the history as a number")


def test_paths_survive_a_host_that_reports_almost_nothing(agent_snapshot: dict[str, Any]) -> None:
    """Keys present, values None: the card must render `n/a` without a KeyError."""
    snapshot = copy.deepcopy(agent_snapshot)
    snapshot["cpu_temperature_celsius"] = None
    snapshot["filesystems"] = []
    snapshot["network"]["interfaces"] = []
    snapshot["gpus"][0]["busy_pct"] = None
    snapshot["gpus"][0]["vram_total_bytes"] = None

    payload = parse_host_snapshot("driveripper", snapshot, make_collector(),
                                  now=1_000.0).payload
    assert_card_contract(payload)
    for path in GPU_METRICS:
        dig(payload["gpus"][0], path)

    assert payload["cpu"]["temp_c"] is None and payload["cpu"]["severity"]["temp_c"] is None
    assert payload["disks"]["worst_used_pct"] is None
    assert payload["disks"]["worst_mount"] is None
    assert payload["disks"]["severity"]["worst_used_pct"] is None
    assert payload["network"]["rx_bytes_per_s"] is None
    gpu = payload["gpus"][0]
    assert gpu["busy_pct"] is None
    assert gpu["vram_used_pct"] is None, "no total, so no percentage — not 0 %"
    assert gpu["mem_used_pct"] is None
    assert gpu["temp_c"] is not None, "the card still has its temperature to print"


# --------------------------------------------------------------------------- #
# What the cards drop stays in the database
# --------------------------------------------------------------------------- #
def test_visuals_the_page_drops_are_still_stored(agent_snapshot: dict[str, Any]) -> None:
    """Per-core / per-mount / per-interface / sensor / process data leaves the page.

    It must still be *sampled*: PLAN.md's *Dropped visuals* anchor says the page stops
    drawing what the operator stopped asking for while history keeps the series, so the
    decision stays reversible.
    """
    collector = make_collector()
    parse_host_snapshot("driveripper", agent_snapshot, collector, now=1_000.0)
    result = parse_host_snapshot("driveripper", _advance(agent_snapshot), collector,
                                 now=1_010.0)
    stored = {sample.metric for sample in result.samples}

    assert result.payload["cpu"]["usage_pct"] == pytest.approx(10.0), "the counters moved"
    cores = {metric for metric in stored if metric.startswith("cpu.usage_pct@core")}
    logical_cpus = result.payload["info"]["cpu_cores_logical"]
    assert len(cores) == logical_cpus, "one series per logical CPU, plotted or not"
    assert "cpu.usage_pct@core0" in stored and f"cpu.usage_pct@core{logical_cpus - 1}" in stored
    assert "disk.used_pct@/" in stored, "per-mount usage is still stored; the card shows worst"
    assert any(metric.startswith("net.rx_bytes_per_s@") for metric in stored), (
        "per-interface throughput is still stored; the card shows the totals")
    assert "temp.max_c" in stored, "the sensor list is not on the card but stays in history"
    assert "host.processes" in stored
    # swap is not a card metric either; it is absent here only because this capture has no
    # swap device (swap_total_bytes == 0), not because storing it stopped
    assert result.payload["memory"]["swap_used_pct"] is None
    assert "mem.swap_used_pct" not in stored


# --------------------------------------------------------------------------- #
# The same contract over HTTP
# --------------------------------------------------------------------------- #
def test_snapshot_endpoint_serves_the_card_paths(api_client: TestClient) -> None:
    """The cards read `/api/system/snapshot`, so the contract is checked on that body too."""
    hosts = api_client.get("/api/system/snapshot").json()["hosts"]
    assert len(hosts) == 1
    assert_card_contract(hosts[0])


def test_history_endpoint_still_answers_for_a_per_core_metric(api_client: TestClient) -> None:
    """`cpu.usage_pct@core7` is off the page but still queryable — dropping the plot is
    not dropping the series."""
    store = api_client.app.state.ctx.store  # type: ignore[attr-defined]
    store.add("host", "driveripper", "cpu.usage_pct@core7", 42.0, time.time())
    body = api_client.get("/api/system/history",
                          params={"metrics": "cpu.usage_pct@core7", "hosts": "driveripper",
                                  "range": "1h"}).json()
    assert body["series"], "the per-core series must still be retrievable"
    assert [point["value"] for point in body["series"][0]["points"]] == [42.0]