dashboards/scripts/capture-fixtures.sh

#!/usr/bin/env bash
# Re-record ground-truth captures into tests/fixtures/.
#
#   ./scripts/capture-fixtures.sh                # dials HOST_TARGETS from .env.dev, directly
#   ./scripts/capture-fixtures.sh --with-llm     # also scrape the live engines' /metrics
#                                                 # (gufo /slots is captured without the flag)
#   SMART_HOSTS="deskwork-ops driveripper-ops" ./scripts/capture-fixtures.sh
#                                                # also capture smartctl per physical disk
#
# Fixtures are verbatim, not hand-written: a fixture you typed is a test of your memory.
# When a host's kernel or an engine's metric names change, re-run this and read the diff.
# Exception on record: tests/fixtures/smartctl_reference_*.txt are reference shapes, not
# field captures (no host could reach a block device when they were written). The SMART_HOSTS
# pass below writes hardware truth beside them as smartctl_<host>_<dev>.txt, so the reference
# shapes stay what they are: synthetic edge cases with verbatim labels.
set -uo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
FIX="$ROOT/tests/fixtures"
mkdir -p "$FIX"

# Agents come from the same env the app reads: HOST_TARGETS in .env.dev, `name=url` pairs
# (space or comma separated — the shape config.parse_targets accepts), the bearer token in
# the URL userinfo used as-is. Nothing is hard-coded here, so a capture can never come from
# a different topology than the dashboard polls.
if [ -f "$ROOT/.env.dev" ]; then
  set -a; . "$ROOT/.env.dev"; set +a
fi
HOSTS=()
targets="${HOST_TARGETS:-}"
for entry in ${targets//,/ }; do   # space or comma separated, like config.parse_targets
  HOSTS+=("$entry")
done
# Engine URLs come from the same env too, so this script carries no deployment
# detail of its own — and never records a tokenised URL into a fixture by accident.
# Engine entries are `<name> <url>` pairs; the name is the fixture suffix and the env
# var that dials the engine (gufo / vllm / radiance — a vLLM derivative that also
# answers /stats and /v1/models, captured in the same pass).
ENGINES=()
[ -n "${GUFO_BASE_URL:-}" ] && ENGINES+=("gufo ${GUFO_BASE_URL%/}/metrics")
[ -n "${VLLM_BASE_URL:-}" ] && ENGINES+=("vllm ${VLLM_BASE_URL%/}/metrics")
[ -n "${RADIANCE_BASE_URL:-}" ] && ENGINES+=("radiance ${RADIANCE_BASE_URL%/}/metrics")

failed=0

if [ ${#HOSTS[@]} -eq 0 ]; then
  echo "  (skipped agent captures — HOST_TARGETS unset; see .env.dev.example)"
fi
for entry in "${HOSTS[@]}"; do
  name=${entry%%=*}; url=${entry#*=}
  # The failure line prints the address without its userinfo: a token belongs in the
  # dial, not in the terminal scrollback or whatever shells this output into.
  if curl -fsS --max-time 8 "${url%/}/snapshot" -o "$FIX/agent_${name}.json"; then
    echo "  ✓ agent_${name}.json  ($(wc -c <"$FIX/agent_${name}.json") bytes)"
  else
    echo "  ✗ agent_${name}.json — no agent answered at ${url#*@} (deploy/install-agent.sh? firewall?)"
    failed=1
  fi
done

# --- UPSes (NUT upsd) ---------------------------------------------------------
# The two real UPSes disagree in ways a hand-written VAR table cannot: the APC's
# `battery.date` is the factory placeholder `2001/09/25` (its `battery.mfr.date` is the real
# one), the CyberPower answers `battery.mfr.date: CPS`, and neither exports
# `battery.temperature` or `battery.charger.status`. `tests/test_ups_contract.py` pins the
# UPS hero row against these replies, so they are recorded from the wire, not typed from
# memory (AGENTS.md rule 7). Skipped when NUT_HOST is unset, so a laptop with no upsd
# reachable still re-records the rest.
if [ -n "${NUT_HOST:-}" ]; then
  if python3 - "$FIX" "$NUT_HOST" "${NUT_PORT:-3493}" <<'PY'
import pathlib
import socket
import sys

fix, host, port = pathlib.Path(sys.argv[1]), sys.argv[2], int(sys.argv[3])
HEADER = ("# Verbatim upsd reply (LIST UPS / LIST VAR / LIST CMD), recorded by\n"
          "# ./scripts/capture-fixtures.sh — one `VAR`/`CMD` line per protocol line.\n"
          "# Volatile values (ups.load, battery.runtime, the voltages) move every poll,\n"
          "# so a diff here on re-capture is expected; the contract asserts shape, units\n"
          "# and which variables the drivers omit, never those numbers.")


class Upsd:
    """Raw socket on purpose: the fixture is the wire, not a parse of it."""

    def __init__(self, host: str, port: int) -> None:
        self.handle = socket.create_connection((host, port), timeout=8).makefile("rwb")

    def line(self) -> str:
        raw = self.handle.readline()
        if not raw:
            raise EOFError("upsd closed the connection")
        return raw.decode(errors="replace").rstrip("\r\n")

    def ask(self, command: str) -> list[str]:
        self.handle.write((command + "\n").encode())
        self.handle.flush()
        first = self.line()
        if not first.startswith("BEGIN LIST"):
            raise RuntimeError(f"{command!r} -> {first!r}")
        rows: list[str] = []
        while True:
            row = self.line()
            if row.startswith("END LIST"):
                return rows
            rows.append(row)


upsd = Upsd(host, port)
for row in upsd.ask("LIST UPS"):
    if not row.startswith("UPS"):
        continue
    name = row.split()[1].strip('"')
    body = [HEADER, row,
            *upsd.ask(f'LIST VAR "{name}"'),
            *upsd.ask(f'LIST CMD "{name}"')]
    path = fix / f"nut_{name}.vars"
    path.write_text("\n".join(body) + "\n")
    print(f"  ✓ nut_{name}.vars ({len(body) - 1} protocol lines)")
PY
  then
    : # per-UPS lines above
  else
    echo "  ✗ nut_*.vars — upsd at $NUT_HOST:${NUT_PORT:-3493} unreachable (down? firewall? NUT_PORT is the port upsd really listens on)"
    failed=1
  fi
else
  echo "  (skipped UPS captures — NUT_HOST unset; see .env.dev.example)"
fi

# --- SMART (smartctl -i -A -H per physical disk) --------------------------------
# The phase-09 `/disks` page parses these. smartctl must be able to READ the block
# devices, and a fleet grants that one of two ways: the agent user in the `disk` group
# (`sudo usermod -aG disk <user>`, what deploy/install-agent.sh does) or a sudoers rule
# for smartctl alone (what this fleet has). Plain smartctl is tried first because that is
# exactly what agent/host_agent.py shells — if the capture needs sudo and the agent does
# not have it, the fixtures would read healthier than the fleet does. When plain smartctl
# answers "Permission denied" the capture retries under `sudo -n`. A drive that answers
# neither way is deliberately NOT recorded (a good fixture is never replaced by a dead
# reader's output).
# Identifiers never go in the repo: Serial Number, LU WWN Device Id and the NVMe namespace
# EUI-64 (which encodes the serial) are replaced with SCRUBBED* markers; everything else is
# verbatim, and tests/test_agent.py fails the day this stops happening.
# Skipped when SMART_HOSTS is unset, so a laptop without fleet ssh still re-records the
# rest. Whole disks only, the same virtual-prefix skip agent/host_agent.py applies —
# partitions and dm-* would double-capture the same platter.
if [ -n "${SMART_HOSTS:-}" ]; then
  for host in $SMART_HOSTS; do
    name="${host%-ops}"   # fixtures are named for the host, not the ssh alias, like agent_*.json
    devs="$(ssh -o BatchMode=yes -o ConnectTimeout=8 "$host" \
      'for d in /sys/block/*; do n=${d##*/}; case "$n" in loop*|ram*|zram*|sr*|dm-*|md*) ;; *) echo "$n";; esac; done' 2>/dev/null)"
    if [ -z "$devs" ]; then
      echo "  ✗ smartctl_${name}_* — ssh did not answer (is $host an ssh alias?)"
      failed=1
      continue
    fi
    for dev in $devs; do
      tmp="$FIX/.smartctl_${name}_${dev}.txt.tmp"
      ssh -o BatchMode=yes -o ConnectTimeout=8 "$host" "smartctl -i -A -H /dev/$dev" >"$tmp" 2>&1
      rc=$?
      if grep -qi "Permission denied" "$tmp"; then
        ssh -o BatchMode=yes -o ConnectTimeout=8 "$host" "sudo -n smartctl -i -A -H /dev/$dev" >"$tmp" 2>&1
        rc=$?
      fi
      # A drive that cannot answer health at all (no SMART support, virtual HBA tricks)
      # is the same fixture contract as an unreadable one: the parse tests require model +
      # health in every recorded capture, so do not record what cannot be parsed.
      if [ ! -s "$tmp" ] || grep -qiE "Permission denied|failed to open|not found|does not support SMART|SMART support is: Unavailable|not smart-capable" "$tmp"; then
        rm -f "$tmp"
        echo "  ✗ smartctl_${name}_${dev} — no readable SMART (rc $rc): install smartmontools and grant this user the disk group or a smartctl sudo rule"
        failed=1
      else
        sed -Ei \
          -e 's/^([[:space:]]*Serial [Nn]umber:)[[:space:]]*.*/\1 SCRUBBEDSERIAL/' \
          -e 's/^([[:space:]]*LU WWN Device Id:)[[:space:]]*.*/\1 SCRUBBEDWWN/' \
          -e 's/^([[:space:]]*(Namespace [0-9]+ )?IEEE EUI-?64:)[[:space:]]*.*/\1 SCRUBBEDEUI64/' \
          "$tmp"
        { echo "# Verbatim smartctl -i -A -H /dev/$dev from $name, recorded by"
          echo "# ./scripts/capture-fixtures.sh — Serial/WWN/EUI-64 values are SCRUBBED,"
          echo "# every other byte is what the drive answered. Volatile rows (the"
          echo "# Temperature raws, Local Time) move; the parse tests read shape."
          cat "$tmp"; } > "$FIX/smartctl_${name}_${dev}.txt"
        rm -f "$tmp"
        echo "  ✓ smartctl_${name}_${dev}.txt"
      fi
    done
  done
else
  echo "  (skipped SMART captures — SMART_HOSTS unset; see .env.dev.example)"
fi

# --- gufo /slots (per-slot rows) ---------------------------------------------
# `tests/slots_contract.py` pins the shape of the `/llm` slot grid (task 04) against
# `tests/fixtures/slots_gufo.json`, so this capture is NOT behind `--with-llm`: it is one
# small JSON document and the hermetic contract test needs it on disk. `/v1/slots` is the
# documented alias, tried second — the same order `collectors.fetch_engine` and
# `llm_monitor`'s passthrough use, so a server that only exposes the alias is still
# capturable. Skipped when GUFO_BASE_URL is unset, like the UPS captures. An idle engine
# still answers one row per parallel slot; that idle shape is what the grid must handle.
if [ -n "${GUFO_BASE_URL:-}" ]; then
  gufo_base="${GUFO_BASE_URL%/}"
  slots_tmp="$FIX/.slots_gufo.json.tmp"
  if curl -fsS --max-time 8 "${gufo_base}/slots" -o "$slots_tmp" ||
     curl -fsS --max-time 8 "${gufo_base}/v1/slots" -o "$slots_tmp"; then
    mv "$slots_tmp" "$FIX/slots_gufo.json"
    echo "  ✓ slots_gufo.json ($(wc -c <"$FIX/slots_gufo.json") bytes)"
  else
    rm -f "$slots_tmp"   # never overwrite a good fixture with a dead engine's empty file
    echo "  ✗ slots_gufo.json — gufo answered neither /slots nor /v1/slots at $gufo_base"
    failed=1
  fi
else
  echo "  (skipped gufo /slots capture — GUFO_BASE_URL unset; see .env.dev.example)"
fi

if [ "${1:-}" = "--with-llm" ]; then
  if [ ${#ENGINES[@]} -eq 0 ]; then
    echo "  ! no GUFO_BASE_URL / VLLM_BASE_URL / RADIANCE_BASE_URL set (see .env.dev) — nothing to capture"
    failed=1
  fi
  for entry in "${ENGINES[@]}"; do
    name=${entry%% *}; url=${entry#* }
    if curl -fsS --max-time 20 "$url" -o "$FIX/metrics_${name}.txt"; then
      # Radiance's capture carries the script's verbatim header, like the other fixtures —
      # gufo's and vLLM's metrics files stay raw, as before.
      if [ "$name" = "radiance" ]; then
        { echo "# Verbatim /metrics reply from the radiance engine, recorded by"
          echo "# ./scripts/capture-fixtures.sh — the engine's own Prometheus text"
          echo "# format: the radiance: native series + the vllm: compatibility family."
          echo "# Volatile values (counters, gauges) move every poll, so a diff here"
          echo "# on re-capture is expected; the contract asserts the series-name set"
          echo "# against RADIANCE_SERIES/RADIANCE_HISTOGRAMS/RADIANCE_OPTIONAL in"
          echo "# sources/llm.py — never the numbers (the nut_*.vars precedent)."
          cat "$FIX/metrics_${name}.txt"; } > "$FIX/.metrics_radiance.txt.tmp" &&
          mv "$FIX/.metrics_radiance.txt.tmp" "$FIX/metrics_${name}.txt"
      fi
      echo "  ✓ metrics_${name}.txt ($(wc -l <"$FIX/metrics_${name}.txt") lines)"
    else
      echo "  ✗ metrics_${name}.txt — $url unreachable (VPN? engine down?)"
      failed=1
    fi
    # radiance is a vLLM derivative and answers two more read-only endpoints: /stats
    # (the engine's own JSON snapshot: cards[], requests[], draft_positions[], ...)
    # and /v1/models (the model's capabilities — the only place the engine states what
    # it can do). Both are recorded verbatim in the same pass the gufo /slots JSON
    # document is, with the same verbatim header; the hermetic shape tests in
    # tests/test_llm_source.py need them on disk.
    if [ "$name" = "radiance" ]; then
      base=${url%/metrics}
      stats_tmp="$FIX/.stats_radiance.json.tmp"
      models_tmp="$FIX/.models_radiance.json.tmp"
      if curl -fsS --max-time 10 "${base}/stats" -o "$stats_tmp"; then
        info=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(d.get('in_flight', 0), len(d.get('requests', [])))" "$stats_tmp" 2>/dev/null || echo '? 0')
        inflight=${info%% *}; rows=${info##* }
        { echo "# Verbatim /stats reply from the radiance engine, recorded by"
          echo "# ./scripts/capture-fixtures.sh — one JSON document, the engine's own"
          echo "# shape. Volatile values (uptime_s, now_ms, the counters, kv_*) move"
          echo "# every poll, so a diff here on re-capture is expected; the contract"
          echo "# asserts the top-level key set and the cards[]/requests[]/"
          echo "# draft_positions[] shape — never those numbers (the nut_*.vars"
          echo "# precedent)."
          echo "# in_flight was ${inflight} at capture (requests[] has ${rows} row(s))."
          cat "$stats_tmp"; } > "$FIX/.stats_radiance.json.tmp2" &&
          mv "$FIX/.stats_radiance.json.tmp2" "$FIX/stats_radiance.json"
        rm -f "$stats_tmp"
        if [ "$inflight" = "0" ]; then
          echo "  (requests[] was empty — re-run with a request in flight to record its shape)"
        fi
        echo "  ✓ stats_radiance.json ($(wc -c <"$FIX/stats_radiance.json") bytes)"
      else
        rm -f "$stats_tmp"   # never overwrite a good fixture with a dead engine's empty file
        echo "  ✗ stats_radiance.json — ${base}/stats unreachable"
        failed=1
      fi
      if curl -fsS --max-time 10 "${base}/v1/models" -o "$models_tmp"; then
        { echo "# Verbatim /v1/models reply from the radiance engine, recorded by"
          echo "# ./scripts/capture-fixtures.sh — one JSON document. The model id and"
          echo "# max_model_len are stable; created is an epoch timestamp that moves."
          echo "# The contract asserts the object/data shape — never the numbers."
          cat "$models_tmp"; } > "$FIX/.models_radiance.json.tmp2" &&
          mv "$FIX/.models_radiance.json.tmp2" "$FIX/models_radiance.json"
        rm -f "$models_tmp"
        echo "  ✓ models_radiance.json ($(wc -c <"$FIX/models_radiance.json") bytes)"
      else
        rm -f "$models_tmp"   # never overwrite a good fixture with a dead engine's empty file
        echo "  ✗ models_radiance.json — ${base}/v1/models unreachable"
        failed=1
      fi
    fi
  done
else
  echo "  (skipped engine /metrics captures — pass --with-llm when the engines are reachable)"
fi

# Secrets never belong in a fixture: engines can expose /metrics behind a token in the URL,
# and a hostname you use at home is a hostname in a public repo.
if grep -rlE "Authorization|Bearer |://[^/]*:[^/@]*@" "$FIX" >/dev/null 2>&1; then
  echo "  ! fixture contains what looks like a credential — scrub it before committing"
  failed=1
fi

# Three things leak out of a real capture that nobody notices: LUKS/partition UUIDs (which
# identify *your* disks, and appear in `device` strings as /dev/mapper/luks-<uuid>), the
# FQDN a host thinks it has, and a UPS's `ups.serial`/`device.serial` (which identifies the
# physical unit, and is echoed back in the card's variable dump). They are replaced
# deterministically so re-capturing does not churn the fixture, and the tests keep the same
# shapes to assert against. A fourth belongs only to the /slots capture: a busy slot's
# `prompt` field is *the user's in-flight request text* — personal data the slot grid never
# renders — so any non-empty prompt value is blanked, keeping the key (the shape) without
# the content.
python3 - "$FIX" <<'PY'
import pathlib
import re
import sys

uuid_re = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
SYNTHETIC = "00000000-0000-4000-8000-"   # already-scrubbed markers: re-running must not churn
fix = pathlib.Path(sys.argv[1])
changed = []
# upsd quotes a field only when it has to, so both `VAR "apc" "ups.serial" "x"` and
# `VAR apc ups.serial "x"` are real wire shapes; the value is always quoted.
serial_re = re.compile(r'(^VAR\s+\S+\s+"?(?:device|ups)\.serial"?\s+")([^"\n]*)(")', re.M)
# smartctl identities: the serial identifies the physical unit and the LU WWN encodes
# the vendor's serial — both scrubbed, same deterministic markers as the UPS serials.
# "Local Time is:" is capture time, not device truth: normalised so re-captures do not
# churn the diff.
smart_serial_re = re.compile(r"(?im)^(Serial Number:\s+)(\S.*?)(\s*)$")
smart_wwn_re = re.compile(r"(?im)^(LU WWN Device Id:\s+)(\S.*?)(\s*)$")
smart_time_re = re.compile(r"(?im)^Local Time is:.*$")
for path in sorted(fix.iterdir()):
    if path.suffix not in (".json", ".txt", ".vars"):
        continue
    text = path.read_text()
    seen: dict[str, str] = {}
    serials: dict[str, str] = {}

    def swap(match: re.Match[str]) -> str:
        value = match.group(0)
        if value.startswith(SYNTHETIC):
            return value
        if value not in seen:
            seen[value] = f"00000000-0000-4000-8000-{len(seen) + 1:012d}"
        return seen[value]

    seen_wwns: dict[str, str] = {}
    text = uuid_re.sub(swap, text)
    # "hostname": "gw.lab.example.com" -> "gw": the dashboard's host key is the target name
    # from HOST_TARGETS, never whatever the box calls itself
    text = re.sub(r'("hostname": "[^"]*?)\.[^"]*(")', r"\1\2", text)
    if path.name.startswith("slots_"):
        # only non-empty values are replaced, so an idle capture does not churn
        text = re.sub(r'("prompt"\s*:\s*)"(?:[^"\\]|\\.)+"', r'\1""', text)

    def scrub_serial(match: re.Match[str]) -> str:
        value = match.group(2)
        body, pad = value.rstrip(), value[len(value.rstrip()):]
        if body.startswith("SCRUBBEDSERIAL"):
            return match.group(0)
        if body not in serials:
            serials[body] = f"SCRUBBEDSERIAL{len(serials) + 1}"
        # trailing spaces are a real quirk of the apc driver — keep the padding verbatim
        return match.group(1) + serials[body] + pad + match.group(3)

    text = serial_re.sub(scrub_serial, text)
    if path.name.startswith("smartctl_"):
        def scrub_field(match: re.Match[str], prefix: str, store: dict[str, str]) -> str:
            body = match.group(2).strip()
            if body.startswith(prefix):
                return match.group(0)
            if body not in store:
                store[body] = f"{prefix}{len(store) + 1}"
            return match.group(1) + store[body] + match.group(3)

        text = smart_serial_re.sub(lambda m: scrub_field(m, "SCRUBBEDSERIAL", serials), text)
        text = smart_wwn_re.sub(lambda m: scrub_field(m, "SCRUBBEDWWN", seen_wwns), text)
        text = smart_time_re.sub("Local Time is:      (time of capture removed by scripts/capture-fixtures.sh)", text)
    if text != path.read_text():
        path.write_text(text)
        changed.append(path.name)
print(f"  scrubbed identifiers in: {', '.join(changed)}" if changed else "  fixtures carry no identifiers")
PY

exit "$failed"