Add homelab dashboards: system, LLM, and UPS on one domain

Three dashboards served by one FastAPI app with a SQLite history store (raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana, no CDN, no build step. Nothing is fetched from the internet at runtime. * system: a stdlib-only agent (agent/host_agent.py) reports /proc and /sys counters; the dashboard turns them into rates over a labelled window, so every number says how it was measured. * llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house exposition parser. Measured delta/elapsed throughput beats a native per-request gauge, and each value carries its provenance. * ups: a NUT wire-protocol client plus health checks that name what they looked at. Commands are gated by NUT_ALLOW_ACTIONS, DASH_READ_ONLY and a DANGER confirmation, re-checked server-side. Missing values render as n/a rather than 0, and an unconfigured source answers 404 {"configured": false} without dialling anything, so "not deployed here" never looks like an outage. 253 tests at 94% coverage against captured real fixtures; ruff and pyright clean. LLM-Generated: true
2026-10-16 17:20:45 -04:00
commit c4bb374ccc
56 changed files with 13018 additions and 0 deletions
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# Re-record ground-truth captures into tests/fixtures/.
+#
+#   ./scripts/capture-fixtures.sh                # needs the tunnels up: scripts/dev-tunnels.sh start
+#   ./scripts/capture-fixtures.sh --with-llm     # also scrape the live engines' /metrics
+#
+# Fixtures are verbatim, not hand-written: a fixture you typed is a test of your memory.
+# When a host's kernel or an engine's metric names change, re-run this and read the diff.
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+FIX="$ROOT/tests/fixtures"
+mkdir -p "$FIX"
+
+HOSTS=(driveripper:9121 deskwork:9122 gateway:9123)
+# Engine URLs come from the same env the app uses, so this script carries no deployment
+# detail of its own — and never records a tokenised URL into a fixture by accident.
+if [ -f "$ROOT/.env.dev" ]; then
+  set -a; . "$ROOT/.env.dev"; set +a
+fi
+ENGINES=()
+[ -n "${GUFO_BASE_URL:-}" ] && ENGINES+=("gufo:${GUFO_BASE_URL%/}/metrics")
+[ -n "${VLLM_BASE_URL:-}" ] && ENGINES+=("vllm:${VLLM_BASE_URL%/}/metrics")
+
+failed=0
+
+for entry in "${HOSTS[@]}"; do
+  name=${entry%%:*}; port=${entry##*:}
+  url="http://127.0.0.1:${port}/snapshot"
+  if curl -fsS --max-time 8 "$url" -o "$FIX/agent_${name}.json"; then
+    echo "  ✓ agent_${name}.json  ($(wc -c <"$FIX/agent_${name}.json") bytes)"
+  else
+    echo "  ✗ agent_${name}.json — no agent on $url (scripts/dev-tunnels.sh start?)"
+    failed=1
+  fi
+done
+
+if [ "${1:-}" = "--with-llm" ]; then
+  if [ ${#ENGINES[@]} -eq 0 ]; then
+    echo "  ! no GUFO_BASE_URL / VLLM_BASE_URL set (see .env.dev) — nothing to capture"
+    failed=1
+  fi
+  for entry in "${ENGINES[@]}"; do
+    name=${entry%%:*}; url=${entry#*:}
+    if curl -fsS --max-time 20 "$url" -o "$FIX/metrics_${name}.txt"; then
+      echo "  ✓ metrics_${name}.txt ($(wc -l <"$FIX/metrics_${name}.txt") lines)"
+    else
+      echo "  ✗ metrics_${name}.txt — $url unreachable (VPN? engine down?)"
+      failed=1
+    fi
+  done
+else
+  echo "  (skipped engine /metrics captures — pass --with-llm when the engines are reachable)"
+fi
+
+# Secrets never belong in a fixture: engines can expose /metrics behind a token in the URL,
+# and a hostname you use at home is a hostname in a public repo.
+if grep -rlE "Authorization|Bearer |://[^/]*:[^/@]*@" "$FIX" >/dev/null 2>&1; then
+  echo "  ! fixture contains what looks like a credential — scrub it before committing"
+  failed=1
+fi
+
+# Two things leak out of a real capture that nobody notices: LUKS/partition UUIDs (which
+# identify *your* disks, and appear in `device` strings as /dev/mapper/luks-<uuid>) and the
+# FQDN a host thinks it has. They are replaced deterministically so re-capturing does not
+# churn the fixture, and the tests keep the same shapes to assert against.
+python3 - "$FIX" <<'PY'
+import pathlib
+import re
+import sys
+
+uuid_re = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
+SYNTHETIC = "00000000-0000-4000-8000-"   # already-scrubbed markers: re-running must not churn
+fix = pathlib.Path(sys.argv[1])
+changed = []
+for path in sorted(fix.iterdir()):
+    if path.suffix not in (".json", ".txt"):
+        continue
+    text = path.read_text()
+    seen: dict[str, str] = {}
+
+    def swap(match: re.Match[str]) -> str:
+        value = match.group(0)
+        if value.startswith(SYNTHETIC):
+            return value
+        if value not in seen:
+            seen[value] = f"00000000-0000-4000-8000-{len(seen) + 1:012d}"
+        return seen[value]
+
+    text = uuid_re.sub(swap, text)
+    # "hostname": "gw.lab.example.com" -> "gw": the dashboard's host key is the target name
+    # from HOST_TARGETS, never whatever the box calls itself
+    text = re.sub(r'("hostname": "[^"]*?)\.[^"]*(")', r"\1\2", text)
+    if text != path.read_text():
+        path.write_text(text)
+        changed.append(path.name)
+print(f"  scrubbed identifiers in: {', '.join(changed)}" if changed else "  fixtures carry no identifiers")
+PY
+
+exit "$failed"
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Capture a REAL node_exporter exposition from each monitored host into tests/fixtures/.
+#
+# Runs node_exporter in the foreground on the remote host for a few seconds, scrapes it
+# over loopback there, and saves the text. Nothing is installed as a service and no port
+# is opened: the only network hop is ssh.
+#
+#   ./scripts/capture-node-exporter.sh                 # all hosts
+#   ./scripts/capture-node-exporter.sh driveripper-ops # one host
+#
+# Why: metric names must be verified against the deployed exporter (see AGENTS.md), never
+# guessed from the upstream README.
+set -uo pipefail
+
+VERSION="${NODE_EXPORTER_VERSION:-1.12.1}"
+OUT="$(cd "$(dirname "$0")/.." && pwd)/tests/fixtures"
+HOSTS=("${@:-driveripper-ops deskwork-ops gateway-ops}")
+PORT=9199
+ARGS="--collector.drm --collector.thermal_zone --collector.cpufreq --collector.hwmon --web.listen-address=127.0.0.1:${PORT}"
+
+mkdir -p "$OUT"
+command -v ssh >/dev/null || { echo "ssh not found"; exit 1; }
+
+for host in $HOSTS; do
+  echo "=== $host"
+  ssh -n -o ConnectTimeout=10 "$host" "set -e
+    mkdir -p \$HOME/exporters && cd \$HOME/exporters
+    if [ ! -x node_exporter ]; then
+      curl -sSL -o ne.tgz https://github.com/prometheus/node_exporter/releases/download/v${VERSION}/node_exporter-${VERSION}.linux-amd64.tar.gz
+      tar xzf ne.tgz && mv node_exporter-${VERSION}.linux-amd64/node_exporter .
+      rm -rf ne.tgz node_exporter-${VERSION}.linux-amd64
+    fi
+    timeout 8 ./node_exporter ${ARGS} >/dev/null 2>&1 &
+    sleep 3
+    curl -s --max-time 5 127.0.0.1:${PORT}/metrics
+  " > "$OUT/node_exporter_${host}.txt" 2>/dev/null
+  lines=$(wc -l < "$OUT/node_exporter_${host}.txt")
+  if [ "$lines" -lt 20 ]; then
+    echo "    FAILED (only $lines lines) — check ssh access to $host"
+    rm -f "$OUT/node_exporter_${host}.txt"
+    continue
+  fi
+  echo "    saved $lines lines -> tests/fixtures/node_exporter_${host}.txt"
+  echo "    gpu: $(grep -cE '^node_gpu' "$OUT/node_exporter_${host}.txt") node_gpu series"
+  grep -oE '^node_gpu[a-z_]*' "$OUT/node_exporter_${host}.txt" | sort -u | sed 's/^/      /' | head -20
+done
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Local development server: start|stop|status|restart|logs|tunnel-check
+#
+#   ./scripts/dev-server.sh restart     # reads .env.dev if present, runs on DASH_PORT
+#
+# The server keeps running after this shell exits (nohup + pidfile) so you can browse it
+# while an agent works. Logs: data/logs/dev.log
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT"
+PIDFILE="$ROOT/data/dev-server.pid"
+LOG="$ROOT/data/logs/dev.log"
+mkdir -p "$ROOT/data/logs"
+
+running() { [ -f "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; }
+
+start() {
+  if running; then
+    echo "already running (pid $(cat "$PIDFILE")) — http://127.0.0.1:${DASH_PORT:-8090}"
+    return 0
+  fi
+  if [ -f "$ROOT/.env.dev" ]; then
+    set -a; . "$ROOT/.env.dev"; set +a
+    echo "loaded .env.dev"
+  fi
+  PORT="${DASH_PORT:-8090}"
+  nohup .venv/bin/dashboards >"$LOG" 2>&1 &
+  echo $! >"$PIDFILE"
+  sleep 3
+  if running && curl -sf -o /dev/null "http://127.0.0.1:${PORT}/api/health"; then
+    echo "started pid $(cat "$PIDFILE") — http://127.0.0.1:${PORT}"
+  else
+    echo "FAILED to start; last lines of $LOG:"
+    tail -20 "$LOG"
+    return 1
+  fi
+}
+
+stop() {
+  if running; then
+    kill "$(cat "$PIDFILE")" 2>/dev/null
+    sleep 1
+    echo "stopped"
+  else
+    echo "not running"
+  fi
+  rm -f "$PIDFILE"
+}
+
+case "${1:-start}" in
+  start) start ;;
+  stop) stop ;;
+  restart) stop; start ;;
+  status)
+    if running; then
+      echo "running pid $(cat "$PIDFILE")"
+      curl -s "http://127.0.0.1:${DASH_PORT:-8090}/api/health" \
+        | .venv/bin/python -c "import json,sys; d=json.load(sys.stdin); print('status:', d['status']); [print(' ', s['key'], 'ok' if s['ok'] else 'DOWN', (s['error'] or '')[:90]) for s in d['sources']]"
+    else
+      echo "not running"
+    fi ;;
+  logs) tail -n 40 "$LOG" ;;
+  *) echo "usage: $0 start|stop|restart|status|logs"; exit 2 ;;
+esac
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Dev-only: run agent/host_agent.py on each monitored host and forward it to localhost,
+# so `make dev` on the laptop sees live fleet data without opening a single port.
+#
+#   ./scripts/dev-tunnels.sh start|stop|status|restart
+#
+# Each tunnel is  ssh -L <local>:127.0.0.1:<remote-port>  <host> 'python3 agent/host_agent.py'
+# and restarts itself if the link drops. Production is the opposite: deploy/install-agent.sh
+# installs a systemd unit that binds the LAN address with a bearer token, and the dashboard
+# dials it directly.
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+RUN="$ROOT/data/tunnels"
+LOGS="$ROOT/data/logs"
+# Where the agent lives on each host. `~` is written into the generated script literally so
+# the REMOTE shell expands it (a local $HOME would be the wrong path over there).
+AGENT_REMOTE_DIR="${AGENT_REMOTE_DIR:-~/lab-agent}"
+AGENT_COMMAND="${AGENT_COMMAND:-python3 host_agent.py}"
+
+# Targets come from .env.dev (see .env.dev.example) — never hardcoded here, so this file
+# stays portable and holds no hostnames of ours.
+[ -f "$ROOT/.env.dev" ] && . "$ROOT/.env.dev"
+DEV_TUNNELS="${DEV_TUNNELS:-}"
+if [ -z "$DEV_TUNNELS" ]; then
+  echo "no DEV_TUNNELS set — copy .env.dev.example to .env.dev first"; exit 2
+fi
+TUNNELS=()
+for entry in $DEV_TUNNELS; do
+  # 9121:driveripper-ops:9120 -> "9121 driveripper-ops 9120"
+  TUNNELS+=("$(echo "$entry" | tr ':' ' ')")
+done
+
+mkdir -p "$RUN" "$LOGS"
+
+start() {
+  for spec in "${TUNNELS[@]}"; do
+    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
+    pidfile="$RUN/$host.pid"
+    if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
+      echo "  $host already tunnelling on :$local (pid $(cat "$pidfile"))"
+      continue
+    fi
+    # A previous agent survives the ssh session that started it, and the next agent then
+    # dies on bind() with "address already in use" — which looks like a broken agent rather
+    # than a leaked process. Clear it in a SEPARATE ssh session: putting pkill in the same
+    # command line as the agent makes pkill match that very command line and shoot itself.
+    cat > "$RUN/$host.sh" <<EOF
+#!/usr/bin/env bash
+# generated by scripts/dev-tunnels.sh — do not edit, it is rewritten on every start
+while true; do
+  ssh -n -o BatchMode=yes "$host" \
+      "pkill -f '[h]ost_agent.py --host 127.0.0.1 --port $remote'" >/dev/null 2>&1
+  sleep 1
+  ssh -nT -o ServerAliveInterval=30 -o ServerAliveCountMax=4 -o ExitOnForwardFailure=yes \
+      -L $local:127.0.0.1:$remote "$host" \
+      "cd $AGENT_REMOTE_DIR && exec $AGENT_COMMAND --host 127.0.0.1 --port $remote"
+  echo "[tunnel] $host link dropped at \$(date -Is) (exit \$?); retrying in 5s"
+  sleep 5
+done
+EOF
+  chmod +x "$RUN/$host.sh"
+    nohup "$RUN/$host.sh" >>"$LOGS/tunnel-$host.log" 2>&1 &
+    echo $! >"$pidfile"
+    echo "  $host -> http://127.0.0.1:$local (pid $!)"
+  done
+}
+
+stop() {
+  for spec in "${TUNNELS[@]}"; do
+    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
+    pidfile="$RUN/$host.pid"
+    [ -f "$pidfile" ] || continue
+    pid=$(cat "$pidfile")
+    pkill -P "$pid" 2>/dev/null
+    kill "$pid" 2>/dev/null
+    # -N would mean "no remote command", so the forward is kept by a plain ssh running the
+    # agent; kill any leftover forward for this host by pattern, not just by parent.
+    pkill -f "ssh -nT.*-L $local:127.0.0.1:$remote $host" 2>/dev/null
+    # The remote agent outlives its ssh session, and the next one then dies on bind() with
+    # "address already in use" — which looks like a broken agent, not a leaked process.
+    timeout 10 ssh -n "$host" "pkill -f '[h]ost_agent.py --host 127.0.0.1 --port $remote'" \
+      >/dev/null 2>&1
+    rm -f "$pidfile"
+    echo "  stopped $host"
+  done
+}
+
+status() {
+  for spec in "${TUNNELS[@]}"; do
+    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
+    body=$(curl -s --max-time 4 "http://127.0.0.1:$local/health" || true)
+    if [ -n "$body" ]; then
+      echo "  OK   $host http://127.0.0.1:$local $body"
+    else
+      echo "  DOWN $host http://127.0.0.1:$local"
+    fi
+  done
+}
+
+case "${1:-start}" in
+  start) start ;;
+  stop) stop ;;
+  restart) stop; sleep 1; start ;;
+  status) status ;;
+  *) echo "usage: $0 start|stop|status|restart"; exit 2 ;;
+esac