dashboards/deploy/install-agent.sh

#!/usr/bin/env bash
# Install agent/host_agent.py as a systemd service on a monitored host.
#
#   sudo ./deploy/install-agent.sh                       # loopback only (read it locally)
#   sudo ./deploy/install-agent.sh --bind 10.0.0.5       # serve the LAN, token required
#   sudo ./deploy/install-agent.sh --cert-bundle agent.bundle   # serve TLS (phase 15)
#   ./deploy/install-agent.sh --status                   # unit state + /health, no root
#   sudo ./deploy/install-agent.sh --uninstall
#
# No root shell? `deploy/agent-sudoers` is the single sudoers line that lets the fleet's
# unprivileged `ops` account run this script as root. Run it without root and it prints that
# exact line with this script's resolved path, then exits 1.
#
# The token is stored in a file only root and the agent's own service user can read, and is
# passed to the agent as --token-file, never on the command line: argv is world-readable
# through `ps`, and this token guards hostname, mounts, disk models, GPU model and process
# names. Re-running the installer reuses the existing token — use --rotate-token when the old
# one leaked.
#
# The certificate bundle (--cert-bundle <file>, fetched by the operator from the dashboard's
# GET /api/certs/<name>/bundle or the /certs page) is the TLS install: one PEM holding this
# host's server certificate and key, the dashboard's root CA, a `# BUNDLE-REV <n>` line and
# the revocation denylist. This script never dials the dashboard for it. The bundle is
# validated before anything is touched — a bundle the agent could not load must stop the
# install with its reason, not leave a unit that exits at startup — and then installed to
# /etc/dashboards-agent.certs, the token-file idiom: the agent opens the bundle *itself* at
# startup and re-reads it when its stamp changes (that is how a re-deployed revocation list
# lands without a restart), so the mode must let the agent's own service user read it and no
# other account. The unit then runs https with --cert-bundle and a RuntimeDirectory for the
# PEM blocks the agent unpacks (it deletes the key once the SSL context holds it), and
# --status probes /health over TLS with the bundle's CA and no client certificate — the path
# the agent exempts for probes. Re-running without the flag leaves an installed bundle in
# place, the token-reuse idiom.
#
# The agent is read-only by construction (it answers GET and writes nothing), runs as its
# own system user, and gets the usual systemd lockdown. It still needs to read /proc, /sys
# and statvfs(), which the hardening below leaves alone.
#
# Disk health (`smartctl -i -A -H`, the `/disks` page) is the one add-on that needs more
# than /proc and /sys: smartctl must open the disk device nodes. The installer therefore
# adds the agent user to the `disk` group when one exists, and the unit device-allows the
# disk nodes instead of PrivateDevices. This is raw *read* access to the disks — the
# standard arrangement for SMART monitoring, granted deliberately — and its absence is a
# documented degradation, never an install failure: no smartmontools (or no group) simply
# means every disk reports `smart: null` with an explanatory error entry.
set -euo pipefail

SERVICE="dashboards-agent"
AGENT_USER="dashboards-agent"
AGENT_DIR="/usr/local/lib/dashboards-agent"
AGENT_TARGET="$AGENT_DIR/host_agent.py"
TOKEN_FILE="/etc/dashboards-agent.token"
CERT_FILE="/etc/dashboards-agent.certs"
UNIT="/etc/systemd/system/$SERVICE.service"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
AGENT_SOURCE="$SCRIPT_DIR/../agent/host_agent.py"
# Absolute, symlink-resolved path of this script: the sudoers grant in deploy/agent-sudoers
# is matched on a path, so the non-root hint below prints the line that actually matches
# wherever the script happens to live (the canonical staging path is
# /home/ops/agent-install/deploy/install-agent.sh).
SCRIPT_PATH="$SCRIPT_DIR/$(basename "$0")"
if command -v realpath >/dev/null 2>&1; then
  SCRIPT_PATH="$(realpath "$0" 2>/dev/null || printf '%s' "$SCRIPT_PATH")"
fi
SUDOERS_SOURCE="$(dirname "$SCRIPT_PATH")/agent-sudoers"

BIND="127.0.0.1"
PORT="9120"
TOKEN="${AGENT_TOKEN:-}"
ROTATE=0
BUNDLE_SRC=""
MODE="install"

usage() {
  # the header comment *is* the help text, so it cannot drift from the flags
  awk 'NR > 1 { if (/^#/) { sub(/^# ?/, ""); print } else { exit } }' "$0"
  exit "${1:-0}"
}

while [ $# -gt 0 ]; do
  case "$1" in
    --bind) BIND="$2"; shift 2 ;;
    --port) PORT="$2"; shift 2 ;;
    --token) TOKEN="$2"; shift 2 ;;
    --rotate-token) ROTATE=1; shift ;;
    --cert-bundle) BUNDLE_SRC="$2"; shift 2 ;;
    --status) MODE="status"; shift ;;
    --uninstall) MODE="uninstall"; shift ;;
    -h|--help) usage 0 ;;
    *) echo "unknown argument: $1" >&2; usage 2 ;;
  esac
done

need_root() {
  if [ "$(id -u)" -ne 0 ]; then
    # The dev workflow deploys over ssh as the unprivileged `ops` account, so "run as root"
    # is not an action the operator can take. Print the one sudoers line that makes this
    # script installable — the same line as deploy/agent-sudoers, with this script's own
    # resolved path — plus the two-stage install it documents, then exit 1 as before.
    cat >&2 <<HINT
$SCRIPT_PATH has to run as root: it writes $UNIT,
creates the $AGENT_USER user and its disk-group membership, and stores
the token in $TOKEN_FILE (and, with --cert-bundle, the certificate bundle in $CERT_FILE).

Add this single sudoers line on this host (it is the whole grant; see $SUDOERS_SOURCE):

ops ALL=(root) NOPASSWD: $SCRIPT_PATH

Install it the two-stage way every sudoers change here uses, as root — the command-scoped
ops sudo on these hosts does not include visudo, so this step is the operator's:

  sudo visudo -cf $SUDOERS_SOURCE
  sudo install -m 0440 -o root -g root $SUDOERS_SOURCE /etc/sudoers.d/dashboards-agent
  sudo visudo -cf /etc/sudoers.d/dashboards-agent

Then run this same command again:

  sudo $SCRIPT_PATH $*
HINT
    exit 1
  fi
}

status() {
  systemctl status "$SERVICE" --no-pager 2>&1 || true
  echo
  if [ -r "$UNIT" ] && grep -q -- '--cert-bundle' "$UNIT"; then
    # TLS install (phase 15): probe over https with the bundle's root CA and *no* client
    # certificate. The agent's CERT_OPTIONAL handshake plus the /health exemption is what
    # makes this possible — the installer holds no client identity, and a service that
    # cannot health-check is a dead install. The CA is the bundle's last certificate block
    # (the format is positional: agent cert, agent key, root CA). Chain-to-our-root is the
    # claim; the leaf names the dial addresses rather than loopback, so the name check is
    # off — forging the chain would still need the root key. python3 is the unit's own
    # interpreter: there is no external TLS binary to install on a monitored host.
    printf 'health: '
    python3 - "$BIND" "$PORT" "$CERT_FILE" <<'PY' || echo "(no answer)"
import re
import ssl
import sys
import urllib.request

host, port, bundle = sys.argv[1], sys.argv[2], sys.argv[3]
try:
    with open(bundle, encoding="utf-8", errors="replace") as handle:
        text = handle.read()
except OSError as exc:
    print(f"(cannot read {bundle}: {exc.strerror or exc} — run --status as root)")
    raise SystemExit(0)
blocks = re.findall(r"-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", text, re.DOTALL)
if len(blocks) < 2:
    print(f"(no root CA block in {bundle})")
    raise SystemExit(0)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.check_hostname = False
context.verify_mode = ssl.CERT_REQUIRED
context.load_verify_locations(cadata=blocks[-1])
try:
    with urllib.request.urlopen(
        f"https://{host}:{port}/health", timeout=4, context=context
    ) as response:
        print(response.read().decode("utf-8", "replace").strip())
except Exception as exc:  # the probe reports, it never propagates
    print(f"(no answer: {exc})")
PY
  elif command -v curl >/dev/null 2>&1; then
    # token mode: the plain probe, unchanged
    printf 'health: '
    curl -s --max-time 4 "http://$BIND:$PORT/health" || echo "(no answer)"
    echo
  fi
}

uninstall() {
  need_root --uninstall
  systemctl disable --now "$SERVICE" 2>/dev/null || true
  rm -f "$UNIT" "$TOKEN_FILE" "$CERT_FILE" "$AGENT_TARGET"
  rmdir "$AGENT_DIR" 2>/dev/null || true
  systemctl daemon-reload
  echo "removed $SERVICE (the $AGENT_USER user was left in place; remove it with userdel)"
}

new_token() {
  # urlsafe and no padding: the token goes into HOST_TARGETS as URL userinfo
  # (`name=http://TOKEN@host:9120`), where '+' and '=' are annoying to quote.
  python3 -c 'import secrets; print(secrets.token_urlsafe(32).rstrip("="))'
}

validate_bundle() {
  # The `resolve_bundle` idiom from agent/host_agent.py, run before anything is touched:
  # a bundle the agent could not load must stop the install with its reason, not leave a
  # unit that exits at startup. Shape only, by text — the agent parses the same way (it is
  # stdlib; a monitored host has no cryptography) and the dashboard is the only signer, so
  # a second copy of this shape check would only ever disagree about nothing. The three PEM
  # blocks are positional: agent certificate, agent key, root CA certificate.
  if ! command -v python3 >/dev/null 2>&1; then
    echo "--cert-bundle needs python3 (3.10 or newer) to be validated" >&2
    exit 2
  fi
  if python3 - "$1" <<'PY' 1>&2
import re
import sys

path = sys.argv[1]
try:
    with open(path, encoding="utf-8", errors="replace") as handle:
        text = handle.read()
except OSError as exc:
    print(f"--cert-bundle {path} cannot be read: {exc.strerror or exc}")
    raise SystemExit(1)
if not text.strip():
    print(f"--cert-bundle {path} is empty")
    raise SystemExit(1)
revision = None
for line in text.splitlines():
    stripped = line.strip()
    if stripped.startswith("# BUNDLE-REV "):
        fields = stripped[len("# BUNDLE-REV "):].split()
        if fields:
            try:
                revision = int(fields[0])
            except ValueError:
                revision = None
        break
if revision is None:
    print(f"--cert-bundle {path} has no `# BUNDLE-REV <n>` line — that line is how the"
          " dashboard tells the bundle it issued from a stale one, so a bundle without it"
          " is not one this app signed")
    raise SystemExit(1)
blocks = re.findall(r"-----BEGIN [A-Z0-9 ]+-----.*?-----END [A-Z0-9 ]+-----", text, re.DOTALL)
certificates = [b for b in blocks if b.startswith("-----BEGIN CERTIFICATE-----")]
keys = [b for b in blocks if not b.startswith("-----BEGIN CERTIFICATE-----")]
if len(certificates) != 2 or len(keys) != 1:
    print(f"--cert-bundle {path} must hold exactly the agent certificate, the agent key and"
          f" the root CA certificate — found {len(certificates)} certificates and"
          f" {len(keys)} keys. Re-download it from GET /api/certs/<name>/bundle")
    raise SystemExit(1)
PY
  then
    return 0
  fi
  exit 2
}

# NOT named `install`: bash resolves a function before an external command of the same name,
# so an `install()` here turns the `install -d` / `install -m` calls below into infinite
# recursion instead of creating $AGENT_DIR. Phase 10 task 02 found exactly that on a live
# host: the run looped on the SMART block forever and installed nothing.
do_install() {
  # Validate before need_root: a bad bundle is refused with its own reason whether or not
  # this shell is root, so a wrong file says "wrong file" and not "add a sudoers line".
  if [ -n "$BUNDLE_SRC" ]; then
    validate_bundle "$BUNDLE_SRC"
  fi
  need_root --bind "$BIND" --port "$PORT"
  [ -f "$AGENT_SOURCE" ] || { echo "no agent next to this script: $AGENT_SOURCE" >&2; exit 1; }
  if ! command -v python3 >/dev/null 2>&1; then
    echo "python3 is required on the monitored host (3.10 or newer)" >&2
    exit 1
  fi
  # The agent needs no privileges, so give it a user with nothing to lose.
  if ! id "$AGENT_USER" >/dev/null 2>&1; then
    useradd --system --home-dir / --shell /usr/sbin/nologin "$AGENT_USER"
  fi

  # SMART (phase 09): the usual distro answer for reading drive health is the `disk`
  # group — no sudoers entry, no root. Re-runs are idempotent.
  if getent group disk >/dev/null 2>&1; then
    if id -nG "$AGENT_USER" 2>/dev/null | tr ' ' '\n' | grep -qx disk; then
      echo "  $AGENT_USER is already in the disk group (SMART reads enabled)"
    else
      usermod -aG disk "$AGENT_USER"
      echo "  added $AGENT_USER to the disk group — raw device reads, which is what smartctl needs"
    fi
  else
    echo "  no disk group on this host — SMART data will read null (harmless; see README)"
  fi
  if ! command -v smartctl >/dev/null 2>&1; then
    echo "  note: smartctl is not installed — every disk reports smart: null until smartmontools is"
  fi

  install -d -m 0755 "$AGENT_DIR"
  install -m 0755 "$AGENT_SOURCE" "$AGENT_TARGET"

  if [ -f "$TOKEN_FILE" ] && [ "$ROTATE" -eq 0 ] && [ -z "$TOKEN" ]; then
    TOKEN="$(cat "$TOKEN_FILE")"
    echo "reusing the existing token in $TOKEN_FILE (--rotate-token for a new one)"
  elif [ -z "$TOKEN" ]; then
    TOKEN="$(new_token)"
  fi
  # 0640 root:$AGENT_USER. The unit passes `--token-file` as a *path*: systemd does not read
  # the secret for the service, the agent process (running as $AGENT_USER) opens it itself,
  # so a 0600 root-owned file makes the agent exit "cannot read --token-file" and the unit
  # never starts. Group ownership is the smallest grant that lets the service read it, and
  # every other account — ops included — still cannot.
  (umask 0177; printf '%s\n' "$TOKEN" > "$TOKEN_FILE")
  chmod 0640 "$TOKEN_FILE"
  chown root:"$AGENT_USER" "$TOKEN_FILE"

  # The certificate bundle is the token's twin and the mode is the whole design. It holds
  # the agent's *private key*, so every other account — ops included — must not read it;
  # but the agent process (running as $AGENT_USER) opens the bundle itself at startup and
  # re-reads it when its stamp changes (that live re-read is how a re-deployed denylist
  # lands without a restart), so a 0600 root-owned file would fail the unit exactly the way
  # it would fail the token file. 0640 root:$AGENT_USER is the smallest grant that works.
  # An existing bundle survives a re-run that was not given the flag — the token-reuse
  # idiom — and this script never dials the dashboard to fetch one.
  if [ -n "$BUNDLE_SRC" ]; then
    install -m 0640 "$BUNDLE_SRC" "$CERT_FILE"
    chown root:"$AGENT_USER" "$CERT_FILE"
    echo "  installed the certificate bundle in $CERT_FILE"
  elif [ -f "$CERT_FILE" ]; then
    echo "reusing the installed certificate bundle in $CERT_FILE (--cert-bundle <file> for a new one)"
  fi

  # TLS mode: the unit runs https with the installed bundle. The --token-file line stays —
  # the agent skips the token check on the TLS path — so a rollout happens host by host
  # instead of on a flag day.
  TLS_EXEC_ARGS=""
  TLS_UNIT_LINES=""
  if [ -f "$CERT_FILE" ]; then
    TLS_EXEC_ARGS=" --cert-bundle $CERT_FILE"
    TLS_UNIT_LINES="# The TLS bundle unpacks its PEM blocks here: tmpfs, private to the unit,
# gone when it stops. ProtectSystem=strict leaves the agent nothing else writable, and the
# agent deletes the key the moment the SSL context holds it.
RuntimeDirectory=dashboards-agent
RuntimeDirectoryMode=0700"
  fi

  cat > "$UNIT" <<UNIT_FILE
[Unit]
Description=Homelab host agent (read-only system metrics)
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
User=$AGENT_USER
ExecStart=/usr/bin/env python3 $AGENT_TARGET --host $BIND --port $PORT --token-file ${TOKEN_FILE}${TLS_EXEC_ARGS}
Restart=on-failure
RestartSec=5
Environment=PYTHONUNBUFFERED=1
SyslogIdentifier=$SERVICE
$TLS_UNIT_LINES
# The agent reads /proc, /sys and statvfs() and writes nothing, so everything writable can
# be taken away. smartctl is the one thing that needs a real /dev node, so instead of
# PrivateDevices the unit closes all devices and reopens only the disk nodes, read/write
# at the cgroup layer while the code (and smartctl -i -A -H) stays read-only by design.
#
# The pseudo nodes are not decoration: DevicePolicy=strict denies every node that is not
# listed, and systemd needs /dev/null for the service's stdin before it ever execs python —
# without it the unit dies at spawn with "Failed at step STDIN" (status 208), which is how
# the first phase-10 install failed on a live host running systemd 257. /dev/zero, full,
# random and urandom are the rest of the set a PrivateDevices= sandbox would have provided,
# and none of them is a disk.
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=false
DevicePolicy=strict
DeviceAllow=/dev/null rw
DeviceAllow=/dev/zero rw
DeviceAllow=/dev/full rw
DeviceAllow=/dev/random rw
DeviceAllow=/dev/urandom rw
DeviceAllow=/dev/nvme* rw
DeviceAllow=/dev/sd* rw
DeviceAllow=/dev/vd* rw
DeviceAllow=/dev/sg* rw
ProtectSystem=strict
ProtectHome=read-only
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictSUIDSGID=true
RestrictNamespaces=true
RestrictRealtime=true
LockPersonality=true
MemoryDenyWriteExecute=true
CapabilityBoundingSet=
SystemCallFilter=@system-service

[Install]
WantedBy=multi-user.target
UNIT_FILE

  systemctl daemon-reload
  systemctl enable --now "$SERVICE"
  sleep 1
  if systemctl is-active --quiet "$SERVICE"; then
    echo "  $SERVICE is running on $BIND:$PORT"
  else
    echo "  $SERVICE failed to start — last lines:" >&2
    journalctl -u "$SERVICE" -n 20 --no-pager >&2
    exit 1
  fi
  if [ -n "$TLS_EXEC_ARGS" ]; then
    cat <<NEXT

The unit now serves https with the certificate bundle in $CERT_FILE. Point the dashboard at
it over TLS — the bearer token plays no part on the certificate path, so the URL carries
none:

  HOST_TARGETS="hostname=https://$BIND:$PORT"

After a rotate or a root rotation on the dashboard, re-download the bundle and re-run this
script with --cert-bundle; the agent picks a changed denylist up without a restart, but a
new certificate needs one.
NEXT
  else
    cat <<NEXT

Point the dashboard at it (the token is the URL userinfo, so it never needs a variable of
its own, and the API strips it before anything reaches a browser):

  HOST_TARGETS="hostname=http://$TOKEN@$BIND:$PORT"

Paste the token now if this is the first install — it is not printed again without
--rotate-token, and only root and the agent's own service user can read it on this box:

  $TOKEN
NEXT
  fi
}

case "$MODE" in
  install) do_install ;;
  status) status ;;
  uninstall) uninstall ;;
esac