dashboards/deploy/install-agent.sh

#!/usr/bin/env bash
# Install agent/host_agent.py as a systemd service on a monitored host.
#
#   sudo ./deploy/install-agent.sh                       # loopback only (read it locally)
#   sudo ./deploy/install-agent.sh --bind 10.0.0.5       # serve the LAN, token required
#   ./deploy/install-agent.sh --status                   # unit state + /health, no root
#   sudo ./deploy/install-agent.sh --uninstall
#
# No root shell? `deploy/agent-sudoers` is the single sudoers line that lets the fleet's
# unprivileged `ops` account run this script as root. Run it without root and it prints that
# exact line with this script's resolved path, then exits 1.
#
# The token is stored in a file only root and the agent's own service user can read, and is
# passed to the agent as --token-file, never on the command line: argv is world-readable
# through `ps`, and this token guards hostname, mounts, disk models, GPU model and process
# names. Re-running the installer reuses the existing token — use --rotate-token when the old
# one leaked.
#
# The agent is read-only by construction (it answers GET and writes nothing), runs as its
# own system user, and gets the usual systemd lockdown. It still needs to read /proc, /sys
# and statvfs(), which the hardening below leaves alone.
#
# Disk health (`smartctl -i -A -H`, the `/disks` page) is the one add-on that needs more
# than /proc and /sys: smartctl must open the disk device nodes. The installer therefore
# adds the agent user to the `disk` group when one exists, and the unit device-allows the
# disk nodes instead of PrivateDevices. This is raw *read* access to the disks — the
# standard arrangement for SMART monitoring, granted deliberately — and its absence is a
# documented degradation, never an install failure: no smartmontools (or no group) simply
# means every disk reports `smart: null` with an explanatory error entry.
set -euo pipefail

SERVICE="dashboards-agent"
AGENT_USER="dashboards-agent"
AGENT_DIR="/usr/local/lib/dashboards-agent"
AGENT_TARGET="$AGENT_DIR/host_agent.py"
TOKEN_FILE="/etc/dashboards-agent.token"
UNIT="/etc/systemd/system/$SERVICE.service"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
AGENT_SOURCE="$SCRIPT_DIR/../agent/host_agent.py"
# Absolute, symlink-resolved path of this script: the sudoers grant in deploy/agent-sudoers
# is matched on a path, so the non-root hint below prints the line that actually matches
# wherever the script happens to live (the canonical staging path is
# /home/ops/agent-install/deploy/install-agent.sh).
SCRIPT_PATH="$SCRIPT_DIR/$(basename "$0")"
if command -v realpath >/dev/null 2>&1; then
  SCRIPT_PATH="$(realpath "$0" 2>/dev/null || printf '%s' "$SCRIPT_PATH")"
fi
SUDOERS_SOURCE="$(dirname "$SCRIPT_PATH")/agent-sudoers"

BIND="127.0.0.1"
PORT="9120"
TOKEN="${AGENT_TOKEN:-}"
ROTATE=0
MODE="install"

usage() {
  # the header comment *is* the help text, so it cannot drift from the flags
  awk 'NR > 1 { if (/^#/) { sub(/^# ?/, ""); print } else { exit } }' "$0"
  exit "${1:-0}"
}

while [ $# -gt 0 ]; do
  case "$1" in
    --bind) BIND="$2"; shift 2 ;;
    --port) PORT="$2"; shift 2 ;;
    --token) TOKEN="$2"; shift 2 ;;
    --rotate-token) ROTATE=1; shift ;;
    --status) MODE="status"; shift ;;
    --uninstall) MODE="uninstall"; shift ;;
    -h|--help) usage 0 ;;
    *) echo "unknown argument: $1" >&2; usage 2 ;;
  esac
done

need_root() {
  if [ "$(id -u)" -ne 0 ]; then
    # The dev workflow deploys over ssh as the unprivileged `ops` account, so "run as root"
    # is not an action the operator can take. Print the one sudoers line that makes this
    # script installable — the same line as deploy/agent-sudoers, with this script's own
    # resolved path — plus the two-stage install it documents, then exit 1 as before.
    cat >&2 <<HINT
$SCRIPT_PATH has to run as root: it writes $UNIT,
creates the $AGENT_USER user and its disk-group membership, and stores
the token in $TOKEN_FILE.

Add this single sudoers line on this host (it is the whole grant; see $SUDOERS_SOURCE):

ops ALL=(root) NOPASSWD: $SCRIPT_PATH

Install it the two-stage way every sudoers change here uses, as root — the command-scoped
ops sudo on these hosts does not include visudo, so this step is the operator's:

  sudo visudo -cf $SUDOERS_SOURCE
  sudo install -m 0440 -o root -g root $SUDOERS_SOURCE /etc/sudoers.d/dashboards-agent
  sudo visudo -cf /etc/sudoers.d/dashboards-agent

Then run this same command again:

  sudo $SCRIPT_PATH $*
HINT
    exit 1
  fi
}

status() {
  systemctl status "$SERVICE" --no-pager 2>&1 || true
  echo
  if command -v curl >/dev/null 2>&1; then
    printf 'health: '
    curl -s --max-time 4 "http://$BIND:$PORT/health" || echo "(no answer)"
    echo
  fi
}

uninstall() {
  need_root --uninstall
  systemctl disable --now "$SERVICE" 2>/dev/null || true
  rm -f "$UNIT" "$TOKEN_FILE" "$AGENT_TARGET"
  rmdir "$AGENT_DIR" 2>/dev/null || true
  systemctl daemon-reload
  echo "removed $SERVICE (the $AGENT_USER user was left in place; remove it with userdel)"
}

new_token() {
  # urlsafe and no padding: the token goes into HOST_TARGETS as URL userinfo
  # (`name=http://TOKEN@host:9120`), where '+' and '=' are annoying to quote.
  python3 -c 'import secrets; print(secrets.token_urlsafe(32).rstrip("="))'
}

# NOT named `install`: bash resolves a function before an external command of the same name,
# so an `install()` here turns the `install -d` / `install -m` calls below into infinite
# recursion instead of creating $AGENT_DIR. Phase 10 task 02 found exactly that on a live
# host: the run looped on the SMART block forever and installed nothing.
do_install() {
  need_root --bind "$BIND" --port "$PORT"
  [ -f "$AGENT_SOURCE" ] || { echo "no agent next to this script: $AGENT_SOURCE" >&2; exit 1; }
  if ! command -v python3 >/dev/null 2>&1; then
    echo "python3 is required on the monitored host (3.10 or newer)" >&2
    exit 1
  fi
  # The agent needs no privileges, so give it a user with nothing to lose.
  if ! id "$AGENT_USER" >/dev/null 2>&1; then
    useradd --system --home-dir / --shell /usr/sbin/nologin "$AGENT_USER"
  fi

  # SMART (phase 09): the usual distro answer for reading drive health is the `disk`
  # group — no sudoers entry, no root. Re-runs are idempotent.
  if getent group disk >/dev/null 2>&1; then
    if id -nG "$AGENT_USER" 2>/dev/null | tr ' ' '\n' | grep -qx disk; then
      echo "  $AGENT_USER is already in the disk group (SMART reads enabled)"
    else
      usermod -aG disk "$AGENT_USER"
      echo "  added $AGENT_USER to the disk group — raw device reads, which is what smartctl needs"
    fi
  else
    echo "  no disk group on this host — SMART data will read null (harmless; see README)"
  fi
  if ! command -v smartctl >/dev/null 2>&1; then
    echo "  note: smartctl is not installed — every disk reports smart: null until smartmontools is"
  fi

  install -d -m 0755 "$AGENT_DIR"
  install -m 0755 "$AGENT_SOURCE" "$AGENT_TARGET"

  if [ -f "$TOKEN_FILE" ] && [ "$ROTATE" -eq 0 ] && [ -z "$TOKEN" ]; then
    TOKEN="$(cat "$TOKEN_FILE")"
    echo "reusing the existing token in $TOKEN_FILE (--rotate-token for a new one)"
  elif [ -z "$TOKEN" ]; then
    TOKEN="$(new_token)"
  fi
  # 0640 root:$AGENT_USER. The unit passes `--token-file` as a *path*: systemd does not read
  # the secret for the service, the agent process (running as $AGENT_USER) opens it itself,
  # so a 0600 root-owned file makes the agent exit "cannot read --token-file" and the unit
  # never starts. Group ownership is the smallest grant that lets the service read it, and
  # every other account — ops included — still cannot.
  (umask 0177; printf '%s\n' "$TOKEN" > "$TOKEN_FILE")
  chmod 0640 "$TOKEN_FILE"
  chown root:"$AGENT_USER" "$TOKEN_FILE"

  cat > "$UNIT" <<UNIT_FILE
[Unit]
Description=Homelab host agent (read-only system metrics)
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
User=$AGENT_USER
ExecStart=/usr/bin/env python3 $AGENT_TARGET --host $BIND --port $PORT --token-file $TOKEN_FILE
Restart=on-failure
RestartSec=5
Environment=PYTHONUNBUFFERED=1
SyslogIdentifier=$SERVICE
# The agent reads /proc, /sys and statvfs() and writes nothing, so everything writable can
# be taken away. smartctl is the one thing that needs a real /dev node, so instead of
# PrivateDevices the unit closes all devices and reopens only the disk nodes, read/write
# at the cgroup layer while the code (and smartctl -i -A -H) stays read-only by design.
#
# The pseudo nodes are not decoration: DevicePolicy=strict denies every node that is not
# listed, and systemd needs /dev/null for the service's stdin before it ever execs python —
# without it the unit dies at spawn with "Failed at step STDIN" (status 208), which is how
# the first phase-10 install failed on a live host running systemd 257. /dev/zero, full,
# random and urandom are the rest of the set a PrivateDevices= sandbox would have provided,
# and none of them is a disk.
NoNewPrivileges=true
PrivateTmp=true
PrivateDevices=false
DevicePolicy=strict
DeviceAllow=/dev/null rw
DeviceAllow=/dev/zero rw
DeviceAllow=/dev/full rw
DeviceAllow=/dev/random rw
DeviceAllow=/dev/urandom rw
DeviceAllow=/dev/nvme* rw
DeviceAllow=/dev/sd* rw
DeviceAllow=/dev/vd* rw
DeviceAllow=/dev/sg* rw
ProtectSystem=strict
ProtectHome=read-only
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
RestrictSUIDSGID=true
RestrictNamespaces=true
RestrictRealtime=true
LockPersonality=true
MemoryDenyWriteExecute=true
CapabilityBoundingSet=
SystemCallFilter=@system-service

[Install]
WantedBy=multi-user.target
UNIT_FILE

  systemctl daemon-reload
  systemctl enable --now "$SERVICE"
  sleep 1
  if systemctl is-active --quiet "$SERVICE"; then
    echo "  $SERVICE is running on $BIND:$PORT"
  else
    echo "  $SERVICE failed to start — last lines:" >&2
    journalctl -u "$SERVICE" -n 20 --no-pager >&2
    exit 1
  fi
  cat <<NEXT

Point the dashboard at it (the token is the URL userinfo, so it never needs a variable of
its own, and the API strips it before anything reaches a browser):

  HOST_TARGETS="hostname=http://$TOKEN@$BIND:$PORT"

Paste the token now if this is the first install — it is not printed again without
--rotate-token, and only root and the agent's own service user can read it on this box:

  $TOKEN
NEXT
}

case "$MODE" in
  install) do_install ;;
  status) status ;;
  uninstall) uninstall ;;
esac