dashboards/scripts/dev-tunnels.sh

#!/usr/bin/env bash
# Dev-only: reach live services on each host from the laptop without opening a single port.
#
#   ./scripts/dev-tunnels.sh start|stop|status|restart
#
# DEV_TUNNELS  starts agent/host_agent.py remotely and forwards it:
#              ssh -L <local>:127.0.0.1:<rport> <host> 'python3 host_agent.py --port <rport>'
# DEV_FORWARDS forwards a service that is already listening remotely (e.g. upsd on 3493):
#              nothing is started over there, the port just appears on localhost.
#
# Both supervise themselves: a dropped link retries every 5s. Production is the opposite —
# deploy/install-agent.sh installs a systemd unit that binds the LAN address with a bearer
# token, and the dashboard dials it directly.
set -uo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
RUN="$ROOT/data/tunnels"
LOGS="$ROOT/data/logs"
# Where the agent lives on each host. `~` is written into the generated script literally so
# the REMOTE shell expands it (a local $HOME would be the wrong path over there).
AGENT_REMOTE_DIR="${AGENT_REMOTE_DIR:-~/lab-agent}"
AGENT_COMMAND="${AGENT_COMMAND:-python3 host_agent.py}"

# Targets come from .env.dev (see .env.dev.example) — never hardcoded here, so this file
# stays portable and holds no hostnames of ours.
[ -f "$ROOT/.env.dev" ] && . "$ROOT/.env.dev"
DEV_TUNNELS="${DEV_TUNNELS:-}"
DEV_FORWARDS="${DEV_FORWARDS:-}"
if [ -z "$DEV_TUNNELS" ] && [ -z "$DEV_FORWARDS" ]; then
  echo "no DEV_TUNNELS or DEV_FORWARDS set — copy .env.dev.example to .env.dev first"; exit 2
fi
TUNNELS=()
for entry in $DEV_TUNNELS; do
  # 9121:driveripper-ops:9120 -> "9121 driveripper-ops 9120"
  TUNNELS+=("$(echo "$entry" | tr ':' ' ')")
done
FORWARDS=()
for entry in $DEV_FORWARDS; do
  FORWARDS+=("$(echo "$entry" | tr ':' ' ')")
done

mkdir -p "$RUN" "$LOGS"

start() {
  for spec in "${TUNNELS[@]}"; do
    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
    # pidfiles are named by host AND port: one host can carry an agent tunnel plus an upsd
    # forward. The older host-only name is cleared so a restart does not double-bind.
    pidfile="$RUN/$host-$local.pid"
    if [ -f "$RUN/$host.pid" ] && kill -0 "$(cat "$RUN/$host.pid")" 2>/dev/null; then
      pkill -P "$(cat "$RUN/$host.pid")" 2>/dev/null
      kill "$(cat "$RUN/$host.pid")" 2>/dev/null
    fi
    rm -f "$RUN/$host.pid"
    if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
      echo "  $host already tunnelling on :$local (pid $(cat "$pidfile"))"
      continue
    fi
    # A previous agent survives the ssh session that started it, and the next agent then
    # dies on bind() with "address already in use" — which looks like a broken agent rather
    # than a leaked process. Clear it in a SEPARATE ssh session: putting pkill in the same
    # command line as the agent makes pkill match that very command line and shoot itself.
    cat > "$RUN/$host.sh" <<EOF
#!/usr/bin/env bash
# generated by scripts/dev-tunnels.sh — do not edit, it is rewritten on every start
while true; do
  ssh -n -o BatchMode=yes "$host" \
      "pkill -f '[h]ost_agent.py --host 127.0.0.1 --port $remote'" >/dev/null 2>&1
  sleep 1
  ssh -nT -o ServerAliveInterval=30 -o ServerAliveCountMax=4 -o ExitOnForwardFailure=yes \
      -L $local:127.0.0.1:$remote "$host" \
      "cd $AGENT_REMOTE_DIR && exec $AGENT_COMMAND --host 127.0.0.1 --port $remote"
  echo "[tunnel] $host link dropped at \$(date -Is) (exit \$?); retrying in 5s"
  sleep 5
done
EOF
  chmod +x "$RUN/$host.sh"
    nohup "$RUN/$host.sh" >>"$LOGS/tunnel-$host.log" 2>&1 &
    echo $! >"$pidfile"
    echo "  $host -> http://127.0.0.1:$local (pid $!)"
  done

  # Plain forwards: `-N` because there is no remote command to run, and nothing to clean up
  # over there when the link goes.
  for spec in "${FORWARDS[@]}"; do
    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
    pidfile="$RUN/$host-$local.pid"
    if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
      echo "  $host:$remote already forwarded on :$local (pid $(cat "$pidfile"))"
      continue
    fi
    cat > "$RUN/$host-$local.sh" <<EOF
#!/usr/bin/env bash
# generated by scripts/dev-tunnels.sh — do not edit, it is rewritten on every start
while true; do
  ssh -n -N -o ServerAliveInterval=30 -o ServerAliveCountMax=4 -o ExitOnForwardFailure=yes \
      -L $local:127.0.0.1:$remote "$host"
  echo "[forward] $host:$remote dropped at \$(date -Is) (exit \$?); retrying in 5s"
  sleep 5
done
EOF
    chmod +x "$RUN/$host-$local.sh"
    nohup "$RUN/$host-$local.sh" >>"$LOGS/tunnel-$host-$local.log" 2>&1 &
    echo $! >"$pidfile"
    echo "  $host:$remote -> 127.0.0.1:$local (pid $!)"
  done
}

stop() {
  for spec in "${TUNNELS[@]}"; do
    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
    pidfile="$RUN/$host-$local.pid"
    if [ -f "$RUN/$host.pid" ]; then
      pkill -P "$(cat "$RUN/$host.pid")" 2>/dev/null
      kill "$(cat "$RUN/$host.pid")" 2>/dev/null
      rm -f "$RUN/$host.pid"
    fi
    [ -f "$pidfile" ] || continue
    pid=$(cat "$pidfile")
    pkill -P "$pid" 2>/dev/null
    kill "$pid" 2>/dev/null
    # -N would mean "no remote command", so the forward is kept by a plain ssh running the
    # agent; kill any leftover forward for this host by pattern, not just by parent.
    pkill -f "ssh -nT.*-L $local:127.0.0.1:$remote $host" 2>/dev/null
    # The remote agent outlives its ssh session, and the next one then dies on bind() with
    # "address already in use" — which looks like a broken agent, not a leaked process.
    timeout 10 ssh -n "$host" "pkill -f '[h]ost_agent.py --host 127.0.0.1 --port $remote'" \
      >/dev/null 2>&1
    rm -f "$pidfile"
    echo "  stopped $host"
  done

  for spec in "${FORWARDS[@]}"; do
    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
    pidfile="$RUN/$host-$local.pid"
    [ -f "$pidfile" ] || continue
    pkill -P "$(cat "$pidfile")" 2>/dev/null
    kill "$(cat "$pidfile")" 2>/dev/null
    pkill -f "ssh -n -N.*-L $local:127.0.0.1:$remote $host" 2>/dev/null
    rm -f "$pidfile" "$RUN/$host-$local.sh"
    echo "  stopped forward $host:$remote (:$local)"
  done
}

status() {
  for spec in "${TUNNELS[@]}"; do
    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
    body=$(curl -s --max-time 4 "http://127.0.0.1:$local/health" || true)
    if [ -n "$body" ]; then
      echo "  OK   $host http://127.0.0.1:$local $body"
    else
      echo "  DOWN $host http://127.0.0.1:$local"
    fi
  done

  for spec in "${FORWARDS[@]}"; do
    read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
    # no HTTP behind these (upsd speaks its own line protocol), so the check is a bare TCP
    # handshake — enough to tell "the tunnel is up" from "nothing listens over there".
    if timeout 3 bash -c "exec 3<>/dev/tcp/127.0.0.1/$local" 2>/dev/null; then
      echo "  OK   $host:$remote 127.0.0.1:$local"
    else
      echo "  DOWN $host:$remote 127.0.0.1:$local"
    fi
  done
}

case "${1:-start}" in
  start) start ;;
  stop) stop ;;
  restart) stop; sleep 1; start ;;
  status) status ;;
  *) echo "usage: $0 start|stop|status|restart"; exit 2 ;;
esac