#!/usr/bin/env bash # Dev-only: reach live services on each host from the laptop without opening a single port. # # ./scripts/dev-tunnels.sh start|stop|status|restart # # DEV_TUNNELS starts agent/host_agent.py remotely and forwards it: # ssh -L :127.0.0.1: 'python3 host_agent.py --port ' # DEV_FORWARDS forwards a service that is already listening remotely (e.g. upsd on 3493): # nothing is started over there, the port just appears on localhost. # # Both supervise themselves: a dropped link retries every 5s. Production is the opposite — # deploy/install-agent.sh installs a systemd unit that binds the LAN address with a bearer # token, and the dashboard dials it directly. set -uo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" RUN="$ROOT/data/tunnels" LOGS="$ROOT/data/logs" # Where the agent lives on each host. `~` is written into the generated script literally so # the REMOTE shell expands it (a local $HOME would be the wrong path over there). AGENT_REMOTE_DIR="${AGENT_REMOTE_DIR:-~/lab-agent}" AGENT_COMMAND="${AGENT_COMMAND:-python3 host_agent.py}" # Targets come from .env.dev (see .env.dev.example) — never hardcoded here, so this file # stays portable and holds no hostnames of ours. [ -f "$ROOT/.env.dev" ] && . "$ROOT/.env.dev" DEV_TUNNELS="${DEV_TUNNELS:-}" DEV_FORWARDS="${DEV_FORWARDS:-}" if [ -z "$DEV_TUNNELS" ] && [ -z "$DEV_FORWARDS" ]; then echo "no DEV_TUNNELS or DEV_FORWARDS set — copy .env.dev.example to .env.dev first"; exit 2 fi TUNNELS=() for entry in $DEV_TUNNELS; do # 9121:driveripper-ops:9120 -> "9121 driveripper-ops 9120" TUNNELS+=("$(echo "$entry" | tr ':' ' ')") done FORWARDS=() for entry in $DEV_FORWARDS; do FORWARDS+=("$(echo "$entry" | tr ':' ' ')") done mkdir -p "$RUN" "$LOGS" start() { for spec in "${TUNNELS[@]}"; do read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" # pidfiles are named by host AND port: one host can carry an agent tunnel plus an upsd # forward. The older host-only name is cleared so a restart does not double-bind. pidfile="$RUN/$host-$local.pid" if [ -f "$RUN/$host.pid" ] && kill -0 "$(cat "$RUN/$host.pid")" 2>/dev/null; then pkill -P "$(cat "$RUN/$host.pid")" 2>/dev/null kill "$(cat "$RUN/$host.pid")" 2>/dev/null fi rm -f "$RUN/$host.pid" if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then echo " $host already tunnelling on :$local (pid $(cat "$pidfile"))" continue fi # A previous agent survives the ssh session that started it, and the next agent then # dies on bind() with "address already in use" — which looks like a broken agent rather # than a leaked process. Clear it in a SEPARATE ssh session: putting pkill in the same # command line as the agent makes pkill match that very command line and shoot itself. cat > "$RUN/$host.sh" </dev/null 2>&1 sleep 1 ssh -nT -o ServerAliveInterval=30 -o ServerAliveCountMax=4 -o ExitOnForwardFailure=yes \ -L $local:127.0.0.1:$remote "$host" \ "cd $AGENT_REMOTE_DIR && exec $AGENT_COMMAND --host 127.0.0.1 --port $remote" echo "[tunnel] $host link dropped at \$(date -Is) (exit \$?); retrying in 5s" sleep 5 done EOF chmod +x "$RUN/$host.sh" nohup "$RUN/$host.sh" >>"$LOGS/tunnel-$host.log" 2>&1 & echo $! >"$pidfile" echo " $host -> http://127.0.0.1:$local (pid $!)" done # Plain forwards: `-N` because there is no remote command to run, and nothing to clean up # over there when the link goes. for spec in "${FORWARDS[@]}"; do read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" pidfile="$RUN/$host-$local.pid" if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then echo " $host:$remote already forwarded on :$local (pid $(cat "$pidfile"))" continue fi cat > "$RUN/$host-$local.sh" <>"$LOGS/tunnel-$host-$local.log" 2>&1 & echo $! >"$pidfile" echo " $host:$remote -> 127.0.0.1:$local (pid $!)" done } stop() { for spec in "${TUNNELS[@]}"; do read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" pidfile="$RUN/$host-$local.pid" if [ -f "$RUN/$host.pid" ]; then pkill -P "$(cat "$RUN/$host.pid")" 2>/dev/null kill "$(cat "$RUN/$host.pid")" 2>/dev/null rm -f "$RUN/$host.pid" fi [ -f "$pidfile" ] || continue pid=$(cat "$pidfile") pkill -P "$pid" 2>/dev/null kill "$pid" 2>/dev/null # -N would mean "no remote command", so the forward is kept by a plain ssh running the # agent; kill any leftover forward for this host by pattern, not just by parent. pkill -f "ssh -nT.*-L $local:127.0.0.1:$remote $host" 2>/dev/null # The remote agent outlives its ssh session, and the next one then dies on bind() with # "address already in use" — which looks like a broken agent, not a leaked process. timeout 10 ssh -n "$host" "pkill -f '[h]ost_agent.py --host 127.0.0.1 --port $remote'" \ >/dev/null 2>&1 rm -f "$pidfile" echo " stopped $host" done for spec in "${FORWARDS[@]}"; do read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" pidfile="$RUN/$host-$local.pid" [ -f "$pidfile" ] || continue pkill -P "$(cat "$pidfile")" 2>/dev/null kill "$(cat "$pidfile")" 2>/dev/null pkill -f "ssh -n -N.*-L $local:127.0.0.1:$remote $host" 2>/dev/null rm -f "$pidfile" "$RUN/$host-$local.sh" echo " stopped forward $host:$remote (:$local)" done } status() { for spec in "${TUNNELS[@]}"; do read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" body=$(curl -s --max-time 4 "http://127.0.0.1:$local/health" || true) if [ -n "$body" ]; then echo " OK $host http://127.0.0.1:$local $body" else echo " DOWN $host http://127.0.0.1:$local" fi done for spec in "${FORWARDS[@]}"; do read -r local host remote <<<"$(echo "$spec" | tr -s ' ')" # no HTTP behind these (upsd speaks its own line protocol), so the check is a bare TCP # handshake — enough to tell "the tunnel is up" from "nothing listens over there". if timeout 3 bash -c "exec 3<>/dev/tcp/127.0.0.1/$local" 2>/dev/null; then echo " OK $host:$remote 127.0.0.1:$local" else echo " DOWN $host:$remote 127.0.0.1:$local" fi done } case "${1:-start}" in start) start ;; stop) stop ;; restart) stop; sleep 1; start ;; status) status ;; *) echo "usage: $0 start|stop|status|restart"; exit 2 ;; esac