ducoterra
c4bb374ccc
Three dashboards served by one FastAPI app with a SQLite history store
(raw / 1m / 1h) and hand-written SVG charts: no Prometheus, no Grafana,
no CDN, no build step. Nothing is fetched from the internet at runtime.
* system: a stdlib-only agent (agent/host_agent.py) reports /proc and
/sys counters; the dashboard turns them into rates over a labelled
window, so every number says how it was measured.
* llm: gufo/llama.cpp and vLLM /metrics are parsed by an in-house
exposition parser. Measured delta/elapsed throughput beats a native
per-request gauge, and each value carries its provenance.
* ups: a NUT wire-protocol client plus health checks that name what
they looked at. Commands are gated by NUT_ALLOW_ACTIONS,
DASH_READ_ONLY and a DANGER confirmation, re-checked server-side.
Missing values render as n/a rather than 0, and an unconfigured source
answers 404 {"configured": false} without dialling anything, so "not
deployed here" never looks like an outage.
253 tests at 94% coverage against captured real fixtures; ruff and
pyright clean.
LLM-Generated: true
108 lines
4.2 KiB
Bash
Executable File
108 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Dev-only: run agent/host_agent.py on each monitored host and forward it to localhost,
|
|
# so `make dev` on the laptop sees live fleet data without opening a single port.
|
|
#
|
|
# ./scripts/dev-tunnels.sh start|stop|status|restart
|
|
#
|
|
# Each tunnel is ssh -L <local>:127.0.0.1:<remote-port> <host> 'python3 agent/host_agent.py'
|
|
# and restarts itself if the link drops. Production is the opposite: deploy/install-agent.sh
|
|
# installs a systemd unit that binds the LAN address with a bearer token, and the dashboard
|
|
# dials it directly.
|
|
set -uo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
RUN="$ROOT/data/tunnels"
|
|
LOGS="$ROOT/data/logs"
|
|
# Where the agent lives on each host. `~` is written into the generated script literally so
|
|
# the REMOTE shell expands it (a local $HOME would be the wrong path over there).
|
|
AGENT_REMOTE_DIR="${AGENT_REMOTE_DIR:-~/lab-agent}"
|
|
AGENT_COMMAND="${AGENT_COMMAND:-python3 host_agent.py}"
|
|
|
|
# Targets come from .env.dev (see .env.dev.example) — never hardcoded here, so this file
|
|
# stays portable and holds no hostnames of ours.
|
|
[ -f "$ROOT/.env.dev" ] && . "$ROOT/.env.dev"
|
|
DEV_TUNNELS="${DEV_TUNNELS:-}"
|
|
if [ -z "$DEV_TUNNELS" ]; then
|
|
echo "no DEV_TUNNELS set — copy .env.dev.example to .env.dev first"; exit 2
|
|
fi
|
|
TUNNELS=()
|
|
for entry in $DEV_TUNNELS; do
|
|
# 9121:driveripper-ops:9120 -> "9121 driveripper-ops 9120"
|
|
TUNNELS+=("$(echo "$entry" | tr ':' ' ')")
|
|
done
|
|
|
|
mkdir -p "$RUN" "$LOGS"
|
|
|
|
start() {
|
|
for spec in "${TUNNELS[@]}"; do
|
|
read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
|
|
pidfile="$RUN/$host.pid"
|
|
if [ -f "$pidfile" ] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
|
|
echo " $host already tunnelling on :$local (pid $(cat "$pidfile"))"
|
|
continue
|
|
fi
|
|
# A previous agent survives the ssh session that started it, and the next agent then
|
|
# dies on bind() with "address already in use" — which looks like a broken agent rather
|
|
# than a leaked process. Clear it in a SEPARATE ssh session: putting pkill in the same
|
|
# command line as the agent makes pkill match that very command line and shoot itself.
|
|
cat > "$RUN/$host.sh" <<EOF
|
|
#!/usr/bin/env bash
|
|
# generated by scripts/dev-tunnels.sh — do not edit, it is rewritten on every start
|
|
while true; do
|
|
ssh -n -o BatchMode=yes "$host" \
|
|
"pkill -f '[h]ost_agent.py --host 127.0.0.1 --port $remote'" >/dev/null 2>&1
|
|
sleep 1
|
|
ssh -nT -o ServerAliveInterval=30 -o ServerAliveCountMax=4 -o ExitOnForwardFailure=yes \
|
|
-L $local:127.0.0.1:$remote "$host" \
|
|
"cd $AGENT_REMOTE_DIR && exec $AGENT_COMMAND --host 127.0.0.1 --port $remote"
|
|
echo "[tunnel] $host link dropped at \$(date -Is) (exit \$?); retrying in 5s"
|
|
sleep 5
|
|
done
|
|
EOF
|
|
chmod +x "$RUN/$host.sh"
|
|
nohup "$RUN/$host.sh" >>"$LOGS/tunnel-$host.log" 2>&1 &
|
|
echo $! >"$pidfile"
|
|
echo " $host -> http://127.0.0.1:$local (pid $!)"
|
|
done
|
|
}
|
|
|
|
stop() {
|
|
for spec in "${TUNNELS[@]}"; do
|
|
read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
|
|
pidfile="$RUN/$host.pid"
|
|
[ -f "$pidfile" ] || continue
|
|
pid=$(cat "$pidfile")
|
|
pkill -P "$pid" 2>/dev/null
|
|
kill "$pid" 2>/dev/null
|
|
# -N would mean "no remote command", so the forward is kept by a plain ssh running the
|
|
# agent; kill any leftover forward for this host by pattern, not just by parent.
|
|
pkill -f "ssh -nT.*-L $local:127.0.0.1:$remote $host" 2>/dev/null
|
|
# The remote agent outlives its ssh session, and the next one then dies on bind() with
|
|
# "address already in use" — which looks like a broken agent, not a leaked process.
|
|
timeout 10 ssh -n "$host" "pkill -f '[h]ost_agent.py --host 127.0.0.1 --port $remote'" \
|
|
>/dev/null 2>&1
|
|
rm -f "$pidfile"
|
|
echo " stopped $host"
|
|
done
|
|
}
|
|
|
|
status() {
|
|
for spec in "${TUNNELS[@]}"; do
|
|
read -r local host remote <<<"$(echo "$spec" | tr -s ' ')"
|
|
body=$(curl -s --max-time 4 "http://127.0.0.1:$local/health" || true)
|
|
if [ -n "$body" ]; then
|
|
echo " OK $host http://127.0.0.1:$local $body"
|
|
else
|
|
echo " DOWN $host http://127.0.0.1:$local"
|
|
fi
|
|
done
|
|
}
|
|
|
|
case "${1:-start}" in
|
|
start) start ;;
|
|
stop) stop ;;
|
|
restart) stop; sleep 1; start ;;
|
|
status) status ;;
|
|
*) echo "usage: $0 start|stop|status|restart"; exit 2 ;;
|
|
esac
|