From 3cdbb405abee1c1b78554b639f234f20beac5f8d Mon Sep 17 00:00:00 2001 From: ducoterra Date: Fri, 16 Oct 2026 09:06:32 -0400 Subject: [PATCH] add WireGuard wg0 watchdog units for always-on SSH When SSH to a Pi targets a WireGuard tunnel address, reachability depends on three things that each fail independently: the unit starting at boot, the NAT mapping staying open, and the peer hostname still resolving. Rebooting onto a network with no route to the endpoint previously meant no way back in. - wg-quick@wg0.service.d/10-boot-resilience.conf: retry a failed boot-time up forever (Restart=on-failure, RestartSec=15, StartLimitIntervalSec=0) and cap endpoint resolution (WG_ENDPOINT_RESOLUTION_RETRIES=60, TimeoutStartSec=120) instead of hanging on the stock infinity. - wg0-watchdog: POSIX sh healer, oldest symptom first - missing interface or peer restarts the unit, a missing PersistentKeepalive is set directly, a stale handshake re-points the peer at a freshly resolved endpoint, and a very stale one restarts. Restarts are gap-limited so a broken network cannot make the unit flap, endpoint resolution falls back to the kernel endpoint and then to a last-known-good cache (tunnel DNS may only be reachable through the tunnel), and nothing it did not create is ever removed. Supports --dry-run. - wg0-watchdog.service/.timer: oneshot on a 60s timer, first check 30s after boot, Persistent=true to catch up after downtime, TimeoutStartSec=120 and Nice=10 so a restart cannot wedge the watchdog. - wg0.conf.template: documents /etc/wireguard/wg0.conf with placeholders only. Deliberately omits the DNS key so name resolution does not depend on the tunnel (Debian wg-quick has no `DNS = none`), and sets PersistentKeepalive 25 to hold the NAT mapping for inbound SSH. - wireguard.md: runbook for the placeholder table, failure modes, deploy, and verification. The real wg0.conf and key material never enter the repo. LLM-Generated: true --- ...uick@wg0.service.d-10-boot-resilience.conf | 17 ++ .../device_raspberry_pi/systemd/wg0-watchdog | 151 ++++++++++++++++ .../systemd/wg0-watchdog.service | 12 ++ .../systemd/wg0-watchdog.timer | 14 ++ .../systemd/wg0.conf.template | 20 +++ .../device_raspberry_pi/systemd/wireguard.md | 162 ++++++++++++++++++ 6 files changed, 376 insertions(+) create mode 100644 active/device_raspberry_pi/systemd/wg-quick@wg0.service.d-10-boot-resilience.conf create mode 100755 active/device_raspberry_pi/systemd/wg0-watchdog create mode 100644 active/device_raspberry_pi/systemd/wg0-watchdog.service create mode 100644 active/device_raspberry_pi/systemd/wg0-watchdog.timer create mode 100644 active/device_raspberry_pi/systemd/wg0.conf.template create mode 100644 active/device_raspberry_pi/systemd/wireguard.md diff --git a/active/device_raspberry_pi/systemd/wg-quick@wg0.service.d-10-boot-resilience.conf b/active/device_raspberry_pi/systemd/wg-quick@wg0.service.d-10-boot-resilience.conf new file mode 100644 index 0000000..e3d9262 --- /dev/null +++ b/active/device_raspberry_pi/systemd/wg-quick@wg0.service.d-10-boot-resilience.conf @@ -0,0 +1,17 @@ +# /etc/systemd/system/wg-quick@wg0.service.d/10-boot-resilience.conf +# The stock unit already has After=/Wants=network-online.target nss-lookup.target. +# This makes a boot-time failure (DHCP/DNS not ready yet, endpoint host offline) +# retry forever instead of leaving the tunnel down until someone logs in. +[Unit] +# Unlimited retries: do not let the start rate limiter give up on us. +StartLimitIntervalSec=0 + +[Service] +# Fail fast instead of hanging forever on endpoint resolution (stock value is +# infinity), then let Restart= try again. Same variable as the stock unit, +# later assignment wins. +Environment=WG_ENDPOINT_RESOLUTION_RETRIES=60 +TimeoutStartSec=120 +TimeoutStopSec=30 +Restart=on-failure +RestartSec=15 diff --git a/active/device_raspberry_pi/systemd/wg0-watchdog b/active/device_raspberry_pi/systemd/wg0-watchdog new file mode 100755 index 0000000..2023f0f --- /dev/null +++ b/active/device_raspberry_pi/systemd/wg0-watchdog @@ -0,0 +1,151 @@ +#!/bin/sh +# wg0-watchdog - keep the WireGuard client tunnel up so SSH to the tunnel +# address works after boot, after roaming to a new network, and after the NAT +# mapping or endpoint IP goes stale. +# +# Decisions, oldest symptom first: +# interface or peer gone -> restart wg-quick@wg0.service +# no PersistentKeepalive in kernel -> set it (covers a config edited without reload) +# handshake older than STALE -> re-point the peer at the endpoint +# (re-resolves the hostname, forces a handshake) +# handshake older than STALE_RESTART-> restart wg-quick@wg0.service +# +# Restarts are gap-limited so a broken network cannot make the unit flap. +# Never removes peers, addresses, or routes it did not create. + +set -eu + +IFACE="${WG_WD_IFACE:-wg0}" +UNIT="${WG_WD_UNIT:-wg-quick@wg0.service}" +CONF="${WG_WD_CONF:-/etc/wireguard/$IFACE.conf}" +KEEPALIVE="${WG_WD_KEEPALIVE:-25}" +STALE="${WG_WD_STALE:-120}" # seconds: poke the endpoint +STALE_RESTART="${WG_WD_STALE_RESTART:-300}" # seconds: restart the unit +MIN_RESTART_GAP="${WG_WD_MIN_RESTART_GAP:-180}" # seconds between restarts +STATE_DIR="${WG_WD_STATE_DIR:-/var/lib/wg0-watchdog}" + +DRY=0 +[ "${1:-}" = "--dry-run" ] && DRY=1 + +now=$(date +%s) +log() { printf 'wg0-watchdog: %s\n' "$*"; } +run() { + if [ "$DRY" -eq 1 ]; then + printf 'wg0-watchdog: DRY-RUN would run:' + printf ' %s' "$@" + printf '\n' + else + "$@" + fi +} + +read_state() { cat "$1" 2>/dev/null || echo 0; } +write_state() { + if [ "$DRY" -eq 1 ]; then + return 0 + fi + mkdir -p "$STATE_DIR" + printf '%s\n' "$2" > "$1" +} + +restart_unit() { + reason=$1 + last=$(read_state "$STATE_DIR/last-restart") + case "$last" in ''|*[!0-9]*) last=0 ;; esac + since=$((now - last)) + if [ "$since" -lt "$MIN_RESTART_GAP" ]; then + log "restart of $UNIT suppressed ($reason); last restart ${since}s ago" + return 0 + fi + log "restarting $UNIT ($reason)" + write_state "$STATE_DIR/last-restart" "$now" + # --no-block: a wg-quick up stuck on endpoint resolution must not wedge the + # watchdog; the next run reports whether the restart worked. + run systemctl restart --no-block "$UNIT" +} + +# --- 1. interface / peer present ------------------------------------------- +if ! ip link show "$IFACE" >/dev/null 2>&1; then + restart_unit "interface $IFACE is missing" + exit 0 +fi + +peer=$(wg show "$IFACE" peers 2>/dev/null | head -n 1) +if [ -z "${peer:-}" ]; then + restart_unit "no peer configured on $IFACE" + exit 0 +fi + +# --- 2. persistent keepalive present --------------------------------------- +live_ka=$(wg show "$IFACE" persistent-keepalive 2>/dev/null | awk -v p="$peer" '$1 == p {print $2}') +[ -n "${live_ka:-}" ] || live_ka=0 +if [ "$live_ka" != "$KEEPALIVE" ]; then + log "persistent-keepalive is $live_ka, setting $KEEPALIVE" + run wg set "$IFACE" peer "$peer" persistent-keepalive "$KEEPALIVE" +fi + +# --- 3. handshake age ------------------------------------------------------- +hs=$(wg show "$IFACE" latest-handshakes 2>/dev/null | awk -v p="$peer" '$1 == p {print $2}') +case "${hs:-}" in ''|*[!0-9]*) hs=0 ;; esac +if [ "$hs" -eq 0 ]; then + age=999999 +else + age=$((now - hs)) +fi + +# endpoint currently in use by the kernel, and the cached last-known-good one +live_ep=$(wg show "$IFACE" endpoints 2>/dev/null | awk -v p="$peer" '$1 == p {print $2}') +cache_file="$STATE_DIR/endpoint" +cached_ep="" +[ -f "$cache_file" ] && cached_ep=$(cat "$cache_file" 2>/dev/null || true) + +if [ "$age" -lt "$STALE" ]; then + if [ -n "$live_ep" ]; then + write_state "$cache_file" "$live_ep" + fi + log "healthy: handshake ${age}s ago, endpoint ${live_ep:-unknown}, keepalive ${KEEPALIVE}s" + exit 0 +fi + +log "stale: last handshake ${age}s ago" + +# Prefer the endpoint from the config file, freshly resolved. Fall back to the +# kernel's current endpoint, then to the cache - important because a DNS server +# from the tunnel config may be reachable only through this very tunnel. +target_ep="" +conf_ep=$(awk -F= 'tolower($1) ~ /^[[:space:]]*endpoint[[:space:]]*$/ { gsub(/[[:space:]]/, "", $2); print $2 }' "$CONF" 2>/dev/null | tail -n 1) +if [ -n "${conf_ep:-}" ]; then + host=${conf_ep%:*} + port=${conf_ep##*:} + case "$host" in + *[!0-9.]*) + resolved=$(getent hosts "$host" 2>/dev/null | awk '{print $1; exit}' || true) + ;; + *) + resolved=$host + ;; + esac + if [ -n "${resolved:-}" ]; then + target_ep="$resolved:$port" + else + log "could not resolve $host (tunnel DNS?), falling back" + fi +fi +[ -n "$target_ep" ] || target_ep=${live_ep:-${cached_ep:-}} + +if [ "$age" -ge "$STALE_RESTART" ]; then + restart_unit "handshake ${age}s old" + exit 0 +fi + +if [ -z "$target_ep" ]; then + log "no endpoint to re-point at; will retry next run" + exit 0 +fi + +if [ "$target_ep" = "$live_ep" ]; then + log "endpoint already $target_ep; forcing a handshake" +fi +log "re-pointing peer at $target_ep" +run wg set "$IFACE" peer "$peer" endpoint "$target_ep" +exit 0 diff --git a/active/device_raspberry_pi/systemd/wg0-watchdog.service b/active/device_raspberry_pi/systemd/wg0-watchdog.service new file mode 100644 index 0000000..91aaf3e --- /dev/null +++ b/active/device_raspberry_pi/systemd/wg0-watchdog.service @@ -0,0 +1,12 @@ +[Unit] +Description=Keep the WireGuard wg0 tunnel reachable for SSH +Documentation=man:wg-quick(8) + +[Service] +Type=oneshot +# Never let a hung systemctl restart inside the script wedge the watchdog. +TimeoutStartSec=120 +ExecStart=/usr/local/sbin/wg0-watchdog +StandardOutput=journal +StandardError=journal +Nice=10 diff --git a/active/device_raspberry_pi/systemd/wg0-watchdog.timer b/active/device_raspberry_pi/systemd/wg0-watchdog.timer new file mode 100644 index 0000000..da80599 --- /dev/null +++ b/active/device_raspberry_pi/systemd/wg0-watchdog.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Check the WireGuard wg0 tunnel every minute and heal it if needed + +[Timer] +# First check shortly after boot, then every minute. Persistent=true catches up +# if the machine was suspended/offline. +OnBootSec=30s +OnUnitActiveSec=60s +AccuracySec=15s +RandomizedDelaySec=5s +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/active/device_raspberry_pi/systemd/wg0.conf.template b/active/device_raspberry_pi/systemd/wg0.conf.template new file mode 100644 index 0000000..58bc7f7 --- /dev/null +++ b/active/device_raspberry_pi/systemd/wg0.conf.template @@ -0,0 +1,20 @@ +# wg0.conf template — /etc/wireguard/wg0.conf on the host, mode 0600 root:root. +# Never commit a real copy: this file holds the tunnel private key. +# Generate the key with `umask 077; wg genkey | tee privatekey | wg pubkey`. + +[Interface] +PrivateKey = +Address = +# Deliberately no `DNS` key: any resolver listed here is pushed as a system +# upstream while the tunnel is up, which makes all name resolution depend on the +# tunnel. Omitting it keeps the LAN's DHCP resolvers. Debian's wg-quick has no +# `DNS = none` - a non-IP value becomes a search domain. + +[Peer] +PublicKey = +AllowedIPs = +Endpoint = +# Required when the client is behind NAT and must be reachable inbound (SSH): +# without it the mapping expires and the server cannot initiate. 25s survives +# typical 30-60s NAT UDP timeouts. +PersistentKeepalive = 25 diff --git a/active/device_raspberry_pi/systemd/wireguard.md b/active/device_raspberry_pi/systemd/wireguard.md new file mode 100644 index 0000000..6ce45ad --- /dev/null +++ b/active/device_raspberry_pi/systemd/wireguard.md @@ -0,0 +1,162 @@ +# Runbook: WireGuard client that survives reboot, NAT, and roaming + +For a Pi whose SSH address is a **WireGuard tunnel address** rather than a LAN +address: the physical interface is ordinary DHCP and the Pi moves between +networks, while the tunnel peer is a reachable server. Reachability then depends +on three separate things, each of which fails independently — the unit starting +at boot, the NAT mapping staying open, and the peer hostname still resolving. + +Placeholders used below: + +| Placeholder | Meaning | +| --- | --- | +| `$HOST` | `root@
` | +| `` | this Pi's tunnel address, e.g. `/32` | +| `` | network routed through the tunnel, e.g. `/` | +| `` | tunnel server endpoint | +| `` | the server's WireGuard public key | + +## Files on the host + +| Path | Purpose | +| --- | --- | +| `/etc/wireguard/wg0.conf` | tunnel config, mode `0600` — see `wg0.conf.template`, never stored in this repo | +| `/etc/systemd/system/wg-quick@wg0.service.d/10-boot-resilience.conf` | retry a failed boot-time `up`: `Restart=on-failure`, `RestartSec=15`, `StartLimitIntervalSec=0`, `WG_ENDPOINT_RESOLUTION_RETRIES=60`, `TimeoutStartSec=120` | +| `/usr/local/sbin/wg0-watchdog` | heals the tunnel once a minute | +| `/etc/systemd/system/wg0-watchdog.service` | oneshot that runs the script | +| `/etc/systemd/system/wg0-watchdog.timer` | `OnBootSec=30s`, `OnUnitActiveSec=60s`, `Persistent=true` | +| `/var/lib/wg0-watchdog/endpoint` | last-known-good endpoint IP, used when DNS is unavailable | + +## Failure modes this closes + +1. **No `PersistentKeepalive`.** Behind NAT the server can only send to the Pi + while the mapping lives, so once it expires the tunnel looks configured but + inbound SSH times out. `PersistentKeepalive = 25` punches UDP state often + enough for typical 30-60s NAT timeouts. +2. **A boot-time failure is permanent.** `wg-quick up` fails outright if the + endpoint hostname does not resolve yet or the server is down, and the unit is + not retried — so the tunnel stays dead until someone logs in locally. + `Restart=on-failure` with `StartLimitIntervalSec=0` retries forever; + `WG_ENDPOINT_RESOLUTION_RETRIES` is lowered from the stock `infinity` + (which hangs the unit indefinitely) so a retry can actually happen. +3. **Nothing heals a stale tunnel.** Roaming to a new network keeps the old + endpoint IP in the kernel, and a peer whose endpoint is unreachable simply + stops handshaking without the unit ever noticing, because the unit already + "succeeded". The watchdog acts on handshake age. + +## Install + +```bash +HOST=root@ +D=$PWD # run the snippets from a checkout of this directory + +ssh "$HOST" 'install -d -m 0755 /usr/local/sbin /etc/systemd/system/wg-quick@wg0.service.d' +tar -C "$D" -cf - wg0-watchdog wg0-watchdog.service wg0-watchdog.timer \ + 'wg-quick@wg0.service.d-10-boot-resilience.conf' | \ +ssh "$HOST" 'tar -xf - -C /root && + install -m 0755 /root/wg0-watchdog /usr/local/sbin/ && + install -m 0644 /root/wg0-watchdog.service /root/wg0-watchdog.timer /etc/systemd/system/ && + install -m 0644 "/root/wg-quick@wg0.service.d-10-boot-resilience.conf" \ + /etc/systemd/system/wg-quick@wg0.service.d/10-boot-resilience.conf && + rm -f /root/wg0-watchdog /root/wg0-watchdog.service /root/wg0-watchdog.timer \ + "/root/wg-quick@wg0.service.d-10-boot-resilience.conf" && + systemctl daemon-reload && systemctl enable --now wg-quick@wg0.service wg0-watchdog.timer' +``` + +Keep `PersistentKeepalive = 25` in `[Peer]` (see `wg0.conf.template`) and check +the config file is `0600` — a world-readable `wg0.conf` exposes the tunnel +private key to every local account: + +```bash +ssh "$HOST" 'ls -l /etc/wireguard/wg0.conf; wg show wg0 persistent-keepalive' +``` + +## Watchdog behaviour + +Oldest symptom first, one action per run: + +1. interface or peer gone → `systemctl restart --no-block wg-quick@wg0.service` +2. no `PersistentKeepalive` in the kernel → `wg set ... persistent-keepalive ` + (covers a config edited without a reload) +3. handshake older than `WG_WD_STALE` (default 120s) → re-point the peer at the + endpoint, preferring a fresh resolution of the hostname from the config file, + then the kernel's current endpoint, then the cached IP +4. handshake older than `WG_WD_STALE_RESTART` (default 300s) → restart the unit + +Restarts are gap-limited to one per `WG_WD_MIN_RESTART_GAP` (default 180s) so a +dead network cannot make the unit flap. `--no-block` keeps a hung `wg-quick up` +from wedging the watchdog. Peers, addresses, and routes are only ever added. +All thresholds and the interface/state paths are environment overrides, and +`--dry-run` prints the decision without acting. + +The endpoint resolution order matters when the tunnel config also pushes a DNS +server: that resolver is reachable only through the tunnel, so a dead tunnel can +mean the endpoint hostname cannot be resolved exactly when it is needed — hence +the cached-IP fallback. + +## DNS + +A `DNS = ` line in the interface section is pushed through +`resolvconf`/`systemd-resolved` while the tunnel is up, which can silently make +*all* name resolution on the host depend on the tunnel. If the Pi must resolve +names on whatever network it lands on, omit the `DNS` key entirely and let the +LAN's DHCP resolvers stand. + +Two gotchas: + +- Debian's `wg-quick` does **not** implement `DNS = none` — any non-IP value + becomes a search domain, so the line has to be absent, not set to `none`. +- A record already pushed by a previous `wg-quick up` survives the config edit; + withdraw it (`resolvconf -d `, name visible in `resolvconf --list`) and + confirm the local caching resolver's upstreams before believing it is gone. + +## Taking wg0 down on purpose + +Stop the watchdog first or the tunnel comes straight back: + +```bash +ssh "$HOST" 'systemctl stop wg0-watchdog.timer wg-quick@wg0.service' +ssh "$HOST" 'systemctl start wg0-watchdog.timer' # resume +``` + +## Check on it + +```bash +ssh "$HOST" 'wg show wg0 + systemctl status wg-quick@wg0 --no-pager | head -6 + systemctl list-timers "wg0-watchdog*" --no-pager | head -3 + journalctl -u wg0-watchdog.service -b --no-pager | tail -15' +``` + +Dry-run the decision logic without touching the tunnel: + +```bash +ssh "$HOST" 'WG_WD_STATE_DIR=/tmp/wd WG_WD_STALE=0 /usr/local/sbin/wg0-watchdog --dry-run; rm -rf /tmp/wd' +``` + +Validate an edited config without disturbing the running tunnel: + +```bash +ssh "$HOST" 'wg-quick strip /etc/wireguard/wg0.conf | sed -E "s/^(PrivateKey[[:space:]]*=).*/\1 [REDACTED]/"' +``` + +## Diagnosis notes (generic, worth checking on any host) + +- `wg-quick@` can be `enabled` yet `inactive` while the interface exists + and passes traffic — it may have been created by hand or taken over by + NetworkManager as an assumed profile (`nmcli dev status` shows such a device + as `connected (externally)`). Nothing owns it: `systemctl reload` fails with + "unit not active", so apply changes with `wg set` or a full restart, and let + `systemctl restart` take ownership before trusting a reboot. +- `Restart=on-failure` does work for `Type=oneshot` on systemd 257 — worth + proving on a new distribution with a throwaway unit under + `/run/systemd/system` before relying on it: `Type=oneshot` + + `ExecStart=/bin/false`, then check `systemctl show -p NRestarts`. +- `systemd-analyze verify ` catches mistyped unit keys, and it only warns — + read the output instead of trusting the exit status. +- NetworkManager does not fight `wg-quick` as long as no keyfile exists for the + interface and its assumed profile keeps `autoconnect no`; check + `/etc/NetworkManager/system-connections/` before assuming that. +- The real end-to-end test is a reboot: DHCP → endpoint resolution → + `wg-quick up` → inbound SSH through the tunnel. `systemd-analyze + critical-chain ssh.service` after boot shows whether the ordering held.