#!/bin/sh # wall-watch.sh — know within minutes that the mirror wall stopped watching. # # Run ON THE MIRROR SERVER HOST from mirror-wall-watch.timer (once a minute). # It reads the service's own /healthz over loopback and posts to ntfy when no # viewer is attached, or when the service stops answering at all — and again when # the wall comes back. # # Why it exists: on 2026-11-02 the reference wall hard-hung at 14:03Z. The # service knew the instant it happened — /healthz reported viewers:0 — and said # nothing, because nothing was listening. The failure was eventually found by # walking up to the panel, which was in another room of another building. # # /healthz is the right probe precisely because it is exempt from the address # gate (AGENTS.md §3): a loopback check works even when MIRROR_ALLOW_CIDRS # excludes loopback, which is also why the container healthcheck can work. # # Config: /etc/mirror-wall-watch.env (root-owned 0600 — it holds the ntfy token) # WALL_HEALTH_URL default https://127.0.0.1/healthz # WALL_EXPECT_VIEWERS default 1; set 0 to stop alerting on "nobody watching" # WALL_THRESHOLD default 3 consecutive bad checks (~3 min) before shouting # WALL_RENOTIFY_SECS default 3600; keeps shouting while it stays down # WALL_STATUS_URL optional; the URL a notification opens # NTFY_URL NTFY_TOKEN NTFY_TOPIC # # Manual run (the timer runs it the same way): # sudo sh /usr/local/libexec/mirror-wall-watch.sh # sudo sh -c 'WALL_HEALTH_URL=https://127.0.0.1/nope WALL_THRESHOLD=1 sh /usr/local/libexec/mirror-wall-watch.sh' # fire a test alert # # Honest limit: this runs on the server, so it cannot tell you the *server* is # down. That is a different alert, and it belongs to whatever watches this host. set -eu CONF="${WALL_WATCH_CONF:-/etc/mirror-wall-watch.env}" STATE_DIR="${WALL_STATE_DIR:-/var/lib/mirror-wall-watch}" STATE_FILE="$STATE_DIR/state" log() { printf '%s\n' "$*" # Under systemd stdout already IS the journal, so writing to /dev/log as well # puts two copies of every check in it. Keep the syslog trail for interactive # runs, where stdout is a terminal and nothing else would record it. [ -t 1 ] && logger -t mirror-wall-watch -- "$*" 2>/dev/null return 0 } die() { log "error: $*"; exit 1; } command -v curl >/dev/null 2>&1 || die "curl is required" # Root is not inherently required, only write access to the state directory — but # the default lives next to root-only config, so a non-root run must say where it # is going to fail rather than fail at the interesting moment. if ! install -d -m 0700 "$STATE_DIR" 2>/dev/null; then die "cannot create $STATE_DIR — run me as root, or set WALL_STATE_DIR somewhere writable" fi # Apply KEY=VALUE from the config file, but only for variables the caller did not # already set. An explicit environment must win over the file — the same precedence # systemd gives Environment= over EnvironmentFile= — otherwise "run the checker # against a different URL for a moment" silently does nothing, which is exactly the # kind of thing you only discover when the alert you were testing never arrives. load_conf() { [ -r "$1" ] || return 0 while IFS= read -r line || [ -n "$line" ]; do case "$line" in ''|'#'*) continue ;; esac key=${line%%=*} val=${line#*=} key=$(printf '%s' "$key" | tr -d '[:space:]') val=$(printf '%s' "$val" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') case "$val" in '\"'*'"') val=${val#\"}; val=${val%\"} ;; esac # Only sane shell identifiers get assigned. case "$key" in ''|*[!A-Za-z0-9_]*|[0-9]*) continue ;; esac eval "cur=\${$key:-}" [ -n "$cur" ] || eval "$key=\$val" done <"$1" } load_conf "$CONF" : "${WALL_HEALTH_URL:=https://127.0.0.1/healthz}" : "${WALL_EXPECT_VIEWERS:=1}" : "${WALL_THRESHOLD:=3}" : "${WALL_RENOTIFY_SECS:=3600}" # A drill is an alert fired on purpose to prove the path works. It must never be # mistaken for an incident, so it says so in the title, the body and the priority # — an alerting tool whose test page looks like a real page trains people to # ignore it. : "${WALL_DRILL:=0}" if [ "$WALL_DRILL" = "1" ]; then DRILL_TAG="[drill] " DRILL_NOTE="> **This is a drill.** Someone ran the test command on \`$(hostname)\`; nothing is wrong with the wall. What it checked is the path: probe -> ntfy -> your phone." DOWN_PRIORITY=3 else DRILL_TAG="" DRILL_NOTE="" DOWN_PRIORITY=4 fi # Posts one notification. Returns non-zero instead of exiting: a failed alert # must not stop the state file being written, or the next run loses the counter # and the wall pages once a minute forever. notify() { prio=$1 tags=$2 title=$3 body=$4 if [ -z "${NTFY_URL:-}" ] || [ -z "${NTFY_TOPIC:-}" ]; then log "error: ntfy is not configured (NTFY_URL/NTFY_TOPIC in $CONF)" return 1 fi # Collect the headers in the positional parameters, and only the ones that # have values, so ntfy never receives an empty Authorization or X-Click. set -- -H "X-Priority: $prio" -H "X-Tags: $tags" -H "X-Title: $title" -H "X-Markdown: yes" [ -n "${NTFY_TOKEN:-}" ] && set -- "$@" -H "Authorization: Bearer $NTFY_TOKEN" [ -n "${WALL_STATUS_URL:-}" ] && set -- "$@" -H "X-Click: $WALL_STATUS_URL" if curl -fsS -m 10 "$@" -d "$body" "$NTFY_URL/$NTFY_TOPIC" >/dev/null 2>&1; then log "notified (priority $prio): $title" else log "error: could not reach $NTFY_URL — the alert did not go out" return 1 fi } # ---- read the service's own truth ------------------------------------------------- probe="" viewers="" if probe=$(curl -fsS -m 8 -k "$WALL_HEALTH_URL" 2>/dev/null); then if command -v jq >/dev/null 2>&1; then viewers=$(printf '%s' "$probe" | jq -r '.viewers // empty' 2>/dev/null || :) else viewers=$(printf '%s' "$probe" | sed -n 's/.*"viewers"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' | head -1) fi # 200 but no parseable viewer count is a broken answer, not a healthy one. [ -n "$viewers" ] || probe="" fi if [ -z "$probe" ]; then condition="unreachable" summary="the mirror service did not answer \`$WALL_HEALTH_URL\`" elif [ "$WALL_EXPECT_VIEWERS" -gt 0 ] && [ "$viewers" -lt "$WALL_EXPECT_VIEWERS" ]; then condition="quiet" summary="the service is up but **nothing is watching** (viewers=$viewers)" else condition="ok" summary="viewers=$viewers" fi # ---- state machine ---------------------------------------------------------------- state="ok" bad=0 last_alert=0 if [ -r "$STATE_FILE" ]; then # shellcheck disable=SC1090 . "$STATE_FILE" 2>/dev/null || : fi now=$(date +%s) NOTIFY_FAILED=0 if [ "$condition" = "ok" ]; then if [ "$state" != "ok" ]; then notify 3 "heavy_check_mark,recovered" "${DRILL_TAG}Wall back" \ "**${DRILL_TAG}The mirror wall is watching again.** was: \`$state\` now: $summary host: \`$(hostname)\` · $(date -u '+%Y-%m-%d %H:%M:%SZ') $DRILL_NOTE" || NOTIFY_FAILED=1 fi state="ok"; bad=0; last_alert=0 else if [ "$state" = "$condition" ]; then bad=$((bad + 1)); else bad=1; fi if [ "$bad" -ge "$WALL_THRESHOLD" ] && { [ "$state" != "$condition" ] || [ $((now - last_alert)) -ge "$WALL_RENOTIFY_SECS" ]; }; then hint="Recover it: \`ssh \` then \`sudo sh /usr/local/share/mirror-kiosk/check-kiosk.sh\`; if ssh itself will not open a session the box is hung, not offline — the watchdog should reset it within a minute of this, and if it does not, power-cycle it." [ "$condition" = "unreachable" ] && hint="Check the service first (\`systemctl status\`, container logs) before the wall: this host may be the thing that is broken." notify "$DOWN_PRIORITY" "rotating_light,wall-down" "${DRILL_TAG}Wall down" \ "**${DRILL_TAG}Mirror wall: $condition** $summary \`\`\` $probe \`\`\` $hint host: \`$(hostname)\` · $(date -u '+%Y-%m-%d %H:%M:%SZ') · bad check #$bad $DRILL_NOTE" || NOTIFY_FAILED=1 # Only bank the re-notify timestamp if it actually went out, so a ntfy # outage is retried on the next tick rather than swallowed. [ "$NOTIFY_FAILED" -eq 0 ] && last_alert=$now fi state="$condition" fi umask 077 cat >"$STATE_FILE" <