mirror/docs/wall-watch/wall-watch.sh

#!/bin/sh
# wall-watch.sh — know within minutes that the mirror wall stopped watching.
#
# Run ON THE MIRROR SERVER HOST from mirror-wall-watch.timer (once a minute).
# It reads the service's own /healthz over loopback and posts to ntfy when no
# viewer is attached, or when the service stops answering at all — and again when
# the wall comes back.
#
# Why it exists: on 2026-11-02 the reference wall hard-hung at 14:03Z. The
# service knew the instant it happened — /healthz reported viewers:0 — and said
# nothing, because nothing was listening. The failure was eventually found by
# walking up to the panel, which was in another room of another building.
#
# /healthz is the right probe precisely because it is exempt from the address
# gate (AGENTS.md §3): a loopback check works even when MIRROR_ALLOW_CIDRS
# excludes loopback, which is also why the container healthcheck can work.
#
# Config: /etc/mirror-wall-watch.env (root-owned 0600 — it holds the ntfy token)
#   WALL_HEALTH_URL       default https://127.0.0.1/healthz
#   WALL_EXPECT_VIEWERS   default 1; set 0 to stop alerting on "nobody watching"
#   WALL_THRESHOLD        default 3 consecutive bad checks (~3 min) before shouting
#   WALL_RENOTIFY_SECS    default 3600; keeps shouting while it stays down
#   WALL_STATUS_URL       optional; the URL a notification opens
#   NTFY_URL NTFY_TOKEN NTFY_TOPIC
#
# Manual run (the timer runs it the same way):
#   sudo sh /usr/local/libexec/mirror-wall-watch.sh
#   sudo sh -c 'WALL_HEALTH_URL=https://127.0.0.1/nope WALL_THRESHOLD=1 sh /usr/local/libexec/mirror-wall-watch.sh'   # fire a test alert
#
# Honest limit: this runs on the server, so it cannot tell you the *server* is
# down. That is a different alert, and it belongs to whatever watches this host.

set -eu

CONF="${WALL_WATCH_CONF:-/etc/mirror-wall-watch.env}"
STATE_DIR="${WALL_STATE_DIR:-/var/lib/mirror-wall-watch}"
STATE_FILE="$STATE_DIR/state"

log() {
    printf '%s\n' "$*"
    # Under systemd stdout already IS the journal, so writing to /dev/log as well
    # puts two copies of every check in it. Keep the syslog trail for interactive
    # runs, where stdout is a terminal and nothing else would record it.
    [ -t 1 ] && logger -t mirror-wall-watch -- "$*" 2>/dev/null
    return 0
}
die() { log "error: $*"; exit 1; }

command -v curl >/dev/null 2>&1 || die "curl is required"
# Root is not inherently required, only write access to the state directory — but
# the default lives next to root-only config, so a non-root run must say where it
# is going to fail rather than fail at the interesting moment.
if ! install -d -m 0700 "$STATE_DIR" 2>/dev/null; then
    die "cannot create $STATE_DIR — run me as root, or set WALL_STATE_DIR somewhere writable"
fi

# Apply KEY=VALUE from the config file, but only for variables the caller did not
# already set. An explicit environment must win over the file — the same precedence
# systemd gives Environment= over EnvironmentFile= — otherwise "run the checker
# against a different URL for a moment" silently does nothing, which is exactly the
# kind of thing you only discover when the alert you were testing never arrives.
load_conf() {
    [ -r "$1" ] || return 0
    while IFS= read -r line || [ -n "$line" ]; do
        case "$line" in ''|'#'*) continue ;; esac
        key=${line%%=*}
        val=${line#*=}
        key=$(printf '%s' "$key" | tr -d '[:space:]')
        val=$(printf '%s' "$val" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
        case "$val" in '\"'*'"') val=${val#\"}; val=${val%\"} ;; esac
        # Only sane shell identifiers get assigned.
        case "$key" in ''|*[!A-Za-z0-9_]*|[0-9]*) continue ;; esac
        eval "cur=\${$key:-}"
        [ -n "$cur" ] || eval "$key=\$val"
    done <"$1"
}
load_conf "$CONF"

: "${WALL_HEALTH_URL:=https://127.0.0.1/healthz}"
: "${WALL_EXPECT_VIEWERS:=1}"
: "${WALL_THRESHOLD:=3}"
: "${WALL_RENOTIFY_SECS:=3600}"

# A drill is an alert fired on purpose to prove the path works. It must never be
# mistaken for an incident, so it says so in the title, the body and the priority
# — an alerting tool whose test page looks like a real page trains people to
# ignore it.
: "${WALL_DRILL:=0}"
if [ "$WALL_DRILL" = "1" ]; then
    DRILL_TAG="[drill] "
    DRILL_NOTE="> **This is a drill.** Someone ran the test command on \`$(hostname)\`; nothing is wrong with the wall. What it checked is the path: probe -> ntfy -> your phone."
    DOWN_PRIORITY=3
else
    DRILL_TAG=""
    DRILL_NOTE=""
    DOWN_PRIORITY=4
fi

# Posts one notification. Returns non-zero instead of exiting: a failed alert
# must not stop the state file being written, or the next run loses the counter
# and the wall pages once a minute forever.
notify() {
    prio=$1 tags=$2 title=$3 body=$4
    if [ -z "${NTFY_URL:-}" ] || [ -z "${NTFY_TOPIC:-}" ]; then
        log "error: ntfy is not configured (NTFY_URL/NTFY_TOPIC in $CONF)"
        return 1
    fi

    # Collect the headers in the positional parameters, and only the ones that
    # have values, so ntfy never receives an empty Authorization or X-Click.
    set -- -H "X-Priority: $prio" -H "X-Tags: $tags" -H "X-Title: $title" -H "X-Markdown: yes"
    [ -n "${NTFY_TOKEN:-}" ] && set -- "$@" -H "Authorization: Bearer $NTFY_TOKEN"
    [ -n "${WALL_STATUS_URL:-}" ] && set -- "$@" -H "X-Click: $WALL_STATUS_URL"

    if curl -fsS -m 10 "$@" -d "$body" "$NTFY_URL/$NTFY_TOPIC" >/dev/null 2>&1; then
        log "notified (priority $prio): $title"
    else
        log "error: could not reach $NTFY_URL — the alert did not go out"
        return 1
    fi
}

# ---- read the service's own truth -------------------------------------------------
probe=""
viewers=""
if probe=$(curl -fsS -m 8 -k "$WALL_HEALTH_URL" 2>/dev/null); then
    if command -v jq >/dev/null 2>&1; then
        viewers=$(printf '%s' "$probe" | jq -r '.viewers // empty' 2>/dev/null || :)
    else
        viewers=$(printf '%s' "$probe" |
            sed -n 's/.*"viewers"[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' | head -1)
    fi
    # 200 but no parseable viewer count is a broken answer, not a healthy one.
    [ -n "$viewers" ] || probe=""
fi

if [ -z "$probe" ]; then
    condition="unreachable"
    summary="the mirror service did not answer \`$WALL_HEALTH_URL\`"
elif [ "$WALL_EXPECT_VIEWERS" -gt 0 ] && [ "$viewers" -lt "$WALL_EXPECT_VIEWERS" ]; then
    condition="quiet"
    summary="the service is up but **nothing is watching** (viewers=$viewers)"
else
    condition="ok"
    summary="viewers=$viewers"
fi

# ---- state machine ----------------------------------------------------------------
state="ok"
bad=0
last_alert=0
if [ -r "$STATE_FILE" ]; then
    # shellcheck disable=SC1090
    . "$STATE_FILE" 2>/dev/null || :
fi
now=$(date +%s)

NOTIFY_FAILED=0

if [ "$condition" = "ok" ]; then
    if [ "$state" != "ok" ]; then
        notify 3 "heavy_check_mark,recovered" "${DRILL_TAG}Wall back" \
"**${DRILL_TAG}The mirror wall is watching again.**

was: \`$state\`
now: $summary
host: \`$(hostname)\` · $(date -u '+%Y-%m-%d %H:%M:%SZ')

$DRILL_NOTE" || NOTIFY_FAILED=1
    fi
    state="ok"; bad=0; last_alert=0
else
    if [ "$state" = "$condition" ]; then bad=$((bad + 1)); else bad=1; fi
    if [ "$bad" -ge "$WALL_THRESHOLD" ] &&
       { [ "$state" != "$condition" ] || [ $((now - last_alert)) -ge "$WALL_RENOTIFY_SECS" ]; }; then
        hint="Recover it: \`ssh <wall>\` then \`sudo sh /usr/local/share/mirror-kiosk/check-kiosk.sh\`; if ssh itself will not open a session the box is hung, not offline — the watchdog should reset it within a minute of this, and if it does not, power-cycle it."
        [ "$condition" = "unreachable" ] &&
            hint="Check the service first (\`systemctl status\`, container logs) before the wall: this host may be the thing that is broken."
        notify "$DOWN_PRIORITY" "rotating_light,wall-down" "${DRILL_TAG}Wall down" \
"**${DRILL_TAG}Mirror wall: $condition**

$summary

\`\`\`
$probe
\`\`\`

$hint

host: \`$(hostname)\` · $(date -u '+%Y-%m-%d %H:%M:%SZ') · bad check #$bad

$DRILL_NOTE" || NOTIFY_FAILED=1
        # Only bank the re-notify timestamp if it actually went out, so a ntfy
        # outage is retried on the next tick rather than swallowed.
        [ "$NOTIFY_FAILED" -eq 0 ] && last_alert=$now
    fi
    state="$condition"
fi

umask 077
cat >"$STATE_FILE" <<EOF
state=$state
bad=$bad
last_alert=$last_alert
EOF

# One greppable line per run, so `journalctl -u mirror-wall-watch` reads as a
# timeline of the wall's health rather than as silence.
log "$condition: $summary (state=$state bad=$bad threshold=$WALL_THRESHOLD)"

# A needed-but-undeliverable alert fails the unit, so `systemctl show`
# mirror-wall-watch.service shows why nobody got paged.
exit "$NOTIFY_FAILED"