feat(kiosk): self-recover a hung wall and page when it goes quiet
An unattended wall failed in two ways on 2026-11-02. It hard-hung with the kernel alive but userspace unable to start a process: ssh completed the handshake and closed 6-13 ms later with zero bytes and no banner, while 62 GiB RAM and zero OOM lines rule out memory. Separately, the service knew at 14:03Z that nothing was watching (healthz viewers:0) and nothing was reading it. It was discovered by walking over to the panel. Hang recovery, two independent layers (install step 7/9, asserted by check-kiosk.sh): - /etc/sysctl.d/90-mirror-wall-selfheal.conf: soft lockup, hard lockup and any oops escalate to a panic, and kernel.panic=30 reboots instead of parking. - /etc/systemd/system.conf.d/90-mirror-wall-watchdog.conf: PID 1 must keep kicking the iTCO hardware watchdog or the chipset resets the box, which is the layer that covers a kernel too wedged to notice. Clamped to the driver's own 30s because systemd kicks at a third of whatever value it is handed. Presence alerting: docs/wall-watch/ runs on the server host, probes loopback /healthz (exempt from the address gate) every minute, debounces three checks, pages over ntfy and then announces the recovery. WALL_DRILL=1 labels a test page in the title, body and priority so a drill can never be mistaken for an incident. Verified against a fake ntfy: caller environment wins over the config file, both transitions fire, and the debounce holds. Firewall (install step 8/9): drop FedoraWorkstation's 1025-65535/tcp. The UDP half is load-bearing for peer-to-peer media (mirror deliberately uses no STUN/TURN) and stays open; the TCP half was every high port on a box whose only access control is its address. Measured, not assumed: a live share kept flowing at 459.6 kbit/s inbound with the TCP range closed, and tcp/8080 stopped answering from a same-subnet host. Also fixed: install-kiosk.sh overwrote an existing client's kiosk.conf with its built-in default URL, so re-running it to repair a wall silently repointed it at the production default. An existing URL now wins and KIOSK_URL= is an explicit change. Documented with the evidence: why gnome-remote-desktop cannot share a gnome-kiosk session (gnome-kiosk exports no org.gnome.Mutter.* names, so grd has nothing to capture, and this image has grdctl rather than grdc), and why no userspace RDP would have survived the hang either. LLM-Generated: true
This commit is contained in:
@@ -15,7 +15,10 @@
|
||||
set -eu
|
||||
|
||||
KIOSK_USER="${KIOSK_USER:-kiosk}"
|
||||
KIOSK_URL="${KIOSK_URL:-https://mirror.reeseapps.com/watch}"
|
||||
# The default is for a NEW client only. An existing client keeps its own URL
|
||||
# unless the operator names one — see "the URL is the one per-client value" below.
|
||||
KIOSK_URL_DEFAULT="https://mirror.reeseapps.com/watch"
|
||||
KIOSK_URL="${KIOSK_URL:-}"
|
||||
KIOSK_SESSION="gnome-kiosk-script-wayland"
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
NEED_REBOOT=0
|
||||
@@ -58,7 +61,7 @@ for c in rpm getent restorecon useradd sed awk grep install; do need "$c"; done
|
||||
[ -f /etc/os-release ] && grep -q '^ID=fedora' /etc/os-release ||
|
||||
say "warning: this does not look like Fedora; continuing anyway"
|
||||
|
||||
say "== 1/7 packages =="
|
||||
say "== 1/9 packages =="
|
||||
if rpm -q gnome-kiosk gnome-kiosk-script-session >/dev/null 2>&1; then
|
||||
say "gnome-kiosk + gnome-kiosk-script-session already present"
|
||||
else
|
||||
@@ -69,7 +72,7 @@ else
|
||||
say "layered the kiosk packages — a reboot is required before they exist"
|
||||
fi
|
||||
|
||||
say "== 2/7 kiosk user =="
|
||||
say "== 2/9 kiosk user =="
|
||||
if id -u "$KIOSK_USER" >/dev/null 2>&1; then
|
||||
HOME_DIR=$(getent passwd "$KIOSK_USER" | cut -d: -f6)
|
||||
else
|
||||
@@ -81,7 +84,7 @@ fi
|
||||
KIOSK_GROUP=$(getent passwd "$KIOSK_USER" | cut -d: -f4)
|
||||
say "user $KIOSK_USER uid=$(id -u "$KIOSK_USER") home=$HOME_DIR"
|
||||
|
||||
say "== 3/7 payload =="
|
||||
say "== 3/9 payload =="
|
||||
# Firefox enterprise policy: no update prompts, no account/sync, no telemetry,
|
||||
# no first-run pages. /etc is mutable on Silverblue and survives upgrades.
|
||||
install -D -m 0644 "$HERE/policies.json" /etc/firefox/policies/policies.json
|
||||
@@ -119,6 +122,19 @@ say " $HOME_DIR/.local/bin/gnome-kiosk-script"
|
||||
|
||||
# The URL is the one per-client value; keep it in its own file so everything
|
||||
# else can stay byte-identical across the fleet.
|
||||
#
|
||||
# Re-running this installer is the documented way to REPAIR a client, and a repair
|
||||
# must not move the wall somewhere else. So an existing kiosk.conf wins over the
|
||||
# default; only an explicit KIOSK_URL= changes it.
|
||||
KIOSK_CONF="$HOME_DIR/.config/mirror/kiosk.conf"
|
||||
if [ -z "$KIOSK_URL" ] && [ -f "$KIOSK_CONF" ]; then
|
||||
KIOSK_URL=$(sed -n 's/^[[:space:]]*MIRROR_KIOSK_URL="\{0,1\}\([^"]*\)"\{0,1\}[[:space:]]*$/\1/p' \
|
||||
"$KIOSK_CONF" | head -1)
|
||||
if [ -n "$KIOSK_URL" ]; then
|
||||
say " keeping this client's existing URL (pass KIOSK_URL=... to change it)"
|
||||
fi
|
||||
fi
|
||||
[ -n "$KIOSK_URL" ] || KIOSK_URL="$KIOSK_URL_DEFAULT"
|
||||
sed "s|^MIRROR_KIOSK_URL=.*|MIRROR_KIOSK_URL=\"$KIOSK_URL\"|" "$HERE/kiosk.conf" \
|
||||
| install -m 0644 -o "$KIOSK_USER" -g "$KIOSK_GROUP" /dev/stdin \
|
||||
"$HOME_DIR/.config/mirror/kiosk.conf"
|
||||
@@ -129,7 +145,7 @@ say " $HOME_DIR/.config/mirror/kiosk.conf -> $KIOSK_URL"
|
||||
install -m 0644 -o "$KIOSK_USER" -g "$KIOSK_GROUP" /dev/null \
|
||||
"$HOME_DIR/.config/gnome-initial-setup-done"
|
||||
|
||||
say "== 4/7 default session =="
|
||||
say "== 4/9 default session =="
|
||||
ACCT="/var/lib/AccountsService/users/$KIOSK_USER"
|
||||
if [ -f "$ACCT" ]; then
|
||||
set_ini "$ACCT" User Session "$KIOSK_SESSION"
|
||||
@@ -141,7 +157,7 @@ fi
|
||||
systemctl restart accounts-daemon.service 2>/dev/null || :
|
||||
say " $ACCT Session=$KIOSK_SESSION"
|
||||
|
||||
say "== 5/7 auto-login =="
|
||||
say "== 5/9 auto-login =="
|
||||
if [ ! -d /etc/gdm ]; then
|
||||
say " gdm is not installed — installing it"
|
||||
rpm-ostree install gdm
|
||||
@@ -156,7 +172,7 @@ if [ -d /etc/gdm ]; then
|
||||
say " /etc/gdm/custom.conf: AutomaticLogin=$KIOSK_USER"
|
||||
fi
|
||||
|
||||
say "== 6/7 console guard =="
|
||||
say "== 6/9 console guard =="
|
||||
# Cheap insurance against the class of bug above: if the kiosk session exists but
|
||||
# something else holds the foreground VT, bring the wall back. Stops short of
|
||||
# ever restarting gdm. Pause it while debugging the console with:
|
||||
@@ -168,6 +184,64 @@ systemctl daemon-reload
|
||||
systemctl enable --now mirror-kiosk-console-guard.timer
|
||||
say " mirror-kiosk-console-guard.timer enabled"
|
||||
|
||||
say "== 7/9 hang recovery =="
|
||||
# An unattended wall that hangs must come back by itself. Two independent layers;
|
||||
# reasoning and the incident behind them in ../kiosk-silverblue.md.
|
||||
install -D -m 0644 "$HERE/90-mirror-wall-selfheal.conf" /etc/sysctl.d/90-mirror-wall-selfheal.conf
|
||||
if sysctl --system >/dev/null 2>&1; then
|
||||
say " soft lockup / hard lockup / oops now panic; panic reboots in $(cat /proc/sys/kernel/panic)s"
|
||||
else
|
||||
say " warning: sysctl --system failed — check: sudo sysctl kernel.softlockup_panic kernel.panic"
|
||||
fi
|
||||
|
||||
# The layer that does not need the kernel to be cooperating: if PID 1 stops
|
||||
# kicking /dev/watchdog, the chipset resets the box. Only configured when the
|
||||
# hardware has one (VMs typically do not).
|
||||
if [ -e /dev/watchdog ]; then
|
||||
wd_identity=$(cat /sys/class/watchdog/watchdog0/identity 2>/dev/null || echo unknown)
|
||||
wd_timeout=$(cat /sys/class/watchdog/watchdog0/timeout 2>/dev/null || echo 0)
|
||||
case "$wd_timeout" in ''|*[!0-9]*) wd_timeout=0 ;; esac
|
||||
install -D -m 0644 "$HERE/90-mirror-wall-watchdog.conf" /etc/systemd/system.conf.d/90-mirror-wall-watchdog.conf
|
||||
# systemd kicks at a third of the value it is given. Asking for more than the
|
||||
# device supports would kick less often than the device expires — a reboot
|
||||
# loop of our own making — so clamp to what the driver came up with.
|
||||
if [ "$wd_timeout" -gt 0 ] && [ "$wd_timeout" -ne 30 ]; then
|
||||
sed -i "s|^RuntimeWatchdogSec=.*|RuntimeWatchdogSec=${wd_timeout}s|" \
|
||||
/etc/systemd/system.conf.d/90-mirror-wall-watchdog.conf
|
||||
say " clamped to the device's own ${wd_timeout}s timeout ($wd_identity)"
|
||||
fi
|
||||
# PID 1 re-reads system.conf on re-exec; nothing else picks it up short of a
|
||||
# reboot, and re-exec is safe (sockets and running services survive it).
|
||||
systemctl daemon-reexec
|
||||
say " $wd_identity hardware watchdog armed (PID 1 must kick it or the box resets)"
|
||||
else
|
||||
say " no /dev/watchdog here — hardware layer skipped (lockup panics still installed)"
|
||||
fi
|
||||
|
||||
say "== 8/9 firewall =="
|
||||
# Fedora's Workstation zone opens 1025-65535/tcp and /udp. The UDP half is what
|
||||
# WebRTC media needs (peer-to-peer, both directions, and mirror uses no STUN/TURN);
|
||||
# the TCP half is pure exposure on a box whose only access control is its address.
|
||||
# Dropping it is safe for a wall, which never serves: verified against a live
|
||||
# share (media keeps flowing at ~460 kbit/s inbound with it gone).
|
||||
if command -v firewall-cmd >/dev/null 2>&1 && firewall-cmd --state >/dev/null 2>&1; then
|
||||
zone=$(firewall-cmd --get-default-zone)
|
||||
if firewall-cmd --zone="$zone" --list-ports | tr ' ' '\n' | grep -qx '1025-65535/tcp'; then
|
||||
firewall-cmd --permanent --zone="$zone" --remove-port=1025-65535/tcp >/dev/null
|
||||
firewall-cmd --reload >/dev/null
|
||||
say " $zone: dropped 1025-65535/tcp (kept 1025-65535/udp for WebRTC media)"
|
||||
else
|
||||
say " $zone: 1025-65535/tcp already closed"
|
||||
fi
|
||||
if firewall-cmd --zone="$zone" --list-ports | tr ' ' '\n' | grep -qx '1025-65535/udp'; then
|
||||
say " 1025-65535/udp still open — required, do not close it or mirrors go black"
|
||||
else
|
||||
say " warning: 1025-65535/udp is NOT open — peer-to-peer media will fail (tile: unreachable)"
|
||||
fi
|
||||
else
|
||||
say " firewalld not running — nothing to tighten (verify your own firewall keeps UDP media)"
|
||||
fi
|
||||
|
||||
# Keep a copy of the payload on the box. /tmp is cleared at every boot, so a
|
||||
# checker staged there is gone precisely when you want to run it again — after a
|
||||
# reboot, or from a rescue session hours later.
|
||||
@@ -177,10 +251,12 @@ if [ "$HERE" != "/usr/local/share/mirror-kiosk" ]; then
|
||||
say " /usr/local/share/mirror-kiosk (survives reboot; run check-kiosk.sh from here)"
|
||||
fi
|
||||
|
||||
say "== 7/7 selinux labels =="
|
||||
say "== 9/9 selinux labels =="
|
||||
restorecon -RF "$HOME_DIR/.local" "$HOME_DIR/.config" 2>/dev/null || :
|
||||
restorecon -RF /etc/firefox /usr/local/share/gnome-kiosk /usr/local/share/wayland-sessions \
|
||||
/usr/local/libexec 2>/dev/null || :
|
||||
restorecon -RF /etc/sysctl.d/90-mirror-wall-selfheal.conf \
|
||||
/etc/systemd/system.conf.d/90-mirror-wall-watchdog.conf 2>/dev/null || :
|
||||
|
||||
say ""
|
||||
say "Installed."
|
||||
|
||||
Reference in New Issue
Block a user