deploy: non-fatal service restarts; mosquitto password_file safeguard and post-check probe

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9

deploy.sh

@@ -28,6 +28,10 @@
 #         VANLINK_HOSTNAME=<name>          # test override: host name used by --apply-preflight (default `hostname -s`; ignored when run as root)
 #         VANLINK_FAILOVER_STATE=<file>    # test override: van-failover state file read by --post-check (default /run/van-failover/state.json)
 #         VANLINK_SYSLEDS=<dir>            # test override: sysfs LED dir read by --post-check (default /sys/class/leds; ignored when run as root)
+#         VANLINK_MOSQUITTO_CONF=<file>    # test override: mosquitto.conf read by --post-check / --mosquitto-pwfile-check (default /etc/mosquitto/mosquitto.conf; conf.d is taken
+#                                    # beside it; ignored when run as root)
+#         VANLINK_MOSQUITTO_USER=<name>    # test override: the broker's unprivileged user (default mosquitto; ignored when run as root)
+#         ./deploy.sh --mosquitto-pwfile-check [mosquitto.conf]  # read-only: prints the broker's password_file when the mosquitto user cannot read it, else nothing (exit 0)
 #         VANLINK_APPLY_STATUS=<file>      # test override: apply-network status file read by --post-check (default /var/lib/vanlink/apply-network.status)
 #         VANLINK_WAN_NETPLAN=<file>       # test override: installed WAN netplan compared by nm_wan_state_netplan (default /etc/netplan/50-van-wan.yaml)
 #         VANLINK_NM_DIR=<dir>             # test override: where existing NM *.nmconnection files are looked up (default /etc/NetworkManager/system-connections;
@@ -256,6 +260,22 @@ classify_radio() {  # stdin: `iw dev <if> info` text -> beaconing (has an ssid l
     echo silent
 }
 
+mosquitto_conf() {  # path of mosquitto.conf; the test knob is honoured only when not root
+    local c=/etc/mosquitto/mosquitto.conf
+    [ "$(id -u)" = 0 ] || c=${VANLINK_MOSQUITTO_CONF:-$c}
+    echo "$c"
+}
+
+mosquitto_pwfile_problem() {  # [mosquitto.conf] — prints the password_file path when it exists but the mosquitto user cannot read it; else nothing
+    local conf=${1:-$(mosquitto_conf)} user=mosquitto pw
+    [ "$(id -u)" = 0 ] || user=${VANLINK_MOSQUITTO_USER:-$user}
+    pw=$({ cat "$conf" "$(dirname "$conf")"/conf.d/*.conf 2>/dev/null || true; } | awk '/^[[:space:]]*password_file[[:space:]]/ { print $2 }' | tail -n1) || pw=
+    [ -n "$pw" ] && [ -e "$pw" ] || return 0
+    id "$user" >/dev/null 2>&1 && command -v runuser >/dev/null 2>&1 || return 0
+    runuser -u "$user" -- test -r "$pw" >/dev/null 2>&1 || echo "$pw"
+    return 0
+}
+
 post_check() {  # read-only live checks; every probe is guarded, warns instead of aborting
     set_units
     echo "== post-deploy check =="
@@ -342,6 +362,15 @@ post_check() {  # read-only live checks; every probe is guarded, warns instead o
         refused) warn "post-deploy: van-router-mqtt is refused by the MQTT broker — add credentials to /etc/van-router-mqtt/credentials.json (see the service journal)" ;;
         connected) echo "   ok    van-router-mqtt: MQTT connection ok" ;;
     esac
+    # g2b. mosquitto (the base router-status broker): unit active, password file readable by the broker's user
+    local mu mpw
+    mu=$(systemctl show -p ActiveState --value mosquitto 2>/dev/null) || mu=
+    if [ "$mu" = active ]; then echo "   ok    unit mosquitto active"
+    else warn "post-deploy: unit mosquitto is not active (ActiveState=${mu:-?})"; fi
+    mpw=$(mosquitto_pwfile_problem)
+    if [ -n "$mpw" ]; then
+        warn "post-deploy: mosquitto password file $mpw is not readable by the mosquitto user — the broker will fail to start on its next restart (fix: chown mosquitto:mosquitto $mpw; chmod 0600 $mpw)"
+    else echo "   ok    mosquitto password file readable (or none configured)"; fi
     if [ "$ENABLE_LEDS" = 1 ]; then
         # g3. front LEDs (van-leds): each configured LED exists; the static ones carry the expected trigger.
         #     The WAN LED follows the uplink, so its state is only reported, never warned about.
@@ -782,6 +811,9 @@ case "${1:-}" in
             [ -e "$path" ] && echo "REMOVE   $path"
         done || true
         exit 0 ;;
+    --mosquitto-pwfile-check)  # ./deploy.sh --mosquitto-pwfile-check [mosquitto.conf] — print the password_file the mosquitto user cannot read (read-only)
+        mosquitto_pwfile_problem "${2:-}"
+        exit 0 ;;
     --redact)   # ./deploy.sh --redact — stdin through redact_secrets (no config, no root)
         redact_secrets
         exit 0 ;;
@@ -1232,8 +1264,14 @@ systemctl daemon-reexec
 # ignore-auto-dns only takes effect on a connection's next activation though —
 # an already-up WAN keeps its currently-applied DNS until it reconnects (or
 # reboot), deliberately not forced here (reconnecting a WAN flaps it).
-systemctl restart systemd-resolved
+systemctl restart systemd-resolved || warn "systemd-resolved failed to restart (journalctl -xeu systemd-resolved)"
-systemctl restart mosquitto
+# A password_file the mosquitto user cannot read (seen: root:root 0700) makes the restart fail and the broker stay down.
+mq_pw=$(mosquitto_pwfile_problem /etc/mosquitto/mosquitto.conf)
+if [ -n "$mq_pw" ]; then
+    chown mosquitto:mosquitto "$mq_pw" && chmod 0600 "$mq_pw"
+    warn "mosquitto password file $mq_pw was not readable by the mosquitto user — fixed ownership (mosquitto:mosquitto 0600)"
+fi
+systemctl restart mosquitto || warn "mosquitto failed to restart (journalctl -xeu mosquitto) — MQTT (Home Assistant, Li3, router status) is down"
 # networkd here owns only the AP radios + bridge + wired LAN port (no real uplink);
 # its wait-online would just stall network-online.target. NM-wait-online covers WANs.
 systemctl mask systemd-networkd-wait-online.service >/dev/null 2>&1 || true
@@ -1251,18 +1289,18 @@ fi
 if [ "$ENABLE_FRIGATE" = 1 ]; then
     systemctl start frigate || warn "frigate failed to start (podman/quadlet — check journalctl -u frigate)"
 fi
-systemctl restart van-thermal
+systemctl restart van-thermal || warn "van-thermal failed to restart (journalctl -xeu van-thermal)"
 if [ "$ENABLE_NVME_WATCH" = 1 ]; then
-    systemctl restart van-nvme-watch
+    systemctl restart van-nvme-watch || warn "van-nvme-watch failed to restart (journalctl -xeu van-nvme-watch)"
 fi
-systemctl restart van-sms-watch
+systemctl restart van-sms-watch || warn "van-sms-watch failed to restart (journalctl -xeu van-sms-watch)"
-systemctl restart van-gps-owntracks
+systemctl restart van-gps-owntracks || warn "van-gps-owntracks failed to restart (journalctl -xeu van-gps-owntracks)"
 if [ "$ENABLE_LI3" = 1 ]; then
-    systemctl restart van-li3-battery
+    systemctl restart van-li3-battery || warn "van-li3-battery failed to restart (journalctl -xeu van-li3-battery)"
 fi
-systemctl restart van-router-mqtt
+systemctl restart van-router-mqtt || warn "van-router-mqtt failed to restart (journalctl -xeu van-router-mqtt)"
 if [ "$ENABLE_LEDS" = 1 ]; then
-    systemctl restart van-leds
+    systemctl restart van-leds || warn "van-leds failed to restart (journalctl -xeu van-leds)"
 fi
 # Pick up unmanaged-devices changes so NM releases/keeps the right interfaces.
 nmcli general reload 2>/dev/null || systemctl reload NetworkManager 2>/dev/null || true
@@ -1276,7 +1314,7 @@ done
 # retrying and claims it the moment it enumerates (post_check reports absent radios).
 systemctl restart hostapd hostapd-2g || true
 # AP watchdogs last, after hostapd is back up (they only ever restart a wedged hostapd)
-systemctl restart van-ap-watchdog van-ap-watchdog-2g
+systemctl restart van-ap-watchdog van-ap-watchdog-2g || warn "an AP watchdog failed to restart (journalctl -xeu van-ap-watchdog)"
 networkctl reload 2>/dev/null || true
 
 post_check

tests/check-templates.sh

@@ -1049,4 +1049,39 @@ o=$(pl "$L" hosts/wan.conf); { ! echo "$o" | grep -q 'LED'; } && ok "22e: ENABLE
 grep -qF 'need["van-leds"]="ENABLE_LEDS"' tests/check-templates.sh && ok "22f: check 6 gates van-leds" || bad "22f: check 6 lacks van-leds"
 rm -rf "$t22"
 
+# 23. mosquitto: a non-fatal restart and the password_file safeguard
+if grep -nE '^[[:space:]]*systemctl restart mosquitto' deploy.sh | grep -qv '|| warn'; then bad "23a: bare 'systemctl restart mosquitto' (set -e would abort the deploy)"; else ok "23a: every systemctl restart mosquitto has || warn"; fi
+if awk '/^set_units\(\)/,0' deploy.sh | sed -n '/^echo "== apply =="/,/^post_check$/p' | grep -E '^[[:space:]]*systemctl restart ' | grep -v '||' | grep -qv 'systemd-networkd$'; then bad "23a: bare systemctl restart in the apply section"; else ok "23a: only the networkd restart is fatal in the apply section"; fi
+t23=$(mktemp -d); mkdir -p "$t23/etc/conf.d" "$t23/stub" "$t23/nostub"
+echo secret > "$t23/pw"
+printf '# password_file /nope\nlistener 1883\npassword_file %s\n' "$t23/pw" > "$t23/etc/mosquitto.conf"
+stubru() { printf '#!/bin/sh\n# runuser -u USER -- test -r FILE\nexit %s\n' "$1" > "$t23/stub/runuser"; chmod +x "$t23/stub/runuser"; }
+mq() { PATH="$t23/stub:/usr/bin:/bin" VANLINK_MOSQUITTO_USER=root ./deploy.sh --mosquitto-pwfile-check "$@" 2>&1; }
+stubru 1; o=$(mq "$t23/etc/mosquitto.conf")
+[ "$o" = "$t23/pw" ] && ok "23b: unreadable password file is reported (path only)" || bad "23b: unreadable: got '$o'"
+stubru 0; o=$(mq "$t23/etc/mosquitto.conf"); [ -z "$o" ] && ok "23b: readable -> nothing" || bad "23b: readable: got '$o'"
+stubru 1; printf 'listener 1883\n#password_file %s\n' "$t23/pw" > "$t23/etc/mosquitto.conf"; o=$(mq "$t23/etc/mosquitto.conf")
+[ -z "$o" ] && ok "23b: commented password_file / none -> nothing" || bad "23b: none: got '$o'"
+echo "password_file $t23/pw" > "$t23/etc/conf.d/x.conf"; o=$(mq "$t23/etc/mosquitto.conf")
+[ "$o" = "$t23/pw" ] && ok "23b: password_file in conf.d is found" || bad "23b: conf.d: got '$o'"
+rm "$t23/etc/conf.d/x.conf"; printf 'password_file %s\n' "$t23/gone" > "$t23/etc/mosquitto.conf"; o=$(mq "$t23/etc/mosquitto.conf")
+[ -z "$o" ] && ok "23b: nonexistent password file -> nothing" || bad "23b: nonexistent: got '$o'"
+printf 'password_file %s\n' "$t23/pw" > "$t23/etc/mosquitto.conf"
+o=$(PATH="/usr/bin:/bin" VANLINK_MOSQUITTO_USER=root ./deploy.sh --mosquitto-pwfile-check "$t23/etc/mosquitto.conf" 2>&1)
+[ -z "$o" ] && ok "23b: runuser missing -> nothing" || bad "23b: no runuser: got '$o'"
+o=$(PATH="$t23/stub:/usr/bin:/bin" VANLINK_MOSQUITTO_USER=no-such-user-x ./deploy.sh --mosquitto-pwfile-check "$t23/etc/mosquitto.conf" 2>&1)
+[ -z "$o" ] && ok "23b: unknown user -> nothing" || bad "23b: no user: got '$o'"
+pcm() { PATH="$t23/stub:/usr/bin:/bin" VANLINK_MOSQUITTO_CONF="$t23/etc/mosquitto.conf" VANLINK_MOSQUITTO_USER=root POSTCHECK_WAIT=0 ./deploy.sh --post-check hosts/van.conf 2>&1 || true; }
+stubru 1; o=$(pcm)
+echo "$o" | grep -qF -- "-> post-deploy: mosquitto password file $t23/pw is not readable by the mosquitto user — the broker will fail to start on its next restart (fix: chown mosquitto:mosquitto $t23/pw; chmod 0600 $t23/pw)" && ok "23c: unreadable -> warn" || bad "23c: unreadable no warn"
+echo "$o" | grep -q 'secret' && bad "23c: password file content leaked" || ok "23c: no file content in output"
+stubru 0; o=$(pcm)
+{ echo "$o" | grep -qF 'ok    mosquitto password file readable' && ! echo "$o" | grep -qF 'mosquitto password file'"$t23"; } && ok "23c: readable -> ok" || bad "23c: readable"
+: > "$t23/etc/mosquitto.conf"; stubru 1; o=$(pcm)
+{ echo "$o" | grep -qF 'ok    mosquitto password file readable (or none configured)' && ! echo "$o" | grep -q 'is not readable by the mosquitto user'; } && ok "23c: no password_file -> ok" || bad "23c: no password_file"
+printf 'password_file %s\n' "$t23/pw" > "$t23/etc/mosquitto.conf"
+o=$(PATH="/usr/bin:/bin" VANLINK_MOSQUITTO_CONF="$t23/etc/mosquitto.conf" VANLINK_MOSQUITTO_USER=root POSTCHECK_WAIT=0 ./deploy.sh --post-check hosts/van.conf 2>&1 || true)
+! echo "$o" | grep -q 'is not readable by the mosquitto user' && ok "23c: runuser missing -> no warn" || bad "23c: runuser missing warned"
+rm -rf "$t23"
+
 exit "$fail"