deploy.sh: post-deploy check, --post-check, host-aware verify list

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9

deploy.sh

@@ -5,6 +5,7 @@
 # Usage:  cd ~/vanlink && sudo ./deploy.sh
 #         ./deploy.sh --check [conf]       # validate conf + render all templates, no install
 #         ./deploy.sh --diff [conf]        # what would change in /etc
+#         ./deploy.sh --post-check [conf]  # read-only live checks (units, br0, LAN, NAT, dnsmasq, radios, uplink); exit 1 on any warning
 #         ./deploy.sh render <file>        # preview a templated file on stdout
 #         ./deploy.sh --redact             # filter stdin, masking secret values (used by --diff; for tests)
 # Idempotent. Netplan (wlan0 + Starlink NIC = NM-managed WANs; eth0 is a LAN
@@ -99,6 +100,90 @@ template_files() {  # every repo file that carries an @TOKEN@ placeholder
         --exclude='*.bak' --exclude='*.md' --exclude=deploy.sh . | sed 's|^\./||' | sort
 }
 
+# Collected during a deploy and persisted to WARNINGS_FILE so the Cockpit vanrouter page
+# can surface deploy-time issues (missing deps, unedited example configs, drift)
+# without someone having to remember to scroll back through deploy output.
+WARNINGS=()
+WARNINGS_FILE=/var/lib/vanlink/deploy-warnings.json
+warn() {  # warn <message...> — prints "   -> <message>" (as before) and records it
+    echo "   -> $*"
+    WARNINGS+=("$*")
+}
+
+# The units a deploy enables (and post_check expects active). Needs the host
+# config loaded. The `if` guards sit in column 0 on purpose: tests/check-templates.sh
+# check 6 requires flag-gated unit names to appear only inside such guards.
+set_units() {
+ENABLE_UNITS=(regdomain.service van-br0-setup.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-modem-watch van-sms-watch van-gps-owntracks van-router-mqtt)
+EXTRA_CHECK_UNITS=()   # started by deploy but not in the `enable` list
+if [ "$ENABLE_NVME_WATCH" = 1 ]; then
+    ENABLE_UNITS+=(van-nvme-watch)
+fi
+if [ "$ENABLE_LI3" = 1 ]; then
+    ENABLE_UNITS+=(van-li3-battery)
+fi
+if [ "$ENABLE_HA" = 1 ]; then
+    EXTRA_CHECK_UNITS+=(homeassistant esphome)
+fi
+if [ "$ENABLE_FRIGATE" = 1 ]; then
+    EXTRA_CHECK_UNITS+=(frigate)
+fi
+}
+
+post_check() {  # read-only live checks; every probe is guarded, warns instead of aborting
+    set_units
+    echo "== post-deploy check =="
+    local u pending=() still t i n out
+    local -A radio=([hostapd]=$WIFI_5G_IFACE [hostapd-2g]=$WIFI_2G_IFACE)
+    # a. units (up to POSTCHECK_WAIT s for slow starters; AP units are judged under (f))
+    for u in "${ENABLE_UNITS[@]}" "${EXTRA_CHECK_UNITS[@]}"; do
+        if [ -n "${radio[$u]:-}" ] && ! ip link show dev "${radio[$u]}" >/dev/null 2>&1; then continue; fi
+        pending+=("$u")
+    done
+    for ((t = 0; t <= ${POSTCHECK_WAIT:-15}; t++)); do
+        still=()
+        for u in "${pending[@]}"; do
+            systemctl is-active --quiet "$u" 2>/dev/null || still+=("$u")
+        done
+        [ ${#still[@]} -eq 0 ] && break
+        [ "$t" -lt "${POSTCHECK_WAIT:-15}" ] && sleep 1
+    done
+    for u in "${pending[@]}"; do
+        if [[ " ${still[*]:-} " == *" $u "* ]]; then warn "post-deploy: unit $u is not active"
+        else echo "   ok    unit $u active"; fi
+    done
+    # b. bridge address
+    if ip -4 -br addr show dev br0 2>/dev/null | grep -qF "$GATEWAY_IP/$GATEWAY_PREFIX"; then
+        echo "   ok    br0 has $GATEWAY_IP/$GATEWAY_PREFIX"
+    else warn "post-deploy: br0 does not have $GATEWAY_IP/$GATEWAY_PREFIX"; fi
+    # c. LAN ports picked up our .network files (not netplan's catch-all)
+    for i in "$LAN_IFACE:05-van-lan.network" "${ONBOARD_LAN_IFACE:-}:06-van-lan-eth0.network"; do
+        n=${i%%:*}; [ -n "$n" ] || continue
+        if ! ip link show dev "$n" >/dev/null 2>&1; then warn "post-deploy: LAN port $n is absent"
+        elif networkctl status "$n" 2>/dev/null | grep -qF "/etc/systemd/network/${i#*:}"; then
+            echo "   ok    $n uses ${i#*:}"
+        else warn "post-deploy: $n is not using /etc/systemd/network/${i#*:} (another .network file shadows it?)"; fi
+    done
+    # d. NAT
+    if nft list ruleset 2>/dev/null | grep -F "ip saddr $GATEWAY_NETWORK" | grep -q masquerade; then
+        echo "   ok    NAT masquerade for $GATEWAY_NETWORK"
+    else warn "post-deploy: no masquerade rule for ip saddr $GATEWAY_NETWORK (nft list ruleset; needs root to read)"; fi
+    # e. DHCP
+    if ss -ulnH 2>/dev/null | awk '{print $4}' | grep -qE ':67$'; then
+        echo "   ok    dnsmasq bound to UDP 67"
+    else warn "post-deploy: nothing bound to UDP port 67 (dnsmasq DHCP)"; fi
+    # f. AP radios
+    for i in "$WIFI_5G_IFACE" "$WIFI_2G_IFACE"; do
+        if out=$(iw dev "$i" info 2>/dev/null); then
+            echo "   ok    radio $i: $(echo "$out" | grep -E 'ssid|channel' | sed 's/^[[:space:]]*//' | tr '\n' ';')"
+        else warn "post-deploy: radio $i not present (its hostapd keeps retrying until it appears)"; fi
+    done
+    # g. uplink
+    if ! ip route show default 2>/dev/null | grep -q .; then warn "post-deploy: no default route"
+    elif ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; then echo "   ok    uplink: default route, 1.1.1.1 reachable"
+    else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi
+}
+
 case "${1:-}" in
     --templates)
         template_files
@@ -144,6 +229,16 @@ case "${1:-}" in
     --redact)   # ./deploy.sh --redact — stdin through redact_secrets (no config, no root)
         redact_secrets
         exit 0 ;;
+    --post-check)   # ./deploy.sh --post-check [conf] — live checks, no root, no install; exit 1 on any warning
+        load_conf "${2:-$VANLINK_CONF}"
+        post_check
+        if [ ${#WARNINGS[@]} -gt 0 ]; then
+            echo
+            echo "== warnings =="
+            for w in "${WARNINGS[@]}"; do echo "   -> $w"; done
+            exit 1
+        fi
+        exit 0 ;;
     render)
         [ -n "${2:-}" ] || { echo "Usage: $0 render <file>"; exit 1; }
         load_conf "$VANLINK_CONF"
@@ -183,19 +278,8 @@ OPTIONAL_PACKAGES=(
 for pkg in "${REQUIRED_PACKAGES[@]}"; do
     dpkg -s "$pkg" >/dev/null 2>&1 || { echo "MISSING REQUIRED: $pkg (apt install $pkg)"; exit 1; }
 done
-# Note: warn() is defined below; we'll check optional packages after it's defined
 
-# Collected below and persisted to WARNINGS_FILE so the Cockpit vanrouter page
+# Check optional packages
-# can surface deploy-time issues (missing deps, unedited example configs, drift)
-# without someone having to remember to scroll back through deploy output.
-WARNINGS=()
-WARNINGS_FILE=/var/lib/vanlink/deploy-warnings.json
-warn() {  # warn <message...> — prints "   -> <message>" (as before) and records it
-    echo "   -> $*"
-    WARNINGS+=("$*")
-}
-
-# Check optional packages now that warn() is defined
 for pkg in "${OPTIONAL_PACKAGES[@]}"; do
     if ! dpkg -s "$pkg" >/dev/null 2>&1; then
         warn "optional package not installed: $pkg (apt install $pkg)"
@@ -467,13 +551,7 @@ systemctl mask systemd-networkd-wait-online.service >/dev/null 2>&1 || true
 systemctl unmask hostapd >/dev/null 2>&1 || true
 # Disable system dnsmasq.service to avoid port 53 conflicts — van-ap-dnsmasq owns AP DHCP
 systemctl disable dnsmasq.service >/dev/null 2>&1 || true
-ENABLE_UNITS=(regdomain.service van-br0-setup.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-modem-watch van-sms-watch van-gps-owntracks van-router-mqtt)
+set_units
-if [ "$ENABLE_NVME_WATCH" = 1 ]; then
-    ENABLE_UNITS+=(van-nvme-watch)
-fi
-if [ "$ENABLE_LI3" = 1 ]; then
-    ENABLE_UNITS+=(van-li3-battery)
-fi
 systemctl enable "${ENABLE_UNITS[@]}" >/dev/null 2>&1 || true
 # bluetooth: host BlueZ serves the onboard hci0 to the HA container over D-Bus
 systemctl enable --now bluetooth >/dev/null 2>&1 || true
@@ -502,14 +580,14 @@ systemctl restart systemd-networkd
 for u in van-ap-dnsmasq nftables van-failover van-wlan-watchdog; do
     systemctl restart "$u" || warn "$u failed to restart (journalctl -xeu $u)"
 done
-# The AP radios live on the USB hub and may be absent; the start then fails but
+# An AP radio may be absent; the start then fails but Restart=always keeps
-# Restart=always keeps retrying and claims them the moment they enumerate.
+# retrying and claims it the moment it enumerates (post_check reports absent radios).
-systemctl restart hostapd hostapd-2g \
+systemctl restart hostapd hostapd-2g || true
-    || echo "   -> hostapd(-2g) waiting for AP radios (USB hub not plugged in)" 
 # AP watchdogs last, after hostapd is back up (they only ever restart a wedged hostapd)
 systemctl restart van-ap-watchdog van-ap-watchdog-2g
 networkctl reload 2>/dev/null || true
 
+post_check
 write_warnings
 
 if [ ${#WARNINGS[@]} -gt 0 ]; then
@@ -519,14 +597,16 @@ if [ ${#WARNINGS[@]} -gt 0 ]; then
 fi
 
 echo
-echo "Deployed. NOTE: until the USB hub (AP radios + LAN/Starlink adapters) is"
+echo "Deployed host $(hostname -s) with $(readlink -f "$VANLINK_CONF")."
-echo "plugged in, hostapd/hostapd-2g just retry every 5s — that is by design."
+echo "Verify further:"
-echo "Verify (with hub present):"
-echo "  iw dev $WIFI_5G_IFACE info | grep -E 'ssid|channel|width'"
-echo "  iw dev $WIFI_2G_IFACE info | grep -E 'ssid|channel|width'"
 echo "  cat /run/van-failover/state.json"
 echo "  cat /run/van-thermal/state.json"
-echo "  cat /run/van-nvme-watch/state.json"
+echo "  journalctl -u van-router-mqtt -n 20"
-echo "  journalctl -u van-li3-battery -n 20"
+echo "  journalctl -u van-sms-watch -n 20"
-echo "  journalctl -u van-router-mqtt -n 20 ; mosquitto_sub -h localhost -t 'van/router/#' -v -C 8"
+if [ "$ENABLE_NVME_WATCH" = 1 ]; then
-echo "  journalctl -u van-sms-watch -n 20 ; tail -f /var/log/van-sms.jsonl"
+    echo "  cat /run/van-nvme-watch/state.json"
+fi
+if [ "$ENABLE_LI3" = 1 ]; then
+    echo "  journalctl -u van-li3-battery -n 20"
+fi
+echo "  ./deploy.sh --post-check     # re-run the checks above"

tests/check-templates.sh

@@ -169,4 +169,18 @@ else
     bad "--redact lost keys or diff markers"; echo "$red" | sed 's/^/       /'
 fi
 
+# 11. --post-check: no root, warns on this (non-router) machine, exits 1; host-aware; no stale text
+pc=$(POSTCHECK_WAIT=0 ./deploy.sh --post-check hosts/van.conf 2>&1) && rc=0 || rc=$?
+if [ "$rc" = 1 ] && echo "$pc" | grep -q '^== post-deploy check ==$' && echo "$pc" | grep -q -- '-> post-deploy:'; then
+    ok "--post-check hosts/van.conf: heading, warnings, exit 1"
+else
+    bad "--post-check hosts/van.conf (exit $rc)"; echo "$pc" | sed 's/^/       /'
+fi
+tmpconf=$(mktemp); trap 'rm -f "$empty" "$tmpconf"' EXIT
+grep -q '^ENABLE_NVME_WATCH=0' hosts/wan.conf && cp hosts/wan.conf "$tmpconf"
+pc=$(POSTCHECK_WAIT=0 ./deploy.sh --post-check "$tmpconf" 2>&1) || true
+if echo "$pc" | grep -q 'van-nvme-watch'; then bad "--post-check mentions van-nvme-watch with ENABLE_NVME_WATCH=0"
+else ok "--post-check skips van-nvme-watch when ENABLE_NVME_WATCH=0"; fi
+if grep -q 'USB hub' deploy.sh; then bad "stale 'USB hub' text remains in deploy.sh"; else ok "no stale 'USB hub' text in deploy.sh"; fi
+
 exit "$fail"