post_check: shared bounded retry for br0/LAN/NAT/DHCP probes, pipefail-safe matching

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9
This commit is contained in:
2026-09-23 17:23:36 -04:00
co-authored by Claude Sonnet 5
parent 5b43add677
commit 6142935e29
2 changed files with 38 additions and 26 deletions
+35 -24
View File
@@ -6,6 +6,7 @@
# ./deploy.sh --check [conf] # validate conf + render all templates, no install
# ./deploy.sh --diff [conf] # what would change in /etc
# ./deploy.sh --post-check [conf] # read-only live checks (units, br0, LAN, NAT, dnsmasq, radios, uplink); exit 1 on any warning
# # POSTCHECK_WAIT=<s>: seconds retried probes may wait in total (default 15; 0 = single attempt)
# ./deploy.sh render <file> # preview a templated file on stdout
# ./deploy.sh --redact # filter stdin, masking secret values (used by --diff; for tests)
# Idempotent. Netplan (wlan0 + Starlink NIC = NM-managed WANs; eth0 is a LAN
@@ -133,53 +134,63 @@ fi
post_check() { # read-only live checks; every probe is guarded, warns instead of aborting
set_units
echo "== post-deploy check =="
local u pending=() still t i n out
local u pending=() still=() n f out
local -A radio=([hostapd]=$WIFI_5G_IFACE [hostapd-2g]=$WIFI_2G_IFACE)
# a. units (up to POSTCHECK_WAIT s for slow starters; AP units are judged under (f))
# One shared deadline: probes are retried ~1/s until it passes (0 = single attempt),
# since a restart just before may not have settled yet.
local deadline=$((SECONDS + ${POSTCHECK_WAIT:-15}))
retry() { until "$@"; do [ "$SECONDS" -lt "$deadline" ] || return 1; sleep 1; done; }
# a. units (AP units are judged under (f))
for u in "${ENABLE_UNITS[@]}" "${EXTRA_CHECK_UNITS[@]}"; do
if [ -n "${radio[$u]:-}" ] && ! ip link show dev "${radio[$u]}" >/dev/null 2>&1; then continue; fi
pending+=("$u")
done
for ((t = 0; t <= ${POSTCHECK_WAIT:-15}; t++)); do
still=()
for u in "${pending[@]}"; do
systemctl is-active --quiet "$u" 2>/dev/null || still+=("$u")
still=("${pending[@]}")
while :; do
local next=()
for u in "${still[@]}"; do
systemctl is-active --quiet "$u" 2>/dev/null || next+=("$u")
done
still=("${next[@]}")
[ ${#still[@]} -eq 0 ] && break
[ "$t" -lt "${POSTCHECK_WAIT:-15}" ] && sleep 1
[ "$SECONDS" -lt "$deadline" ] || break
sleep 1
done
for u in "${pending[@]}"; do
if [[ " ${still[*]:-} " == *" $u "* ]]; then warn "post-deploy: unit $u is not active"
else echo " ok unit $u active"; fi
done
# b. bridge address
if ip -4 -br addr show dev br0 2>/dev/null | grep -qF "$GATEWAY_IP/$GATEWAY_PREFIX"; then
echo " ok br0 has $GATEWAY_IP/$GATEWAY_PREFIX"
has_br0_addr() { local o; o=$(ip -4 -br addr show dev br0 2>/dev/null) || true; [[ $o == *"$GATEWAY_IP/$GATEWAY_PREFIX"* ]]; }
if retry has_br0_addr; then echo " ok br0 has $GATEWAY_IP/$GATEWAY_PREFIX"
else warn "post-deploy: br0 does not have $GATEWAY_IP/$GATEWAY_PREFIX"; fi
# c. LAN ports picked up our .network files (not netplan's catch-all)
for i in "$LAN_IFACE:05-van-lan.network" "${ONBOARD_LAN_IFACE:-}:06-van-lan-eth0.network"; do
n=${i%%:*}; [ -n "$n" ] || continue
uses_file() { local o; o=$(networkctl status "$1" 2>/dev/null) || true; [[ $o == *"/etc/systemd/network/$2"* ]]; }
for f in "$LAN_IFACE:05-van-lan.network" "${ONBOARD_LAN_IFACE:-}:06-van-lan-eth0.network"; do
n=${f%%:*}; [ -n "$n" ] || continue
if ! ip link show dev "$n" >/dev/null 2>&1; then warn "post-deploy: LAN port $n is absent"
elif networkctl status "$n" 2>/dev/null | grep -qF "/etc/systemd/network/${i#*:}"; then
echo " ok $n uses ${i#*:}"
else warn "post-deploy: $n is not using /etc/systemd/network/${i#*:} (another .network file shadows it?)"; fi
elif retry uses_file "$n" "${f#*:}"; then echo " ok $n uses ${f#*:}"
else warn "post-deploy: $n is not using /etc/systemd/network/${f#*:} (another .network file shadows it?)"; fi
done
# d. NAT
if nft list ruleset 2>/dev/null | grep -F "ip saddr $GATEWAY_NETWORK" | grep -q masquerade; then
echo " ok NAT masquerade for $GATEWAY_NETWORK"
else warn "post-deploy: no masquerade rule for ip saddr $GATEWAY_NETWORK (nft list ruleset; needs root to read)"; fi
has_nat() { local o; o=$(nft list ruleset 2>/dev/null) || true
[[ $o == *"ip saddr $GATEWAY_NETWORK"*masquerade* ]]; }
if retry has_nat; then echo " ok NAT masquerade for $GATEWAY_NETWORK"
elif [ "$(id -u)" != 0 ]; then warn "post-deploy: no masquerade rule for ip saddr $GATEWAY_NETWORK found (nft list ruleset needs root to read)"
else warn "post-deploy: NAT rule for $GATEWAY_NETWORK not found"; fi
# e. DHCP
if ss -ulnH 2>/dev/null | awk '{print $4}' | grep -qE ':67$'; then
echo " ok dnsmasq bound to UDP 67"
has_dhcp() { local o; o=$(ss -ulnH 2>/dev/null | awk '{print $4}') || true; [[ $'\n'$o == *:67$'\n'* || $o == *:67 ]]; }
if retry has_dhcp; then echo " ok dnsmasq bound to UDP 67"
else warn "post-deploy: nothing bound to UDP port 67 (dnsmasq DHCP)"; fi
# f. AP radios
for i in "$WIFI_5G_IFACE" "$WIFI_2G_IFACE"; do
if out=$(iw dev "$i" info 2>/dev/null); then
echo " ok radio $i: $(echo "$out" | grep -E 'ssid|channel' | sed 's/^[[:space:]]*//' | tr '\n' ';')"
else warn "post-deploy: radio $i not present (its hostapd keeps retrying until it appears)"; fi
for n in "$WIFI_5G_IFACE" "$WIFI_2G_IFACE"; do
if out=$(iw dev "$n" info 2>/dev/null); then
echo " ok radio $n: $(echo "$out" | grep -E 'ssid|channel' | sed 's/^[[:space:]]*//' | tr '\n' ';')"
else warn "post-deploy: radio $n not present (its hostapd keeps retrying until it appears)"; fi
done
# g. uplink
if ! ip route show default 2>/dev/null | grep -q .; then warn "post-deploy: no default route"
out=$(ip route show default 2>/dev/null) || true
if [ -z "$out" ]; then warn "post-deploy: no default route"
elif ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; then echo " ok uplink: default route, 1.1.1.1 reachable"
else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi
}