From cefca06dac99f0ebf0f552d9c738cdd2ed5f47f2 Mon Sep 17 00:00:00 2001 From: Andreas Wrede Date: Wed, 23 Sep 2026 18:55:04 -0400 Subject: [PATCH] deploy: apply helper gateway from DHCP routers, trap/timeouts, running status, per-stamp files Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9 --- deploy.sh | 116 +++++++++++++++++++++++++++++---------- tests/check-templates.sh | 79 ++++++++++++++++++++++---- 2 files changed, 154 insertions(+), 41 deletions(-) diff --git a/deploy.sh b/deploy.sh index a05bb93..f1b2f8e 100755 --- a/deploy.sh +++ b/deploy.sh @@ -12,7 +12,9 @@ # ./deploy.sh --render-wan-netplan # rendered /etc/netplan/50-van-wan.yaml (ethernet part + wifis stanza when WIFI_WAN_IFACE is set) # ./deploy.sh --wan-plan [conf] # what the NetworkManager WAN provisioning would do (read-only, no root) # ./deploy.sh --apply-network [--force] # root: a detached helper installs the WAN netplan, applies it, gates on uplink health (2 stages) and rolls back -# # automatically on failure; status in /var/lib/vanlink/apply-network.status. No-op when already applied unless --force. +# # automatically on failure; status in /var/lib/vanlink/apply-network.status. No-op when already applied, and refused after a +# # running/rollback-failed status, unless --force. +# ./deploy.sh --apply-status-check # test seam: yes|no, would this status file (running|rollback-failed) block --apply-network # ./deploy.sh --print-apply-helper [conf] # print the detached apply/rollback helper script (read-only, no root; for review and tests) # # test-only flags: --backup --netplan-dir --log --status --sysnet --rendered --deadlines "s1 s2 rc" --stamp # ./deploy.sh render # preview a templated file on stdout @@ -70,6 +72,7 @@ load_conf() { # load_conf — source and validate a host config *) die "$f: $flag must be 0 or 1 (got '${!flag}')" ;; esac done + [[ $STARLINK_IFACE =~ ^[A-Za-z0-9._-]+$ ]] || die "$f: STARLINK_IFACE '$STARLINK_IFACE' has characters outside [A-Za-z0-9._-]" if [ "$ENABLE_NVME_WATCH" = 1 ]; then { [ -n "${NVME_DEVICE:-}" ] && [ -n "${NVME_FS_DEVICE:-}" ]; } \ || die "$f: ENABLE_NVME_WATCH=1 needs NVME_DEVICE and NVME_FS_DEVICE" @@ -279,14 +282,31 @@ post_check() { # read-only live checks; every probe is guarded, warns instead o else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi } -# --apply-network time budgets (seconds), in one place. The helper's worst case is -# apply + stage 1 + restart + stage 2 + rollback (apply + restart + recheck); the -# foreground waits that long plus a margin. van-failover starts optimistic and -# probes every WAN at once (probe_timeout 3 s), enforcing the route in its first -# loop, so a healthy default route shows up within seconds of a restart; the 120 s -# stage 2 deadline is a wide margin for a slow Starlink/cellular probe. -AN_APPLY_T=90 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 -AN_WAIT=$(( 2 * AN_APPLY_T + 2 * AN_RESTART_T + AN_S1 + AN_S2 + AN_RC + 30 )) +# --apply-network time budgets (seconds), in one place; all derived from the AN_* constants. +# forward path AN_FWD = generate + apply + stage 1 + restart + stage 2 +# rollback path AN_RB = generate + apply + restart + connectivity recheck +# helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their +# deadline by a few nmcli/ping timeouts) +# foreground waits AN_WAIT = AN_HELPER_MAX + 30 +# transient unit: RuntimeMaxSec = AN_HELPER_MAX + 60 (backstop; systemd then sends TERM and the +# helper's trap runs the restore), TimeoutStopSec = AN_RB + 30 (time the restore may take). +# van-failover starts optimistic and probes every WAN at once (probe_timeout 3 s), enforcing the +# route in its first loop, so a healthy default route shows up within seconds of a restart; the +# 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe. +AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60 +AN_FWD=$(( AN_GEN_T + AN_APPLY_T + AN_S1 + AN_RESTART_T + AN_S2 )) +AN_RB=$(( AN_GEN_T + AN_APPLY_T + AN_RESTART_T + AN_RC )) +AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP )) +AN_WAIT=$(( AN_HELPER_MAX + 30 )) +AN_RUNMAX=$(( AN_HELPER_MAX + 60 )) +AN_STOP_T=$(( AN_RB + 30 )) + +apply_status_blocks_run() { # apply_status_blocks_run -> yes|no: a run that is/was in flight or failed to roll back blocks a new one + case "$(cut -d' ' -f1 "$1" 2>/dev/null || true)" in + running|rollback-failed) echo yes ;; + *) echo no ;; + esac +} # apply_helper_script [netplan-dir] [log] [status-file] [sysnet-dir] [rendered-file] [deadlines "s1 s2 rc"] [run-stamp] # The script the detached unit runs. Paths are parameters only so tests can bake @@ -301,10 +321,11 @@ apply_helper_script() { cat </dev/null && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; } gate1() { [ -e "\$sn/$STARLINK_IFACE" ] || return 0 - [ "\$(nmcli -g GENERAL.STATE device show '$STARLINK_IFACE' 2>/dev/null)" = "100 (connected)" ] || return 1 - case "\$(nmcli -g GENERAL.CONNECTION device show '$STARLINK_IFACE' 2>/dev/null)" in netplan-*) ;; *) return 1 ;; esac - a=\$(nmcli -g IP4.ADDRESS device show '$STARLINK_IFACE' 2>/dev/null) - g=\$(nmcli -g IP4.GATEWAY device show '$STARLINK_IFACE' 2>/dev/null) - [ -n "\$a" ] && [ "\$a" != -- ] && [ -n "\$g" ] && [ "\$g" != -- ] || return 1 + [ "\$(timeout $AN_NM_T nmcli -g GENERAL.STATE device show '$STARLINK_IFACE' 2>/dev/null)" = "100 (connected)" ] || return 1 + case "\$(timeout $AN_NM_T nmcli -g GENERAL.CONNECTION device show '$STARLINK_IFACE' 2>/dev/null)" in netplan-*) ;; *) return 1 ;; esac + a=\$(timeout $AN_NM_T nmcli -g IP4.ADDRESS device show '$STARLINK_IFACE' 2>/dev/null) + [ -n "\$a" ] && [ "\$a" != -- ] || return 1 + g=\$(timeout $AN_NM_T nmcli -g IP4.GATEWAY device show '$STARLINK_IFACE' 2>/dev/null) + case "\$g" in + ''|--) g=\$(timeout $AN_NM_T nmcli -g DHCP4.OPTION device show '$STARLINK_IFACE' 2>/dev/null | tr '|' '\\n' | sed -n 's/^ *routers = *//p' | head -1 | cut -d' ' -f1) ;; + esac + [ -n "\$g" ] && [ "\$g" != -- ] || return 1 ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1 } -restore() { # restore : put the pre-run netplan back, re-check connectivity, always exit 1 +route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; } +in_restore=0 +restore() { # restore : put the pre-run netplan back, re-check connectivity, always exit 1 (runs once) + [ "\$in_restore" = 0 ] || exit 1 + in_restore=1 + trap '' TERM INT HUP local rb=ok c=down echo "apply-network FAILED (\$1) - restoring \$bak" + route_log if [ -d "\$bak" ]; then rm -f "\$nd"/* "\$nd"/.[!.]* cp -a "\$bak"/. "\$nd"/ || rb=CP-FAILED else + # No backup to restore from: at least undo this helper's own two changes. rb=BACKUP-MISSING + rm -f "\$nd/50-van-wan.yaml" + [ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml" fi - netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; } + timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; } timeout $AN_RESTART_T systemctl restart van-failover || true end=\$((SECONDS+$d3)) while :; do @@ -342,23 +376,29 @@ restore() { # restore : put the pre-run netplan back, re-check connecti [ "\$SECONDS" -ge "\$end" ] && break sleep 1 done + rm -f "\$rf" if [ "\$rb" = ok ] && [ "\$c" = ok ]; then echo "apply-network ROLLED BACK (restored, connectivity ok)" status rolled-back "\$1" elif [ "\$rb" = ok ]; then echo "apply-network ROLLED BACK (restored, but no connectivity - check the uplink/cable)" - status rolled-back restored-no-connectivity + status rolled-back "restored-no-connectivity; \$1" else echo "apply-network ROLLBACK FAILED: \$rb / connectivity \$c" status rollback-failed "\$1; \$rb / connectivity \$c" - [ "\$c" = ok ] || { echo "rollback failed and no connectivity - rebooting (boot uses the files on disk)"; systemctl reboot; } + # Reboot (boot uses the files on disk) only when a restore was attempted and failed, and there is no connectivity. + case "\$rb" in + CP-FAILED|APPLY-FAILED) [ "\$c" = ok ] || { echo "rollback failed and no connectivity - rebooting"; systemctl reboot; } ;; + esac fi exit 1 } +status running "\$bak" +trap 'restore terminated' TERM INT HUP applied=0 if install -m0600 "\$rf" "\$nd/50-van-wan.yaml" \\ && { [ ! -f "\$nd/10-dhcp-all-interfaces.yaml" ] || mv "\$nd/10-dhcp-all-interfaces.yaml" "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled"; } \\ - && netplan generate && timeout $AN_APPLY_T netplan apply; then + && timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply; then applied=1 else echo "netplan install/generate/apply failed" @@ -381,7 +421,10 @@ while :; do [ "\$SECONDS" -ge "\$end" ] && break sleep 1 done +route_log [ "\$s2" = 1 ] || restore "stage 2: no default route or internet" +trap - TERM INT HUP +rm -f "\$rf" status ok echo "apply-network OK" exit 0 @@ -396,29 +439,39 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c dpkg-query -W -f='${Status}' network-manager 2>/dev/null | grep -q 'install ok installed' \ || die "network-manager is not installed — run ./deploy.sh first" systemctl is-active --quiet NetworkManager || die "NetworkManager is not running — handing it the uplink port would flap it" + systemctl cat van-failover >/dev/null 2>&1 || die "van-failover.service does not exist — run ./deploy.sh first (stage 2 of the gate needs it)" systemctl is-active --quiet vanlink-apply-network && die "an apply is already running — see /var/log/vanlink-apply-network.log" + local log=/var/log/vanlink-apply-network.log st=/var/lib/vanlink/apply-network.status + if [ "$force" = 0 ] && [ "$(apply_status_blocks_run "$st")" = yes ]; then + die "the last apply did not finish cleanly: $(cat "$st" 2>/dev/null) — inspect $st and $log, check the backup dir named there, then re-run with --force" + fi if [ "$force" = 0 ] && [ "$(nm_wan_state_netplan)" = matches ] && [ ! -e /etc/netplan/10-dhcp-all-interfaces.yaml ]; then echo "nothing to apply (WAN netplan already matches; use --force to re-apply)" return 0 fi - local rendered=/run/vanlink-50-van-wan.yaml stamp bak log=/var/log/vanlink-apply-network.log st=/var/lib/vanlink/apply-network.status + local stamp helper rendered bak pt + stamp=$(date +%Y%m%d-%H%M%S)-$$ + rendered=/run/vanlink-50-van-wan-$stamp.yaml + helper=/run/vanlink-apply-network-$stamp.sh (umask 077; render_wan_netplan > "$rendered") || die "render failed" check_rendered ap/50-van-wan.yaml "$rendered" || { rm -f "$rendered"; exit 1; } - stamp=$(date +%Y%m%d-%H%M%S) bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || die "cannot create backup dir" cp -a /etc/netplan/. "$bak/" || die "backup failed — nothing changed" if [ ! -e /var/backups/vanlink-netplan-orig ] && [ ! -e /etc/netplan/50-van-wan.yaml ]; then - install -d -m0700 /var/backups/vanlink-netplan-orig - cp -a /etc/netplan/. /var/backups/vanlink-netplan-orig/ || { rm -rf /var/backups/vanlink-netplan-orig; die "pristine backup failed — nothing changed"; } + pt=/var/backups/vanlink-netplan-orig.tmp.$$ + rm -rf "$pt" + install -d -m0700 "$pt" + cp -a /etc/netplan/. "$pt/" || { rm -rf "$pt"; die "pristine backup failed — nothing changed"; } + mv "$pt" /var/backups/vanlink-netplan-orig fi install -d -m0755 /var/lib/vanlink rm -f "$st" - apply_helper_script "$bak" /etc/netplan "$log" "$st" /sys/class/net "$rendered" "$AN_S1 $AN_S2 $AN_RC" "$stamp" > /run/vanlink-apply-network.sh - chmod 0700 /run/vanlink-apply-network.sh + apply_helper_script "$bak" /etc/netplan "$log" "$st" /sys/class/net "$rendered" "$AN_S1 $AN_S2 $AN_RC" "$stamp" > "$helper" + chmod 0700 "$helper" echo "backup: $bak log: $log" echo "applying from a detached unit (survives an SSH drop); waiting up to $AN_WAIT s ..." systemctl reset-failed vanlink-apply-network 2>/dev/null || true - systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec=900 /bin/bash /run/vanlink-apply-network.sh >/dev/null \ + systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec="$AN_RUNMAX" -p TimeoutStopSec="$AN_STOP_T" /bin/bash "$helper" >/dev/null \ || die "systemd-run failed — nothing was changed (backup: $bak)" SECONDS=0 while systemctl is-active --quiet vanlink-apply-network && [ "$SECONDS" -lt "$AN_WAIT" ]; do sleep 1; done @@ -432,7 +485,7 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c case "$verdict" in ok) return 0 ;; rolled-back|rollback-failed) echo "apply-network: $(cat "$st") (previous config: $bak)" >&2; return 1 ;; - *) echo "apply-network: no status file — see $log (previous config: $bak)" >&2; return 1 ;; + *) echo "apply-network: no final status — see $log (previous config: $bak)" >&2; return 1 ;; esac } @@ -485,6 +538,9 @@ case "${1:-}" in load_conf "$VANLINK_CONF" apply_network "${2:-}" exit $? ;; + --apply-status-check) # test seam: --apply-status-check prints yes|no (does that status block a new --apply-network run?) + apply_status_blocks_run "${2:?status file}" + exit 0 ;; --print-apply-helper) # [conf] the helper script for review; test-only flags: --backup D --netplan-dir D --log L --status S --sysnet N --rendered R --deadlines "s1 s2 rc" --stamp T shift; pc= a_bak=/var/backups/vanlink-netplan-EXAMPLE a_nd= a_log= a_st= a_sn= a_rf= a_dl= a_stamp= while [ $# -gt 0 ]; do diff --git a/tests/check-templates.sh b/tests/check-templates.sh index 87b8db1..866e838 100755 --- a/tests/check-templates.sh +++ b/tests/check-templates.sh @@ -259,7 +259,9 @@ rm -rf "$nmd" "$nme" # 18. the detached apply helper: syntax, then behaviour against stubbed system commands in a sandbox h=$(VANLINK_CONF=hosts/van.conf ./deploy.sh --print-apply-helper 2>&1) || { bad "--print-apply-helper failed"; echo "$h" | sed 's/^/ /'; h=; } -if [ -n "$h" ]; then +if [ -n "$h" ] && [ "$(id -u)" = 0 ]; then + bad "check 18 refuses to run the helper sandbox as root" +elif [ -n "$h" ]; then echo "$h" | bash -n && ok "apply helper: bash -n" || bad "apply helper: bash -n" echo "$h" | grep -q 'enP2p33s0' && ok "apply helper waits on the Starlink port" || bad "apply helper does not reference STARLINK_IFACE" echo "$h" | grep -q 'systemctl restart van-failover' && ok "apply helper restarts van-failover" || bad "apply helper lacks van-failover restart" @@ -269,6 +271,10 @@ if [ -n "$h" ]; then done cat >> "$stubs/netplan" <<'STUB' if [ "$1" = apply ]; then + if [ -e "${STUB_HOLD:-/nonexistent}" ]; then + : > "$STUB_SB/applying" + while [ -e "$STUB_HOLD" ]; do /bin/sleep 0.05; done + fi n=$(cat "$STUB_SB/napply" 2>/dev/null || echo 0); echo $((n + 1)) > "$STUB_SB/napply" [ "${STUB_FAIL_APPLY:-}" = first ] && [ "$n" = 0 ] && exit 1 fi @@ -279,11 +285,14 @@ case "$2" in GENERAL.STATE) echo "${NM_STATE-100 (connected)}" ;; GENERAL.CONNECTION) echo "${NM_CONN-netplan-enP2p33s0}" ;; IP4.ADDRESS) echo "${NM_ADDR-192.168.100.10/24}" ;; - IP4.GATEWAY) echo "${NM_GW-192.168.100.1}" ;; + IP4.GATEWAY) echo "${NM_GW-}" ;; + # never-default: IP4.GATEWAY is empty, the router is only in the DHCP lease options (nmcli -g joins them with " | ") + DHCP4.OPTION) echo "${NM_DHCP-requested_routers = 1 | routers = 192.168.100.1 | subnet_mask = 255.255.255.0}" ;; esac exit 0 STUB cat >> "$stubs/ip" <<'STUB' +if [ "$2" = get ]; then echo "1.1.1.1 via 192.168.100.1 dev x"; exit 0; fi case "${STUB_ROUTE:-yes}" in yes) echo "default via 192.168.100.1 dev x" ;; after-rollback) [ -e "$STUB_ND/50-van-wan.yaml" ] || echo "default via 192.168.100.1 dev x" ;; @@ -302,17 +311,23 @@ STUB for c in netplan nmcli ip ping systemctl reboot sleep timeout; do [ -x "$stubs/$c" ] || stubs_ok=0; done [ "$stubs_ok" = 1 ] && ok "apply helper sandbox: all system commands stubbed" || bad "apply helper sandbox: stub missing" - an_run() { # an_run [VAR=val ...] — fresh sandbox, run the helper; sets sb rc + an_prep() { # fresh sandbox + generated helper; sets sb sb=$(mktemp -d); mkdir -p "$sb/netplan" "$sb/sysnet" printf 'armbian\n' > "$sb/netplan/10-dhcp-all-interfaces.yaml"; chmod 0600 "$sb/netplan/10-dhcp-all-interfaces.yaml" printf 'old\n' > "$sb/netplan/60-other.yaml" cp -a "$sb/netplan" "$sb/bak" [ -n "${AN_NOPORT:-}" ] || mkdir "$sb/sysnet/enP2p33s0" [ -z "${AN_NOBAK:-}" ] || rm -rf "$sb/bak" + [ -z "${AN_BAKBAD:-}" ] || chmod 000 "$sb/bak/60-other.yaml" printf 'network:\n version: 2\n' > "$sb/rendered.yaml" VANLINK_CONF=hosts/van.conf ./deploy.sh --print-apply-helper --backup "$sb/bak" --netplan-dir "$sb/netplan" \ --log "$sb/log" --status "$sb/status" --sysnet "$sb/sysnet" --rendered "$sb/rendered.yaml" \ --deadlines "1 1 1" --stamp T > "$sb/helper.sh" + # never run a helper that does not point at the sandbox + grep -qF "nd='$sb/netplan'" "$sb/helper.sh" && [ "$(id -u)" != 0 ] || { bad "helper does not bake the sandbox netplan dir (or root) - refusing to run"; return 1; } + } + an_run() { # an_run [VAR=val ...] — fresh sandbox, run the helper; sets sb rc + an_prep || { rc=99; return 0; } env PATH="$stubs:/usr/bin:/bin" STUB_CALLS="$sb/calls" STUB_SB="$sb" STUB_ND="$sb/netplan" "$@" bash "$sb/helper.sh" && rc=0 || rc=$? } a_is() { [ "$2" = "$3" ] && ok "apply helper $1" || bad "apply helper $1 (got '$2', want '$3')"; } @@ -320,6 +335,8 @@ STUB # (1) success an_run a_is "success: exit 0" "$rc" 0; a_is "success: status" "$(st_word)" ok + grep -q '^ping -c1 -W2 -I enP2p33s0 192.168.100.1$' "$sb/calls" && ok "apply helper success: gateway from the DHCP routers option" || bad "apply helper success: routers option not used" + grep -q 'route to 1.1.1.1:' "$sb/log" && ok "apply helper success: route logged" || bad "apply helper success: no route log" grep -q 'apply-network OK' "$sb/log" && ok "apply helper success: log OK" || bad "apply helper success: log" a_is "success: 50-van-wan.yaml mode" "$(stat -c %a "$sb/netplan/50-van-wan.yaml" 2>/dev/null)" 600 [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] && [ ! -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && ok "apply helper success: Armbian file moved aside" || bad "apply helper success: Armbian file not moved aside" @@ -335,7 +352,11 @@ STUB # (2b) stage 1 fails: wrong connection name; gateway unpingable an_run NM_CONN="Wired connection 1"; a_is "stage1 conn name: status" "$(st_word)" rolled-back; rm -rf "$sb" an_run STUB_GW_PING=1; a_is "stage1 gateway ping: status" "$(st_word)" rolled-back; rm -rf "$sb" - an_run NM_GW=; a_is "stage1 no gateway: status" "$(st_word)" rolled-back; rm -rf "$sb" + an_run NM_DHCP="requested_routers = 1 | subnet_mask = 255.255.255.0"; a_is "stage1 no IP4.GATEWAY and no routers option: status" "$(st_word)" rolled-back; rm -rf "$sb" + an_run NM_DHCP="routers = 192.168.100.1 192.168.100.2"; a_is "routers option with two routers: status" "$(st_word)" ok + grep -q '^ping -c1 -W2 -I enP2p33s0 192.168.100.1$' "$sb/calls" && ok "apply helper: pinged the first router" || bad "apply helper: first router not pinged"; rm -rf "$sb" + an_run NM_GW=192.168.100.9 NM_DHCP=; a_is "IP4.GATEWAY present: status" "$(st_word)" ok + grep -q '^ping -c1 -W2 -I enP2p33s0 192.168.100.9$' "$sb/calls" && ok "apply helper: pinged IP4.GATEWAY" || bad "apply helper: IP4.GATEWAY not pinged"; rm -rf "$sb" # (3) netplan apply fails, port absent AN_NOPORT=1 an_run STUB_FAIL_APPLY=first a_is "apply failure: exit 1" "$rc" 1; a_is "apply failure: not ok" "$(st_word)" rolled-back @@ -343,12 +364,19 @@ STUB rm -rf "$sb" # (3b) port absent and apply ok: stage 1 needs only the apply AN_NOPORT=1 an_run; a_is "port absent, apply ok: status" "$(st_word)" ok; rm -rf "$sb" - # (4) rollback cp fails (backup dir missing) - AN_NOBAK=1 an_run NM_STATE=bad - a_is "rollback cp failure: exit 1" "$rc" 1; a_is "rollback cp failure: status" "$(st_word)" rollback-failed - grep -q 'ROLLBACK FAILED' "$sb/log" && ok "apply helper rollback failure: loud log" || bad "apply helper rollback failure: log" - grep -q '^reboot\|systemctl reboot' "$sb/calls" && bad "apply helper rollback failure with connectivity: must not reboot" || ok "apply helper rollback failure with connectivity: no reboot" + # (4) backup dir missing: undo own changes, report failure, never reboot into the new config + AN_NOBAK=1 an_run NM_STATE=bad STUB_ROUTE=none + a_is "backup missing: exit 1" "$rc" 1; a_is "backup missing: status" "$(st_word)" rollback-failed + grep -q 'ROLLBACK FAILED: BACKUP-MISSING' "$sb/log" && ok "apply helper backup missing: loud log" || bad "apply helper backup missing: log" + grep -q '^systemctl reboot' "$sb/calls" && bad "apply helper backup missing: must not reboot" || ok "apply helper backup missing: no reboot" + [ ! -e "$sb/netplan/50-van-wan.yaml" ] && [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && [ ! -e "$sb/netplan/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] && ok "apply helper backup missing: own changes undone" || bad "apply helper backup missing: own changes not undone" rm -rf "$sb" + # (4b) rollback cp fails (unreadable backup file) but connectivity is fine: loud, no reboot + AN_BAKBAD=1 an_run NM_STATE=bad + a_is "cp failure: status" "$(st_word)" rollback-failed + grep -q 'ROLLBACK FAILED: CP-FAILED' "$sb/log" && ok "apply helper cp failure: loud log" || bad "apply helper cp failure: log" + grep -q '^systemctl reboot' "$sb/calls" && bad "apply helper cp failure with connectivity: must not reboot" || ok "apply helper cp failure with connectivity: no reboot" + chmod 600 "$sb/bak/60-other.yaml"; rm -rf "$sb" # (5) stage 2 fails: no default route until rolled back an_run STUB_ROUTE=after-rollback a_is "stage2: exit 1" "$rc" 1; a_is "stage2: status" "$(st_word)" rolled-back @@ -357,17 +385,46 @@ STUB # (6) clean rollback but no connectivity (uplink unplugged): must NOT reboot an_run STUB_ROUTE=none a_is "no connectivity after clean rollback: exit" "$rc" 1 - a_is "no connectivity after clean rollback: status" "$(cat "$sb/status" 2>/dev/null | cut -d' ' -f1,3)" "rolled-back restored-no-connectivity" + a_is "no connectivity after clean rollback: status" "$(cut -d' ' -f1,3 "$sb/status" 2>/dev/null)" "rolled-back restored-no-connectivity;" + grep -q 'restored-no-connectivity; stage 2' "$sb/status" && ok "apply helper: original reason kept in status" || bad "apply helper: reason lost from status" grep -q 'restored, but no connectivity' "$sb/log" && ok "apply helper: no-connectivity log" || bad "apply helper: no-connectivity log" grep -q '^systemctl reboot' "$sb/calls" && bad "apply helper: must not reboot after a clean rollback" || ok "apply helper: no reboot after a clean rollback" [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && [ ! -e "$sb/netplan/50-van-wan.yaml" ] && ok "apply helper: clean rollback restored files" || bad "apply helper: clean rollback files" rm -rf "$sb" # (7) rollback cp fails AND no connectivity: last-resort reboot - AN_NOBAK=1 an_run STUB_ROUTE=none + AN_BAKBAD=1 an_run STUB_ROUTE=none a_is "failed rollback, no connectivity: status" "$(st_word)" rollback-failed grep -q '^systemctl reboot' "$sb/calls" && ok "apply helper: reboots when the rollback failed and connectivity is absent" || bad "apply helper: no reboot after failed rollback" rm -rf "$sb" + chmod 600 "$sb/bak/60-other.yaml" 2>/dev/null || true; rm -rf "$sb" + # (8) TERM while a slow netplan apply is running: the trap restores, once + an_prep && { + : > "$sb/hold" + env PATH="$stubs:/usr/bin:/bin" STUB_CALLS="$sb/calls" STUB_SB="$sb" STUB_ND="$sb/netplan" STUB_HOLD="$sb/hold" bash "$sb/helper.sh" & + hp=$! + for _ in $(seq 1 100); do [ -e "$sb/applying" ] && break; /bin/sleep 0.05; done + [ -e "$sb/applying" ] && ok "apply helper TERM: helper reached the slow apply" || bad "apply helper TERM: never reached the apply" + a_is "TERM: status while running" "$(cut -d' ' -f1,3 "$sb/status" 2>/dev/null)" "running $sb/bak" + kill -TERM "$hp"; rm -f "$sb/hold" + wait "$hp" && rc=0 || rc=$? + a_is "TERM: exit 1" "$rc" 1 + a_is "TERM: status" "$(cut -d' ' -f1,3 "$sb/status" 2>/dev/null)" "rolled-back terminated" + [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && [ ! -e "$sb/netplan/50-van-wan.yaml" ] && ok "apply helper TERM: restored" || bad "apply helper TERM: not restored" + [ "$(grep -c 'restoring' "$sb/log")" = 1 ] && ok "apply helper TERM: restore ran once" || bad "apply helper TERM: restore count" + rm -rf "$sb" + } rm -rf "$stubs" fi +# 18b. status file gate and STARLINK_IFACE validation +gs=$(mktemp -d) +for w in running rollback-failed ok rolled-back; do + echo "$w 2026-01-01T00:00:00+00:00 x" > "$gs/s" + case $w in running|rollback-failed) want=yes ;; *) want=no ;; esac + [ "$(./deploy.sh --apply-status-check "$gs/s")" = "$want" ] && ok "apply-status-check: $w -> $want" || bad "apply-status-check: $w should be $want" +done +[ "$(./deploy.sh --apply-status-check "$gs/none")" = no ] && ok "apply-status-check: no file -> no" || bad "apply-status-check: no file" +sed 's/^STARLINK_IFACE=.*/STARLINK_IFACE="a'"'"'b"/' hosts/van.conf > "$gs/bad.conf" +if ./deploy.sh --check "$gs/bad.conf" >/dev/null 2>&1; then bad "STARLINK_IFACE with a quote must be rejected"; else ok "STARLINK_IFACE with a quote is rejected"; fi +rm -rf "$gs" exit "$fail"