andreas/vanlink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy: apply helper gateway from DHCP routers, trap/timeouts, running status, per-stamp files

Browse Source 
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9
This commit is contained in:
andreas
2026-09-23 18:55:04 -04:00
co-authored by Claude Sonnet 5
parent ca7d22384b
commit cefca06dac
2 changed files with 154 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy.sh
+86 -30
View File
@@ -12,7 +12,9 @@
# ./deploy.sh --render-wan-netplan # rendered /etc/netplan/50-van-wan.yaml (ethernet part + wifis stanza when WIFI_WAN_IFACE is set) # ./deploy.sh --render-wan-netplan # rendered /etc/netplan/50-van-wan.yaml (ethernet part + wifis stanza when WIFI_WAN_IFACE is set)
# ./deploy.sh --wan-plan [conf] # what the NetworkManager WAN provisioning would do (read-only, no root) # ./deploy.sh --wan-plan [conf] # what the NetworkManager WAN provisioning would do (read-only, no root)
# ./deploy.sh --apply-network [--force] # root: a detached helper installs the WAN netplan, applies it, gates on uplink health (2 stages) and rolls back # ./deploy.sh --apply-network [--force] # root: a detached helper installs the WAN netplan, applies it, gates on uplink health (2 stages) and rolls back
# # automatically on failure; status in /var/lib/vanlink/apply-network.status. No-op when already applied unless --force. # # automatically on failure; status in /var/lib/vanlink/apply-network.status. No-op when already applied, and refused after a
# # running/rollback-failed status, unless --force.
# ./deploy.sh --apply-status-check <file> # test seam: yes|no, would this status file (running|rollback-failed) block --apply-network
# ./deploy.sh --print-apply-helper [conf] # print the detached apply/rollback helper script (read-only, no root; for review and tests) # ./deploy.sh --print-apply-helper [conf] # print the detached apply/rollback helper script (read-only, no root; for review and tests)
# # test-only flags: --backup --netplan-dir --log --status --sysnet --rendered --deadlines "s1 s2 rc" --stamp # # test-only flags: --backup --netplan-dir --log --status --sysnet --rendered --deadlines "s1 s2 rc" --stamp
# ./deploy.sh render <file> # preview a templated file on stdout # ./deploy.sh render <file> # preview a templated file on stdout
@@ -70,6 +72,7 @@ load_conf() { # load_conf <file> — source and validate a host config
*) die "$f: $flag must be 0 or 1 (got '${!flag}')" ;; *) die "$f: $flag must be 0 or 1 (got '${!flag}')" ;;
esac esac
done done
[[ $STARLINK_IFACE =~ ^[A-Za-z0-9._-]+$ ]] || die "$f: STARLINK_IFACE '$STARLINK_IFACE' has characters outside [A-Za-z0-9._-]"
if [ "$ENABLE_NVME_WATCH" = 1 ]; then if [ "$ENABLE_NVME_WATCH" = 1 ]; then
{ [ -n "${NVME_DEVICE:-}" ] && [ -n "${NVME_FS_DEVICE:-}" ]; } \ { [ -n "${NVME_DEVICE:-}" ] && [ -n "${NVME_FS_DEVICE:-}" ]; } \
|| die "$f: ENABLE_NVME_WATCH=1 needs NVME_DEVICE and NVME_FS_DEVICE" || die "$f: ENABLE_NVME_WATCH=1 needs NVME_DEVICE and NVME_FS_DEVICE"
@@ -279,14 +282,31 @@ post_check() { # read-only live checks; every probe is guarded, warns instead o
else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi
} }
# --apply-network time budgets (seconds), in one place. The helper's worst case is # --apply-network time budgets (seconds), in one place; all derived from the AN_* constants.
# apply + stage 1 + restart + stage 2 + rollback (apply + restart + recheck); the # forward path AN_FWD = generate + apply + stage 1 + restart + stage 2
# foreground waits that long plus a margin. van-failover starts optimistic and # rollback path AN_RB = generate + apply + restart + connectivity recheck
# probes every WAN at once (probe_timeout 3 s), enforcing the route in its first # helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their
# loop, so a healthy default route shows up within seconds of a restart; the 120 s # deadline by a few nmcli/ping timeouts)
# stage 2 deadline is a wide margin for a slow Starlink/cellular probe. # foreground waits AN_WAIT = AN_HELPER_MAX + 30
AN_APPLY_T=90 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 # transient unit: RuntimeMaxSec = AN_HELPER_MAX + 60 (backstop; systemd then sends TERM and the
AN_WAIT=$(( 2 * AN_APPLY_T + 2 * AN_RESTART_T + AN_S1 + AN_S2 + AN_RC + 30 )) # helper's trap runs the restore), TimeoutStopSec = AN_RB + 30 (time the restore may take).
# van-failover starts optimistic and probes every WAN at once (probe_timeout 3 s), enforcing the
# route in its first loop, so a healthy default route shows up within seconds of a restart; the
# 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe.
AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60
AN_FWD=$(( AN_GEN_T + AN_APPLY_T + AN_S1 + AN_RESTART_T + AN_S2 ))
AN_RB=$(( AN_GEN_T + AN_APPLY_T + AN_RESTART_T + AN_RC ))
AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP ))
AN_WAIT=$(( AN_HELPER_MAX + 30 ))
AN_RUNMAX=$(( AN_HELPER_MAX + 60 ))
AN_STOP_T=$(( AN_RB + 30 ))
apply_status_blocks_run() { # apply_status_blocks_run <status-file> -> yes|no: a run that is/was in flight or failed to roll back blocks a new one
case "$(cut -d' ' -f1 "$1" 2>/dev/null || true)" in
running|rollback-failed) echo yes ;;
*) echo no ;;
esac
}
# apply_helper_script <backup-dir> [netplan-dir] [log] [status-file] [sysnet-dir] [rendered-file] [deadlines "s1 s2 rc"] [run-stamp] # apply_helper_script <backup-dir> [netplan-dir] [log] [status-file] [sysnet-dir] [rendered-file] [deadlines "s1 s2 rc"] [run-stamp]
# The script the detached unit runs. Paths are parameters only so tests can bake # The script the detached unit runs. Paths are parameters only so tests can bake
@@ -301,10 +321,11 @@ apply_helper_script() {
cat <<HELPER cat <<HELPER
#!/bin/bash #!/bin/bash
# Generated by deploy.sh --apply-network; runs detached. Installs the rendered WAN netplan, # Generated by deploy.sh --apply-network; runs detached. Installs the rendered WAN netplan,
# applies it, verifies the uplink in two stages and restores the pre-run netplan on any failure. # applies it, verifies the uplink in two stages and restores the pre-run netplan on any failure
# (including TERM/INT/HUP, e.g. systemd stopping the unit at its RuntimeMaxSec).
# Stage 1 (${d1}s): the apply succeeded and, if the Starlink port exists, NM shows it connected # Stage 1 (${d1}s): the apply succeeded and, if the Starlink port exists, NM shows it connected
# via a netplan-* profile with an IPv4 address whose gateway answers on-link (the profile is # via a netplan-* profile with an IPv4 address whose gateway (IP4.GATEWAY, else the DHCP
# never-default; van-failover owns default routes). # routers option: never-default hides IP4.GATEWAY) answers on-link.
# Stage 2 (${d2}s): van-failover restarted; default route and internet ping. # Stage 2 (${d2}s): van-failover restarted; default route and internet ping.
bak='$bak' bak='$bak'
nd='$nd' nd='$nd'
@@ -318,23 +339,36 @@ echo "== apply-network $stamp \$(date -Is) (backup: \$bak)"
conn_ok() { ip route show default | grep . >/dev/null && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; } conn_ok() { ip route show default | grep . >/dev/null && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; }
gate1() { gate1() {
[ -e "\$sn/$STARLINK_IFACE" ] || return 0 [ -e "\$sn/$STARLINK_IFACE" ] || return 0
[ "\$(nmcli -g GENERAL.STATE device show '$STARLINK_IFACE' 2>/dev/null)" = "100 (connected)" ] || return 1 [ "\$(timeout $AN_NM_T nmcli -g GENERAL.STATE device show '$STARLINK_IFACE' 2>/dev/null)" = "100 (connected)" ] || return 1
case "\$(nmcli -g GENERAL.CONNECTION device show '$STARLINK_IFACE' 2>/dev/null)" in netplan-*) ;; *) return 1 ;; esac case "\$(timeout $AN_NM_T nmcli -g GENERAL.CONNECTION device show '$STARLINK_IFACE' 2>/dev/null)" in netplan-*) ;; *) return 1 ;; esac
a=\$(nmcli -g IP4.ADDRESS device show '$STARLINK_IFACE' 2>/dev/null) a=\$(timeout $AN_NM_T nmcli -g IP4.ADDRESS device show '$STARLINK_IFACE' 2>/dev/null)
g=\$(nmcli -g IP4.GATEWAY device show '$STARLINK_IFACE' 2>/dev/null) [ -n "\$a" ] && [ "\$a" != -- ] || return 1
[ -n "\$a" ] && [ "\$a" != -- ] && [ -n "\$g" ] && [ "\$g" != -- ] || return 1 g=\$(timeout $AN_NM_T nmcli -g IP4.GATEWAY device show '$STARLINK_IFACE' 2>/dev/null)
case "\$g" in
''|--) g=\$(timeout $AN_NM_T nmcli -g DHCP4.OPTION device show '$STARLINK_IFACE' 2>/dev/null | tr '|' '\\n' | sed -n 's/^ *routers = *//p' | head -1 | cut -d' ' -f1) ;;
esac
[ -n "\$g" ] && [ "\$g" != -- ] || return 1
ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1 ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1
} }
restore() { # restore <reason>: put the pre-run netplan back, re-check connectivity, always exit 1 route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; }
in_restore=0
restore() { # restore <reason>: put the pre-run netplan back, re-check connectivity, always exit 1 (runs once)
[ "\$in_restore" = 0 ] || exit 1
in_restore=1
trap '' TERM INT HUP
local rb=ok c=down local rb=ok c=down
echo "apply-network FAILED (\$1) - restoring \$bak" echo "apply-network FAILED (\$1) - restoring \$bak"
route_log
if [ -d "\$bak" ]; then if [ -d "\$bak" ]; then
rm -f "\$nd"/* "\$nd"/.[!.]* rm -f "\$nd"/* "\$nd"/.[!.]*
cp -a "\$bak"/. "\$nd"/ || rb=CP-FAILED cp -a "\$bak"/. "\$nd"/ || rb=CP-FAILED
else else
# No backup to restore from: at least undo this helper's own two changes.
rb=BACKUP-MISSING rb=BACKUP-MISSING
rm -f "\$nd/50-van-wan.yaml"
[ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml"
fi fi
netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; } timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; }
timeout $AN_RESTART_T systemctl restart van-failover || true timeout $AN_RESTART_T systemctl restart van-failover || true
end=\$((SECONDS+$d3)) end=\$((SECONDS+$d3))
while :; do while :; do
@@ -342,23 +376,29 @@ restore() { # restore <reason>: put the pre-run netplan back, re-check connecti
[ "\$SECONDS" -ge "\$end" ] && break [ "\$SECONDS" -ge "\$end" ] && break
sleep 1 sleep 1
done done
rm -f "\$rf"
if [ "\$rb" = ok ] && [ "\$c" = ok ]; then if [ "\$rb" = ok ] && [ "\$c" = ok ]; then
echo "apply-network ROLLED BACK (restored, connectivity ok)" echo "apply-network ROLLED BACK (restored, connectivity ok)"
status rolled-back "\$1" status rolled-back "\$1"
elif [ "\$rb" = ok ]; then elif [ "\$rb" = ok ]; then
echo "apply-network ROLLED BACK (restored, but no connectivity - check the uplink/cable)" echo "apply-network ROLLED BACK (restored, but no connectivity - check the uplink/cable)"
status rolled-back restored-no-connectivity status rolled-back "restored-no-connectivity; \$1"
else else
echo "apply-network ROLLBACK FAILED: \$rb / connectivity \$c" echo "apply-network ROLLBACK FAILED: \$rb / connectivity \$c"
status rollback-failed "\$1; \$rb / connectivity \$c" status rollback-failed "\$1; \$rb / connectivity \$c"
[ "\$c" = ok ] || { echo "rollback failed and no connectivity - rebooting (boot uses the files on disk)"; systemctl reboot; } # Reboot (boot uses the files on disk) only when a restore was attempted and failed, and there is no connectivity.
case "\$rb" in
CP-FAILED|APPLY-FAILED) [ "\$c" = ok ] || { echo "rollback failed and no connectivity - rebooting"; systemctl reboot; } ;;
esac
fi fi
exit 1 exit 1
} }
status running "\$bak"
trap 'restore terminated' TERM INT HUP
applied=0 applied=0
if install -m0600 "\$rf" "\$nd/50-van-wan.yaml" \\ if install -m0600 "\$rf" "\$nd/50-van-wan.yaml" \\
&& { [ ! -f "\$nd/10-dhcp-all-interfaces.yaml" ] || mv "\$nd/10-dhcp-all-interfaces.yaml" "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled"; } \\ && { [ ! -f "\$nd/10-dhcp-all-interfaces.yaml" ] || mv "\$nd/10-dhcp-all-interfaces.yaml" "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled"; } \\
&& netplan generate && timeout $AN_APPLY_T netplan apply; then && timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply; then
applied=1 applied=1
else else
echo "netplan install/generate/apply failed" echo "netplan install/generate/apply failed"
@@ -381,7 +421,10 @@ while :; do
[ "\$SECONDS" -ge "\$end" ] && break [ "\$SECONDS" -ge "\$end" ] && break
sleep 1 sleep 1
done done
route_log
[ "\$s2" = 1 ] || restore "stage 2: no default route or internet" [ "\$s2" = 1 ] || restore "stage 2: no default route or internet"
trap - TERM INT HUP
rm -f "\$rf"
status ok status ok
echo "apply-network OK" echo "apply-network OK"
exit 0 exit 0
@@ -396,29 +439,39 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c
dpkg-query -W -f='${Status}' network-manager 2>/dev/null | grep -q 'install ok installed' \ dpkg-query -W -f='${Status}' network-manager 2>/dev/null | grep -q 'install ok installed' \
|| die "network-manager is not installed — run ./deploy.sh first" || die "network-manager is not installed — run ./deploy.sh first"
systemctl is-active --quiet NetworkManager || die "NetworkManager is not running — handing it the uplink port would flap it" systemctl is-active --quiet NetworkManager || die "NetworkManager is not running — handing it the uplink port would flap it"
systemctl cat van-failover >/dev/null 2>&1 || die "van-failover.service does not exist — run ./deploy.sh first (stage 2 of the gate needs it)"
systemctl is-active --quiet vanlink-apply-network && die "an apply is already running — see /var/log/vanlink-apply-network.log" systemctl is-active --quiet vanlink-apply-network && die "an apply is already running — see /var/log/vanlink-apply-network.log"
local log=/var/log/vanlink-apply-network.log st=/var/lib/vanlink/apply-network.status
if [ "$force" = 0 ] && [ "$(apply_status_blocks_run "$st")" = yes ]; then
die "the last apply did not finish cleanly: $(cat "$st" 2>/dev/null) — inspect $st and $log, check the backup dir named there, then re-run with --force"
fi
if [ "$force" = 0 ] && [ "$(nm_wan_state_netplan)" = matches ] && [ ! -e /etc/netplan/10-dhcp-all-interfaces.yaml ]; then if [ "$force" = 0 ] && [ "$(nm_wan_state_netplan)" = matches ] && [ ! -e /etc/netplan/10-dhcp-all-interfaces.yaml ]; then
echo "nothing to apply (WAN netplan already matches; use --force to re-apply)" echo "nothing to apply (WAN netplan already matches; use --force to re-apply)"
return 0 return 0
fi fi
local rendered=/run/vanlink-50-van-wan.yaml stamp bak log=/var/log/vanlink-apply-network.log st=/var/lib/vanlink/apply-network.status local stamp helper rendered bak pt
stamp=$(date +%Y%m%d-%H%M%S)-$$
rendered=/run/vanlink-50-van-wan-$stamp.yaml
helper=/run/vanlink-apply-network-$stamp.sh
(umask 077; render_wan_netplan > "$rendered") || die "render failed" (umask 077; render_wan_netplan > "$rendered") || die "render failed"
check_rendered ap/50-van-wan.yaml "$rendered" || { rm -f "$rendered"; exit 1; } check_rendered ap/50-van-wan.yaml "$rendered" || { rm -f "$rendered"; exit 1; }
stamp=$(date +%Y%m%d-%H%M%S)
bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || die "cannot create backup dir" bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || die "cannot create backup dir"
cp -a /etc/netplan/. "$bak/" || die "backup failed — nothing changed" cp -a /etc/netplan/. "$bak/" || die "backup failed — nothing changed"
if [ ! -e /var/backups/vanlink-netplan-orig ] && [ ! -e /etc/netplan/50-van-wan.yaml ]; then if [ ! -e /var/backups/vanlink-netplan-orig ] && [ ! -e /etc/netplan/50-van-wan.yaml ]; then
install -d -m0700 /var/backups/vanlink-netplan-orig pt=/var/backups/vanlink-netplan-orig.tmp.$$
cp -a /etc/netplan/. /var/backups/vanlink-netplan-orig/ || { rm -rf /var/backups/vanlink-netplan-orig; die "pristine backup failed — nothing changed"; } rm -rf "$pt"
install -d -m0700 "$pt"
cp -a /etc/netplan/. "$pt/" || { rm -rf "$pt"; die "pristine backup failed — nothing changed"; }
mv "$pt" /var/backups/vanlink-netplan-orig
fi fi
install -d -m0755 /var/lib/vanlink install -d -m0755 /var/lib/vanlink
rm -f "$st" rm -f "$st"
apply_helper_script "$bak" /etc/netplan "$log" "$st" /sys/class/net "$rendered" "$AN_S1 $AN_S2 $AN_RC" "$stamp" > /run/vanlink-apply-network.sh apply_helper_script "$bak" /etc/netplan "$log" "$st" /sys/class/net "$rendered" "$AN_S1 $AN_S2 $AN_RC" "$stamp" > "$helper"
chmod 0700 /run/vanlink-apply-network.sh chmod 0700 "$helper"
echo "backup: $bak log: $log" echo "backup: $bak log: $log"
echo "applying from a detached unit (survives an SSH drop); waiting up to $AN_WAIT s ..." echo "applying from a detached unit (survives an SSH drop); waiting up to $AN_WAIT s ..."
systemctl reset-failed vanlink-apply-network 2>/dev/null || true systemctl reset-failed vanlink-apply-network 2>/dev/null || true
systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec=900 /bin/bash /run/vanlink-apply-network.sh >/dev/null \ systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec="$AN_RUNMAX" -p TimeoutStopSec="$AN_STOP_T" /bin/bash "$helper" >/dev/null \
|| die "systemd-run failed — nothing was changed (backup: $bak)" || die "systemd-run failed — nothing was changed (backup: $bak)"
SECONDS=0 SECONDS=0
while systemctl is-active --quiet vanlink-apply-network && [ "$SECONDS" -lt "$AN_WAIT" ]; do sleep 1; done while systemctl is-active --quiet vanlink-apply-network && [ "$SECONDS" -lt "$AN_WAIT" ]; do sleep 1; done
@@ -432,7 +485,7 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c
case "$verdict" in case "$verdict" in
ok) return 0 ;; ok) return 0 ;;
rolled-back|rollback-failed) echo "apply-network: $(cat "$st") (previous config: $bak)" >&2; return 1 ;; rolled-back|rollback-failed) echo "apply-network: $(cat "$st") (previous config: $bak)" >&2; return 1 ;;
*) echo "apply-network: no status file — see $log (previous config: $bak)" >&2; return 1 ;; *) echo "apply-network: no final status — see $log (previous config: $bak)" >&2; return 1 ;;
esac esac
} }
@@ -485,6 +538,9 @@ case "${1:-}" in
load_conf "$VANLINK_CONF" load_conf "$VANLINK_CONF"
apply_network "${2:-}" apply_network "${2:-}"
exit $? ;; exit $? ;;
--apply-status-check) # test seam: --apply-status-check <status-file> prints yes|no (does that status block a new --apply-network run?)
apply_status_blocks_run "${2:?status file}"
exit 0 ;;
--print-apply-helper) # [conf] the helper script for review; test-only flags: --backup D --netplan-dir D --log L --status S --sysnet N --rendered R --deadlines "s1 s2 rc" --stamp T --print-apply-helper) # [conf] the helper script for review; test-only flags: --backup D --netplan-dir D --log L --status S --sysnet N --rendered R --deadlines "s1 s2 rc" --stamp T
shift; pc= a_bak=/var/backups/vanlink-netplan-EXAMPLE a_nd= a_log= a_st= a_sn= a_rf= a_dl= a_stamp= shift; pc= a_bak=/var/backups/vanlink-netplan-EXAMPLE a_nd= a_log= a_st= a_sn= a_rf= a_dl= a_stamp=
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
tests/check-templates.sh
+68 -11
View File
@@ -259,7 +259,9 @@ rm -rf "$nmd" "$nme"
# 18. the detached apply helper: syntax, then behaviour against stubbed system commands in a sandbox # 18. the detached apply helper: syntax, then behaviour against stubbed system commands in a sandbox
h=$(VANLINK_CONF=hosts/van.conf ./deploy.sh --print-apply-helper 2>&1) || { bad "--print-apply-helper failed"; echo "$h" | sed 's/^/ /'; h=; } h=$(VANLINK_CONF=hosts/van.conf ./deploy.sh --print-apply-helper 2>&1) || { bad "--print-apply-helper failed"; echo "$h" | sed 's/^/ /'; h=; }
if [ -n "$h" ]; then if [ -n "$h" ] && [ "$(id -u)" = 0 ]; then
bad "check 18 refuses to run the helper sandbox as root"
elif [ -n "$h" ]; then
echo "$h" | bash -n && ok "apply helper: bash -n" || bad "apply helper: bash -n" echo "$h" | bash -n && ok "apply helper: bash -n" || bad "apply helper: bash -n"
echo "$h" | grep -q 'enP2p33s0' && ok "apply helper waits on the Starlink port" || bad "apply helper does not reference STARLINK_IFACE" echo "$h" | grep -q 'enP2p33s0' && ok "apply helper waits on the Starlink port" || bad "apply helper does not reference STARLINK_IFACE"
echo "$h" | grep -q 'systemctl restart van-failover' && ok "apply helper restarts van-failover" || bad "apply helper lacks van-failover restart" echo "$h" | grep -q 'systemctl restart van-failover' && ok "apply helper restarts van-failover" || bad "apply helper lacks van-failover restart"
@@ -269,6 +271,10 @@ if [ -n "$h" ]; then
done done
cat >> "$stubs/netplan" <<'STUB' cat >> "$stubs/netplan" <<'STUB'
if [ "$1" = apply ]; then if [ "$1" = apply ]; then
if [ -e "${STUB_HOLD:-/nonexistent}" ]; then
: > "$STUB_SB/applying"
while [ -e "$STUB_HOLD" ]; do /bin/sleep 0.05; done
fi
n=$(cat "$STUB_SB/napply" 2>/dev/null || echo 0); echo $((n + 1)) > "$STUB_SB/napply" n=$(cat "$STUB_SB/napply" 2>/dev/null || echo 0); echo $((n + 1)) > "$STUB_SB/napply"
[ "${STUB_FAIL_APPLY:-}" = first ] && [ "$n" = 0 ] && exit 1 [ "${STUB_FAIL_APPLY:-}" = first ] && [ "$n" = 0 ] && exit 1
fi fi
@@ -279,11 +285,14 @@ case "$2" in
GENERAL.STATE) echo "${NM_STATE-100 (connected)}" ;; GENERAL.STATE) echo "${NM_STATE-100 (connected)}" ;;
GENERAL.CONNECTION) echo "${NM_CONN-netplan-enP2p33s0}" ;; GENERAL.CONNECTION) echo "${NM_CONN-netplan-enP2p33s0}" ;;
IP4.ADDRESS) echo "${NM_ADDR-192.168.100.10/24}" ;; IP4.ADDRESS) echo "${NM_ADDR-192.168.100.10/24}" ;;
IP4.GATEWAY) echo "${NM_GW-192.168.100.1}" ;; IP4.GATEWAY) echo "${NM_GW-}" ;;
# never-default: IP4.GATEWAY is empty, the router is only in the DHCP lease options (nmcli -g joins them with " | ")
DHCP4.OPTION) echo "${NM_DHCP-requested_routers = 1 | routers = 192.168.100.1 | subnet_mask = 255.255.255.0}" ;;
esac esac
exit 0 exit 0
STUB STUB
cat >> "$stubs/ip" <<'STUB' cat >> "$stubs/ip" <<'STUB'
if [ "$2" = get ]; then echo "1.1.1.1 via 192.168.100.1 dev x"; exit 0; fi
case "${STUB_ROUTE:-yes}" in case "${STUB_ROUTE:-yes}" in
yes) echo "default via 192.168.100.1 dev x" ;; yes) echo "default via 192.168.100.1 dev x" ;;
after-rollback) [ -e "$STUB_ND/50-van-wan.yaml" ] || echo "default via 192.168.100.1 dev x" ;; after-rollback) [ -e "$STUB_ND/50-van-wan.yaml" ] || echo "default via 192.168.100.1 dev x" ;;
@@ -302,17 +311,23 @@ STUB
for c in netplan nmcli ip ping systemctl reboot sleep timeout; do [ -x "$stubs/$c" ] || stubs_ok=0; done for c in netplan nmcli ip ping systemctl reboot sleep timeout; do [ -x "$stubs/$c" ] || stubs_ok=0; done
[ "$stubs_ok" = 1 ] && ok "apply helper sandbox: all system commands stubbed" || bad "apply helper sandbox: stub missing" [ "$stubs_ok" = 1 ] && ok "apply helper sandbox: all system commands stubbed" || bad "apply helper sandbox: stub missing"
an_run() { # an_run [VAR=val ...] — fresh sandbox, run the helper; sets sb rc an_prep() { # fresh sandbox + generated helper; sets sb
sb=$(mktemp -d); mkdir -p "$sb/netplan" "$sb/sysnet" sb=$(mktemp -d); mkdir -p "$sb/netplan" "$sb/sysnet"
printf 'armbian\n' > "$sb/netplan/10-dhcp-all-interfaces.yaml"; chmod 0600 "$sb/netplan/10-dhcp-all-interfaces.yaml" printf 'armbian\n' > "$sb/netplan/10-dhcp-all-interfaces.yaml"; chmod 0600 "$sb/netplan/10-dhcp-all-interfaces.yaml"
printf 'old\n' > "$sb/netplan/60-other.yaml" printf 'old\n' > "$sb/netplan/60-other.yaml"
cp -a "$sb/netplan" "$sb/bak" cp -a "$sb/netplan" "$sb/bak"
[ -n "${AN_NOPORT:-}" ] || mkdir "$sb/sysnet/enP2p33s0" [ -n "${AN_NOPORT:-}" ] || mkdir "$sb/sysnet/enP2p33s0"
[ -z "${AN_NOBAK:-}" ] || rm -rf "$sb/bak" [ -z "${AN_NOBAK:-}" ] || rm -rf "$sb/bak"
[ -z "${AN_BAKBAD:-}" ] || chmod 000 "$sb/bak/60-other.yaml"
printf 'network:\n version: 2\n' > "$sb/rendered.yaml" printf 'network:\n version: 2\n' > "$sb/rendered.yaml"
VANLINK_CONF=hosts/van.conf ./deploy.sh --print-apply-helper --backup "$sb/bak" --netplan-dir "$sb/netplan" \ VANLINK_CONF=hosts/van.conf ./deploy.sh --print-apply-helper --backup "$sb/bak" --netplan-dir "$sb/netplan" \
--log "$sb/log" --status "$sb/status" --sysnet "$sb/sysnet" --rendered "$sb/rendered.yaml" \ --log "$sb/log" --status "$sb/status" --sysnet "$sb/sysnet" --rendered "$sb/rendered.yaml" \
--deadlines "1 1 1" --stamp T > "$sb/helper.sh" --deadlines "1 1 1" --stamp T > "$sb/helper.sh"
# never run a helper that does not point at the sandbox
grep -qF "nd='$sb/netplan'" "$sb/helper.sh" && [ "$(id -u)" != 0 ] || { bad "helper does not bake the sandbox netplan dir (or root) - refusing to run"; return 1; }
}
an_run() { # an_run [VAR=val ...] — fresh sandbox, run the helper; sets sb rc
an_prep || { rc=99; return 0; }
env PATH="$stubs:/usr/bin:/bin" STUB_CALLS="$sb/calls" STUB_SB="$sb" STUB_ND="$sb/netplan" "$@" bash "$sb/helper.sh" && rc=0 || rc=$? env PATH="$stubs:/usr/bin:/bin" STUB_CALLS="$sb/calls" STUB_SB="$sb" STUB_ND="$sb/netplan" "$@" bash "$sb/helper.sh" && rc=0 || rc=$?
} }
a_is() { [ "$2" = "$3" ] && ok "apply helper $1" || bad "apply helper $1 (got '$2', want '$3')"; } a_is() { [ "$2" = "$3" ] && ok "apply helper $1" || bad "apply helper $1 (got '$2', want '$3')"; }
@@ -320,6 +335,8 @@ STUB
# (1) success # (1) success
an_run an_run
a_is "success: exit 0" "$rc" 0; a_is "success: status" "$(st_word)" ok a_is "success: exit 0" "$rc" 0; a_is "success: status" "$(st_word)" ok
grep -q '^ping -c1 -W2 -I enP2p33s0 192.168.100.1$' "$sb/calls" && ok "apply helper success: gateway from the DHCP routers option" || bad "apply helper success: routers option not used"
grep -q 'route to 1.1.1.1:' "$sb/log" && ok "apply helper success: route logged" || bad "apply helper success: no route log"
grep -q 'apply-network OK' "$sb/log" && ok "apply helper success: log OK" || bad "apply helper success: log" grep -q 'apply-network OK' "$sb/log" && ok "apply helper success: log OK" || bad "apply helper success: log"
a_is "success: 50-van-wan.yaml mode" "$(stat -c %a "$sb/netplan/50-van-wan.yaml" 2>/dev/null)" 600 a_is "success: 50-van-wan.yaml mode" "$(stat -c %a "$sb/netplan/50-van-wan.yaml" 2>/dev/null)" 600
[ -e "$sb/netplan/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] && [ ! -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && ok "apply helper success: Armbian file moved aside" || bad "apply helper success: Armbian file not moved aside" [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] && [ ! -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && ok "apply helper success: Armbian file moved aside" || bad "apply helper success: Armbian file not moved aside"
@@ -335,7 +352,11 @@ STUB
# (2b) stage 1 fails: wrong connection name; gateway unpingable # (2b) stage 1 fails: wrong connection name; gateway unpingable
an_run NM_CONN="Wired connection 1"; a_is "stage1 conn name: status" "$(st_word)" rolled-back; rm -rf "$sb" an_run NM_CONN="Wired connection 1"; a_is "stage1 conn name: status" "$(st_word)" rolled-back; rm -rf "$sb"
an_run STUB_GW_PING=1; a_is "stage1 gateway ping: status" "$(st_word)" rolled-back; rm -rf "$sb" an_run STUB_GW_PING=1; a_is "stage1 gateway ping: status" "$(st_word)" rolled-back; rm -rf "$sb"
an_run NM_GW=; a_is "stage1 no gateway: status" "$(st_word)" rolled-back; rm -rf "$sb" an_run NM_DHCP="requested_routers = 1 | subnet_mask = 255.255.255.0"; a_is "stage1 no IP4.GATEWAY and no routers option: status" "$(st_word)" rolled-back; rm -rf "$sb"
an_run NM_DHCP="routers = 192.168.100.1 192.168.100.2"; a_is "routers option with two routers: status" "$(st_word)" ok
grep -q '^ping -c1 -W2 -I enP2p33s0 192.168.100.1$' "$sb/calls" && ok "apply helper: pinged the first router" || bad "apply helper: first router not pinged"; rm -rf "$sb"
an_run NM_GW=192.168.100.9 NM_DHCP=; a_is "IP4.GATEWAY present: status" "$(st_word)" ok
grep -q '^ping -c1 -W2 -I enP2p33s0 192.168.100.9$' "$sb/calls" && ok "apply helper: pinged IP4.GATEWAY" || bad "apply helper: IP4.GATEWAY not pinged"; rm -rf "$sb"
# (3) netplan apply fails, port absent # (3) netplan apply fails, port absent
AN_NOPORT=1 an_run STUB_FAIL_APPLY=first AN_NOPORT=1 an_run STUB_FAIL_APPLY=first
a_is "apply failure: exit 1" "$rc" 1; a_is "apply failure: not ok" "$(st_word)" rolled-back a_is "apply failure: exit 1" "$rc" 1; a_is "apply failure: not ok" "$(st_word)" rolled-back
@@ -343,12 +364,19 @@ STUB
rm -rf "$sb" rm -rf "$sb"
# (3b) port absent and apply ok: stage 1 needs only the apply # (3b) port absent and apply ok: stage 1 needs only the apply
AN_NOPORT=1 an_run; a_is "port absent, apply ok: status" "$(st_word)" ok; rm -rf "$sb" AN_NOPORT=1 an_run; a_is "port absent, apply ok: status" "$(st_word)" ok; rm -rf "$sb"
# (4) rollback cp fails (backup dir missing) # (4) backup dir missing: undo own changes, report failure, never reboot into the new config
AN_NOBAK=1 an_run NM_STATE=bad AN_NOBAK=1 an_run NM_STATE=bad STUB_ROUTE=none
a_is "rollback cp failure: exit 1" "$rc" 1; a_is "rollback cp failure: status" "$(st_word)" rollback-failed a_is "backup missing: exit 1" "$rc" 1; a_is "backup missing: status" "$(st_word)" rollback-failed
grep -q 'ROLLBACK FAILED' "$sb/log" && ok "apply helper rollback failure: loud log" || bad "apply helper rollback failure: log" grep -q 'ROLLBACK FAILED: BACKUP-MISSING' "$sb/log" && ok "apply helper backup missing: loud log" || bad "apply helper backup missing: log"
grep -q '^reboot\|systemctl reboot' "$sb/calls" && bad "apply helper rollback failure with connectivity: must not reboot" || ok "apply helper rollback failure with connectivity: no reboot" grep -q '^systemctl reboot' "$sb/calls" && bad "apply helper backup missing: must not reboot" || ok "apply helper backup missing: no reboot"
[ ! -e "$sb/netplan/50-van-wan.yaml" ] && [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && [ ! -e "$sb/netplan/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] && ok "apply helper backup missing: own changes undone" || bad "apply helper backup missing: own changes not undone"
rm -rf "$sb" rm -rf "$sb"
# (4b) rollback cp fails (unreadable backup file) but connectivity is fine: loud, no reboot
AN_BAKBAD=1 an_run NM_STATE=bad
a_is "cp failure: status" "$(st_word)" rollback-failed
grep -q 'ROLLBACK FAILED: CP-FAILED' "$sb/log" && ok "apply helper cp failure: loud log" || bad "apply helper cp failure: log"
grep -q '^systemctl reboot' "$sb/calls" && bad "apply helper cp failure with connectivity: must not reboot" || ok "apply helper cp failure with connectivity: no reboot"
chmod 600 "$sb/bak/60-other.yaml"; rm -rf "$sb"
# (5) stage 2 fails: no default route until rolled back # (5) stage 2 fails: no default route until rolled back
an_run STUB_ROUTE=after-rollback an_run STUB_ROUTE=after-rollback
a_is "stage2: exit 1" "$rc" 1; a_is "stage2: status" "$(st_word)" rolled-back a_is "stage2: exit 1" "$rc" 1; a_is "stage2: status" "$(st_word)" rolled-back
@@ -357,17 +385,46 @@ STUB
# (6) clean rollback but no connectivity (uplink unplugged): must NOT reboot # (6) clean rollback but no connectivity (uplink unplugged): must NOT reboot
an_run STUB_ROUTE=none an_run STUB_ROUTE=none
a_is "no connectivity after clean rollback: exit" "$rc" 1 a_is "no connectivity after clean rollback: exit" "$rc" 1
a_is "no connectivity after clean rollback: status" "$(cat "$sb/status" 2>/dev/null | cut -d' ' -f1,3)" "rolled-back restored-no-connectivity" a_is "no connectivity after clean rollback: status" "$(cut -d' ' -f1,3 "$sb/status" 2>/dev/null)" "rolled-back restored-no-connectivity;"
grep -q 'restored-no-connectivity; stage 2' "$sb/status" && ok "apply helper: original reason kept in status" || bad "apply helper: reason lost from status"
grep -q 'restored, but no connectivity' "$sb/log" && ok "apply helper: no-connectivity log" || bad "apply helper: no-connectivity log" grep -q 'restored, but no connectivity' "$sb/log" && ok "apply helper: no-connectivity log" || bad "apply helper: no-connectivity log"
grep -q '^systemctl reboot' "$sb/calls" && bad "apply helper: must not reboot after a clean rollback" || ok "apply helper: no reboot after a clean rollback" grep -q '^systemctl reboot' "$sb/calls" && bad "apply helper: must not reboot after a clean rollback" || ok "apply helper: no reboot after a clean rollback"
[ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && [ ! -e "$sb/netplan/50-van-wan.yaml" ] && ok "apply helper: clean rollback restored files" || bad "apply helper: clean rollback files" [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && [ ! -e "$sb/netplan/50-van-wan.yaml" ] && ok "apply helper: clean rollback restored files" || bad "apply helper: clean rollback files"
rm -rf "$sb" rm -rf "$sb"
# (7) rollback cp fails AND no connectivity: last-resort reboot # (7) rollback cp fails AND no connectivity: last-resort reboot
AN_NOBAK=1 an_run STUB_ROUTE=none AN_BAKBAD=1 an_run STUB_ROUTE=none
a_is "failed rollback, no connectivity: status" "$(st_word)" rollback-failed a_is "failed rollback, no connectivity: status" "$(st_word)" rollback-failed
grep -q '^systemctl reboot' "$sb/calls" && ok "apply helper: reboots when the rollback failed and connectivity is absent" || bad "apply helper: no reboot after failed rollback" grep -q '^systemctl reboot' "$sb/calls" && ok "apply helper: reboots when the rollback failed and connectivity is absent" || bad "apply helper: no reboot after failed rollback"
rm -rf "$sb" rm -rf "$sb"
chmod 600 "$sb/bak/60-other.yaml" 2>/dev/null || true; rm -rf "$sb"
# (8) TERM while a slow netplan apply is running: the trap restores, once
an_prep && {
: > "$sb/hold"
env PATH="$stubs:/usr/bin:/bin" STUB_CALLS="$sb/calls" STUB_SB="$sb" STUB_ND="$sb/netplan" STUB_HOLD="$sb/hold" bash "$sb/helper.sh" &
hp=$!
for _ in $(seq 1 100); do [ -e "$sb/applying" ] && break; /bin/sleep 0.05; done
[ -e "$sb/applying" ] && ok "apply helper TERM: helper reached the slow apply" || bad "apply helper TERM: never reached the apply"
a_is "TERM: status while running" "$(cut -d' ' -f1,3 "$sb/status" 2>/dev/null)" "running $sb/bak"
kill -TERM "$hp"; rm -f "$sb/hold"
wait "$hp" && rc=0 || rc=$?
a_is "TERM: exit 1" "$rc" 1
a_is "TERM: status" "$(cut -d' ' -f1,3 "$sb/status" 2>/dev/null)" "rolled-back terminated"
[ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && [ ! -e "$sb/netplan/50-van-wan.yaml" ] && ok "apply helper TERM: restored" || bad "apply helper TERM: not restored"
[ "$(grep -c 'restoring' "$sb/log")" = 1 ] && ok "apply helper TERM: restore ran once" || bad "apply helper TERM: restore count"
rm -rf "$sb"
}
rm -rf "$stubs" rm -rf "$stubs"
fi fi
# 18b. status file gate and STARLINK_IFACE validation
gs=$(mktemp -d)
for w in running rollback-failed ok rolled-back; do
echo "$w 2026-01-01T00:00:00+00:00 x" > "$gs/s"
case $w in running|rollback-failed) want=yes ;; *) want=no ;; esac
[ "$(./deploy.sh --apply-status-check "$gs/s")" = "$want" ] && ok "apply-status-check: $w -> $want" || bad "apply-status-check: $w should be $want"
done
[ "$(./deploy.sh --apply-status-check "$gs/none")" = no ] && ok "apply-status-check: no file -> no" || bad "apply-status-check: no file"
sed 's/^STARLINK_IFACE=.*/STARLINK_IFACE="a'"'"'b"/' hosts/van.conf > "$gs/bad.conf"
if ./deploy.sh --check "$gs/bad.conf" >/dev/null 2>&1; then bad "STARLINK_IFACE with a quote must be rejected"; else ok "STARLINK_IFACE with a quote is rejected"; fi
rm -rf "$gs"
exit "$fail" exit "$fail"