From 50ac006894ac4d8d8c4eacada50fed69f2f023b6 Mon Sep 17 00:00:00 2001 From: Andreas Wrede Date: Wed, 23 Sep 2026 19:01:25 -0400 Subject: [PATCH] deploy: apply helper hardening (signal ordering, timeout -k, self-cleanup, checked pristine mv) Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9 --- deploy.sh | 30 ++++++++++++++++-------------- tests/check-templates.sh | 11 +++++++++-- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/deploy.sh b/deploy.sh index f1b2f8e..b2e3772 100755 --- a/deploy.sh +++ b/deploy.sh @@ -283,8 +283,9 @@ post_check() { # read-only live checks; every probe is guarded, warns instead o } # --apply-network time budgets (seconds), in one place; all derived from the AN_* constants. -# forward path AN_FWD = generate + apply + stage 1 + restart + stage 2 -# rollback path AN_RB = generate + apply + restart + connectivity recheck +# (netplan generate/apply run under timeout -k AN_KILL_T: +AN_KILL_T each in the worst case) +# forward path AN_FWD = generate + apply + 2*kill + stage 1 + restart + stage 2 +# rollback path AN_RB = generate + apply + 2*kill + restart + connectivity recheck # helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their # deadline by a few nmcli/ping timeouts) # foreground waits AN_WAIT = AN_HELPER_MAX + 30 @@ -293,9 +294,9 @@ post_check() { # read-only live checks; every probe is guarded, warns instead o # van-failover starts optimistic and probes every WAN at once (probe_timeout 3 s), enforcing the # route in its first loop, so a healthy default route shows up within seconds of a restart; the # 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe. -AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60 -AN_FWD=$(( AN_GEN_T + AN_APPLY_T + AN_S1 + AN_RESTART_T + AN_S2 )) -AN_RB=$(( AN_GEN_T + AN_APPLY_T + AN_RESTART_T + AN_RC )) +AN_KILL_T=10 AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60 +AN_FWD=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_S1 + AN_RESTART_T + AN_S2 )) +AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_RESTART_T + AN_RC )) AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP )) AN_WAIT=$(( AN_HELPER_MAX + 30 )) AN_RUNMAX=$(( AN_HELPER_MAX + 60 )) @@ -334,6 +335,7 @@ st='$st' sn='$sn' rf='$rf' exec >>"\$log" 2>&1 +trap 'rm -f "\$0"' EXIT status() { mkdir -p "\$(dirname "\$st")"; echo "\$1 \$(date -Is)\${2:+ \$2}" > "\$st"; } echo "== apply-network $stamp \$(date -Is) (backup: \$bak)" conn_ok() { ip route show default | grep . >/dev/null && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; } @@ -353,9 +355,9 @@ gate1() { route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; } in_restore=0 restore() { # restore : put the pre-run netplan back, re-check connectivity, always exit 1 (runs once) + trap '' TERM INT HUP [ "\$in_restore" = 0 ] || exit 1 in_restore=1 - trap '' TERM INT HUP local rb=ok c=down echo "apply-network FAILED (\$1) - restoring \$bak" route_log @@ -368,7 +370,7 @@ restore() { # restore : put the pre-run netplan back, re-check connecti rm -f "\$nd/50-van-wan.yaml" [ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml" fi - timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; } + timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; } timeout $AN_RESTART_T systemctl restart van-failover || true end=\$((SECONDS+$d3)) while :; do @@ -398,7 +400,7 @@ trap 'restore terminated' TERM INT HUP applied=0 if install -m0600 "\$rf" "\$nd/50-van-wan.yaml" \\ && { [ ! -f "\$nd/10-dhcp-all-interfaces.yaml" ] || mv "\$nd/10-dhcp-all-interfaces.yaml" "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled"; } \\ - && timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply; then + && timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply; then applied=1 else echo "netplan install/generate/apply failed" @@ -423,9 +425,9 @@ while :; do done route_log [ "\$s2" = 1 ] || restore "stage 2: no default route or internet" -trap - TERM INT HUP rm -f "\$rf" status ok +trap - TERM INT HUP echo "apply-network OK" exit 0 HELPER @@ -455,14 +457,14 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c helper=/run/vanlink-apply-network-$stamp.sh (umask 077; render_wan_netplan > "$rendered") || die "render failed" check_rendered ap/50-van-wan.yaml "$rendered" || { rm -f "$rendered"; exit 1; } - bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || die "cannot create backup dir" - cp -a /etc/netplan/. "$bak/" || die "backup failed — nothing changed" + bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || { rm -f "$rendered"; die "cannot create backup dir"; } + cp -a /etc/netplan/. "$bak/" || { rm -f "$rendered"; die "backup failed — nothing changed"; } if [ ! -e /var/backups/vanlink-netplan-orig ] && [ ! -e /etc/netplan/50-van-wan.yaml ]; then pt=/var/backups/vanlink-netplan-orig.tmp.$$ rm -rf "$pt" install -d -m0700 "$pt" - cp -a /etc/netplan/. "$pt/" || { rm -rf "$pt"; die "pristine backup failed — nothing changed"; } - mv "$pt" /var/backups/vanlink-netplan-orig + cp -a /etc/netplan/. "$pt/" || { rm -rf "$pt"; rm -f "$rendered"; die "pristine backup failed — nothing changed"; } + mv "$pt" /var/backups/vanlink-netplan-orig || { rm -rf "$pt"; rm -f "$rendered"; die "pristine backup could not be moved into place — nothing changed"; } fi install -d -m0755 /var/lib/vanlink rm -f "$st" @@ -472,7 +474,7 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c echo "applying from a detached unit (survives an SSH drop); waiting up to $AN_WAIT s ..." systemctl reset-failed vanlink-apply-network 2>/dev/null || true systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec="$AN_RUNMAX" -p TimeoutStopSec="$AN_STOP_T" /bin/bash "$helper" >/dev/null \ - || die "systemd-run failed — nothing was changed (backup: $bak)" + || { rm -f "$helper" "$rendered"; die "systemd-run failed — nothing was changed (backup: $bak)"; } SECONDS=0 while systemctl is-active --quiet vanlink-apply-network && [ "$SECONDS" -lt "$AN_WAIT" ]; do sleep 1; done awk -v s="== apply-network $stamp" 'index($0, s) == 1 { p = 1 } p' "$log" 2>/dev/null | tail -n 8 || true diff --git a/tests/check-templates.sh b/tests/check-templates.sh index 866e838..8cb0f8a 100755 --- a/tests/check-templates.sh +++ b/tests/check-templates.sh @@ -304,7 +304,7 @@ case " $* " in *" -I "*) exit "${STUB_GW_PING:-0}" ;; esac exit "${STUB_NET_PING:-0}" STUB printf '#!/bin/bash\nexec /bin/sleep 0.05\n' > "$stubs/sleep" - printf '#!/bin/bash\nshift\nexec "$@"\n' > "$stubs/timeout" + printf '#!/bin/bash\n[ "$1" = -k ] && shift 2\nshift\nexec "$@"\n' > "$stubs/timeout" printf '#!/bin/bash\nexit 0\n' >> "$stubs/systemctl" chmod +x "$stubs"/* stubs_ok=1 @@ -336,6 +336,7 @@ STUB an_run a_is "success: exit 0" "$rc" 0; a_is "success: status" "$(st_word)" ok grep -q '^ping -c1 -W2 -I enP2p33s0 192.168.100.1$' "$sb/calls" && ok "apply helper success: gateway from the DHCP routers option" || bad "apply helper success: routers option not used" + [ ! -e "$sb/helper.sh" ] && ok "apply helper success: helper removed itself" || bad "apply helper success: helper file left behind" grep -q 'route to 1.1.1.1:' "$sb/log" && ok "apply helper success: route logged" || bad "apply helper success: no route log" grep -q 'apply-network OK' "$sb/log" && ok "apply helper success: log OK" || bad "apply helper success: log" a_is "success: 50-van-wan.yaml mode" "$(stat -c %a "$sb/netplan/50-van-wan.yaml" 2>/dev/null)" 600 @@ -396,7 +397,6 @@ STUB a_is "failed rollback, no connectivity: status" "$(st_word)" rollback-failed grep -q '^systemctl reboot' "$sb/calls" && ok "apply helper: reboots when the rollback failed and connectivity is absent" || bad "apply helper: no reboot after failed rollback" rm -rf "$sb" - chmod 600 "$sb/bak/60-other.yaml" 2>/dev/null || true; rm -rf "$sb" # (8) TERM while a slow netplan apply is running: the trap restores, once an_prep && { : > "$sb/hold" @@ -414,6 +414,13 @@ STUB rm -rf "$sb" } rm -rf "$stubs" + # 18a. ordering that a behavioural test cannot reach (a signal at an exact instant), checked on the text + r1=$(echo "$h" | grep -A2 '^restore() {' | sed -n 2p) + [ "$r1" = " trap '' TERM INT HUP" ] && ok "apply helper: restore() ignores signals before its guard" || bad "apply helper: restore() must start with trap '' (got '$r1')" + so=$(echo "$h" | grep -n '^status ok$' | cut -d: -f1); tr=$(echo "$h" | grep -n '^trap - TERM INT HUP$' | cut -d: -f1) + [ -n "$so" ] && [ -n "$tr" ] && [ "$so" -lt "$tr" ] && ok "apply helper: status ok is written before the trap is removed" || bad "apply helper: status ok must precede trap removal" + echo "$h" | grep -q "^trap 'rm -f \"\$0\"' EXIT$" && ok "apply helper removes itself on exit" || bad "apply helper lacks the self-remove EXIT trap" + echo "$h" | grep -q 'timeout -k 10 30 netplan generate' && echo "$h" | grep -q 'timeout -k 10 90 netplan apply' && ok "apply helper: netplan under timeout -k" || bad "apply helper: netplan lacks timeout -k" fi # 18b. status file gate and STARLINK_IFACE validation gs=$(mktemp -d)