deploy: apply helper hardening (signal ordering, timeout -k, self-cleanup, checked pristine mv)
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9
This commit is contained in:
Claude Sonnet 5
@@ -283,8 +283,9 @@ post_check() { # read-only live checks; every probe is guarded, warns instead o
|
||||
}
|
||||
|
||||
# --apply-network time budgets (seconds), in one place; all derived from the AN_* constants.
|
||||
# forward path AN_FWD = generate + apply + stage 1 + restart + stage 2
|
||||
# rollback path AN_RB = generate + apply + restart + connectivity recheck
|
||||
# (netplan generate/apply run under timeout -k AN_KILL_T: +AN_KILL_T each in the worst case)
|
||||
# forward path AN_FWD = generate + apply + 2*kill + stage 1 + restart + stage 2
|
||||
# rollback path AN_RB = generate + apply + 2*kill + restart + connectivity recheck
|
||||
# helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their
|
||||
# deadline by a few nmcli/ping timeouts)
|
||||
# foreground waits AN_WAIT = AN_HELPER_MAX + 30
|
||||
@@ -293,9 +294,9 @@ post_check() { # read-only live checks; every probe is guarded, warns instead o
|
||||
# van-failover starts optimistic and probes every WAN at once (probe_timeout 3 s), enforcing the
|
||||
# route in its first loop, so a healthy default route shows up within seconds of a restart; the
|
||||
# 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe.
|
||||
AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60
|
||||
AN_FWD=$(( AN_GEN_T + AN_APPLY_T + AN_S1 + AN_RESTART_T + AN_S2 ))
|
||||
AN_RB=$(( AN_GEN_T + AN_APPLY_T + AN_RESTART_T + AN_RC ))
|
||||
AN_KILL_T=10 AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60
|
||||
AN_FWD=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_S1 + AN_RESTART_T + AN_S2 ))
|
||||
AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_RESTART_T + AN_RC ))
|
||||
AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP ))
|
||||
AN_WAIT=$(( AN_HELPER_MAX + 30 ))
|
||||
AN_RUNMAX=$(( AN_HELPER_MAX + 60 ))
|
||||
@@ -334,6 +335,7 @@ st='$st'
|
||||
sn='$sn'
|
||||
rf='$rf'
|
||||
exec >>"\$log" 2>&1
|
||||
trap 'rm -f "\$0"' EXIT
|
||||
status() { mkdir -p "\$(dirname "\$st")"; echo "\$1 \$(date -Is)\${2:+ \$2}" > "\$st"; }
|
||||
echo "== apply-network $stamp \$(date -Is) (backup: \$bak)"
|
||||
conn_ok() { ip route show default | grep . >/dev/null && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; }
|
||||
@@ -353,9 +355,9 @@ gate1() {
|
||||
route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; }
|
||||
in_restore=0
|
||||
restore() { # restore <reason>: put the pre-run netplan back, re-check connectivity, always exit 1 (runs once)
|
||||
trap '' TERM INT HUP
|
||||
[ "\$in_restore" = 0 ] || exit 1
|
||||
in_restore=1
|
||||
trap '' TERM INT HUP
|
||||
local rb=ok c=down
|
||||
echo "apply-network FAILED (\$1) - restoring \$bak"
|
||||
route_log
|
||||
@@ -368,7 +370,7 @@ restore() { # restore <reason>: put the pre-run netplan back, re-check connecti
|
||||
rm -f "\$nd/50-van-wan.yaml"
|
||||
[ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml"
|
||||
fi
|
||||
timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; }
|
||||
timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; }
|
||||
timeout $AN_RESTART_T systemctl restart van-failover || true
|
||||
end=\$((SECONDS+$d3))
|
||||
while :; do
|
||||
@@ -398,7 +400,7 @@ trap 'restore terminated' TERM INT HUP
|
||||
applied=0
|
||||
if install -m0600 "\$rf" "\$nd/50-van-wan.yaml" \\
|
||||
&& { [ ! -f "\$nd/10-dhcp-all-interfaces.yaml" ] || mv "\$nd/10-dhcp-all-interfaces.yaml" "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled"; } \\
|
||||
&& timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply; then
|
||||
&& timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply; then
|
||||
applied=1
|
||||
else
|
||||
echo "netplan install/generate/apply failed"
|
||||
@@ -423,9 +425,9 @@ while :; do
|
||||
done
|
||||
route_log
|
||||
[ "\$s2" = 1 ] || restore "stage 2: no default route or internet"
|
||||
trap - TERM INT HUP
|
||||
rm -f "\$rf"
|
||||
status ok
|
||||
trap - TERM INT HUP
|
||||
echo "apply-network OK"
|
||||
exit 0
|
||||
HELPER
|
||||
@@ -455,14 +457,14 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c
|
||||
helper=/run/vanlink-apply-network-$stamp.sh
|
||||
(umask 077; render_wan_netplan > "$rendered") || die "render failed"
|
||||
check_rendered ap/50-van-wan.yaml "$rendered" || { rm -f "$rendered"; exit 1; }
|
||||
bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || die "cannot create backup dir"
|
||||
cp -a /etc/netplan/. "$bak/" || die "backup failed — nothing changed"
|
||||
bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || { rm -f "$rendered"; die "cannot create backup dir"; }
|
||||
cp -a /etc/netplan/. "$bak/" || { rm -f "$rendered"; die "backup failed — nothing changed"; }
|
||||
if [ ! -e /var/backups/vanlink-netplan-orig ] && [ ! -e /etc/netplan/50-van-wan.yaml ]; then
|
||||
pt=/var/backups/vanlink-netplan-orig.tmp.$$
|
||||
rm -rf "$pt"
|
||||
install -d -m0700 "$pt"
|
||||
cp -a /etc/netplan/. "$pt/" || { rm -rf "$pt"; die "pristine backup failed — nothing changed"; }
|
||||
mv "$pt" /var/backups/vanlink-netplan-orig
|
||||
cp -a /etc/netplan/. "$pt/" || { rm -rf "$pt"; rm -f "$rendered"; die "pristine backup failed — nothing changed"; }
|
||||
mv "$pt" /var/backups/vanlink-netplan-orig || { rm -rf "$pt"; rm -f "$rendered"; die "pristine backup could not be moved into place — nothing changed"; }
|
||||
fi
|
||||
install -d -m0755 /var/lib/vanlink
|
||||
rm -f "$st"
|
||||
@@ -472,7 +474,7 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c
|
||||
echo "applying from a detached unit (survives an SSH drop); waiting up to $AN_WAIT s ..."
|
||||
systemctl reset-failed vanlink-apply-network 2>/dev/null || true
|
||||
systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec="$AN_RUNMAX" -p TimeoutStopSec="$AN_STOP_T" /bin/bash "$helper" >/dev/null \
|
||||
|| die "systemd-run failed — nothing was changed (backup: $bak)"
|
||||
|| { rm -f "$helper" "$rendered"; die "systemd-run failed — nothing was changed (backup: $bak)"; }
|
||||
SECONDS=0
|
||||
while systemctl is-active --quiet vanlink-apply-network && [ "$SECONDS" -lt "$AN_WAIT" ]; do sleep 1; done
|
||||
awk -v s="== apply-network $stamp" 'index($0, s) == 1 { p = 1 } p' "$log" 2>/dev/null | tail -n 8 || true
|
||||
|
||||
Reference in New Issue
Block a user