deploy: apply helper gateway from DHCP routers, trap/timeouts, running status, per-stamp files
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9
This commit is contained in:
Claude Sonnet 5
@@ -12,7 +12,9 @@
|
||||
# ./deploy.sh --render-wan-netplan # rendered /etc/netplan/50-van-wan.yaml (ethernet part + wifis stanza when WIFI_WAN_IFACE is set)
|
||||
# ./deploy.sh --wan-plan [conf] # what the NetworkManager WAN provisioning would do (read-only, no root)
|
||||
# ./deploy.sh --apply-network [--force] # root: a detached helper installs the WAN netplan, applies it, gates on uplink health (2 stages) and rolls back
|
||||
# # automatically on failure; status in /var/lib/vanlink/apply-network.status. No-op when already applied unless --force.
|
||||
# # automatically on failure; status in /var/lib/vanlink/apply-network.status. No-op when already applied, and refused after a
|
||||
# # running/rollback-failed status, unless --force.
|
||||
# ./deploy.sh --apply-status-check <file> # test seam: yes|no, would this status file (running|rollback-failed) block --apply-network
|
||||
# ./deploy.sh --print-apply-helper [conf] # print the detached apply/rollback helper script (read-only, no root; for review and tests)
|
||||
# # test-only flags: --backup --netplan-dir --log --status --sysnet --rendered --deadlines "s1 s2 rc" --stamp
|
||||
# ./deploy.sh render <file> # preview a templated file on stdout
|
||||
@@ -70,6 +72,7 @@ load_conf() { # load_conf <file> — source and validate a host config
|
||||
*) die "$f: $flag must be 0 or 1 (got '${!flag}')" ;;
|
||||
esac
|
||||
done
|
||||
[[ $STARLINK_IFACE =~ ^[A-Za-z0-9._-]+$ ]] || die "$f: STARLINK_IFACE '$STARLINK_IFACE' has characters outside [A-Za-z0-9._-]"
|
||||
if [ "$ENABLE_NVME_WATCH" = 1 ]; then
|
||||
{ [ -n "${NVME_DEVICE:-}" ] && [ -n "${NVME_FS_DEVICE:-}" ]; } \
|
||||
|| die "$f: ENABLE_NVME_WATCH=1 needs NVME_DEVICE and NVME_FS_DEVICE"
|
||||
@@ -279,14 +282,31 @@ post_check() { # read-only live checks; every probe is guarded, warns instead o
|
||||
else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi
|
||||
}
|
||||
|
||||
# --apply-network time budgets (seconds), in one place. The helper's worst case is
|
||||
# apply + stage 1 + restart + stage 2 + rollback (apply + restart + recheck); the
|
||||
# foreground waits that long plus a margin. van-failover starts optimistic and
|
||||
# probes every WAN at once (probe_timeout 3 s), enforcing the route in its first
|
||||
# loop, so a healthy default route shows up within seconds of a restart; the 120 s
|
||||
# stage 2 deadline is a wide margin for a slow Starlink/cellular probe.
|
||||
AN_APPLY_T=90 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60
|
||||
AN_WAIT=$(( 2 * AN_APPLY_T + 2 * AN_RESTART_T + AN_S1 + AN_S2 + AN_RC + 30 ))
|
||||
# --apply-network time budgets (seconds), in one place; all derived from the AN_* constants.
|
||||
# forward path AN_FWD = generate + apply + stage 1 + restart + stage 2
|
||||
# rollback path AN_RB = generate + apply + restart + connectivity recheck
|
||||
# helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their
|
||||
# deadline by a few nmcli/ping timeouts)
|
||||
# foreground waits AN_WAIT = AN_HELPER_MAX + 30
|
||||
# transient unit: RuntimeMaxSec = AN_HELPER_MAX + 60 (backstop; systemd then sends TERM and the
|
||||
# helper's trap runs the restore), TimeoutStopSec = AN_RB + 30 (time the restore may take).
|
||||
# van-failover starts optimistic and probes every WAN at once (probe_timeout 3 s), enforcing the
|
||||
# route in its first loop, so a healthy default route shows up within seconds of a restart; the
|
||||
# 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe.
|
||||
AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60
|
||||
AN_FWD=$(( AN_GEN_T + AN_APPLY_T + AN_S1 + AN_RESTART_T + AN_S2 ))
|
||||
AN_RB=$(( AN_GEN_T + AN_APPLY_T + AN_RESTART_T + AN_RC ))
|
||||
AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP ))
|
||||
AN_WAIT=$(( AN_HELPER_MAX + 30 ))
|
||||
AN_RUNMAX=$(( AN_HELPER_MAX + 60 ))
|
||||
AN_STOP_T=$(( AN_RB + 30 ))
|
||||
|
||||
apply_status_blocks_run() { # apply_status_blocks_run <status-file> -> yes|no: a run that is/was in flight or failed to roll back blocks a new one
|
||||
case "$(cut -d' ' -f1 "$1" 2>/dev/null || true)" in
|
||||
running|rollback-failed) echo yes ;;
|
||||
*) echo no ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# apply_helper_script <backup-dir> [netplan-dir] [log] [status-file] [sysnet-dir] [rendered-file] [deadlines "s1 s2 rc"] [run-stamp]
|
||||
# The script the detached unit runs. Paths are parameters only so tests can bake
|
||||
@@ -301,10 +321,11 @@ apply_helper_script() {
|
||||
cat <<HELPER
|
||||
#!/bin/bash
|
||||
# Generated by deploy.sh --apply-network; runs detached. Installs the rendered WAN netplan,
|
||||
# applies it, verifies the uplink in two stages and restores the pre-run netplan on any failure.
|
||||
# applies it, verifies the uplink in two stages and restores the pre-run netplan on any failure
|
||||
# (including TERM/INT/HUP, e.g. systemd stopping the unit at its RuntimeMaxSec).
|
||||
# Stage 1 (${d1}s): the apply succeeded and, if the Starlink port exists, NM shows it connected
|
||||
# via a netplan-* profile with an IPv4 address whose gateway answers on-link (the profile is
|
||||
# never-default; van-failover owns default routes).
|
||||
# via a netplan-* profile with an IPv4 address whose gateway (IP4.GATEWAY, else the DHCP
|
||||
# routers option: never-default hides IP4.GATEWAY) answers on-link.
|
||||
# Stage 2 (${d2}s): van-failover restarted; default route and internet ping.
|
||||
bak='$bak'
|
||||
nd='$nd'
|
||||
@@ -318,23 +339,36 @@ echo "== apply-network $stamp \$(date -Is) (backup: \$bak)"
|
||||
conn_ok() { ip route show default | grep . >/dev/null && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; }
|
||||
gate1() {
|
||||
[ -e "\$sn/$STARLINK_IFACE" ] || return 0
|
||||
[ "\$(nmcli -g GENERAL.STATE device show '$STARLINK_IFACE' 2>/dev/null)" = "100 (connected)" ] || return 1
|
||||
case "\$(nmcli -g GENERAL.CONNECTION device show '$STARLINK_IFACE' 2>/dev/null)" in netplan-*) ;; *) return 1 ;; esac
|
||||
a=\$(nmcli -g IP4.ADDRESS device show '$STARLINK_IFACE' 2>/dev/null)
|
||||
g=\$(nmcli -g IP4.GATEWAY device show '$STARLINK_IFACE' 2>/dev/null)
|
||||
[ -n "\$a" ] && [ "\$a" != -- ] && [ -n "\$g" ] && [ "\$g" != -- ] || return 1
|
||||
[ "\$(timeout $AN_NM_T nmcli -g GENERAL.STATE device show '$STARLINK_IFACE' 2>/dev/null)" = "100 (connected)" ] || return 1
|
||||
case "\$(timeout $AN_NM_T nmcli -g GENERAL.CONNECTION device show '$STARLINK_IFACE' 2>/dev/null)" in netplan-*) ;; *) return 1 ;; esac
|
||||
a=\$(timeout $AN_NM_T nmcli -g IP4.ADDRESS device show '$STARLINK_IFACE' 2>/dev/null)
|
||||
[ -n "\$a" ] && [ "\$a" != -- ] || return 1
|
||||
g=\$(timeout $AN_NM_T nmcli -g IP4.GATEWAY device show '$STARLINK_IFACE' 2>/dev/null)
|
||||
case "\$g" in
|
||||
''|--) g=\$(timeout $AN_NM_T nmcli -g DHCP4.OPTION device show '$STARLINK_IFACE' 2>/dev/null | tr '|' '\\n' | sed -n 's/^ *routers = *//p' | head -1 | cut -d' ' -f1) ;;
|
||||
esac
|
||||
[ -n "\$g" ] && [ "\$g" != -- ] || return 1
|
||||
ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1
|
||||
}
|
||||
restore() { # restore <reason>: put the pre-run netplan back, re-check connectivity, always exit 1
|
||||
route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; }
|
||||
in_restore=0
|
||||
restore() { # restore <reason>: put the pre-run netplan back, re-check connectivity, always exit 1 (runs once)
|
||||
[ "\$in_restore" = 0 ] || exit 1
|
||||
in_restore=1
|
||||
trap '' TERM INT HUP
|
||||
local rb=ok c=down
|
||||
echo "apply-network FAILED (\$1) - restoring \$bak"
|
||||
route_log
|
||||
if [ -d "\$bak" ]; then
|
||||
rm -f "\$nd"/* "\$nd"/.[!.]*
|
||||
cp -a "\$bak"/. "\$nd"/ || rb=CP-FAILED
|
||||
else
|
||||
# No backup to restore from: at least undo this helper's own two changes.
|
||||
rb=BACKUP-MISSING
|
||||
rm -f "\$nd/50-van-wan.yaml"
|
||||
[ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml"
|
||||
fi
|
||||
netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; }
|
||||
timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; }
|
||||
timeout $AN_RESTART_T systemctl restart van-failover || true
|
||||
end=\$((SECONDS+$d3))
|
||||
while :; do
|
||||
@@ -342,23 +376,29 @@ restore() { # restore <reason>: put the pre-run netplan back, re-check connecti
|
||||
[ "\$SECONDS" -ge "\$end" ] && break
|
||||
sleep 1
|
||||
done
|
||||
rm -f "\$rf"
|
||||
if [ "\$rb" = ok ] && [ "\$c" = ok ]; then
|
||||
echo "apply-network ROLLED BACK (restored, connectivity ok)"
|
||||
status rolled-back "\$1"
|
||||
elif [ "\$rb" = ok ]; then
|
||||
echo "apply-network ROLLED BACK (restored, but no connectivity - check the uplink/cable)"
|
||||
status rolled-back restored-no-connectivity
|
||||
status rolled-back "restored-no-connectivity; \$1"
|
||||
else
|
||||
echo "apply-network ROLLBACK FAILED: \$rb / connectivity \$c"
|
||||
status rollback-failed "\$1; \$rb / connectivity \$c"
|
||||
[ "\$c" = ok ] || { echo "rollback failed and no connectivity - rebooting (boot uses the files on disk)"; systemctl reboot; }
|
||||
# Reboot (boot uses the files on disk) only when a restore was attempted and failed, and there is no connectivity.
|
||||
case "\$rb" in
|
||||
CP-FAILED|APPLY-FAILED) [ "\$c" = ok ] || { echo "rollback failed and no connectivity - rebooting"; systemctl reboot; } ;;
|
||||
esac
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
status running "\$bak"
|
||||
trap 'restore terminated' TERM INT HUP
|
||||
applied=0
|
||||
if install -m0600 "\$rf" "\$nd/50-van-wan.yaml" \\
|
||||
&& { [ ! -f "\$nd/10-dhcp-all-interfaces.yaml" ] || mv "\$nd/10-dhcp-all-interfaces.yaml" "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled"; } \\
|
||||
&& netplan generate && timeout $AN_APPLY_T netplan apply; then
|
||||
&& timeout $AN_GEN_T netplan generate && timeout $AN_APPLY_T netplan apply; then
|
||||
applied=1
|
||||
else
|
||||
echo "netplan install/generate/apply failed"
|
||||
@@ -381,7 +421,10 @@ while :; do
|
||||
[ "\$SECONDS" -ge "\$end" ] && break
|
||||
sleep 1
|
||||
done
|
||||
route_log
|
||||
[ "\$s2" = 1 ] || restore "stage 2: no default route or internet"
|
||||
trap - TERM INT HUP
|
||||
rm -f "\$rf"
|
||||
status ok
|
||||
echo "apply-network OK"
|
||||
exit 0
|
||||
@@ -396,29 +439,39 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c
|
||||
dpkg-query -W -f='${Status}' network-manager 2>/dev/null | grep -q 'install ok installed' \
|
||||
|| die "network-manager is not installed — run ./deploy.sh first"
|
||||
systemctl is-active --quiet NetworkManager || die "NetworkManager is not running — handing it the uplink port would flap it"
|
||||
systemctl cat van-failover >/dev/null 2>&1 || die "van-failover.service does not exist — run ./deploy.sh first (stage 2 of the gate needs it)"
|
||||
systemctl is-active --quiet vanlink-apply-network && die "an apply is already running — see /var/log/vanlink-apply-network.log"
|
||||
local log=/var/log/vanlink-apply-network.log st=/var/lib/vanlink/apply-network.status
|
||||
if [ "$force" = 0 ] && [ "$(apply_status_blocks_run "$st")" = yes ]; then
|
||||
die "the last apply did not finish cleanly: $(cat "$st" 2>/dev/null) — inspect $st and $log, check the backup dir named there, then re-run with --force"
|
||||
fi
|
||||
if [ "$force" = 0 ] && [ "$(nm_wan_state_netplan)" = matches ] && [ ! -e /etc/netplan/10-dhcp-all-interfaces.yaml ]; then
|
||||
echo "nothing to apply (WAN netplan already matches; use --force to re-apply)"
|
||||
return 0
|
||||
fi
|
||||
local rendered=/run/vanlink-50-van-wan.yaml stamp bak log=/var/log/vanlink-apply-network.log st=/var/lib/vanlink/apply-network.status
|
||||
local stamp helper rendered bak pt
|
||||
stamp=$(date +%Y%m%d-%H%M%S)-$$
|
||||
rendered=/run/vanlink-50-van-wan-$stamp.yaml
|
||||
helper=/run/vanlink-apply-network-$stamp.sh
|
||||
(umask 077; render_wan_netplan > "$rendered") || die "render failed"
|
||||
check_rendered ap/50-van-wan.yaml "$rendered" || { rm -f "$rendered"; exit 1; }
|
||||
stamp=$(date +%Y%m%d-%H%M%S)
|
||||
bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || die "cannot create backup dir"
|
||||
cp -a /etc/netplan/. "$bak/" || die "backup failed — nothing changed"
|
||||
if [ ! -e /var/backups/vanlink-netplan-orig ] && [ ! -e /etc/netplan/50-van-wan.yaml ]; then
|
||||
install -d -m0700 /var/backups/vanlink-netplan-orig
|
||||
cp -a /etc/netplan/. /var/backups/vanlink-netplan-orig/ || { rm -rf /var/backups/vanlink-netplan-orig; die "pristine backup failed — nothing changed"; }
|
||||
pt=/var/backups/vanlink-netplan-orig.tmp.$$
|
||||
rm -rf "$pt"
|
||||
install -d -m0700 "$pt"
|
||||
cp -a /etc/netplan/. "$pt/" || { rm -rf "$pt"; die "pristine backup failed — nothing changed"; }
|
||||
mv "$pt" /var/backups/vanlink-netplan-orig
|
||||
fi
|
||||
install -d -m0755 /var/lib/vanlink
|
||||
rm -f "$st"
|
||||
apply_helper_script "$bak" /etc/netplan "$log" "$st" /sys/class/net "$rendered" "$AN_S1 $AN_S2 $AN_RC" "$stamp" > /run/vanlink-apply-network.sh
|
||||
chmod 0700 /run/vanlink-apply-network.sh
|
||||
apply_helper_script "$bak" /etc/netplan "$log" "$st" /sys/class/net "$rendered" "$AN_S1 $AN_S2 $AN_RC" "$stamp" > "$helper"
|
||||
chmod 0700 "$helper"
|
||||
echo "backup: $bak log: $log"
|
||||
echo "applying from a detached unit (survives an SSH drop); waiting up to $AN_WAIT s ..."
|
||||
systemctl reset-failed vanlink-apply-network 2>/dev/null || true
|
||||
systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec=900 /bin/bash /run/vanlink-apply-network.sh >/dev/null \
|
||||
systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec="$AN_RUNMAX" -p TimeoutStopSec="$AN_STOP_T" /bin/bash "$helper" >/dev/null \
|
||||
|| die "systemd-run failed — nothing was changed (backup: $bak)"
|
||||
SECONDS=0
|
||||
while systemctl is-active --quiet vanlink-apply-network && [ "$SECONDS" -lt "$AN_WAIT" ]; do sleep 1; done
|
||||
@@ -432,7 +485,7 @@ apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; c
|
||||
case "$verdict" in
|
||||
ok) return 0 ;;
|
||||
rolled-back|rollback-failed) echo "apply-network: $(cat "$st") (previous config: $bak)" >&2; return 1 ;;
|
||||
*) echo "apply-network: no status file — see $log (previous config: $bak)" >&2; return 1 ;;
|
||||
*) echo "apply-network: no final status — see $log (previous config: $bak)" >&2; return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
@@ -485,6 +538,9 @@ case "${1:-}" in
|
||||
load_conf "$VANLINK_CONF"
|
||||
apply_network "${2:-}"
|
||||
exit $? ;;
|
||||
--apply-status-check) # test seam: --apply-status-check <status-file> prints yes|no (does that status block a new --apply-network run?)
|
||||
apply_status_blocks_run "${2:?status file}"
|
||||
exit 0 ;;
|
||||
--print-apply-helper) # [conf] the helper script for review; test-only flags: --backup D --netplan-dir D --log L --status S --sysnet N --rendered R --deadlines "s1 s2 rc" --stamp T
|
||||
shift; pc= a_bak=/var/backups/vanlink-netplan-EXAMPLE a_nd= a_log= a_st= a_sn= a_rf= a_dl= a_stamp=
|
||||
while [ $# -gt 0 ]; do
|
||||
|
||||
Reference in New Issue
Block a user