diff --git a/deploy.sh b/deploy.sh index edfadb9..c0ff8c7 100755 --- a/deploy.sh +++ b/deploy.sh @@ -11,8 +11,10 @@ # ./deploy.sh --classify-radio # reads `iw dev info` text on stdin, prints beaconing|silent (no ssid line = silent; used by post_check; for tests) # ./deploy.sh --render-wan-netplan # rendered /etc/netplan/50-van-wan.yaml (ethernet part + wifis stanza when WIFI_WAN_IFACE is set) # ./deploy.sh --wan-plan [conf] # what the NetworkManager WAN provisioning would do (read-only, no root) -# ./deploy.sh --apply-network # root: install the WAN netplan, apply it from a detached unit, roll back automatically if the uplink does not return +# ./deploy.sh --apply-network [--force] # root: a detached helper installs the WAN netplan, applies it, gates on uplink health (2 stages) and rolls back +# # automatically on failure; status in /var/lib/vanlink/apply-network.status. No-op when already applied unless --force. # ./deploy.sh --print-apply-helper [conf] # print the detached apply/rollback helper script (read-only, no root; for review and tests) +# # test-only flags: --backup --netplan-dir --log --status --sysnet --rendered --deadlines "s1 s2 rc" --stamp # ./deploy.sh render # preview a templated file on stdout # ./deploy.sh --redact # filter stdin, masking secret values (used by --diff; for tests) # VANLINK_NM_DIR= # test override: where existing NM *.nmconnection files are looked up (default /etc/NetworkManager/system-connections) @@ -277,68 +279,158 @@ post_check() { # read-only live checks; every probe is guarded, warns instead o else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi } -apply_helper_script() { # apply_helper_script — the script the detached unit runs - local bak=$1 - # Unquoted heredoc on purpose: $bak and $STARLINK_IFACE expand now; \$... stay for run time. +# --apply-network time budgets (seconds), in one place. The helper's worst case is +# apply + stage 1 + restart + stage 2 + rollback (apply + restart + recheck); the +# foreground waits that long plus a margin. van-failover starts optimistic and +# probes every WAN at once (probe_timeout 3 s), enforcing the route in its first +# loop, so a healthy default route shows up within seconds of a restart; the 120 s +# stage 2 deadline is a wide margin for a slow Starlink/cellular probe. +AN_APPLY_T=90 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 +AN_WAIT=$(( 2 * AN_APPLY_T + 2 * AN_RESTART_T + AN_S1 + AN_S2 + AN_RC + 30 )) + +# apply_helper_script [netplan-dir] [log] [status-file] [sysnet-dir] [rendered-file] [deadlines "s1 s2 rc"] [run-stamp] +# The script the detached unit runs. Paths are parameters only so tests can bake +# sandbox paths into the text; the production helper reads no environment. +apply_helper_script() { + local bak=$1 nd=${2:-/etc/netplan} log=${3:-/var/log/vanlink-apply-network.log} + local st=${4:-/var/lib/vanlink/apply-network.status} sn=${5:-/sys/class/net} + local rf=${6:-/run/vanlink-50-van-wan.yaml} dl=${7:-"$AN_S1 $AN_S2 $AN_RC"} stamp=${8:-manual} + local d1 d2 d3 + read -r d1 d2 d3 <<< "$dl" + # Unquoted heredoc on purpose: values expand now; \$... stays for run time. cat <>/var/log/vanlink-apply-network.log 2>&1 -echo "== apply-network \$(date -Is) (backup: $bak)" -netplan generate && netplan apply || echo "netplan apply failed" -ok=0 -for i in \$(seq 1 60); do - have_default=0; ip route show default | grep . >/dev/null && have_default=1 - nm_ok=1 - if [ -e /sys/class/net/$STARLINK_IFACE ]; then - nm_ok=0 - nmcli -t -f DEVICE,STATE device 2>/dev/null | grep -Fx '$STARLINK_IFACE:connected' >/dev/null && nm_ok=1 +# Generated by deploy.sh --apply-network; runs detached. Installs the rendered WAN netplan, +# applies it, verifies the uplink in two stages and restores the pre-run netplan on any failure. +# Stage 1 (${d1}s): the apply succeeded and, if the Starlink port exists, NM shows it connected +# via a netplan-* profile with an IPv4 address whose gateway answers on-link (the profile is +# never-default; van-failover owns default routes). +# Stage 2 (${d2}s): van-failover restarted; default route and internet ping. +bak='$bak' +nd='$nd' +log='$log' +st='$st' +sn='$sn' +rf='$rf' +exec >>"\$log" 2>&1 +status() { mkdir -p "\$(dirname "\$st")"; echo "\$1 \$(date -Is)\${2:+ \$2}" > "\$st"; } +echo "== apply-network $stamp \$(date -Is) (backup: \$bak)" +conn_ok() { ip route show default | grep . >/dev/null && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; } +gate1() { + [ -e "\$sn/$STARLINK_IFACE" ] || return 0 + [ "\$(nmcli -g GENERAL.STATE device show '$STARLINK_IFACE' 2>/dev/null)" = "100 (connected)" ] || return 1 + case "\$(nmcli -g GENERAL.CONNECTION device show '$STARLINK_IFACE' 2>/dev/null)" in netplan-*) ;; *) return 1 ;; esac + a=\$(nmcli -g IP4.ADDRESS device show '$STARLINK_IFACE' 2>/dev/null) + g=\$(nmcli -g IP4.GATEWAY device show '$STARLINK_IFACE' 2>/dev/null) + [ -n "\$a" ] && [ "\$a" != -- ] && [ -n "\$g" ] && [ "\$g" != -- ] || return 1 + ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1 +} +restore() { # restore : put the pre-run netplan back, re-check connectivity, always exit 1 + local rb=ok c=down + echo "apply-network FAILED (\$1) - restoring \$bak" + if [ -d "\$bak" ]; then + rm -f "\$nd"/* "\$nd"/.[!.]* + cp -a "\$bak"/. "\$nd"/ || rb=CP-FAILED + else + rb=BACKUP-MISSING fi - if [ \$have_default = 1 ] && [ \$nm_ok = 1 ] && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; then ok=1; break; fi + netplan generate && timeout $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; } + timeout $AN_RESTART_T systemctl restart van-failover || true + end=\$((SECONDS+$d3)) + while :; do + conn_ok && { c=ok; break; } + [ "\$SECONDS" -ge "\$end" ] && break + sleep 1 + done + if [ "\$rb" = ok ] && [ "\$c" = ok ]; then + echo "apply-network ROLLED BACK (restored, connectivity ok)" + status rolled-back "\$1" + else + echo "apply-network ROLLBACK FAILED: \$rb / connectivity \$c" + status rollback-failed "\$1; \$rb / connectivity \$c" + [ "\$c" = ok ] || { echo "no connectivity after rollback - rebooting"; systemctl reboot; } + fi + exit 1 +} +applied=0 +if install -m0600 "\$rf" "\$nd/50-van-wan.yaml" \\ + && { [ ! -f "\$nd/10-dhcp-all-interfaces.yaml" ] || mv "\$nd/10-dhcp-all-interfaces.yaml" "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled"; } \\ + && netplan generate && timeout $AN_APPLY_T netplan apply; then + applied=1 +else + echo "netplan install/generate/apply failed" +fi +s1=0 +if [ "\$applied" = 1 ]; then + end=\$((SECONDS+$d1)) + while :; do + gate1 && { s1=1; break; } + [ "\$SECONDS" -ge "\$end" ] && break + sleep 1 + done +fi +[ "\$s1" = 1 ] || restore "stage 1: apply or uplink gate failed" +timeout $AN_RESTART_T systemctl restart van-failover || restore "van-failover restart failed" +s2=0 +end=\$((SECONDS+$d2)) +while :; do + conn_ok && { s2=1; break; } + [ "\$SECONDS" -ge "\$end" ] && break sleep 1 done -if [ \$ok = 1 ]; then - echo "apply-network OK" -else - echo "apply-network FAILED — restoring $bak" - rm -f /etc/netplan/* - cp -a $bak/. /etc/netplan/ - netplan generate && netplan apply - echo "apply-network ROLLED BACK" - exit 1 -fi +[ "\$s2" = 1 ] || restore "stage 2: no default route or internet" +status ok +echo "apply-network OK" +exit 0 HELPER } -apply_network() { # ./deploy.sh --apply-network — explicit, root; see spec §4 +apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; changes nothing under /etc/netplan itself (the detached helper does); see spec §4 + local force=0 + [ "${1:-}" = --force ] && force=1 [ "$(id -u)" = 0 ] || die "--apply-network needs root (sudo)" [ "$ENABLE_NM_WAN" = 1 ] || die "ENABLE_NM_WAN is 0 in $VANLINK_CONF — nothing to apply" - dpkg -s network-manager >/dev/null 2>&1 || die "network-manager is not installed — run ./deploy.sh first" - local stamp bak tmp - stamp=$(date +%Y%m%d-%H%M%S) - bak=/var/backups/vanlink-netplan-$stamp - install -d -m0700 "$bak" - cp -a /etc/netplan/. "$bak/" - tmp=$(mktemp) - render_wan_netplan > "$tmp" - check_rendered ap/50-van-wan.yaml "$tmp" || { rm -f "$tmp"; exit 1; } - install -m0600 "$tmp" /etc/netplan/50-van-wan.yaml - rm -f "$tmp" - if [ -f /etc/netplan/10-dhcp-all-interfaces.yaml ]; then - mv /etc/netplan/10-dhcp-all-interfaces.yaml /etc/netplan/10-dhcp-all-interfaces.yaml.vanlink-disabled + dpkg-query -W -f='${Status}' network-manager 2>/dev/null | grep -q 'install ok installed' \ + || die "network-manager is not installed — run ./deploy.sh first" + systemctl is-active --quiet NetworkManager || die "NetworkManager is not running — handing it the uplink port would flap it" + systemctl is-active --quiet vanlink-apply-network && die "an apply is already running — see /var/log/vanlink-apply-network.log" + if [ "$force" = 0 ] && [ "$(nm_wan_state_netplan)" = matches ] && [ ! -e /etc/netplan/10-dhcp-all-interfaces.yaml ]; then + echo "nothing to apply (WAN netplan already matches; use --force to re-apply)" + return 0 fi - : > /var/log/vanlink-apply-network.log - apply_helper_script "$bak" > /run/vanlink-apply-network.sh + local rendered=/run/vanlink-50-van-wan.yaml stamp bak log=/var/log/vanlink-apply-network.log st=/var/lib/vanlink/apply-network.status + (umask 077; render_wan_netplan > "$rendered") || die "render failed" + check_rendered ap/50-van-wan.yaml "$rendered" || { rm -f "$rendered"; exit 1; } + stamp=$(date +%Y%m%d-%H%M%S) + bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || die "cannot create backup dir" + cp -a /etc/netplan/. "$bak/" || die "backup failed — nothing changed" + if [ ! -e /var/backups/vanlink-netplan-orig ] && [ ! -e /etc/netplan/50-van-wan.yaml ]; then + install -d -m0700 /var/backups/vanlink-netplan-orig + cp -a /etc/netplan/. /var/backups/vanlink-netplan-orig/ || { rm -rf /var/backups/vanlink-netplan-orig; die "pristine backup failed — nothing changed"; } + fi + install -d -m0755 /var/lib/vanlink + rm -f "$st" + apply_helper_script "$bak" /etc/netplan "$log" "$st" /sys/class/net "$rendered" "$AN_S1 $AN_S2 $AN_RC" "$stamp" > /run/vanlink-apply-network.sh chmod 0700 /run/vanlink-apply-network.sh - echo "backup: $bak log: /var/log/vanlink-apply-network.log" - echo "applying from a detached unit (survives an SSH drop); waiting up to 90 s ..." + echo "backup: $bak log: $log" + echo "applying from a detached unit (survives an SSH drop); waiting up to $AN_WAIT s ..." systemctl reset-failed vanlink-apply-network 2>/dev/null || true - systemd-run --unit=vanlink-apply-network --collect /bin/bash /run/vanlink-apply-network.sh >/dev/null - local t=0 - while systemctl is-active --quiet vanlink-apply-network && [ "$t" -lt 90 ]; do sleep 1; t=$((t + 1)); done - tail -n 6 /var/log/vanlink-apply-network.log 2>/dev/null || true - grep -q 'apply-network OK' /var/log/vanlink-apply-network.log 2>/dev/null \ - || die "apply-network did not report OK — see /var/log/vanlink-apply-network.log (previous config: $bak)" + systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec=900 /bin/bash /run/vanlink-apply-network.sh >/dev/null \ + || die "systemd-run failed — nothing was changed (backup: $bak)" + SECONDS=0 + while systemctl is-active --quiet vanlink-apply-network && [ "$SECONDS" -lt "$AN_WAIT" ]; do sleep 1; done + awk -v s="== apply-network $stamp" 'index($0, s) == 1 { p = 1 } p' "$log" 2>/dev/null | tail -n 8 || true + if systemctl is-active --quiet vanlink-apply-network; then + echo "still running — check $log" + return 2 + fi + local verdict + verdict=$(cut -d' ' -f1 "$st" 2>/dev/null || true) + case "$verdict" in + ok) return 0 ;; + rolled-back|rollback-failed) echo "apply-network: $(cat "$st") (previous config: $bak)" >&2; return 1 ;; + *) echo "apply-network: no status file — see $log (previous config: $bak)" >&2; return 1 ;; + esac } case "${1:-}" in @@ -386,13 +478,31 @@ case "${1:-}" in --redact) # ./deploy.sh --redact — stdin through redact_secrets (no config, no root) redact_secrets exit 0 ;; - --apply-network) # ./deploy.sh --apply-network — root; applies the WAN netplan with automatic rollback + --apply-network) # ./deploy.sh --apply-network [--force] — root; applies the WAN netplan from a detached helper with automatic rollback load_conf "$VANLINK_CONF" - apply_network - exit 0 ;; - --print-apply-helper) # for review/tests: the helper script, with a placeholder backup dir; no root, no changes - load_conf "${2:-$VANLINK_CONF}" - apply_helper_script /var/backups/vanlink-netplan-EXAMPLE + apply_network "${2:-}" + exit $? ;; + --print-apply-helper) # [conf] the helper script for review; test-only flags: --backup D --netplan-dir D --log L --status S --sysnet N --rendered R --deadlines "s1 s2 rc" --stamp T + shift; pc= a_bak=/var/backups/vanlink-netplan-EXAMPLE a_nd= a_log= a_st= a_sn= a_rf= a_dl= a_stamp= + while [ $# -gt 0 ]; do + case "$1" in + --backup|--netplan-dir|--log|--status|--sysnet|--rendered|--deadlines|--stamp) + [ $# -ge 2 ] || die "$1 needs a value" ;; + esac + case "$1" in + --backup) a_bak=$2; shift 2 ;; + --netplan-dir) a_nd=$2; shift 2 ;; + --log) a_log=$2; shift 2 ;; + --status) a_st=$2; shift 2 ;; + --sysnet) a_sn=$2; shift 2 ;; + --rendered) a_rf=$2; shift 2 ;; + --deadlines) a_dl=$2; shift 2 ;; + --stamp) a_stamp=$2; shift 2 ;; + *) pc=$1; shift ;; + esac + done + load_conf "${pc:-$VANLINK_CONF}" + apply_helper_script "$a_bak" "$a_nd" "$a_log" "$a_st" "$a_sn" "$a_rf" "$a_dl" "$a_stamp" exit 0 ;; --render-wan-netplan) load_conf "$VANLINK_CONF"; render_wan_netplan; exit 0 ;; --wan-plan) # ./deploy.sh --wan-plan [conf] — what the NM WAN provisioning would do; changes nothing diff --git a/tests/check-templates.sh b/tests/check-templates.sh index 29cf940..abc3fb6 100755 --- a/tests/check-templates.sh +++ b/tests/check-templates.sh @@ -257,16 +257,109 @@ rm -rf "$nmd" "$nme" # 17c. NM must not auto-create "Wired connection" profiles [ "$(grep -v '^[[:space:]]*#' ap/nm-no-auto-default.conf 2>/dev/null | grep -v '^[[:space:]]*$')" = "$(printf '[main]\nno-auto-default=*')" ] && ok "nm-no-auto-default.conf is [main] + no-auto-default=*" || bad "ap/nm-no-auto-default.conf missing or wrong" -# 18. the detached apply helper: syntax, ordering, rollback branch +# 18. the detached apply helper: syntax, then behaviour against stubbed system commands in a sandbox h=$(VANLINK_CONF=hosts/van.conf ./deploy.sh --print-apply-helper 2>&1) || { bad "--print-apply-helper failed"; echo "$h" | sed 's/^/ /'; h=; } if [ -n "$h" ]; then echo "$h" | bash -n && ok "apply helper: bash -n" || bad "apply helper: bash -n" - echo "$h" | grep -q 'netplan generate' && echo "$h" | grep -q 'netplan apply' && ok "apply helper applies netplan" || bad "apply helper lacks netplan generate/apply" - echo "$h" | grep -q 'restoring' && ok "apply helper has a rollback branch" || bad "apply helper lacks rollback" echo "$h" | grep -q 'enP2p33s0' && ok "apply helper waits on the Starlink port" || bad "apply helper does not reference STARLINK_IFACE" - # rollback must come after the wait loop, and the backup is restored wholesale - a=$(echo "$h" | grep -n 'netplan apply' | head -1 | cut -d: -f1); r=$(echo "$h" | grep -n 'restoring' | head -1 | cut -d: -f1) - [ "$a" -lt "$r" ] && ok "apply helper: apply precedes rollback" || bad "apply helper: rollback ordering" + echo "$h" | grep -q 'systemctl restart van-failover' && ok "apply helper restarts van-failover" || bad "apply helper lacks van-failover restart" + stubs=$(mktemp -d) + for c in netplan nmcli ip ping systemctl reboot; do + printf '#!/bin/bash\necho "%s $*" >> "$STUB_CALLS"\n' "$c" > "$stubs/$c" + done + cat >> "$stubs/netplan" <<'STUB' +if [ "$1" = apply ]; then + n=$(cat "$STUB_SB/napply" 2>/dev/null || echo 0); echo $((n + 1)) > "$STUB_SB/napply" + [ "${STUB_FAIL_APPLY:-}" = first ] && [ "$n" = 0 ] && exit 1 +fi +exit 0 +STUB + cat >> "$stubs/nmcli" <<'STUB' +case "$2" in + GENERAL.STATE) echo "${NM_STATE-100 (connected)}" ;; + GENERAL.CONNECTION) echo "${NM_CONN-netplan-enP2p33s0}" ;; + IP4.ADDRESS) echo "${NM_ADDR-192.168.100.10/24}" ;; + IP4.GATEWAY) echo "${NM_GW-192.168.100.1}" ;; +esac +exit 0 +STUB + cat >> "$stubs/ip" <<'STUB' +case "${STUB_ROUTE:-yes}" in + yes) echo "default via 192.168.100.1 dev x" ;; + after-rollback) [ -e "$STUB_ND/50-van-wan.yaml" ] || echo "default via 192.168.100.1 dev x" ;; +esac +exit 0 +STUB + cat >> "$stubs/ping" <<'STUB' +case " $* " in *" -I "*) exit "${STUB_GW_PING:-0}" ;; esac +exit "${STUB_NET_PING:-0}" +STUB + printf '#!/bin/bash\nexec /bin/sleep 0.05\n' > "$stubs/sleep" + printf '#!/bin/bash\nshift\nexec "$@"\n' > "$stubs/timeout" + printf '#!/bin/bash\nexit 0\n' >> "$stubs/systemctl" + chmod +x "$stubs"/* + stubs_ok=1 + for c in netplan nmcli ip ping systemctl reboot sleep timeout; do [ -x "$stubs/$c" ] || stubs_ok=0; done + [ "$stubs_ok" = 1 ] && ok "apply helper sandbox: all system commands stubbed" || bad "apply helper sandbox: stub missing" + + an_run() { # an_run [VAR=val ...] — fresh sandbox, run the helper; sets sb rc + sb=$(mktemp -d); mkdir -p "$sb/netplan" "$sb/sysnet" + printf 'armbian\n' > "$sb/netplan/10-dhcp-all-interfaces.yaml"; chmod 0600 "$sb/netplan/10-dhcp-all-interfaces.yaml" + printf 'old\n' > "$sb/netplan/60-other.yaml" + cp -a "$sb/netplan" "$sb/bak" + [ -n "${AN_NOPORT:-}" ] || mkdir "$sb/sysnet/enP2p33s0" + [ -z "${AN_NOBAK:-}" ] || rm -rf "$sb/bak" + printf 'network:\n version: 2\n' > "$sb/rendered.yaml" + VANLINK_CONF=hosts/van.conf ./deploy.sh --print-apply-helper --backup "$sb/bak" --netplan-dir "$sb/netplan" \ + --log "$sb/log" --status "$sb/status" --sysnet "$sb/sysnet" --rendered "$sb/rendered.yaml" \ + --deadlines "1 1 1" --stamp T > "$sb/helper.sh" + env PATH="$stubs:/usr/bin:/bin" STUB_CALLS="$sb/calls" STUB_SB="$sb" STUB_ND="$sb/netplan" "$@" bash "$sb/helper.sh" && rc=0 || rc=$? + } + a_is() { [ "$2" = "$3" ] && ok "apply helper $1" || bad "apply helper $1 (got '$2', want '$3')"; } + st_word() { cut -d' ' -f1 "$sb/status" 2>/dev/null; } + # (1) success + an_run + a_is "success: exit 0" "$rc" 0; a_is "success: status" "$(st_word)" ok + grep -q 'apply-network OK' "$sb/log" && ok "apply helper success: log OK" || bad "apply helper success: log" + a_is "success: 50-van-wan.yaml mode" "$(stat -c %a "$sb/netplan/50-van-wan.yaml" 2>/dev/null)" 600 + [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] && [ ! -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && ok "apply helper success: Armbian file moved aside" || bad "apply helper success: Armbian file not moved aside" + grep -q '^systemctl restart van-failover' "$sb/calls" && ok "apply helper success: van-failover restarted" || bad "apply helper success: no van-failover restart" + rm -rf "$sb" + # (2) stage 1 fails: connected (externally) + an_run NM_STATE="100 (connected (externally))" + a_is "stage1: exit 1" "$rc" 1; a_is "stage1: status" "$(st_word)" rolled-back + grep -q 'ROLLED BACK (restored, connectivity ok)' "$sb/log" && ok "apply helper stage1: log ROLLED BACK" || bad "apply helper stage1: log" + a_is "stage1: Armbian file restored mode" "$(stat -c %a "$sb/netplan/10-dhcp-all-interfaces.yaml" 2>/dev/null)" 600 + [ ! -e "$sb/netplan/50-van-wan.yaml" ] && [ ! -e "$sb/netplan/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] && [ -e "$sb/netplan/60-other.yaml" ] && ok "apply helper stage1: netplan dir restored wholesale" || bad "apply helper stage1: netplan dir not restored" + rm -rf "$sb" + # (2b) stage 1 fails: wrong connection name; gateway unpingable + an_run NM_CONN="Wired connection 1"; a_is "stage1 conn name: status" "$(st_word)" rolled-back; rm -rf "$sb" + an_run STUB_GW_PING=1; a_is "stage1 gateway ping: status" "$(st_word)" rolled-back; rm -rf "$sb" + an_run NM_GW=; a_is "stage1 no gateway: status" "$(st_word)" rolled-back; rm -rf "$sb" + # (3) netplan apply fails, port absent + AN_NOPORT=1 an_run STUB_FAIL_APPLY=first + a_is "apply failure: exit 1" "$rc" 1; a_is "apply failure: not ok" "$(st_word)" rolled-back + [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && [ ! -e "$sb/netplan/50-van-wan.yaml" ] && ok "apply helper apply failure: restored" || bad "apply helper apply failure: not restored" + rm -rf "$sb" + # (3b) port absent and apply ok: stage 1 needs only the apply + AN_NOPORT=1 an_run; a_is "port absent, apply ok: status" "$(st_word)" ok; rm -rf "$sb" + # (4) rollback cp fails (backup dir missing) + AN_NOBAK=1 an_run NM_STATE=bad + a_is "rollback cp failure: exit 1" "$rc" 1; a_is "rollback cp failure: status" "$(st_word)" rollback-failed + grep -q 'ROLLBACK FAILED' "$sb/log" && ok "apply helper rollback failure: loud log" || bad "apply helper rollback failure: log" + grep -q '^reboot\|systemctl reboot' "$sb/calls" && bad "apply helper rollback failure with connectivity: must not reboot" || ok "apply helper rollback failure with connectivity: no reboot" + rm -rf "$sb" + # (5) stage 2 fails: no default route until rolled back + an_run STUB_ROUTE=after-rollback + a_is "stage2: exit 1" "$rc" 1; a_is "stage2: status" "$(st_word)" rolled-back + [ ! -e "$sb/netplan/50-van-wan.yaml" ] && [ -e "$sb/netplan/10-dhcp-all-interfaces.yaml" ] && ok "apply helper stage2: restored" || bad "apply helper stage2: not restored" + rm -rf "$sb" + # (6) no connectivity even after rollback: last-resort reboot + an_run STUB_ROUTE=none + a_is "no connectivity: status" "$(st_word)" rollback-failed + grep -q '^systemctl reboot' "$sb/calls" && ok "apply helper: reboots when connectivity is not back after rollback" || bad "apply helper: no reboot" + rm -rf "$sb" + rm -rf "$stubs" fi exit "$fail"