From 4f62b9f770a665e7ca1a0492ed14e0578f17f2bf Mon Sep 17 00:00:00 2001 From: Andreas Wrede Date: Wed, 23 Sep 2026 20:58:29 -0400 Subject: [PATCH] apply-network: reload networkd after each netplan apply; post-check flags a networkd/NM double claim Live R5C run: after the Armbian catch-all .network was removed, networkd kept its in-memory claim on the uplink port (two addresses, extra proto dhcp default route) until `networkctl reload`. The helper now reloads networkd after the forward apply and after the rollback re-apply (not fatal, not gating; logs the port state and a "networkd still manages" note on the forward path), the AN_* time budget includes it, and --post-check warns when the netplan matches but networkd still lists the uplink port configured/configuring. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9 --- README.md | 2 +- deploy.sh | 36 ++++++++++++++++++++++++---- tests/check-templates.sh | 52 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f73483c..dc80410 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ host file matches the hostname — and the refusal to replace a WAN netplan that helper installs the netplan, gates on uplink health and rolls back automatically on failure. Its log is `/var/log/vanlink-apply-network.log` and its status line is in `/var/lib/vanlink/apply-network.status` (`ok|running|rolled-back|rollback-failed`). `./deploy.sh --wan-plan` -shows, read-only, what the provisioning would do. `--post-check` reports the last apply status. Until a host's WAN netplan is applied, the routine deploy also keeps +shows, read-only, what the provisioning would do. `--post-check` reports the last apply status. The apply also reloads networkd (`networkctl reload`) so it drops its stale claim on the uplink port, and `--post-check` warns when networkd and NetworkManager both still claim that port. Until a host's WAN netplan is applied, the routine deploy also keeps NetworkManager off the uplink with `/etc/NetworkManager/conf.d/zz-van-wan-pending.conf` (only while the netplan is *missing*; `--wan-plan` shows it) — the apply removes it. diff --git a/deploy.sh b/deploy.sh index d6998e9..b86cf61 100755 --- a/deploy.sh +++ b/deploy.sh @@ -386,6 +386,16 @@ PY unreadable) echo " ok WAN netplan: cannot read as non-root (run with sudo to verify)" ;; *) warn "post-deploy: WAN netplan $f — run ./deploy.sh --apply-network" ;; esac + # k2. once the WAN netplan is applied NM owns the uplink port; a networkd that still lists it + # configured/configuring holds a stale in-memory claim (cured by networkctl reload). + # Before the apply networkd legitimately owns it: say nothing. + if [ "$f" = matches ] && [ -e "$sysnet/$STARLINK_IFACE" ] && command -v networkctl >/dev/null 2>&1; then + out=$(networkctl --no-pager --no-legend list 2>/dev/null | awk -v i="$STARLINK_IFACE" '$2 == i { print $5; exit }') || out= + case "$out" in + configured|configuring) warn "post-deploy: networkd still manages $STARLINK_IFACE although NetworkManager owns it — run: networkctl reload" ;; + unmanaged) echo " ok networkd leaves $STARLINK_IFACE to NetworkManager" ;; + esac + fi # l. last --apply-network as_f=${VANLINK_APPLY_STATUS:-/var/lib/vanlink/apply-network.status} if [ -r "$as_f" ]; then @@ -402,9 +412,11 @@ PY # --apply-network time budgets (seconds), in one place; all derived from the AN_* constants. # (netplan generate/apply run under timeout -k AN_KILL_T: +AN_KILL_T each in the worst case) -# forward path AN_FWD = NM conf reload + generate + apply + 2*kill + van-br0-setup (unit check + restart) +# networkd reload AN_ND = networkctl reload (timeout AN_RELOAD_T) + settle sleep AN_SETTLE + port state line (AN_NM_T); +# runs once after each netplan apply (forward path and rollback) +# forward path AN_FWD = NM conf reload + generate + apply + 2*kill + AN_ND + van-br0-setup (unit check + restart) # + stage 1 + restart + stage 2 + networkd info line + diagnostics (logged once, on a failed gate) -# rollback path AN_RB = generate + apply + 2*kill + NM conf reload + van-br0-setup (unit check + restart) + restart + connectivity recheck +# rollback path AN_RB = generate + apply + 2*kill + AN_ND + NM conf reload + van-br0-setup (unit check + restart) + restart + connectivity recheck # helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their # deadline by a few nmcli/ping timeouts) # foreground waits AN_WAIT = AN_HELPER_MAX + 30 @@ -414,9 +426,11 @@ PY # route in its first loop, so a healthy default route shows up within seconds of a restart; the # 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe. AN_KILL_T=10 AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60 +AN_RELOAD_T=10 AN_SETTLE=3 +AN_ND=$(( AN_RELOAD_T + AN_SETTLE + AN_NM_T )) AN_DIAG=$(( 5 * AN_NM_T )) # diag(): five commands, each under timeout AN_NM_T -AN_FWD=$(( AN_NM_T + AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_NM_T + AN_RESTART_T + AN_S1 + AN_RESTART_T + AN_S2 + AN_NM_T + AN_DIAG )) -AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_NM_T + AN_NM_T + AN_RESTART_T + AN_RESTART_T + AN_RC )) +AN_FWD=$(( AN_NM_T + AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_ND + AN_NM_T + AN_RESTART_T + AN_S1 + AN_RESTART_T + AN_S2 + AN_NM_T + AN_DIAG )) +AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_ND + AN_NM_T + AN_NM_T + AN_RESTART_T + AN_RESTART_T + AN_RC )) AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP )) AN_WAIT=$(( AN_HELPER_MAX + 30 )) AN_RUNMAX=$(( AN_HELPER_MAX + 60 )) @@ -497,6 +511,18 @@ gate1() { [ -n "\$g" ] && [ "\$g" != -- ] || return 1 ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1 } +nd_reload() { # nd_reload [note]: after a netplan apply networkd may keep a removed file's config in memory; reload it (never fatal, never gating) + command -v networkctl >/dev/null 2>&1 || return 0 + timeout $AN_RELOAD_T networkctl reload || echo "networkctl reload failed (continuing)" + sleep $AN_SETTLE + local l + l=\$(timeout $AN_NM_T networkctl --no-pager --no-legend list 2>/dev/null | awk -v i='$STARLINK_IFACE' '\$2 == i { print; exit }') + echo "networkd after reload: \${l:-$STARLINK_IFACE not listed}" + case " \$l " in + *" configured "*|*" configuring "*) [ -z "\${1:-}" ] || echo "networkd still manages $STARLINK_IFACE (NetworkManager should own it)" ;; + esac + return 0 +} route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; } in_restore=0 restore() { # restore : put the pre-run netplan back, re-check connectivity, always exit 1 (runs once) @@ -519,6 +545,7 @@ restore() { # restore : put the pre-run netplan back, re-check connecti [ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml" fi timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; } + nd_reload # the old config is back: networkd's in-memory state changes the other way [ ! -f "\$ncd/$pc" ] || timeout $AN_NM_T nmcli general reload conf || echo "nmcli general reload conf failed" # the re-apply restarted networkd too, which can drop br0's members (as on the forward path; not gating) if timeout $AN_NM_T systemctl cat van-br0-setup >/dev/null 2>&1; then @@ -567,6 +594,7 @@ else fi s1=0 if [ "\$applied" = 1 ]; then + nd_reload note # the Armbian catch-all .network is gone from disk but networkd still holds it until reloaded # netplan apply restarts networkd, which can drop br0's members: re-run the bridge setup (not gating) if timeout $AN_NM_T systemctl cat van-br0-setup >/dev/null 2>&1; then timeout $AN_RESTART_T systemctl restart van-br0-setup || echo "van-br0-setup restart failed (continuing)" diff --git a/tests/check-templates.sh b/tests/check-templates.sh index f6a1c70..68d3c92 100755 --- a/tests/check-templates.sh +++ b/tests/check-templates.sh @@ -273,7 +273,8 @@ elif [ -n "$h" ]; then done cat >> "$stubs/networkctl" <<'STUB' case " $* " in - *" list "*) echo " 3 enP2p33s0 ether routable configured" ;; + *" list "*) echo " 3 enP2p33s0 ether routable ${STUB_NL_SETUP:-configured}" ;; + *" reload "*) [ -z "${STUB_RELOAD_FAIL:-}" ] || exit 1 ;; *" status "*) echo "STUB-NETWORKCTL-STATUS" ;; esac exit 0 @@ -379,6 +380,11 @@ STUB [ "$(grep -c '^systemctl restart van-br0-setup$' "$sb/calls")" = 1 ] && ok "apply helper success: van-br0-setup restarted once" || bad "apply helper success: van-br0-setup restart count" grep -q '^networkd (info, not gating): *3 enP2p33s0 ether routable configured' "$sb/log" && ok "apply helper success: networkd state of the port logged" || bad "apply helper success: networkd info line missing" grep -q -- '--- diagnostics' "$sb/log" && bad "apply helper success: diagnostics must only be logged on a failed gate" || ok "apply helper success: no failure diagnostics" + l_nr=$(ln_of '^networkctl reload$'); l_s1=$(ln_of '^nmcli -g GENERAL.STATE device') + [ -n "$l_nr" ] && [ -n "$l_ap" ] && [ -n "$l_s1" ] && [ "$l_ap" -lt "$l_nr" ] && [ "$l_nr" -lt "$l_s1" ] && ok "apply helper success: networkctl reload after netplan apply, before the stage-1 probes" || bad "apply helper success: networkctl reload missing or misplaced (apply '$l_ap', reload '$l_nr', stage1 '$l_s1')" + [ "$(grep -c '^networkctl reload$' "$sb/calls")" = 1 ] && ok "apply helper success: networkd reloaded once" || bad "apply helper success: networkctl reload count" + grep -q '^networkd after reload: .*enP2p33s0 ether routable configured' "$sb/log" && ok "apply helper success: port state logged after the reload" || bad "apply helper success: no post-reload port state line" + grep -q '^networkd still manages enP2p33s0' "$sb/log" && ok "apply helper success: 'networkd still manages' noted for a configured port" || bad "apply helper success: still-manages note missing" rm -rf "$sb" # (1b) van-br0-setup unit absent: not restarted, still ok; no pending conf: nothing to reload AN_NOPEND=1 an_run STUB_NO_BR0=1 @@ -386,6 +392,18 @@ STUB grep -q '^systemctl restart van-br0-setup' "$sb/calls" && bad "apply helper: restarted a van-br0-setup that does not exist" || ok "apply helper: van-br0-setup restart skipped when the unit is absent" grep -q '^nmcli general reload conf$' "$sb/calls" && bad "apply helper: reloaded NM conf with no pending conf" || ok "apply helper: no NM conf reload without a pending conf" rm -rf "$sb" + # (1c) after the reload the port is unmanaged: no note. A failing reload changes nothing (logged, not fatal, not gating) + an_run STUB_NL_SETUP=unmanaged + a_is "port unmanaged after reload: status" "$(st_word)" ok + grep -q '^networkd still manages' "$sb/log" && bad "apply helper: 'still manages' noted for an unmanaged port" || ok "apply helper: no 'still manages' note for an unmanaged port" + rm -rf "$sb" + an_run STUB_RELOAD_FAIL=1 + a_is "reload failure: exit 0" "$rc" 0; a_is "reload failure: status" "$(st_word)" ok + grep -q 'networkctl reload failed' "$sb/log" && ok "apply helper: failed reload is logged" || bad "apply helper: failed reload not logged" + rm -rf "$sb" + an_run STUB_RELOAD_FAIL=1 NM_STATE=bad + a_is "reload failure + stage 1 failure: status" "$(st_word)" rolled-back + rm -rf "$sb" # (2) stage 1 fails: connected (externally) an_run NM_STATE="100 (connected (externally))" a_is "stage1: exit 1" "$rc" 1; a_is "stage1: status" "$(st_word)" rolled-back @@ -396,6 +414,9 @@ STUB [ ! -e "$sb/netplan/nm-pending.conf" ] && ok "apply helper stage1: nm-pending.conf not copied into the netplan dir" || bad "apply helper stage1: nm-pending.conf leaked into the netplan dir" l_ap2=$(grep -n '^netplan apply' "$sb/calls" | sed -n 2p | cut -d: -f1 || true); l_rl2=$(grep -n '^nmcli general reload conf$' "$sb/calls" | sed -n 2p | cut -d: -f1 || true) [ -n "$l_ap2" ] && [ -n "$l_rl2" ] && [ "$l_ap2" -lt "$l_rl2" ] && ok "apply helper stage1: NM conf reloaded after the rollback apply" || bad "apply helper stage1: no NM conf reload after the rollback apply (apply '$l_ap2', reload '$l_rl2')" + l_nr2=$(grep -n '^networkctl reload$' "$sb/calls" | sed -n 2p | cut -d: -f1 || true) + [ -n "$l_ap2" ] && [ -n "$l_nr2" ] && [ "$l_ap2" -lt "$l_nr2" ] && ok "apply helper stage1: networkctl reload after the rollback apply" || bad "apply helper stage1: no networkctl reload after the rollback apply (apply '$l_ap2', reload '$l_nr2')" + sed -n '/restoring/,$p' "$sb/log" | grep -q '^networkd still manages' && bad "apply helper rollback: 'still manages' note belongs to the forward path only" || ok "apply helper rollback: no 'still manages' note" for m in '--- diagnostics: stage 1' 'STUB-DIAG-NM' 'routers = 192.168.100.1' 'STUB-NETWORKCTL-STATUS' 'STUB-ADDR inet' 'default via 192.168.100.1 dev enP2p33s0'; do sed -n '/--- diagnostics/,/--- end diagnostics/p' "$sb/log" | grep -qF -- "$m" && ok "apply helper stage1: diagnostics contain '$m'" || bad "apply helper stage1: diagnostics lack '$m'" done @@ -496,6 +517,8 @@ STUB dg=$(echo "$h" | sed -n '/^diag() {/,/^}/p') [ -n "$dg" ] && ! echo "$dg" | grep -E '(^ *|[;|&(] *)(nmcli|networkctl|ip) ' | grep -q . && [ "$(echo "$dg" | grep -c 'timeout 5 ')" -ge 5 ] \ && ok "apply helper: every diagnostic command runs under timeout 5" || bad "apply helper: diag() missing or a diagnostic command without timeout 5" + grep -q '^AN_ND=' deploy.sh && grep -n '^AN_FWD=.*AN_ND' deploy.sh >/dev/null && grep -n '^AN_RB=.*AN_ND' deploy.sh >/dev/null && ok "apply helper: networkd reload time is in both worst cases" || bad "apply helper: AN_ND missing from AN_FWD/AN_RB" + echo "$h" | grep -q 'timeout 10 networkctl reload' && ok "apply helper: networkctl reload under timeout 10" || bad "apply helper: networkctl reload without timeout 10" echo "$h" | grep -q journalctl && bad "apply helper must not call journalctl" || ok "apply helper: no journal calls" fi # 18b. status file gate and STARLINK_IFACE validation @@ -577,6 +600,33 @@ echo "$o" | grep -q -- "-> post-deploy: NetworkManager connection $conn not foun echo "$o" | grep -q -- '-> post-deploy: WAN netplan differs' && ok "19e: netplan differs -> warn" || bad "19e: netplan warn missing" printf 'id=%s\n' "$conn" > "$tc/nm/x.nmconnection" nm | grep -q " ok NetworkManager connection $conn exists" && ok "19e: cellular keyfile scan -> ok" || bad "19e: keyfile scan ok missing" +# 19h. networkd must not still manage the uplink port NM owns (stub networkctl first on PATH) +mkdir -p "$tc/nostub" "$tc/sys3" +STAR=$(sed -n 's/^STARLINK_IFACE=\([^ ]*\).*/\1/p' hosts/van.conf | tr -d '"') +mkdir -p "$tc/sys3/$STAR" +cat > "$tc/stub/networkctl" <<'S' +#!/bin/sh +case "$*" in + *list*) printf ' 1 lo loopback carrier unmanaged\n 2 %sx ether routable configured\n 3 %s ether routable %s\n' "$STUB_STAR" "$STUB_STAR" "$STUB_SETUP" ;; +esac +S +chmod +x "$tc/stub/networkctl" +for x in /usr/bin/* /bin/*; do [ "${x##*/}" = networkctl ] || ln -sf "$x" "$tc/nostub/${x##*/}" 2>/dev/null || true; done +nd() { # nd [netplan-state: matches|missing] + [ "${3:-matches}" = matches ] && cp "$tc/np-ok.yaml" "$tc/np.yaml" || rm -f "$tc/np.yaml" + PATH="$1:/usr/bin:/bin" STUB_STAR="$STAR" STUB_SETUP="$2" STUB_DEV="$tc/dev" STUB_CONN="$tc/conn" VANLINK_NM_DIR="$tc/nm" VANLINK_SYSNET="$tc/sys3" \ + VANLINK_WAN_NETPLAN="$tc/np.yaml" VANLINK_APPLY_STATUS="$tc/none" POSTCHECK_WAIT=0 ./deploy.sh --post-check hosts/van.conf 2>&1 | sed '/^== warnings ==$/,$d' || true +} +VANLINK_CONF=hosts/van.conf ./deploy.sh --render-wan-netplan > "$tc/np-ok.yaml" +W="-> post-deploy: networkd still manages $STAR although NetworkManager owns it — run: networkctl reload" +nd "$tc/stub" configured | grep -qF -- "$W" && ok "19h: matches + configured -> warn" || bad "19h: matches + configured warn missing" +nd "$tc/stub" configuring | grep -qF -- "$W" && ok "19h: matches + configuring -> warn" || bad "19h: matches + configuring warn missing" +o=$(nd "$tc/stub" unmanaged) +echo "$o" | grep -qF " ok networkd leaves $STAR to NetworkManager" && ! echo "$o" | grep -q 'networkd still manages' && ok "19h: matches + unmanaged -> ok" || bad "19h: matches + unmanaged ok line missing" +nd "$tc/stub" configured missing | grep -qE 'networkd (still manages|leaves)' && bad "19h: netplan not matching must print nothing about networkd" || ok "19h: netplan missing + configured -> nothing" +nd "$tc/nostub" configured | grep -qE 'networkd (still manages|leaves)' && bad "19h: no networkctl must print nothing" || ok "19h: no networkctl -> nothing" +rmdir "$tc/sys3/$STAR" +nd "$tc/stub" configured | grep -qE 'networkd (still manages|leaves)' && bad "19h: absent port must print nothing" || ok "19h: uplink port absent -> nothing" # 19g. an installed WAN netplan this user cannot read (the helper installs it 0600) is "unreadable", not "differs" if [ "$(id -u)" = 0 ]; then ok "19g: unreadable-netplan checks skipped (running as root)"