apply-network: reload networkd after each netplan apply; post-check flags a networkd/NM double claim

Live R5C run: after the Armbian catch-all .network was removed, networkd kept its
in-memory claim on the uplink port (two addresses, extra proto dhcp default route)
until `networkctl reload`. The helper now reloads networkd after the forward apply
and after the rollback re-apply (not fatal, not gating; logs the port state and a
"networkd still manages" note on the forward path), the AN_* time budget includes
it, and --post-check warns when the netplan matches but networkd still lists the
uplink port configured/configuring.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9

README.md

@@ -19,7 +19,7 @@ host file matches the hostname — and the refusal to replace a WAN netplan that
 helper installs the netplan, gates on uplink health and rolls back automatically on failure. Its log
 is `/var/log/vanlink-apply-network.log` and its status line is in
 `/var/lib/vanlink/apply-network.status` (`ok|running|rolled-back|rollback-failed`). `./deploy.sh --wan-plan`
-shows, read-only, what the provisioning would do. `--post-check` reports the last apply status. Until a host's WAN netplan is applied, the routine deploy also keeps
+shows, read-only, what the provisioning would do. `--post-check` reports the last apply status. The apply also reloads networkd (`networkctl reload`) so it drops its stale claim on the uplink port, and `--post-check` warns when networkd and NetworkManager both still claim that port. Until a host's WAN netplan is applied, the routine deploy also keeps
 NetworkManager off the uplink with `/etc/NetworkManager/conf.d/zz-van-wan-pending.conf` (only while the netplan is
 *missing*; `--wan-plan` shows it) — the apply removes it.
 

deploy.sh

@@ -386,6 +386,16 @@ PY
             unreadable) echo "   ok    WAN netplan: cannot read as non-root (run with sudo to verify)" ;;
             *) warn "post-deploy: WAN netplan $f — run ./deploy.sh --apply-network" ;;
         esac
+        # k2. once the WAN netplan is applied NM owns the uplink port; a networkd that still lists it
+        #     configured/configuring holds a stale in-memory claim (cured by networkctl reload).
+        #     Before the apply networkd legitimately owns it: say nothing.
+        if [ "$f" = matches ] && [ -e "$sysnet/$STARLINK_IFACE" ] && command -v networkctl >/dev/null 2>&1; then
+            out=$(networkctl --no-pager --no-legend list 2>/dev/null | awk -v i="$STARLINK_IFACE" '$2 == i { print $5; exit }') || out=
+            case "$out" in
+                configured|configuring) warn "post-deploy: networkd still manages $STARLINK_IFACE although NetworkManager owns it — run: networkctl reload" ;;
+                unmanaged) echo "   ok    networkd leaves $STARLINK_IFACE to NetworkManager" ;;
+            esac
+        fi
         # l. last --apply-network
         as_f=${VANLINK_APPLY_STATUS:-/var/lib/vanlink/apply-network.status}
         if [ -r "$as_f" ]; then
@@ -402,9 +412,11 @@ PY
 
 # --apply-network time budgets (seconds), in one place; all derived from the AN_* constants.
 #   (netplan generate/apply run under timeout -k AN_KILL_T: +AN_KILL_T each in the worst case)
-#   forward path  AN_FWD = NM conf reload + generate + apply + 2*kill + van-br0-setup (unit check + restart)
+#   networkd reload AN_ND = networkctl reload (timeout AN_RELOAD_T) + settle sleep AN_SETTLE + port state line (AN_NM_T);
+#     runs once after each netplan apply (forward path and rollback)
+#   forward path  AN_FWD = NM conf reload + generate + apply + 2*kill + AN_ND + van-br0-setup (unit check + restart)
 #                          + stage 1 + restart + stage 2 + networkd info line + diagnostics (logged once, on a failed gate)
-#   rollback path AN_RB  = generate + apply + 2*kill + NM conf reload + van-br0-setup (unit check + restart) + restart + connectivity recheck
+#   rollback path AN_RB  = generate + apply + 2*kill + AN_ND + NM conf reload + van-br0-setup (unit check + restart) + restart + connectivity recheck
 #   helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their
 #     deadline by a few nmcli/ping timeouts)
 #   foreground waits AN_WAIT = AN_HELPER_MAX + 30
@@ -414,9 +426,11 @@ PY
 # route in its first loop, so a healthy default route shows up within seconds of a restart; the
 # 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe.
 AN_KILL_T=10 AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60
+AN_RELOAD_T=10 AN_SETTLE=3
+AN_ND=$(( AN_RELOAD_T + AN_SETTLE + AN_NM_T ))
 AN_DIAG=$(( 5 * AN_NM_T ))   # diag(): five commands, each under timeout AN_NM_T
-AN_FWD=$(( AN_NM_T + AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_NM_T + AN_RESTART_T + AN_S1 + AN_RESTART_T + AN_S2 + AN_NM_T + AN_DIAG ))
+AN_FWD=$(( AN_NM_T + AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_ND + AN_NM_T + AN_RESTART_T + AN_S1 + AN_RESTART_T + AN_S2 + AN_NM_T + AN_DIAG ))
-AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_NM_T + AN_NM_T + AN_RESTART_T + AN_RESTART_T + AN_RC ))
+AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_ND + AN_NM_T + AN_NM_T + AN_RESTART_T + AN_RESTART_T + AN_RC ))
 AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP ))
 AN_WAIT=$(( AN_HELPER_MAX + 30 ))
 AN_RUNMAX=$(( AN_HELPER_MAX + 60 ))
@@ -497,6 +511,18 @@ gate1() {
     [ -n "\$g" ] && [ "\$g" != -- ] || return 1
     ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1
 }
+nd_reload() {  # nd_reload [note]: after a netplan apply networkd may keep a removed file's config in memory; reload it (never fatal, never gating)
+    command -v networkctl >/dev/null 2>&1 || return 0
+    timeout $AN_RELOAD_T networkctl reload || echo "networkctl reload failed (continuing)"
+    sleep $AN_SETTLE
+    local l
+    l=\$(timeout $AN_NM_T networkctl --no-pager --no-legend list 2>/dev/null | awk -v i='$STARLINK_IFACE' '\$2 == i { print; exit }')
+    echo "networkd after reload: \${l:-$STARLINK_IFACE not listed}"
+    case " \$l " in
+        *" configured "*|*" configuring "*) [ -z "\${1:-}" ] || echo "networkd still manages $STARLINK_IFACE (NetworkManager should own it)" ;;
+    esac
+    return 0
+}
 route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; }
 in_restore=0
 restore() {  # restore <reason>: put the pre-run netplan back, re-check connectivity, always exit 1 (runs once)
@@ -519,6 +545,7 @@ restore() {  # restore <reason>: put the pre-run netplan back, re-check connecti
         [ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml"
     fi
     timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; }
+    nd_reload   # the old config is back: networkd's in-memory state changes the other way
     [ ! -f "\$ncd/$pc" ] || timeout $AN_NM_T nmcli general reload conf || echo "nmcli general reload conf failed"
     # the re-apply restarted networkd too, which can drop br0's members (as on the forward path; not gating)
     if timeout $AN_NM_T systemctl cat van-br0-setup >/dev/null 2>&1; then
@@ -567,6 +594,7 @@ else
 fi
 s1=0
 if [ "\$applied" = 1 ]; then
+    nd_reload note   # the Armbian catch-all .network is gone from disk but networkd still holds it until reloaded
     # netplan apply restarts networkd, which can drop br0's members: re-run the bridge setup (not gating)
     if timeout $AN_NM_T systemctl cat van-br0-setup >/dev/null 2>&1; then
         timeout $AN_RESTART_T systemctl restart van-br0-setup || echo "van-br0-setup restart failed (continuing)"

tests/check-templates.sh

@@ -273,7 +273,8 @@ elif [ -n "$h" ]; then
     done
     cat >> "$stubs/networkctl" <<'STUB'
 case " $* " in
-    *" list "*) echo "  3 enP2p33s0 ether routable configured" ;;
+    *" list "*) echo "  3 enP2p33s0 ether routable ${STUB_NL_SETUP:-configured}" ;;
+    *" reload "*) [ -z "${STUB_RELOAD_FAIL:-}" ] || exit 1 ;;
     *" status "*) echo "STUB-NETWORKCTL-STATUS" ;;
 esac
 exit 0
@@ -379,6 +380,11 @@ STUB
     [ "$(grep -c '^systemctl restart van-br0-setup$' "$sb/calls")" = 1 ] && ok "apply helper success: van-br0-setup restarted once" || bad "apply helper success: van-br0-setup restart count"
     grep -q '^networkd (info, not gating): *3 enP2p33s0 ether routable configured' "$sb/log" && ok "apply helper success: networkd state of the port logged" || bad "apply helper success: networkd info line missing"
     grep -q -- '--- diagnostics' "$sb/log" && bad "apply helper success: diagnostics must only be logged on a failed gate" || ok "apply helper success: no failure diagnostics"
+    l_nr=$(ln_of '^networkctl reload$'); l_s1=$(ln_of '^nmcli -g GENERAL.STATE device')
+    [ -n "$l_nr" ] && [ -n "$l_ap" ] && [ -n "$l_s1" ] && [ "$l_ap" -lt "$l_nr" ] && [ "$l_nr" -lt "$l_s1" ] && ok "apply helper success: networkctl reload after netplan apply, before the stage-1 probes" || bad "apply helper success: networkctl reload missing or misplaced (apply '$l_ap', reload '$l_nr', stage1 '$l_s1')"
+    [ "$(grep -c '^networkctl reload$' "$sb/calls")" = 1 ] && ok "apply helper success: networkd reloaded once" || bad "apply helper success: networkctl reload count"
+    grep -q '^networkd after reload: .*enP2p33s0 ether routable configured' "$sb/log" && ok "apply helper success: port state logged after the reload" || bad "apply helper success: no post-reload port state line"
+    grep -q '^networkd still manages enP2p33s0' "$sb/log" && ok "apply helper success: 'networkd still manages' noted for a configured port" || bad "apply helper success: still-manages note missing"
     rm -rf "$sb"
     # (1b) van-br0-setup unit absent: not restarted, still ok; no pending conf: nothing to reload
     AN_NOPEND=1 an_run STUB_NO_BR0=1
@@ -386,6 +392,18 @@ STUB
     grep -q '^systemctl restart van-br0-setup' "$sb/calls" && bad "apply helper: restarted a van-br0-setup that does not exist" || ok "apply helper: van-br0-setup restart skipped when the unit is absent"
     grep -q '^nmcli general reload conf$' "$sb/calls" && bad "apply helper: reloaded NM conf with no pending conf" || ok "apply helper: no NM conf reload without a pending conf"
     rm -rf "$sb"
+    # (1c) after the reload the port is unmanaged: no note. A failing reload changes nothing (logged, not fatal, not gating)
+    an_run STUB_NL_SETUP=unmanaged
+    a_is "port unmanaged after reload: status" "$(st_word)" ok
+    grep -q '^networkd still manages' "$sb/log" && bad "apply helper: 'still manages' noted for an unmanaged port" || ok "apply helper: no 'still manages' note for an unmanaged port"
+    rm -rf "$sb"
+    an_run STUB_RELOAD_FAIL=1
+    a_is "reload failure: exit 0" "$rc" 0; a_is "reload failure: status" "$(st_word)" ok
+    grep -q 'networkctl reload failed' "$sb/log" && ok "apply helper: failed reload is logged" || bad "apply helper: failed reload not logged"
+    rm -rf "$sb"
+    an_run STUB_RELOAD_FAIL=1 NM_STATE=bad
+    a_is "reload failure + stage 1 failure: status" "$(st_word)" rolled-back
+    rm -rf "$sb"
     # (2) stage 1 fails: connected (externally)
     an_run NM_STATE="100 (connected (externally))"
     a_is "stage1: exit 1" "$rc" 1; a_is "stage1: status" "$(st_word)" rolled-back
@@ -396,6 +414,9 @@ STUB
     [ ! -e "$sb/netplan/nm-pending.conf" ] && ok "apply helper stage1: nm-pending.conf not copied into the netplan dir" || bad "apply helper stage1: nm-pending.conf leaked into the netplan dir"
     l_ap2=$(grep -n '^netplan apply' "$sb/calls" | sed -n 2p | cut -d: -f1 || true); l_rl2=$(grep -n '^nmcli general reload conf$' "$sb/calls" | sed -n 2p | cut -d: -f1 || true)
     [ -n "$l_ap2" ] && [ -n "$l_rl2" ] && [ "$l_ap2" -lt "$l_rl2" ] && ok "apply helper stage1: NM conf reloaded after the rollback apply" || bad "apply helper stage1: no NM conf reload after the rollback apply (apply '$l_ap2', reload '$l_rl2')"
+    l_nr2=$(grep -n '^networkctl reload$' "$sb/calls" | sed -n 2p | cut -d: -f1 || true)
+    [ -n "$l_ap2" ] && [ -n "$l_nr2" ] && [ "$l_ap2" -lt "$l_nr2" ] && ok "apply helper stage1: networkctl reload after the rollback apply" || bad "apply helper stage1: no networkctl reload after the rollback apply (apply '$l_ap2', reload '$l_nr2')"
+    sed -n '/restoring/,$p' "$sb/log" | grep -q '^networkd still manages' && bad "apply helper rollback: 'still manages' note belongs to the forward path only" || ok "apply helper rollback: no 'still manages' note"
     for m in '--- diagnostics: stage 1' 'STUB-DIAG-NM' 'routers = 192.168.100.1' 'STUB-NETWORKCTL-STATUS' 'STUB-ADDR inet' 'default via 192.168.100.1 dev enP2p33s0'; do
         sed -n '/--- diagnostics/,/--- end diagnostics/p' "$sb/log" | grep -qF -- "$m" && ok "apply helper stage1: diagnostics contain '$m'" || bad "apply helper stage1: diagnostics lack '$m'"
     done
@@ -496,6 +517,8 @@ STUB
     dg=$(echo "$h" | sed -n '/^diag() {/,/^}/p')
     [ -n "$dg" ] && ! echo "$dg" | grep -E '(^ *|[;|&(] *)(nmcli|networkctl|ip) ' | grep -q . && [ "$(echo "$dg" | grep -c 'timeout 5 ')" -ge 5 ] \
         && ok "apply helper: every diagnostic command runs under timeout 5" || bad "apply helper: diag() missing or a diagnostic command without timeout 5"
+    grep -q '^AN_ND=' deploy.sh && grep -n '^AN_FWD=.*AN_ND' deploy.sh >/dev/null && grep -n '^AN_RB=.*AN_ND' deploy.sh >/dev/null && ok "apply helper: networkd reload time is in both worst cases" || bad "apply helper: AN_ND missing from AN_FWD/AN_RB"
+    echo "$h" | grep -q 'timeout 10 networkctl reload' && ok "apply helper: networkctl reload under timeout 10" || bad "apply helper: networkctl reload without timeout 10"
     echo "$h" | grep -q journalctl && bad "apply helper must not call journalctl" || ok "apply helper: no journal calls"
 fi
 # 18b. status file gate and STARLINK_IFACE validation
@@ -577,6 +600,33 @@ echo "$o" | grep -q -- "-> post-deploy: NetworkManager connection $conn not foun
 echo "$o" | grep -q -- '-> post-deploy: WAN netplan differs' && ok "19e: netplan differs -> warn" || bad "19e: netplan warn missing"
 printf 'id=%s\n' "$conn" > "$tc/nm/x.nmconnection"
 nm | grep -q "   ok    NetworkManager connection $conn exists" && ok "19e: cellular keyfile scan -> ok" || bad "19e: keyfile scan ok missing"
+# 19h. networkd must not still manage the uplink port NM owns (stub networkctl first on PATH)
+mkdir -p "$tc/nostub" "$tc/sys3"
+STAR=$(sed -n 's/^STARLINK_IFACE=\([^ ]*\).*/\1/p' hosts/van.conf | tr -d '"')
+mkdir -p "$tc/sys3/$STAR"
+cat > "$tc/stub/networkctl" <<'S'
+#!/bin/sh
+case "$*" in
+  *list*) printf '  1 lo loopback carrier unmanaged\n  2 %sx ether routable configured\n  3 %s ether routable %s\n' "$STUB_STAR" "$STUB_STAR" "$STUB_SETUP" ;;
+esac
+S
+chmod +x "$tc/stub/networkctl"
+for x in /usr/bin/* /bin/*; do [ "${x##*/}" = networkctl ] || ln -sf "$x" "$tc/nostub/${x##*/}" 2>/dev/null || true; done
+nd() {  # nd <path-dir> <setup> [netplan-state: matches|missing]
+    [ "${3:-matches}" = matches ] && cp "$tc/np-ok.yaml" "$tc/np.yaml" || rm -f "$tc/np.yaml"
+    PATH="$1:/usr/bin:/bin" STUB_STAR="$STAR" STUB_SETUP="$2" STUB_DEV="$tc/dev" STUB_CONN="$tc/conn" VANLINK_NM_DIR="$tc/nm" VANLINK_SYSNET="$tc/sys3" \
+        VANLINK_WAN_NETPLAN="$tc/np.yaml" VANLINK_APPLY_STATUS="$tc/none" POSTCHECK_WAIT=0 ./deploy.sh --post-check hosts/van.conf 2>&1 | sed '/^== warnings ==$/,$d' || true
+}
+VANLINK_CONF=hosts/van.conf ./deploy.sh --render-wan-netplan > "$tc/np-ok.yaml"
+W="-> post-deploy: networkd still manages $STAR although NetworkManager owns it — run: networkctl reload"
+nd "$tc/stub" configured | grep -qF -- "$W" && ok "19h: matches + configured -> warn" || bad "19h: matches + configured warn missing"
+nd "$tc/stub" configuring | grep -qF -- "$W" && ok "19h: matches + configuring -> warn" || bad "19h: matches + configuring warn missing"
+o=$(nd "$tc/stub" unmanaged)
+echo "$o" | grep -qF "   ok    networkd leaves $STAR to NetworkManager" && ! echo "$o" | grep -q 'networkd still manages' && ok "19h: matches + unmanaged -> ok" || bad "19h: matches + unmanaged ok line missing"
+nd "$tc/stub" configured missing | grep -qE 'networkd (still manages|leaves)' && bad "19h: netplan not matching must print nothing about networkd" || ok "19h: netplan missing + configured -> nothing"
+nd "$tc/nostub" configured | grep -qE 'networkd (still manages|leaves)' && bad "19h: no networkctl must print nothing" || ok "19h: no networkctl -> nothing"
+rmdir "$tc/sys3/$STAR"
+nd "$tc/stub" configured | grep -qE 'networkd (still manages|leaves)' && bad "19h: absent port must print nothing" || ok "19h: uplink port absent -> nothing"
 # 19g. an installed WAN netplan this user cannot read (the helper installs it 0600) is "unreadable", not "differs"
 if [ "$(id -u)" = 0 ]; then
     ok "19g: unreadable-netplan checks skipped (running as root)"