andreas/vanlink
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

apply-network: reload networkd after each netplan apply; post-check flags a networkd/NM double claim

Browse Source 
Live R5C run: after the Armbian catch-all .network was removed, networkd kept its
in-memory claim on the uplink port (two addresses, extra proto dhcp default route)
until `networkctl reload`. The helper now reloads networkd after the forward apply
and after the rollback re-apply (not fatal, not gating; logs the port state and a
"networkd still manages" note on the forward path), the AN_* time budget includes
it, and --post-check warns when the netplan matches but networkd still lists the
uplink port configured/configuring.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KqhjP7nud5gdBg25zCiRk9
This commit is contained in:
andreas
2026-09-23 20:58:29 -04:00
co-authored by Claude Sonnet 5
parent 1ce4a30336
commit 4f62b9f770
3 changed files with 84 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -19,7 +19,7 @@ host file matches the hostname — and the refusal to replace a WAN netplan that
helper installs the netplan, gates on uplink health and rolls back automatically on failure. Its log
is `/var/log/vanlink-apply-network.log` and its status line is in
`/var/lib/vanlink/apply-network.status` (`ok|running|rolled-back|rollback-failed`). `./deploy.sh --wan-plan`
shows, read-only, what the provisioning would do. `--post-check` reports the last apply status. Until a host's WAN netplan is applied, the routine deploy also keeps
shows, read-only, what the provisioning would do. `--post-check` reports the last apply status. The apply also reloads networkd (`networkctl reload`) so it drops its stale claim on the uplink port, and `--post-check` warns when networkd and NetworkManager both still claim that port. Until a host's WAN netplan is applied, the routine deploy also keeps
NetworkManager off the uplink with `/etc/NetworkManager/conf.d/zz-van-wan-pending.conf` (only while the netplan is
*missing*; `--wan-plan` shows it) — the apply removes it.
deploy.sh
+32 -4
View File
@@ -386,6 +386,16 @@ PY
unreadable) echo " ok WAN netplan: cannot read as non-root (run with sudo to verify)" ;;
*) warn "post-deploy: WAN netplan $f — run ./deploy.sh --apply-network" ;;
esac
# k2. once the WAN netplan is applied NM owns the uplink port; a networkd that still lists it
# configured/configuring holds a stale in-memory claim (cured by networkctl reload).
# Before the apply networkd legitimately owns it: say nothing.
if [ "$f" = matches ] && [ -e "$sysnet/$STARLINK_IFACE" ] && command -v networkctl >/dev/null 2>&1; then
out=$(networkctl --no-pager --no-legend list 2>/dev/null | awk -v i="$STARLINK_IFACE" '$2 == i { print $5; exit }') || out=
case "$out" in
configured|configuring) warn "post-deploy: networkd still manages $STARLINK_IFACE although NetworkManager owns it — run: networkctl reload" ;;
unmanaged) echo " ok networkd leaves $STARLINK_IFACE to NetworkManager" ;;
esac
fi
# l. last --apply-network
as_f=${VANLINK_APPLY_STATUS:-/var/lib/vanlink/apply-network.status}
if [ -r "$as_f" ]; then
@@ -402,9 +412,11 @@ PY
# --apply-network time budgets (seconds), in one place; all derived from the AN_* constants.
# (netplan generate/apply run under timeout -k AN_KILL_T: +AN_KILL_T each in the worst case)
# forward path AN_FWD = NM conf reload + generate + apply + 2*kill + van-br0-setup (unit check + restart)
# networkd reload AN_ND = networkctl reload (timeout AN_RELOAD_T) + settle sleep AN_SETTLE + port state line (AN_NM_T);
# runs once after each netplan apply (forward path and rollback)
# forward path AN_FWD = NM conf reload + generate + apply + 2*kill + AN_ND + van-br0-setup (unit check + restart)
# + stage 1 + restart + stage 2 + networkd info line + diagnostics (logged once, on a failed gate)
# rollback path AN_RB = generate + apply + 2*kill + NM conf reload + van-br0-setup (unit check + restart) + restart + connectivity recheck
# rollback path AN_RB = generate + apply + 2*kill + AN_ND + NM conf reload + van-br0-setup (unit check + restart) + restart + connectivity recheck
# helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their
# deadline by a few nmcli/ping timeouts)
# foreground waits AN_WAIT = AN_HELPER_MAX + 30
@@ -414,9 +426,11 @@ PY
# route in its first loop, so a healthy default route shows up within seconds of a restart; the
# 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe.
AN_KILL_T=10 AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60
AN_RELOAD_T=10 AN_SETTLE=3
AN_ND=$(( AN_RELOAD_T + AN_SETTLE + AN_NM_T ))
AN_DIAG=$(( 5 * AN_NM_T )) # diag(): five commands, each under timeout AN_NM_T
AN_FWD=$(( AN_NM_T + AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_NM_T + AN_RESTART_T + AN_S1 + AN_RESTART_T + AN_S2 + AN_NM_T + AN_DIAG ))
AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_NM_T + AN_NM_T + AN_RESTART_T + AN_RESTART_T + AN_RC ))
AN_FWD=$(( AN_NM_T + AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_ND + AN_NM_T + AN_RESTART_T + AN_S1 + AN_RESTART_T + AN_S2 + AN_NM_T + AN_DIAG ))
AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_ND + AN_NM_T + AN_NM_T + AN_RESTART_T + AN_RESTART_T + AN_RC ))
AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP ))
AN_WAIT=$(( AN_HELPER_MAX + 30 ))
AN_RUNMAX=$(( AN_HELPER_MAX + 60 ))
@@ -497,6 +511,18 @@ gate1() {
[ -n "\$g" ] && [ "\$g" != -- ] || return 1
ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1
}
nd_reload() { # nd_reload [note]: after a netplan apply networkd may keep a removed file's config in memory; reload it (never fatal, never gating)
command -v networkctl >/dev/null 2>&1 || return 0
timeout $AN_RELOAD_T networkctl reload || echo "networkctl reload failed (continuing)"
sleep $AN_SETTLE
local l
l=\$(timeout $AN_NM_T networkctl --no-pager --no-legend list 2>/dev/null | awk -v i='$STARLINK_IFACE' '\$2 == i { print; exit }')
echo "networkd after reload: \${l:-$STARLINK_IFACE not listed}"
case " \$l " in
*" configured "*|*" configuring "*) [ -z "\${1:-}" ] || echo "networkd still manages $STARLINK_IFACE (NetworkManager should own it)" ;;
esac
return 0
}
route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; }
in_restore=0
restore() { # restore <reason>: put the pre-run netplan back, re-check connectivity, always exit 1 (runs once)
@@ -519,6 +545,7 @@ restore() { # restore <reason>: put the pre-run netplan back, re-check connecti
[ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml"
fi
timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; }
nd_reload # the old config is back: networkd's in-memory state changes the other way
[ ! -f "\$ncd/$pc" ] || timeout $AN_NM_T nmcli general reload conf || echo "nmcli general reload conf failed"
# the re-apply restarted networkd too, which can drop br0's members (as on the forward path; not gating)
if timeout $AN_NM_T systemctl cat van-br0-setup >/dev/null 2>&1; then
@@ -567,6 +594,7 @@ else
fi
s1=0
if [ "\$applied" = 1 ]; then
nd_reload note # the Armbian catch-all .network is gone from disk but networkd still holds it until reloaded
# netplan apply restarts networkd, which can drop br0's members: re-run the bridge setup (not gating)
if timeout $AN_NM_T systemctl cat van-br0-setup >/dev/null 2>&1; then
timeout $AN_RESTART_T systemctl restart van-br0-setup || echo "van-br0-setup restart failed (continuing)"
tests/check-templates.sh
+51 -1
View File
@@ -273,7 +273,8 @@ elif [ -n "$h" ]; then
done
cat >> "$stubs/networkctl" <<'STUB'
case " $* " in
*" list "*) echo " 3 enP2p33s0 ether routable configured" ;;
*" list "*) echo " 3 enP2p33s0 ether routable ${STUB_NL_SETUP:-configured}" ;;
*" reload "*) [ -z "${STUB_RELOAD_FAIL:-}" ] || exit 1 ;;
*" status "*) echo "STUB-NETWORKCTL-STATUS" ;;
esac
exit 0
@@ -379,6 +380,11 @@ STUB
[ "$(grep -c '^systemctl restart van-br0-setup$' "$sb/calls")" = 1 ] && ok "apply helper success: van-br0-setup restarted once" || bad "apply helper success: van-br0-setup restart count"
grep -q '^networkd (info, not gating): *3 enP2p33s0 ether routable configured' "$sb/log" && ok "apply helper success: networkd state of the port logged" || bad "apply helper success: networkd info line missing"
grep -q -- '--- diagnostics' "$sb/log" && bad "apply helper success: diagnostics must only be logged on a failed gate" || ok "apply helper success: no failure diagnostics"
l_nr=$(ln_of '^networkctl reload$'); l_s1=$(ln_of '^nmcli -g GENERAL.STATE device')
[ -n "$l_nr" ] && [ -n "$l_ap" ] && [ -n "$l_s1" ] && [ "$l_ap" -lt "$l_nr" ] && [ "$l_nr" -lt "$l_s1" ] && ok "apply helper success: networkctl reload after netplan apply, before the stage-1 probes" || bad "apply helper success: networkctl reload missing or misplaced (apply '$l_ap', reload '$l_nr', stage1 '$l_s1')"
[ "$(grep -c '^networkctl reload$' "$sb/calls")" = 1 ] && ok "apply helper success: networkd reloaded once" || bad "apply helper success: networkctl reload count"
grep -q '^networkd after reload: .*enP2p33s0 ether routable configured' "$sb/log" && ok "apply helper success: port state logged after the reload" || bad "apply helper success: no post-reload port state line"
grep -q '^networkd still manages enP2p33s0' "$sb/log" && ok "apply helper success: 'networkd still manages' noted for a configured port" || bad "apply helper success: still-manages note missing"
rm -rf "$sb"
# (1b) van-br0-setup unit absent: not restarted, still ok; no pending conf: nothing to reload
AN_NOPEND=1 an_run STUB_NO_BR0=1
@@ -386,6 +392,18 @@ STUB
grep -q '^systemctl restart van-br0-setup' "$sb/calls" && bad "apply helper: restarted a van-br0-setup that does not exist" || ok "apply helper: van-br0-setup restart skipped when the unit is absent"
grep -q '^nmcli general reload conf$' "$sb/calls" && bad "apply helper: reloaded NM conf with no pending conf" || ok "apply helper: no NM conf reload without a pending conf"
rm -rf "$sb"
# (1c) after the reload the port is unmanaged: no note. A failing reload changes nothing (logged, not fatal, not gating)
an_run STUB_NL_SETUP=unmanaged
a_is "port unmanaged after reload: status" "$(st_word)" ok
grep -q '^networkd still manages' "$sb/log" && bad "apply helper: 'still manages' noted for an unmanaged port" || ok "apply helper: no 'still manages' note for an unmanaged port"
rm -rf "$sb"
an_run STUB_RELOAD_FAIL=1
a_is "reload failure: exit 0" "$rc" 0; a_is "reload failure: status" "$(st_word)" ok
grep -q 'networkctl reload failed' "$sb/log" && ok "apply helper: failed reload is logged" || bad "apply helper: failed reload not logged"
rm -rf "$sb"
an_run STUB_RELOAD_FAIL=1 NM_STATE=bad
a_is "reload failure + stage 1 failure: status" "$(st_word)" rolled-back
rm -rf "$sb"
# (2) stage 1 fails: connected (externally)
an_run NM_STATE="100 (connected (externally))"
a_is "stage1: exit 1" "$rc" 1; a_is "stage1: status" "$(st_word)" rolled-back
@@ -396,6 +414,9 @@ STUB
[ ! -e "$sb/netplan/nm-pending.conf" ] && ok "apply helper stage1: nm-pending.conf not copied into the netplan dir" || bad "apply helper stage1: nm-pending.conf leaked into the netplan dir"
l_ap2=$(grep -n '^netplan apply' "$sb/calls" | sed -n 2p | cut -d: -f1 || true); l_rl2=$(grep -n '^nmcli general reload conf$' "$sb/calls" | sed -n 2p | cut -d: -f1 || true)
[ -n "$l_ap2" ] && [ -n "$l_rl2" ] && [ "$l_ap2" -lt "$l_rl2" ] && ok "apply helper stage1: NM conf reloaded after the rollback apply" || bad "apply helper stage1: no NM conf reload after the rollback apply (apply '$l_ap2', reload '$l_rl2')"
l_nr2=$(grep -n '^networkctl reload$' "$sb/calls" | sed -n 2p | cut -d: -f1 || true)
[ -n "$l_ap2" ] && [ -n "$l_nr2" ] && [ "$l_ap2" -lt "$l_nr2" ] && ok "apply helper stage1: networkctl reload after the rollback apply" || bad "apply helper stage1: no networkctl reload after the rollback apply (apply '$l_ap2', reload '$l_nr2')"
sed -n '/restoring/,$p' "$sb/log" | grep -q '^networkd still manages' && bad "apply helper rollback: 'still manages' note belongs to the forward path only" || ok "apply helper rollback: no 'still manages' note"
for m in '--- diagnostics: stage 1' 'STUB-DIAG-NM' 'routers = 192.168.100.1' 'STUB-NETWORKCTL-STATUS' 'STUB-ADDR inet' 'default via 192.168.100.1 dev enP2p33s0'; do
sed -n '/--- diagnostics/,/--- end diagnostics/p' "$sb/log" | grep -qF -- "$m" && ok "apply helper stage1: diagnostics contain '$m'" || bad "apply helper stage1: diagnostics lack '$m'"
done
@@ -496,6 +517,8 @@ STUB
dg=$(echo "$h" | sed -n '/^diag() {/,/^}/p')
[ -n "$dg" ] && ! echo "$dg" | grep -E '(^ *|[;|&(] *)(nmcli|networkctl|ip) ' | grep -q . && [ "$(echo "$dg" | grep -c 'timeout 5 ')" -ge 5 ] \
&& ok "apply helper: every diagnostic command runs under timeout 5" || bad "apply helper: diag() missing or a diagnostic command without timeout 5"
grep -q '^AN_ND=' deploy.sh && grep -n '^AN_FWD=.*AN_ND' deploy.sh >/dev/null && grep -n '^AN_RB=.*AN_ND' deploy.sh >/dev/null && ok "apply helper: networkd reload time is in both worst cases" || bad "apply helper: AN_ND missing from AN_FWD/AN_RB"
echo "$h" | grep -q 'timeout 10 networkctl reload' && ok "apply helper: networkctl reload under timeout 10" || bad "apply helper: networkctl reload without timeout 10"
echo "$h" | grep -q journalctl && bad "apply helper must not call journalctl" || ok "apply helper: no journal calls"
fi
# 18b. status file gate and STARLINK_IFACE validation
@@ -577,6 +600,33 @@ echo "$o" | grep -q -- "-> post-deploy: NetworkManager connection $conn not foun
echo "$o" | grep -q -- '-> post-deploy: WAN netplan differs' && ok "19e: netplan differs -> warn" || bad "19e: netplan warn missing"
printf 'id=%s\n' "$conn" > "$tc/nm/x.nmconnection"
nm | grep -q " ok NetworkManager connection $conn exists" && ok "19e: cellular keyfile scan -> ok" || bad "19e: keyfile scan ok missing"
# 19h. networkd must not still manage the uplink port NM owns (stub networkctl first on PATH)
mkdir -p "$tc/nostub" "$tc/sys3"
STAR=$(sed -n 's/^STARLINK_IFACE=\([^ ]*\).*/\1/p' hosts/van.conf | tr -d '"')
mkdir -p "$tc/sys3/$STAR"
cat > "$tc/stub/networkctl" <<'S'
#!/bin/sh
case "$*" in
*list*) printf ' 1 lo loopback carrier unmanaged\n 2 %sx ether routable configured\n 3 %s ether routable %s\n' "$STUB_STAR" "$STUB_STAR" "$STUB_SETUP" ;;
esac
S
chmod +x "$tc/stub/networkctl"
for x in /usr/bin/* /bin/*; do [ "${x##*/}" = networkctl ] || ln -sf "$x" "$tc/nostub/${x##*/}" 2>/dev/null || true; done
nd() { # nd <path-dir> <setup> [netplan-state: matches|missing]
[ "${3:-matches}" = matches ] && cp "$tc/np-ok.yaml" "$tc/np.yaml" || rm -f "$tc/np.yaml"
PATH="$1:/usr/bin:/bin" STUB_STAR="$STAR" STUB_SETUP="$2" STUB_DEV="$tc/dev" STUB_CONN="$tc/conn" VANLINK_NM_DIR="$tc/nm" VANLINK_SYSNET="$tc/sys3" \
VANLINK_WAN_NETPLAN="$tc/np.yaml" VANLINK_APPLY_STATUS="$tc/none" POSTCHECK_WAIT=0 ./deploy.sh --post-check hosts/van.conf 2>&1 | sed '/^== warnings ==$/,$d' || true
}
VANLINK_CONF=hosts/van.conf ./deploy.sh --render-wan-netplan > "$tc/np-ok.yaml"
W="-> post-deploy: networkd still manages $STAR although NetworkManager owns it — run: networkctl reload"
nd "$tc/stub" configured | grep -qF -- "$W" && ok "19h: matches + configured -> warn" || bad "19h: matches + configured warn missing"
nd "$tc/stub" configuring | grep -qF -- "$W" && ok "19h: matches + configuring -> warn" || bad "19h: matches + configuring warn missing"
o=$(nd "$tc/stub" unmanaged)
echo "$o" | grep -qF " ok networkd leaves $STAR to NetworkManager" && ! echo "$o" | grep -q 'networkd still manages' && ok "19h: matches + unmanaged -> ok" || bad "19h: matches + unmanaged ok line missing"
nd "$tc/stub" configured missing | grep -qE 'networkd (still manages|leaves)' && bad "19h: netplan not matching must print nothing about networkd" || ok "19h: netplan missing + configured -> nothing"
nd "$tc/nostub" configured | grep -qE 'networkd (still manages|leaves)' && bad "19h: no networkctl must print nothing" || ok "19h: no networkctl -> nothing"
rmdir "$tc/sys3/$STAR"
nd "$tc/stub" configured | grep -qE 'networkd (still manages|leaves)' && bad "19h: absent port must print nothing" || ok "19h: uplink port absent -> nothing"
# 19g. an installed WAN netplan this user cannot read (the helper installs it 0600) is "unreadable", not "differs"
if [ "$(id -u)" = 0 ]; then
ok "19g: unreadable-netplan checks skipped (running as root)"