#!/usr/bin/env bash # Deploy vanlink configs/scripts to their system locations. One checkout is # shared by every host; host-specific values come from /etc/vanlink.conf # (symlink to hosts/.conf — see hosts/vanlink.conf.example). # Usage: cd ~/vanlink && sudo ./deploy.sh # ./deploy.sh --check [conf] # validate conf + render all templates, no install # ./deploy.sh --diff [conf] # what would change in /etc # ./deploy.sh --post-check [conf] # read-only live checks (units, br0, LAN, NAT, dnsmasq, radios, uplink); exit 1 on any warning # # POSTCHECK_WAIT=: seconds retried probes may wait in total (default 15; 0 = single attempt) # ./deploy.sh --classify-unit # prints ok|finished|wait|warn (used by post_check; for tests) # ./deploy.sh --classify-radio # reads `iw dev info` text on stdin, prints beaconing|silent (no ssid line = silent; used by post_check; for tests) # ./deploy.sh --render-wan-netplan # rendered /etc/netplan/50-van-wan.yaml (ethernet part + wifis stanza when WIFI_WAN_IFACE is set) # ./deploy.sh --wan-plan [conf] # what the NetworkManager WAN provisioning would do (read-only, no root) # ./deploy.sh --apply-network [--force] # root: a detached helper installs the WAN netplan, applies it, gates on uplink health (2 stages) and rolls back # # automatically on failure; status in /var/lib/vanlink/apply-network.status. No-op when already applied, and refused after a # # running/rollback-failed status, unless --force. # # Refused (unless --force) when the preflight finds a problem, or when an existing # # /etc/netplan/50-van-wan.yaml differs from the rendered one (the diff is shown, secrets redacted). # ./deploy.sh --apply-preflight [conf] # problems --apply-network would refuse on, one per line (none: no output, exit 0): # # STARLINK_IFACE absent in sysfs; /etc/vanlink.conf -> hosts/.conf with != `hostname -s` # ./deploy.sh --apply-status-check # test seam: yes|no, would this status file (running|rollback-failed) block --apply-network # ./deploy.sh --print-apply-helper [conf] # print the detached apply/rollback helper script (read-only, no root; for review and tests) # # test-only flags: --backup --netplan-dir --log --status --sysnet --rendered --nm-conf-dir --deadlines "s1 s2 rc" --stamp # ./deploy.sh render # preview a templated file on stdout # ./deploy.sh --redact # filter stdin, masking secret values (used by --diff; for tests) # VANLINK_CONF= # host config path (default /etc/vanlink.conf); sub-commands taking [conf] use that argument instead # VANLINK_SYSNET= # test override: sysfs net dir used by --post-check and --apply-preflight (default /sys/class/net) # VANLINK_HOSTNAME= # test override: host name used by --apply-preflight (default `hostname -s`; ignored when run as root) # VANLINK_FAILOVER_STATE= # test override: van-failover state file read by --post-check (default /run/van-failover/state.json) # VANLINK_APPLY_STATUS= # test override: apply-network status file read by --post-check (default /var/lib/vanlink/apply-network.status) # VANLINK_WAN_NETPLAN= # test override: installed WAN netplan compared by nm_wan_state_netplan (default /etc/netplan/50-van-wan.yaml) # VANLINK_NM_DIR= # test override: where existing NM *.nmconnection files are looked up (default /etc/NetworkManager/system-connections; # # ignored when run as root) # Idempotent. Netplan (the NM-managed WANs; eth0 is a LAN port on br0) is never # changed by a routine deploy (applying it flaps the uplinks). ENABLE_NM_WAN=0: # applied manually from the reference copy in ap/50-van-wan.yaml, and checked for # drift below. ENABLE_NM_WAN=1: a missing/differing WAN netplan is only warned # about; it is applied by `./deploy.sh --apply-network` alone. set -euo pipefail cd "$(dirname "$(readlink -f "$0")")" # Host-specific values (interface names, addressing, feature flags) live in # /etc/vanlink.conf — a symlink to hosts/.conf in this repo. Config # files below carry @TOKEN@ placeholders substituted from those variables via # render()/install_rendered() — edit hosts/.conf, not the configs. VANLINK_CONF=${VANLINK_CONF:-/etc/vanlink.conf} # Must be defined (non-empty) in the host file; deploy aborts naming any missing. REQUIRED_VARS=( WIFI_5G_IFACE WIFI_2G_IFACE LAN_IFACE STARLINK_IFACE GATEWAY_IP GATEWAY_PREFIX GATEWAY_NETWORK DHCP_RANGE_START DHCP_RANGE_END DNS_RESOLVERS MODEM_USB_VENDOR AP_SSID_5G AP_SSID_2G AP_5G_RADIO ENABLE_COCKPIT ENABLE_LI3 ENABLE_NVME_WATCH ENABLE_HA ENABLE_FRIGATE ENABLE_NM_WAN ) # May be empty or absent; NVME_* are required only when ENABLE_NVME_WATCH=1. OPTIONAL_VARS=( WIFI_WAN_IFACE CELLULAR_CONN_NAME CELLULAR_APN ONBOARD_LAN_IFACE NVME_DEVICE NVME_FS_DEVICE USB_BACKUP_ROOT_PART USB_BACKUP_MOUNT SD_BACKUP_UUID ) # Computed in load_conf; usable as @TOKEN@ like the rest. DERIVED_VARS=(ONBOARD_LAN_UNMANAGED CELLULAR_UUID) TOKEN_VARS=("${REQUIRED_VARS[@]}" "${OPTIONAL_VARS[@]}" "${DERIVED_VARS[@]}") TOKEN_RE='@[A-Z][A-Z0-9_]*@' die() { echo "deploy: $*" >&2; exit 1; } load_conf() { # load_conf — source and validate a host config local f=$1 v flag missing=() [ -e "$f" ] || die "$f not found (missing or dangling symlink). Create it for this host with: sudo ln -sfn \"$PWD/hosts/$(hostname -s).conf\" /etc/vanlink.conf" for v in "${TOKEN_VARS[@]}"; do unset "$v"; done # shellcheck disable=SC1090 . "$f" for v in "${REQUIRED_VARS[@]}"; do [ -n "${!v:-}" ] || missing+=("$v") done [ ${#missing[@]} -eq 0 ] || die "$f is missing required variable(s): ${missing[*]}" for flag in ENABLE_COCKPIT ENABLE_LI3 ENABLE_NVME_WATCH ENABLE_HA ENABLE_FRIGATE ENABLE_NM_WAN; do case ${!flag} in 0|1) ;; *) die "$f: $flag must be 0 or 1 (got '${!flag}')" ;; esac done [[ $STARLINK_IFACE =~ ^[A-Za-z0-9._-]+$ ]] || die "$f: STARLINK_IFACE '$STARLINK_IFACE' has characters outside [A-Za-z0-9._-]" if [ "$ENABLE_NVME_WATCH" = 1 ]; then { [ -n "${NVME_DEVICE:-}" ] && [ -n "${NVME_FS_DEVICE:-}" ]; } \ || die "$f: ENABLE_NVME_WATCH=1 needs NVME_DEVICE and NVME_FS_DEVICE" fi ONBOARD_LAN_UNMANAGED=${ONBOARD_LAN_IFACE:+;interface-name:$ONBOARD_LAN_IFACE} # Stable per connection name; pure bash + sha1sum (coreutils), no uuid-runtime dependency. local h h=$(printf 'vanlink-%s' "${CELLULAR_CONN_NAME:-}" | sha1sum | cut -c1-32) CELLULAR_UUID="${h:0:8}-${h:8:4}-5${h:13:3}-8${h:17:3}-${h:20:12}" } render() { # render -> stdout, with @TOKEN@ placeholders substituted local args=() v for v in "${TOKEN_VARS[@]}"; do args+=(-e "s|@${v}@|${!v-}|g") done sed "${args[@]}" "$1" } # The netplan is two templates: ap/50-van-wan.yaml (ethernet/Starlink) and # ap/50-van-wan-wifi.yaml (the wifis: stanza, appended verbatim only for hosts # with a Wi-Fi uplink, so a host without one gets no empty/invalid wifis: block). # The rendered netplan must stay byte-identical to what live hosts have installed (comments included: # nm_wan_state_netplan compares bytes, and "differs" invites an --apply-network that flaps the uplinks). # Do not edit comments in ap/50-van-wan*.yaml without re-checking `--wan-plan` on every host. render_wan_netplan() { # ap/50-van-wan.yaml, plus the wifis stanza when the host has a Wi-Fi uplink render ap/50-van-wan.yaml if [ -n "${WIFI_WAN_IFACE:-}" ]; then render ap/50-van-wan-wifi.yaml fi } nm_wan_state_netplan() { # missing | unreadable | differs | matches (the installed WAN netplan vs the rendered one) local f=${VANLINK_WAN_NETPLAN:-/etc/netplan/50-van-wan.yaml} if [ ! -f "$f" ]; then echo missing elif [ ! -r "$f" ]; then echo unreadable # installed 0600 by --apply-network; only root can compare it elif diff -q <(render_wan_netplan) "$f" >/dev/null 2>&1; then echo matches else echo differs; fi } nm_pkg_installed() { # true if the network-manager package is installed (dpkg's Status, not just known to dpkg) local s s=$(dpkg-query -W -f='${Status}' network-manager 2>/dev/null) || return 1 [ "$s" = "install ok installed" ] } # Keeps NetworkManager off the uplink port until --apply-network hands it over (ap/nm-uplink-pending.conf). NM_PENDING_CONF=/etc/NetworkManager/conf.d/zz-van-wan-pending.conf # nm_pending_action -> would install | installed | would remove | absent # The routine deploy installs the pending conf only while the WAN netplan is missing (the uplink was never # handed to NM) and removes it once it matches. differs/unreadable: the port was already handed to NM by an # earlier netplan, so the file is left as it is (installing it would take a live uplink away from NM). nm_pending_action() { case "$1" in missing) if [ -e "$2" ]; then echo installed; else echo "would install"; fi ;; matches) if [ -e "$2" ]; then echo "would remove"; else echo absent; fi ;; *) if [ -e "$2" ]; then echo installed; else echo absent; fi ;; esac } cellular_profile_exists() { # true if an NM profile with id=$CELLULAR_CONN_NAME exists (any file name, or known to nmcli) local d=/etc/NetworkManager/system-connections f names [ "$(id -u)" = 0 ] || d=${VANLINK_NM_DIR:-$d} # the test knob is ignored when run as root for f in "$d"/*.nmconnection; do [ -f "$f" ] || continue if grep -Fx "id=${CELLULAR_CONN_NAME}" "$f" >/dev/null 2>&1; then return 0; fi done if command -v nmcli >/dev/null 2>&1; then names=$(nmcli -g NAME connection show 2>/dev/null) || names= if printf '%s\n' "$names" | grep -Fx "${CELLULAR_CONN_NAME}" >/dev/null 2>&1; then return 0; fi fi return 1 } check_rendered() { # check_rendered — fail on a leftover @TOKEN@ local left left=$(grep -oE "$TOKEN_RE" "$2" | sort -u | tr '\n' ' ') || true [ -z "$left" ] && return 0 echo "deploy: $1: unrendered token(s) after render: $left" >&2 return 1 } install_rendered() { # install_rendered [mode] local tmp tmp=$(mktemp) render "$1" > "$tmp" check_rendered "$1" "$tmp" || { rm -f "$tmp"; exit 1; } install -D -m"${3:-0644}" "$tmp" "$2" rm -f "$tmp" } redact_secrets() { # stdin -> stdout; mask values of secret-looking keys, keep key + diff markers sed -E \ -e 's/((pass(word|phrase)|secret|token)[A-Za-z_]*[[:space:]]*=).*/\1/I' \ -e 's/("[A-Za-z_]*(pass(word|phrase)|secret|token|user)[A-Za-z_]*"[[:space:]]*:[[:space:]]*)"[^"]*"/\1""/I' \ -e 's/^([-+ ]?[[:space:]]*(pass(word|phrase)|secret|token)[A-Za-z_]*:[[:space:]]*)[^[:space:]].*/\1/I' } # apply_preflight_problems — one line per reason --apply-network should not run # here (nothing when fine): the uplink port does not exist, or the conf is a hosts/.conf of another host # (checked only when the conf resolves into a hosts/ directory). apply_preflight_problems() { local conf=$1 sn=$2 hn=$3 real [ -e "$sn/$STARLINK_IFACE" ] || echo "uplink port STARLINK_IFACE=$STARLINK_IFACE does not exist ($sn/$STARLINK_IFACE)" real=$(readlink -f "$conf") if [ "$(basename "$(dirname "$real")")" = hosts ] && [ "$(basename "$real" .conf)" != "$hn" ]; then echo "$conf -> $real is not this host's config (hostname $hn)" fi } template_files() { # every repo file that carries an @TOKEN@ placeholder grep -rlE "$TOKEN_RE" --exclude-dir=.git --exclude-dir=__pycache__ --exclude-dir=.superpowers \ --exclude-dir=docs --exclude-dir=tests --exclude-dir=hosts \ --exclude='*.bak' --exclude='*.md' --exclude=deploy.sh . | sed 's|^\./||' | sort } # Collected during a deploy and persisted to WARNINGS_FILE so the Cockpit vanrouter page # can surface deploy-time issues (missing deps, unedited example configs, drift) # without someone having to remember to scroll back through deploy output. WARNINGS=() WARNINGS_FILE=/var/lib/vanlink/deploy-warnings.json warn() { # warn — prints " -> " (as before) and records it echo " -> $*" WARNINGS+=("$*") } # The units a deploy enables (and post_check expects active). Needs the host # config loaded. The `if` guards sit in column 0 on purpose: tests/check-templates.sh # check 6 requires flag-gated unit names to appear only inside such guards. set_units() { ENABLE_UNITS=(regdomain.service van-br0-setup.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-modem-watch van-sms-watch van-gps-owntracks van-router-mqtt) EXTRA_CHECK_UNITS=() # started by deploy but not in the `enable` list if [ "$ENABLE_NVME_WATCH" = 1 ]; then ENABLE_UNITS+=(van-nvme-watch) fi if [ "$ENABLE_LI3" = 1 ]; then ENABLE_UNITS+=(van-li3-battery) fi if [ "$ENABLE_NM_WAN" = 1 ]; then ENABLE_UNITS+=(NetworkManager ModemManager) fi if [ "$ENABLE_HA" = 1 ]; then EXTRA_CHECK_UNITS+=(homeassistant esphome) fi if [ "$ENABLE_FRIGATE" = 1 ]; then EXTRA_CHECK_UNITS+=(frigate) fi } classify_unit() { # classify_unit -> ok | finished | wait | warn case "$1" in active) echo ok ;; activating|reloading) echo wait ;; inactive) if [ "${2:-}" = success ] && [ "${3:-}" = 0 ]; then echo finished; else echo warn; fi ;; *) echo warn ;; esac } classify_radio() { # stdin: `iw dev info` text -> beaconing (has an ssid line) | silent local line while IFS= read -r line || [ -n "$line" ]; do line=${line#"${line%%[![:space:]]*}"} case "$line" in "ssid "*) echo beaconing; return 0 ;; esac done echo silent } post_check() { # read-only live checks; every probe is guarded, warns instead of aborting set_units echo "== post-deploy check ==" local u pending=() still=() n f out local -A radio=([hostapd]=$WIFI_5G_IFACE [hostapd-2g]=$WIFI_2G_IFACE) # One shared deadline: probes are retried ~1/s until it passes (0 = single attempt), # since a restart just before may not have settled yet. local deadline=$((SECONDS + ${POSTCHECK_WAIT:-15})) retry() { until "$@"; do [ "$SECONDS" -lt "$deadline" ] || return 1; sleep 1; done; } # a. units (AP units are judged under (f)) for u in "${ENABLE_UNITS[@]}" "${EXTRA_CHECK_UNITS[@]}"; do if [ -n "${radio[$u]:-}" ] && ! ip link show dev "${radio[$u]}" >/dev/null 2>&1; then continue; fi pending+=("$u") done still=("${pending[@]}") local -A verdict=() detail=() local k v as res st ls while :; do local next=() for u in "${still[@]}"; do as= res= st= ls= while IFS='=' read -r k v; do case "$k" in LoadState) ls=$v ;; ActiveState) as=$v ;; Result) res=$v ;; ExecMainStatus) st=$v ;; esac done < <(systemctl show -p LoadState,ActiveState,Result,ExecMainStatus "$u" 2>/dev/null || true) verdict[$u]=$(classify_unit "$as" "$res" "$st") [ "$ls" = loaded ] || { verdict[$u]=warn; as="${as:-?}(LoadState=${ls:-?})"; } detail[$u]="ActiveState=${as:-?} Result=${res:-?}" case "${verdict[$u]}" in ok|finished) ;; *) next+=("$u") ;; esac done still=("${next[@]}") [ ${#still[@]} -eq 0 ] && break [ "$SECONDS" -lt "$deadline" ] || break sleep 1 done for u in "${pending[@]}"; do case "${verdict[$u]:-warn}" in ok) echo " ok unit $u active" ;; finished) echo " ok unit $u finished (exit 0)" ;; *) warn "post-deploy: unit $u is not active (${detail[$u]:-state unknown})" ;; esac done # b. bridge address has_br0_addr() { local o; o=$(ip -4 -br addr show dev br0 2>/dev/null) || true; [[ $o == *"$GATEWAY_IP/$GATEWAY_PREFIX"* ]]; } if retry has_br0_addr; then echo " ok br0 has $GATEWAY_IP/$GATEWAY_PREFIX" else warn "post-deploy: br0 does not have $GATEWAY_IP/$GATEWAY_PREFIX"; fi # c. LAN ports picked up our .network files (not netplan's catch-all) uses_file() { local o; o=$(networkctl status "$1" 2>/dev/null) || true; [[ $o == *"/etc/systemd/network/$2"* ]]; } for f in "$LAN_IFACE:05-van-lan.network" "${ONBOARD_LAN_IFACE:-}:06-van-lan-eth0.network"; do n=${f%%:*}; [ -n "$n" ] || continue if ! ip link show dev "$n" >/dev/null 2>&1; then warn "post-deploy: LAN port $n is absent" elif retry uses_file "$n" "${f#*:}"; then echo " ok $n uses ${f#*:}" else warn "post-deploy: $n is not using /etc/systemd/network/${f#*:} (another .network file shadows it?)"; fi done # d. NAT has_nat() { local o; o=$(nft list ruleset 2>/dev/null) || true [[ $o == *"ip saddr $GATEWAY_NETWORK"*masquerade* ]]; } if retry has_nat; then echo " ok NAT masquerade for $GATEWAY_NETWORK" elif [ "$(id -u)" != 0 ]; then warn "post-deploy: no masquerade rule for ip saddr $GATEWAY_NETWORK found (nft list ruleset needs root to read)" else warn "post-deploy: NAT rule for $GATEWAY_NETWORK not found"; fi # e. DHCP has_dhcp() { local o; o=$(ss -ulnH 2>/dev/null | awk '{print $4}') || true; [[ $'\n'$o == *:67$'\n'* || $o == *:67 ]]; } if retry has_dhcp; then echo " ok dnsmasq bound to UDP 67" else warn "post-deploy: nothing bound to UDP port 67 (dnsmasq DHCP)"; fi # f. AP radios (present but no ssid = hostapd is not beaconing) radio_beaconing() { out=$(iw dev "$1" info 2>/dev/null) || return 1 [ "$(classify_radio <<<"$out")" = beaconing ]; } for n in "$WIFI_5G_IFACE" "$WIFI_2G_IFACE"; do u=hostapd; [ "$n" = "$WIFI_5G_IFACE" ] || u=hostapd-2g out= if retry radio_beaconing "$n"; then echo " ok radio $n: $(echo "$out" | grep -E 'ssid|channel' | sed 's/^[[:space:]]*//' | tr '\n' ';')" elif [ -z "$out" ]; then warn "post-deploy: radio $n not present (its hostapd keeps retrying until it appears)" else warn "post-deploy: radio $n is present but not beaconing (no ssid) — check: journalctl -u $u -n 30"; fi done # g. uplink out=$(ip route show default 2>/dev/null) || true if [ -z "$out" ]; then warn "post-deploy: no default route" elif ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; then echo " ok uplink: default route, 1.1.1.1 reachable" else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi if [ "$ENABLE_NM_WAN" = 1 ]; then local sysnet=${VANLINK_SYSNET:-/sys/class/net} nmst fst as_f as_l # h. NetworkManager must leave the AP/LAN radios and ports alone nmst=$(nmcli -t -f DEVICE,STATE device 2>/dev/null) || nmst= for n in "$WIFI_5G_IFACE" "$WIFI_2G_IFACE" "$LAN_IFACE" "${ONBOARD_LAN_IFACE:-}"; do [ -n "$n" ] && [ -e "$sysnet/$n" ] || continue if [[ $'\n'$nmst == *$'\n'"$n:unmanaged"* ]]; then echo " ok NetworkManager leaves $n unmanaged" else warn "post-deploy: NetworkManager manages $n (or its state is unknown) — expected unmanaged; check /etc/NetworkManager/conf.d/"; fi done # i. cellular connection if [ -n "${CELLULAR_APN:-}" ]; then if cellular_profile_exists; then echo " ok NetworkManager connection $CELLULAR_CONN_NAME exists" else warn "post-deploy: NetworkManager connection $CELLULAR_CONN_NAME not found"; fi fi # j. van-failover state: "present" = the device has an active NM connection, so an # unplugged/out-of-range WAN is normal; only a device WITH link and present:false is actionable fst=${VANLINK_FAILOVER_STATE:-/run/van-failover/state.json} fo_read() { # parses the state file (needs a valid "updated" and "wans"); sets $out out=$(python3 - "$fst" "$sysnet" 2>/dev/null <<'PY' import json, os, sys, time try: with open(sys.argv[1]) as f: d = json.load(f) time.strptime(d["updated"], "%Y-%m-%dT%H:%M:%S%z") for w in d["wans"]: dev = w.get("device") or "" if not dev or not os.path.exists(os.path.join(sys.argv[2], dev)): continue try: with open(os.path.join(sys.argv[2], dev, "carrier")) as c: if c.read().strip() != "1": continue except Exception: continue print(("ok" if w.get("present") is True else "nocon"), w.get("name", "?"), dev) except Exception: sys.exit(3) PY ) } if ! command -v python3 >/dev/null 2>&1; then warn "post-deploy: python3 not available — cannot read the van-failover state" elif retry fo_read; then while read -r as_f as_l n; do [ -n "$as_f" ] || continue if [ "$as_f" = ok ]; then echo " ok van-failover: WAN $as_l ($n) has an active NetworkManager connection" else warn "post-deploy: $as_l ($n) has link but no active NetworkManager connection — not handed to NM yet (./deploy.sh --apply-network) or its profile is missing"; fi done <<<"$out" else warn "post-deploy: van-failover state $fst missing or unreadable"; fi # k. netplan drift f=$(nm_wan_state_netplan) case "$f" in matches) echo " ok WAN netplan matches" ;; unreadable) echo " ok WAN netplan: cannot read as non-root (run with sudo to verify)" ;; *) warn "post-deploy: WAN netplan $f — run ./deploy.sh --apply-network" ;; esac # k2. once the WAN netplan is applied NM owns the uplink port; a networkd that still lists it # configured/configuring holds a stale in-memory claim (cured by networkctl reload). # Before the apply networkd legitimately owns it: say nothing. if [ "$f" = matches ] && [ -e "$sysnet/$STARLINK_IFACE" ] && command -v networkctl >/dev/null 2>&1; then out=$(networkctl --no-pager --no-legend list 2>/dev/null | awk -v i="$STARLINK_IFACE" '$2 == i { s = $5 } END { print s }') case "$out" in configured|configuring) warn "post-deploy: networkd still manages $STARLINK_IFACE although NetworkManager owns it — run: networkctl reload" ;; unmanaged) echo " ok networkd leaves $STARLINK_IFACE to NetworkManager" ;; esac fi # l. last --apply-network as_f=${VANLINK_APPLY_STATUS:-/var/lib/vanlink/apply-network.status} if [ -r "$as_f" ]; then as_l=$(grep -m1 '[^[:space:]]' "$as_f" 2>/dev/null) || as_l= case "${as_l%% *}" in "") ;; ok) echo " ok last --apply-network: $as_l" ;; running|rolled-back|rollback-failed) warn "post-deploy: last --apply-network status: $as_l — see /var/log/vanlink-apply-network.log" ;; *) warn "post-deploy: unrecognised apply-network status: $as_l" ;; esac fi fi } # --apply-network time budgets (seconds), in one place; all derived from the AN_* constants. # (netplan generate/apply run under timeout -k AN_KILL_T: +AN_KILL_T each in the worst case) # networkd reload AN_ND = networkctl reload (timeout AN_RELOAD_T) + settle sleep AN_SETTLE + port state line (AN_NM_T); # runs once after each netplan apply (forward path and rollback) # forward path AN_FWD = NM conf reload + generate + apply + 2*kill + AN_ND + van-br0-setup (unit check + restart) # + stage 1 + restart + stage 2 + networkd info line + diagnostics (logged once, on a failed gate) # rollback path AN_RB = generate + apply + 2*kill + AN_ND + NM conf reload + van-br0-setup (unit check + restart) + restart + connectivity recheck # helper worst case AN_HELPER_MAX = AN_FWD + AN_RB + AN_SLOP (loop iterations overshoot their # deadline by a few nmcli/ping timeouts) # foreground waits AN_WAIT = AN_HELPER_MAX + 30 # transient unit: RuntimeMaxSec = AN_HELPER_MAX + 60 (backstop; systemd then sends TERM and the # helper's trap runs the restore), TimeoutStopSec = AN_RB + 30 (time the restore may take). # van-failover starts optimistic and probes every WAN at once (probe_timeout 3 s), enforcing the # route in its first loop, so a healthy default route shows up within seconds of a restart; the # 120 s stage 2 deadline is a wide margin for a slow Starlink/cellular probe. AN_KILL_T=10 AN_APPLY_T=90 AN_GEN_T=30 AN_NM_T=5 AN_RESTART_T=30 AN_S1=60 AN_S2=120 AN_RC=60 AN_SLOP=60 AN_RELOAD_T=10 AN_SETTLE=3 AN_ND=$(( AN_RELOAD_T + AN_SETTLE + AN_NM_T )) AN_DIAG=$(( 5 * AN_NM_T )) # diag(): five commands, each under timeout AN_NM_T AN_FWD=$(( AN_NM_T + AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_ND + AN_NM_T + AN_RESTART_T + AN_S1 + AN_RESTART_T + AN_S2 + AN_NM_T + AN_DIAG )) AN_RB=$(( AN_GEN_T + AN_APPLY_T + 2 * AN_KILL_T + AN_ND + AN_NM_T + AN_NM_T + AN_RESTART_T + AN_RESTART_T + AN_RC )) AN_HELPER_MAX=$(( AN_FWD + AN_RB + AN_SLOP )) AN_WAIT=$(( AN_HELPER_MAX + 30 )) AN_RUNMAX=$(( AN_HELPER_MAX + 60 )) AN_STOP_T=$(( AN_RB + 30 )) apply_status_blocks_run() { # apply_status_blocks_run -> yes|no: a run that is/was in flight or failed to roll back blocks a new one case "$(cut -d' ' -f1 "$1" 2>/dev/null || true)" in running|rollback-failed) echo yes ;; *) echo no ;; esac } # apply_helper_script [netplan-dir] [log] [status-file] [sysnet-dir] [rendered-file] [deadlines "s1 s2 rc"] [run-stamp] [nm-conf-dir] # The script the detached unit runs. Paths are parameters only so tests can bake # sandbox paths into the text; the production helper reads no environment. apply_helper_script() { local bak=$1 nd=${2:-/etc/netplan} log=${3:-/var/log/vanlink-apply-network.log} local st=${4:-/var/lib/vanlink/apply-network.status} sn=${5:-/sys/class/net} local rf=${6:-/run/vanlink-50-van-wan.yaml} dl=${7:-"$AN_S1 $AN_S2 $AN_RC"} stamp=${8:-manual} local ncd=${9:-$(dirname "$NM_PENDING_CONF")} pc=${NM_PENDING_CONF##*/} local d1 d2 d3 read -r d1 d2 d3 <<< "$dl" # Unquoted heredoc on purpose: values expand now; \$... stays for run time. cat <>"\$log" 2>&1 trap 'rm -f "\$0"' EXIT status() { mkdir -p "\$(dirname "\$st")"; echo "\$1 \$(date -Is)\${2:+ \$2}" > "\$st"; } echo "== apply-network $stamp \$(date -Is) (backup: \$bak)" conn_ok() { ip route show default | grep . >/dev/null && ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; } gate2() { conn_ok || return 1 [ -e "\$sn/$STARLINK_IFACE" ] || return 0 ip route show default dev '$STARLINK_IFACE' proto static | grep . >/dev/null } diag() { # diag : uplink state for the log; informational, every command bounded and failure-tolerant echo "--- diagnostics: \$1" echo "--- nmcli device show $STARLINK_IFACE" timeout $AN_NM_T nmcli -g GENERAL.STATE,GENERAL.CONNECTION,IP4.ADDRESS,IP4.GATEWAY device show '$STARLINK_IFACE' 2>&1 echo "--- DHCP4 routers" timeout $AN_NM_T nmcli -g DHCP4.OPTION device show '$STARLINK_IFACE' 2>&1 | tr '|' '\\n' | grep -E '^ *routers =' echo "--- networkctl status $STARLINK_IFACE" timeout $AN_NM_T networkctl --no-pager -n 0 status '$STARLINK_IFACE' 2>&1 | head -8 echo "--- ip -4 addr show $STARLINK_IFACE" timeout $AN_NM_T ip -4 addr show '$STARLINK_IFACE' 2>&1 echo "--- ip route" timeout $AN_NM_T ip route 2>&1 echo "--- end diagnostics" return 0 } gate1() { [ -e "\$sn/$STARLINK_IFACE" ] || return 0 [ "\$(timeout $AN_NM_T nmcli -g GENERAL.STATE device show '$STARLINK_IFACE' 2>/dev/null)" = "100 (connected)" ] || return 1 case "\$(timeout $AN_NM_T nmcli -g GENERAL.CONNECTION device show '$STARLINK_IFACE' 2>/dev/null)" in netplan-*) ;; *) return 1 ;; esac a=\$(timeout $AN_NM_T nmcli -g IP4.ADDRESS device show '$STARLINK_IFACE' 2>/dev/null) [ -n "\$a" ] && [ "\$a" != -- ] || return 1 g=\$(timeout $AN_NM_T nmcli -g IP4.GATEWAY device show '$STARLINK_IFACE' 2>/dev/null) case "\$g" in ''|--) g=\$(timeout $AN_NM_T nmcli -g DHCP4.OPTION device show '$STARLINK_IFACE' 2>/dev/null | tr '|' '\\n' | sed -n 's/^ *routers = *//p' | head -1 | cut -d' ' -f1) ;; esac [ -n "\$g" ] && [ "\$g" != -- ] || return 1 ping -c1 -W2 -I '$STARLINK_IFACE' "\$g" >/dev/null 2>&1 } nd_reload() { # nd_reload [note]: after a netplan apply networkd may keep a removed file's config in memory; reload it (never fatal, never gating) command -v networkctl >/dev/null 2>&1 || return 0 timeout $AN_RELOAD_T networkctl reload || echo "networkctl reload failed (continuing)" sleep $AN_SETTLE local l l=\$(timeout $AN_NM_T networkctl --no-pager --no-legend list 2>/dev/null | awk -v i='$STARLINK_IFACE' '\$2 == i { print; exit }') echo "networkd after reload: \${l:-$STARLINK_IFACE not listed}" case " \$l " in *" configured "*|*" configuring "*) [ -z "\${1:-}" ] || echo "networkd still manages $STARLINK_IFACE (NetworkManager should own it)" ;; esac return 0 } route_log() { echo "route to 1.1.1.1: \$(ip route get 1.1.1.1 2>&1 | head -1)"; } in_restore=0 restore() { # restore : put the pre-run netplan back, re-check connectivity, always exit 1 (runs once) trap '' TERM INT HUP [ "\$in_restore" = 0 ] || exit 1 in_restore=1 local rb=ok c=down echo "apply-network FAILED (\$1) - restoring \$bak" route_log if [ -d "\$bak" ]; then rm -f "\$nd"/* "\$nd"/.[!.]* cp -a "\$bak"/. "\$nd"/ || rb=CP-FAILED rm -f "\$nd/nm-pending.conf" # the backed-up NM conf is not a netplan file # the NM conf that kept NM off the uplink goes back too; NM re-reads it after the re-apply below [ ! -f "\$bak/nm-pending.conf" ] || cp -a "\$bak/nm-pending.conf" "\$ncd/$pc" || echo "WARNING: could not restore \$ncd/$pc" else # No backup to restore from: at least undo this helper's own two changes. rb=BACKUP-MISSING rm -f "\$nd/50-van-wan.yaml" [ ! -f "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" ] || mv "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled" "\$nd/10-dhcp-all-interfaces.yaml" fi timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply || { [ "\$rb" != ok ] || rb=APPLY-FAILED; } [ ! -f "\$ncd/$pc" ] || timeout $AN_NM_T nmcli general reload conf || echo "nmcli general reload conf failed" nd_reload # NM has let go of the port; now networkd, with the old config back, claims it (after the NM reload, not before) # the re-apply restarted networkd too, which can drop br0's members (as on the forward path; not gating) if timeout $AN_NM_T systemctl cat van-br0-setup >/dev/null 2>&1; then timeout $AN_RESTART_T systemctl restart van-br0-setup || echo "van-br0-setup restart failed (continuing)" fi timeout $AN_RESTART_T systemctl restart van-failover || true end=\$((SECONDS+$d3)) while :; do conn_ok && { c=ok; break; } [ "\$SECONDS" -ge "\$end" ] && break sleep 1 done rm -f "\$rf" if [ "\$rb" = ok ] && [ "\$c" = ok ]; then echo "apply-network ROLLED BACK (restored, connectivity ok)" status rolled-back "\$1" elif [ "\$rb" = ok ]; then echo "apply-network ROLLED BACK (restored, but no connectivity - check the uplink/cable)" status rolled-back "restored-no-connectivity; \$1" else echo "apply-network ROLLBACK FAILED: \$rb / connectivity \$c" status rollback-failed "\$1; \$rb / connectivity \$c" # Reboot (boot uses the files on disk) only when a restore was attempted and failed, and there is no connectivity. case "\$rb" in CP-FAILED|APPLY-FAILED) [ "\$c" = ok ] || { echo "rollback failed and no connectivity - rebooting"; systemctl reboot; } ;; esac fi exit 1 } pending_off() { # hand the uplink to NM: drop the NM conf that kept NM off it, right before the apply [ -e "\$ncd/$pc" ] || return 0 rm -f "\$ncd/$pc" || return 1 timeout $AN_NM_T nmcli general reload conf || echo "nmcli general reload conf failed (continuing; stage 1 decides)" return 0 } status running "\$bak" trap 'restore terminated' TERM INT HUP applied=0 if install -m0600 "\$rf" "\$nd/50-van-wan.yaml" \\ && { [ ! -f "\$nd/10-dhcp-all-interfaces.yaml" ] || mv "\$nd/10-dhcp-all-interfaces.yaml" "\$nd/10-dhcp-all-interfaces.yaml.vanlink-disabled"; } \\ && pending_off \\ && timeout -k $AN_KILL_T $AN_GEN_T netplan generate && timeout -k $AN_KILL_T $AN_APPLY_T netplan apply; then applied=1 else echo "netplan install/generate/apply failed" fi s1=0 if [ "\$applied" = 1 ]; then nd_reload note # the Armbian catch-all .network is gone from disk but networkd still holds it until reloaded # netplan apply restarts networkd, which can drop br0's members: re-run the bridge setup (not gating) if timeout $AN_NM_T systemctl cat van-br0-setup >/dev/null 2>&1; then timeout $AN_RESTART_T systemctl restart van-br0-setup || echo "van-br0-setup restart failed (continuing)" fi end=\$((SECONDS+$d1)) while :; do gate1 && { s1=1; break; } [ "\$SECONDS" -ge "\$end" ] && break sleep 1 done fi [ "\$s1" = 1 ] || { diag "stage 1"; restore "stage 1: apply or uplink gate failed"; } timeout $AN_RESTART_T systemctl restart van-failover || restore "van-failover restart failed" s2=0 end=\$((SECONDS+$d2)) while :; do gate2 && { s2=1; break; } [ "\$SECONDS" -ge "\$end" ] && break sleep 1 done route_log [ "\$s2" = 1 ] || { diag "stage 2"; restore "stage 2: no default route, no van-failover route on the port, or no internet"; } nwl=\$(timeout $AN_NM_T networkctl --no-pager --no-legend list 2>/dev/null | awk -v i='$STARLINK_IFACE' '\$2 == i { print; exit }') echo "networkd (info, not gating): \${nwl:-$STARLINK_IFACE not listed}" rm -f "\$rf" status ok trap - TERM INT HUP echo "apply-network OK" exit 0 HELPER } apply_network() { # ./deploy.sh --apply-network [--force] — explicit, root; changes nothing under /etc/netplan itself (the detached helper does); see spec §4 local force=0 [ "${1:-}" = --force ] && force=1 [ "$(id -u)" = 0 ] || die "--apply-network needs root (sudo)" [ "$ENABLE_NM_WAN" = 1 ] || die "ENABLE_NM_WAN is 0 in $VANLINK_CONF — nothing to apply" nm_pkg_installed || die "network-manager is not installed — run ./deploy.sh first" systemctl is-active --quiet NetworkManager || die "NetworkManager is not running — handing it the uplink port would flap it" systemctl cat van-failover >/dev/null 2>&1 || die "van-failover.service does not exist — run ./deploy.sh first (stage 2 of the gate needs it)" systemctl is-active --quiet vanlink-apply-network && die "an apply is already running — see /var/log/vanlink-apply-network.log" local log=/var/log/vanlink-apply-network.log st=/var/lib/vanlink/apply-network.status if [ "$force" = 0 ] && [ "$(apply_status_blocks_run "$st")" = yes ]; then die "the last apply did not finish cleanly: $(cat "$st" 2>/dev/null) — inspect $st and $log, check the backup dir named there, then re-run with --force" fi local np_state problems np_state=$(VANLINK_WAN_NETPLAN='' nm_wan_state_netplan) if [ "$force" = 0 ] && [ "$np_state" = matches ] && [ ! -e /etc/netplan/10-dhcp-all-interfaces.yaml ]; then echo "nothing to apply (WAN netplan already matches; use --force to re-apply)" return 0 fi problems=$(apply_preflight_problems "$VANLINK_CONF" /sys/class/net "$(hostname -s)") if [ -n "$problems" ] && [ "$force" = 0 ]; then die "refusing to apply: $problems (fix it, or re-run with --force)" fi if [ "$force" = 0 ] && { [ "$np_state" = differs ] || [ "$np_state" = unreadable ]; }; then echo "/etc/netplan/50-van-wan.yaml exists and differs from the rendered template (secrets redacted):" diff -u /etc/netplan/50-van-wan.yaml <(render_wan_netplan) | redact_secrets | sed 's/^/ /' || true die "refusing to replace an existing, different WAN netplan without --force (review the diff above)" fi local stamp helper rendered bak pt stamp=$(date +%Y%m%d-%H%M%S)-$$ rendered=/run/vanlink-50-van-wan-$stamp.yaml helper=/run/vanlink-apply-network-$stamp.sh (umask 077; render_wan_netplan > "$rendered") || die "render failed" check_rendered ap/50-van-wan.yaml "$rendered" || { rm -f "$rendered"; exit 1; } bak=$(mktemp -d "/var/backups/vanlink-netplan-$stamp.XXXX") || { rm -f "$rendered"; die "cannot create backup dir"; } cp -a /etc/netplan/. "$bak/" || { rm -f "$rendered"; die "backup failed — nothing changed"; } # the NM conf keeping NM off the uplink: the helper removes it right before the apply, a rollback restores it from here if [ -e "$NM_PENDING_CONF" ]; then cp -a "$NM_PENDING_CONF" "$bak/nm-pending.conf" || { rm -f "$rendered"; die "backup of $NM_PENDING_CONF failed — nothing changed"; } fi if [ ! -e /var/backups/vanlink-netplan-orig ] && [ ! -e /etc/netplan/50-van-wan.yaml ]; then pt=/var/backups/vanlink-netplan-orig.tmp.$$ rm -rf "$pt" install -d -m0700 "$pt" cp -a /etc/netplan/. "$pt/" || { rm -rf "$pt"; rm -f "$rendered"; die "pristine backup failed — nothing changed"; } mv "$pt" /var/backups/vanlink-netplan-orig || { rm -rf "$pt"; rm -f "$rendered"; die "pristine backup could not be moved into place — nothing changed"; } fi install -d -m0755 /var/lib/vanlink rm -f "$st" apply_helper_script "$bak" /etc/netplan "$log" "$st" /sys/class/net "$rendered" "$AN_S1 $AN_S2 $AN_RC" "$stamp" "$(dirname "$NM_PENDING_CONF")" > "$helper" chmod 0700 "$helper" echo "backup: $bak log: $log" echo "applying from a detached unit (survives an SSH drop); waiting up to $AN_WAIT s ..." systemctl reset-failed vanlink-apply-network 2>/dev/null || true systemd-run --unit=vanlink-apply-network --collect -p RuntimeMaxSec="$AN_RUNMAX" -p TimeoutStopSec="$AN_STOP_T" /bin/bash "$helper" >/dev/null \ || { rm -f "$helper" "$rendered"; die "systemd-run failed — nothing was changed (backup: $bak)"; } SECONDS=0 while systemctl is-active --quiet vanlink-apply-network && [ "$SECONDS" -lt "$AN_WAIT" ]; do sleep 1; done awk -v s="== apply-network $stamp" 'index($0, s) == 1 { p = 1 } p' "$log" 2>/dev/null | tail -n 8 || true if systemctl is-active --quiet vanlink-apply-network; then echo "still running — check $log" return 2 fi local verdict verdict=$(cut -d' ' -f1 "$st" 2>/dev/null || true) case "$verdict" in ok) return 0 ;; rolled-back|rollback-failed) echo "apply-network: $(cat "$st") (previous config: $bak)" >&2; return 1 ;; *) echo "apply-network: no final status — see $log (previous config: $bak)" >&2; return 1 ;; esac } case "${1:-}" in --templates) template_files exit 0 ;; --check) # ./deploy.sh --check [conf] — validate conf, render every template, install nothing load_conf "${2:-$VANLINK_CONF}" rc=0 tmp=$(mktemp) for f in $(template_files); do render "$f" > "$tmp" check_rendered "$f" "$tmp" || rc=1 done rm -f "$tmp" [ "$rc" -eq 0 ] && echo "check OK: $(template_files | wc -l) templates render cleanly" exit "$rc" ;; --diff) # ./deploy.sh --diff [conf] — what would change in /etc, without installing load_conf "${2:-$VANLINK_CONF}" grep -E '^[[:space:]]*install_rendered +[^" ]+ +/' deploy.sh | while read -r _ src dst _; do if [ ! -e "$dst" ]; then echo "NEW $dst" elif ! diff -q <(render "$src") "$dst" >/dev/null; then echo "CHANGED $dst" diff -u "$dst" <(render "$src") | redact_secrets | sed 's/^/ /' || true fi done # Also lists plain `install` files and `rm -f` targets — these sections may be # flag-gated in the real run, but --diff reports them regardless of the flag. grep -E '^[[:space:]]*install -D -m[0-9]+ +[^" ]+ +/' deploy.sh | while read -r _ _ mode src dst _; do # seed-if-absent secret files (from *.example, or 0600): never diff or print case "$src" in *.example) continue ;; esac [ "$mode" = -m0600 ] && continue if [ ! -e "$dst" ]; then echo "NEW $dst" elif ! cmp -s "$src" "$dst"; then echo "CHANGED $dst" diff -u "$dst" "$src" | redact_secrets | sed 's/^/ /' || true fi done grep -E '^[[:space:]]*rm -f +/' deploy.sh | while read -r _ _ path _; do [ -e "$path" ] && echo "REMOVE $path" done || true exit 0 ;; --redact) # ./deploy.sh --redact — stdin through redact_secrets (no config, no root) redact_secrets exit 0 ;; --apply-network) # ./deploy.sh --apply-network [--force] — root; applies the WAN netplan from a detached helper with automatic rollback load_conf "$VANLINK_CONF" apply_network "${2:-}" exit $? ;; --apply-status-check) # test seam: --apply-status-check prints yes|no (does that status block a new --apply-network run?) apply_status_blocks_run "${2:?status file}" exit 0 ;; --print-apply-helper) # [conf] the helper script for review; test-only flags: --backup D --netplan-dir D --log L --status S --sysnet N --rendered R --nm-conf-dir D --deadlines "s1 s2 rc" --stamp T shift; pc= a_bak=/var/backups/vanlink-netplan-EXAMPLE a_nd= a_log= a_st= a_sn= a_rf= a_dl= a_stamp= a_ncd= while [ $# -gt 0 ]; do case "$1" in --backup|--netplan-dir|--log|--status|--sysnet|--rendered|--nm-conf-dir|--deadlines|--stamp) [ $# -ge 2 ] || die "$1 needs a value" ;; esac case "$1" in --backup) a_bak=$2; shift 2 ;; --netplan-dir) a_nd=$2; shift 2 ;; --log) a_log=$2; shift 2 ;; --status) a_st=$2; shift 2 ;; --sysnet) a_sn=$2; shift 2 ;; --rendered) a_rf=$2; shift 2 ;; --nm-conf-dir) a_ncd=$2; shift 2 ;; --deadlines) a_dl=$2; shift 2 ;; --stamp) a_stamp=$2; shift 2 ;; *) pc=$1; shift ;; esac done load_conf "${pc:-$VANLINK_CONF}" apply_helper_script "$a_bak" "$a_nd" "$a_log" "$a_st" "$a_sn" "$a_rf" "$a_dl" "$a_stamp" "$a_ncd" exit 0 ;; --apply-preflight) # ./deploy.sh --apply-preflight [conf] — the problems --apply-network would refuse on; exit 1 if any c=${2:-$VANLINK_CONF} load_conf "$c" hn=$(hostname -s) [ "$(id -u)" = 0 ] || hn=${VANLINK_HOSTNAME:-$hn} # test knob, ignored as root out=$(apply_preflight_problems "$c" "${VANLINK_SYSNET:-/sys/class/net}" "$hn") [ -z "$out" ] && exit 0 echo "$out" exit 1 ;; --render-wan-netplan) load_conf "$VANLINK_CONF"; render_wan_netplan; exit 0 ;; --wan-plan) # ./deploy.sh --wan-plan [conf] — what the NM WAN provisioning would do; changes nothing load_conf "${2:-$VANLINK_CONF}" echo "wan-plan: ENABLE_NM_WAN=$ENABLE_NM_WAN" if [ "$ENABLE_NM_WAN" = 1 ]; then if [ -n "${WIFI_WAN_IFACE:-}" ]; then echo " wifi WAN: $WIFI_WAN_IFACE"; else echo " wifi WAN: none (wifis stanza omitted)"; fi if nm_pkg_installed; then echo " packages: network-manager installed"; else echo " packages: network-manager would install"; fi np=$(nm_wan_state_netplan) if [ "$np" = unreadable ]; then echo " netplan: /etc/netplan/50-van-wan.yaml unreadable (run with sudo)"; else echo " netplan: /etc/netplan/50-van-wan.yaml $np"; fi echo " uplink NM pending conf: $(nm_pending_action "$np" "$NM_PENDING_CONF")" if [ -f /etc/netplan/10-dhcp-all-interfaces.yaml ]; then echo " armbian: /etc/netplan/10-dhcp-all-interfaces.yaml present -> would move aside"; else echo " armbian: /etc/netplan/10-dhcp-all-interfaces.yaml absent"; fi if [ -z "${CELLULAR_APN:-}" ]; then echo " cellular profile: skipped (no CELLULAR_APN)" elif cellular_profile_exists; then echo " cellular profile: ${CELLULAR_CONN_NAME} apn=${CELLULAR_APN} exists" else echo " cellular profile: ${CELLULAR_CONN_NAME} apn=${CELLULAR_APN} would create" fi fi exit 0 ;; --classify-unit) # ./deploy.sh --classify-unit classify_unit "${2:-}" "${3:-}" "${4:-}" exit 0 ;; --classify-radio) # ./deploy.sh --classify-radio < `iw dev info` text classify_radio exit 0 ;; --post-check) # ./deploy.sh --post-check [conf] — live checks, no root, no install; exit 1 on any warning load_conf "${2:-$VANLINK_CONF}" post_check if [ ${#WARNINGS[@]} -gt 0 ]; then echo echo "== warnings ==" for w in "${WARNINGS[@]}"; do echo " -> $w"; done exit 1 fi exit 0 ;; render) [ -n "${2:-}" ] || { echo "Usage: $0 render "; exit 1; } load_conf "$VANLINK_CONF" render "$2" exit 0 ;; esac [ "$(id -u)" = 0 ] || { echo "Run with sudo (writes to /etc, /usr)."; exit 1; } load_conf "$VANLINK_CONF" echo "== host config: $(readlink -f "$VANLINK_CONF") ==" # Check for essential package dependencies echo "== dependency check ==" REQUIRED_PACKAGES=( hostapd iw nftables dnsmasq systemd-resolved python3-gps python3-bleak modemmanager usb-modeswitch gpsd gpsd-clients rsync mosquitto python3-paho-mqtt mobile-broadband-provider-info zerotier-one ) if [ "$ENABLE_HA" = 1 ]; then REQUIRED_PACKAGES+=(podman) fi OPTIONAL_PACKAGES=( ) missing_pkgs=() for pkg in "${REQUIRED_PACKAGES[@]}"; do dpkg -s "$pkg" >/dev/null 2>&1 || missing_pkgs+=("$pkg") done if [ ${#missing_pkgs[@]} -gt 0 ]; then echo " installing missing required packages: ${missing_pkgs[*]}" apt-get update -qq || echo " -> apt-get update failed (continuing with the cached lists)" DEBIAN_FRONTEND=noninteractive apt-get install -y "${missing_pkgs[@]}" \ || { echo "deploy: apt-get install failed for: ${missing_pkgs[*]}" >&2; exit 1; } fi # Check optional packages for pkg in "${OPTIONAL_PACKAGES[@]}"; do if ! dpkg -s "$pkg" >/dev/null 2>&1; then warn "optional package not installed: $pkg (apt install $pkg)" fi done write_warnings() { install -d -m0755 "$(dirname "$WARNINGS_FILE")" { printf '{\n "deployed": "%s",\n "warnings": [' "$(date -Iseconds)" local first=1 w e for w in "${WARNINGS[@]}"; do [ "$first" = 1 ] || printf ',' first=0 e=${w//\\/\\\\}; e=${e//\"/\\\"} printf '\n "%s"' "$e" done printf '\n ]\n}\n' } > "$WARNINGS_FILE" } # A host file with uncommitted edits means this deploy uses unreviewed values. conf_real=$(readlink -f "$VANLINK_CONF") if git -c safe.directory='*' -C "$(dirname "$conf_real")" status --porcelain -- "$conf_real" 2>/dev/null | grep -q .; then warn "$conf_real has uncommitted changes — commit it so the host config stays reviewable" fi if [ "$ENABLE_NM_WAN" != 1 ]; then echo "== netplan drift check ==" # Netplan is never installed by this script (applying it flaps the uplinks — # see note below), so it's easy to edit ap/50-van-wan.yaml and forget the # manual `netplan apply` step. Warn loudly rather than silently drifting. if [ ! -f /etc/netplan/50-van-wan.yaml ]; then warn "netplan: /etc/netplan/50-van-wan.yaml is missing — repo config was never deployed. Run: sudo ./deploy.sh --render-wan-netplan | sudo tee /etc/netplan/50-van-wan.yaml && sudo netplan generate && sudo netplan apply" elif ! diff -q <(render_wan_netplan) /etc/netplan/50-van-wan.yaml >/dev/null 2>&1; then warn "netplan: /etc/netplan/50-van-wan.yaml differs from ap/50-van-wan.yaml (rendered). Deploy manually: sudo ./deploy.sh --render-wan-netplan | sudo tee /etc/netplan/50-van-wan.yaml && sudo netplan generate && sudo netplan apply" diff -u /etc/netplan/50-van-wan.yaml <(render_wan_netplan) || true fi fi echo "== access point ==" install_rendered ap/hostapd.conf /etc/hostapd/hostapd.conf install -D -m0644 ap/hostapd-restart.conf /etc/systemd/system/hostapd.service.d/restart.conf install -D -m0644 ap/hostapd-regdomain.conf /etc/systemd/system/hostapd.service.d/regdomain.conf install -D -m0644 ap/default-hostapd /etc/default/hostapd install_rendered ap/hostapd-2g.conf /etc/hostapd/hostapd-2g.conf install_rendered ap/hostapd-2g.service /etc/systemd/system/hostapd-2g.service install -D -m0644 ap/hostapd-regdomain.conf /etc/systemd/system/hostapd-2g.service.d/regdomain.conf install_rendered ap/11-van-ap-2g.network /etc/systemd/network/11-van-ap-2g.network install -D -m0644 ap/van-ap-watchdog-2g.service /etc/systemd/system/van-ap-watchdog-2g.service install -D -m0755 ap/van-ap-watchdog /usr/local/sbin/van-ap-watchdog install -D -m0644 ap/van-ap-watchdog.service /etc/systemd/system/van-ap-watchdog.service install_rendered ap/van-ap-dnsmasq.conf /etc/van-ap/dnsmasq.conf install -D -m0644 ap/van-ap-dnsmasq.service /etc/systemd/system/van-ap-dnsmasq.service install_rendered ap/10-van-ap.network /etc/systemd/network/10-van-ap.network install -D -m0644 ap/20-van-br0.netdev /etc/systemd/network/20-van-br0.netdev install_rendered ap/21-van-br0.network /etc/systemd/network/21-van-br0.network # Named 05- (not 22-) so it sorts before Armbian's netplan-generated # /run/systemd/network/10-netplan-all-eth-interfaces.network, which matches # every e* port with DHCP and otherwise wins over this file (first match by # filename), leaving the LAN port un-bridged. install_rendered ap/22-van-lan.network /etc/systemd/network/05-van-lan.network rm -f /etc/systemd/network/22-van-lan.network if [ -n "${ONBOARD_LAN_IFACE:-}" ]; then # 06- sorts before netplan's 10-* catch-all, like 05-van-lan above install_rendered ap/23-van-lan-eth0.network /etc/systemd/network/06-van-lan-eth0.network rm -f /etc/systemd/network/23-van-lan-eth0.network else rm -f /etc/systemd/network/23-van-lan-eth0.network /etc/systemd/network/06-van-lan-eth0.network fi install_rendered ap/van-ap-unmanaged.conf /etc/NetworkManager/conf.d/van-ap-unmanaged.conf # Mask dracut's initramfs-generated catch-all (/run/systemd/network/ # zzzz-dracut-default.network, regenerated every boot): it matches every # unconfigured link, so networkd co-managed the NM-owned WANs — a second # DHCPv4 client on wlan0 and IPv6 that worked only by accident. NM owns WAN # IPv6 now (see ap/50-van-wan.yaml). /etc overrides /run; /dev/null masks. ln -sfn /dev/null /etc/systemd/network/zzzz-dracut-default.network install_rendered ap/nftables.conf /etc/nftables.conf install -D -m0644 ap/regdomain.service /etc/systemd/system/regdomain.service install_rendered ap/van-br0-setup.service /etc/systemd/system/van-br0-setup.service install -D -m0644 ap/80-regdomain.rules /etc/udev/rules.d/80-regdomain.rules install -D -m0644 ap/rtl8188gu-modeswitch.conf /etc/usb-modeswitch.d/0bda:1a2b install -D -m0644 ap/rtw89.conf /etc/modprobe.d/rtw89.conf install -D -m0644 ap/99-van-router.conf /etc/sysctl.d/99-van-router.conf echo "== dns ==" # Disable systemd-resolved DNS listener — dnsmasq handles DNS+DHCP on bridge. # Even localhost-only listening blocks dnsmasq from binding to port 53. install -D -m0644 ap/systemd-resolved-disable.conf /etc/systemd/system/systemd-resolved.service.d/disable.conf # Fixed upstream resolvers (/etc/vanlink.conf's DNS_RESOLVERS), never a WAN's own # DHCP/RA-provided DNS — see the files themselves for the full rationale. # (NM's ipv4/ipv6.ignore-auto-dns can't be set as a config-file connection # default — NM rejects it there — so 60-van-wan-dns enforces this directly # against resolved instead, on every WAN connect/lease event.) # .local (mDNS) is handled separately, scoped to br0 (see 21-van-br0.network # above). ZeroTier-managed DNS (zt.wrede.pvt) is a separate, additive path. install_rendered ap/99-van-router-dns.conf /etc/systemd/resolved.conf.d/99-van-router-dns.conf install -D -m0755 failover/60-van-wan-dns /etc/NetworkManager/dispatcher.d/60-van-wan-dns # ZeroTier-managed DNS for zt.wrede.pvt (see README's "ZeroTier managed DNS"). # allowDNS on the network + the search-domain drop-in are repo-tracked so a # reimage doesn't need the manual `zerotier-cli set ... allowDNS=1` step # remembered by hand; the manager binary itself is a hand-installed .deb # (not in apt) — see https://github.com/zerotier/zerotier-systemd-manager/releases. install -D -m0600 dns/zt-network.local.conf /var/lib/zerotier-one/networks.d/d3ecf5726d041b2a.local.conf install -D -m0644 dns/zt-search.conf /etc/systemd/network/99-ztuga7c2kh.network.d/search.conf dpkg -s zerotier-systemd-manager >/dev/null 2>&1 \ || warn "zerotier-systemd-manager not installed — zt.wrede.pvt won't resolve. Install the arm64 .deb from https://github.com/zerotier/zerotier-systemd-manager/releases" echo "== failover ==" install -D -m0755 failover/van-failover /usr/local/sbin/van-failover install_rendered failover/config.json /etc/van-failover/config.json install -D -m0644 failover/van-failover.service /etc/systemd/system/van-failover.service install -D -m0755 failover/50-disable-eee /etc/NetworkManager/dispatcher.d/50-disable-eee install -D -m0644 failover/99-van-arp.conf /etc/sysctl.d/99-van-arp.conf # Backstop for wlan0's post-boot NM no-secrets wedge (see the script's docstring). install -D -m0755 failover/van-wlan-watchdog /usr/local/sbin/van-wlan-watchdog install_rendered failover/van-wlan-watchdog.service /etc/systemd/system/van-wlan-watchdog.service echo "== cellular modem (GSM/LTE) ==" # NM's gsm.auto-config APN lookup needs this apt-only carrier database; without # it even the right APN can't be auto-detected, and MVNOs (e.g. Koodo, which # isn't listed under its own name — only under host network "Telus Mobility") # often aren't in it anyway, so the modem's NM connection profile may still # need an explicit gsm.apn set by hand regardless. dpkg -s mobile-broadband-provider-info >/dev/null 2>&1 \ || warn "mobile-broadband-provider-info missing (apt install mobile-broadband-provider-info) — GSM APN auto-config will fail" # Backstop for the modem sometimes never enumerating at boot (see the # script's docstring) — detection + Pushover alert only. No automated # recovery: a hub power-cycle was proven not to fix this (only a genuine # physical unplug/replug does), so the modem is now on the Pi's native USB # port rather than through the hub. This just pages if it ever recurs. install -D -m0755 failover/van-modem-watch /usr/local/sbin/van-modem-watch install_rendered failover/van-modem-watch.service /etc/systemd/system/van-modem-watch.service # SMS archive + Pushover relay: polls ModemManager for inbound SMS, appends # each to /var/log/van-sms.jsonl, pages via Pushover, then deletes it from # the modem's own flash storage (small, fills up silently otherwise). MMS # can't be fetched here (see script docstring) — notification-only paging. install -D -m0755 modem/van-sms-watch /usr/local/sbin/van-sms-watch install -D -m0644 modem/van-sms-watch.service /etc/systemd/system/van-sms-watch.service # CLI to send an SMS ad hoc (needs sudo — ModemManager's Messaging actions # are PolicyKit-gated). Not a service, just a tool: sudo van-sms-send install -D -m0755 modem/van-sms-send /usr/local/sbin/van-sms-send if [ "$ENABLE_COCKPIT" = 1 ]; then echo "== cockpit plugin ==" install -d /usr/share/cockpit/vanrouter for f in cockpit/vanrouter/*; do b=$(basename "$f") if [ "$b" = "vanrouter.js" ]; then install_rendered "$f" "/usr/share/cockpit/vanrouter/$b" else install -m0644 "$f" "/usr/share/cockpit/vanrouter/$b" fi done # Bridge fd headroom (Python bridge frees spawn pipes only at GC; 1024 is too tight) install -D -m0644 cockpit/cockpit-session-nofile.conf /etc/systemd/system/cockpit-session@.service.d/nofile.conf fi # The Starlink card queries the dish's gRPC API; grpcurl isn't packaged in apt. command -v grpcurl >/dev/null 2>&1 \ || warn "grpcurl missing (Starlink card will say so): install linux_arm64 binary from github.com/fullstorydev/grpcurl/releases" echo "== router status (MQTT -> Home Assistant, replacing the Cockpit plugin above) ==" # Same status/actions as the cockpit plugin, published via MQTT discovery # instead of a web UI of its own (Cockpit's login was unreliable and it was a # resource hog on this Pi). No secrets in config.json (the local Mosquitto # broker currently allows anonymous connections) so it's rendered + installed # every deploy, same as failover/config.json. install -D -m0755 router-status/van-router-mqtt /usr/local/sbin/van-router-mqtt install_rendered router-status/config.json /etc/van-router-mqtt/config.json install -D -m0644 router-status/van-router-mqtt.service /etc/systemd/system/van-router-mqtt.service python3 -c 'import paho.mqtt' 2>/dev/null \ || warn "python3-paho-mqtt missing (apt install python3-paho-mqtt) — van-router-mqtt won't start" # MQTT broker must listen on all interfaces (not just localhost) for clients on br0 install -D -m0644 router-status/mosquitto-listener.conf /etc/mosquitto/conf.d/van-listener.conf # Restart limit drop-ins: prevent infinite restart loops that trigger hardware watchdog install -D -m0644 router-status/van-gps-owntracks-restart-limit.conf /etc/systemd/system/van-gps-owntracks.service.d/restart-limit.conf echo "== gps (cellular modem GNSS -> gpsd) ==" if dpkg -s gpsd >/dev/null 2>&1; then install -D -m0644 gps/77-modem-gps.rules /etc/udev/rules.d/77-modem-gps.rules install -D -m0644 gps/gpsd.default /etc/default/gpsd udevadm control --reload else warn "gpsd not installed (apt install gpsd gpsd-clients) — skipping GPS setup" fi # OwnTracks publisher: gpsd fix -> MQTT (broker + creds live only on the # system, 0600 — same pattern as pushover.json). install -D -m0755 gps/van-gps-owntracks /usr/local/sbin/van-gps-owntracks install -D -m0644 gps/van-gps-owntracks.service /etc/systemd/system/van-gps-owntracks.service if [ ! -f /etc/van-gps/config.json ]; then install -D -m0600 gps/config.json.example /etc/van-gps/config.json warn "seeded /etc/van-gps/config.json (EDIT IT: add MQTT username + password)" fi python3 -c 'import gps' 2>/dev/null \ || warn "python3-gps missing (apt install python3-gps) — van-gps-owntracks won't start" python3 -c 'import paho.mqtt' 2>/dev/null \ || warn "python3-paho-mqtt missing (apt install python3-paho-mqtt) — van-gps-owntracks won't start" if [ "$ENABLE_LI3" = 1 ]; then echo "== li3 (RV house battery BMS -> MQTT/HA) ==" # NOT the laptop's own AC/battery monitor — that's power/van-battery (different # hardware, different concern). This is the Lithionics Li3 12V LiFePO4 house # battery, over its BLE HM-10 UART module (no pairing) -> Home Assistant MQTT # discovery. Broker + creds live only on the system (0600, seeded from # li3/config.json.example — same pattern as gps/pushover). install -D -m0755 li3/van-li3-battery /usr/local/sbin/van-li3-battery install -D -m0644 li3/van-li3-battery.service /etc/systemd/system/van-li3-battery.service install -D -m0644 li3/van-li3-battery-restart-limit.conf /etc/systemd/system/van-li3-battery.service.d/restart-limit.conf if [ ! -f /etc/van-li3/config.json ]; then install -D -m0600 li3/config.json.example /etc/van-li3/config.json warn "seeded /etc/van-li3/config.json (EDIT IT: add MQTT username + password)" fi python3 -c 'import bleak' 2>/dev/null \ || warn "python3-bleak missing (apt install python3-bleak) — van-li3-battery won't start" fi echo "== thermal monitor ==" install -D -m0755 power/van-thermal /usr/local/sbin/van-thermal install -D -m0644 power/thermal-config.json /etc/van-thermal/config.json install -D -m0644 power/van-thermal.service /etc/systemd/system/van-thermal.service # Pushover secrets live only on the system (0600), never in the repo. Path kept # under /etc/van-battery/ for parity with wayback's van-thermal default. if [ ! -f /etc/van-battery/pushover.json ]; then install -D -m0600 power/pushover.json.example /etc/van-battery/pushover.json warn "seeded /etc/van-battery/pushover.json (EDIT IT: add Pushover token + user)" fi if [ "$ENABLE_NVME_WATCH" = 1 ]; then echo "== nvme watchdog ==" install -D -m0755 power/van-nvme-watch /usr/local/sbin/van-nvme-watch install_rendered power/nvme-watch-config.json /etc/van-nvme-watch/config.json install -D -m0644 power/van-nvme-watch.service /etc/systemd/system/van-nvme-watch.service fi echo "== home assistant ==" # Native HA (Podman Quadlet, replaced the ha_van VM). daemon-reload below # regenerates homeassistant.service; started (not restarted) at the end so a # deploy never bounces HA — after editing the .container, restart it manually. if [ "$ENABLE_HA" = 1 ]; then install -D -m0644 ha/homeassistant.container /etc/containers/systemd/homeassistant.container install -d -m0755 /srv/homeassistant # ESPHome dashboard (sibling container, same rationale as HA above — no # Supervisor/add-on store here). install -D -m0644 ha/esphome.container /etc/containers/systemd/esphome.container install -d -m0755 /srv/esphome fi # Frigate NVR (sibling container, same rationale as HA/ESPHome above). if [ "$ENABLE_FRIGATE" = 1 ]; then install -D -m0644 ha/frigate.container /etc/containers/systemd/frigate.container install -d -m0755 /srv/frigate/config /srv/frigate/storage fi echo "== hardware watchdog ==" install -D -m0644 power/10-vanlink-watchdog.conf /etc/systemd/system.conf.d/10-vanlink-watchdog.conf if [ "$ENABLE_NM_WAN" = 1 ]; then echo "== networkmanager WANs ==" # In place before the package starts NM: no auto-created "Wired connection" profiles. install -D -m0644 ap/nm-no-auto-default.conf /etc/NetworkManager/conf.d/00-van-no-auto-default.conf # Also in place before the package starts NM: NM keeps off the uplink port until --apply-network hands it # over (WAN netplan missing), and the conf goes once the netplan matches; see nm_pending_action. NM picks # up either change at the `nmcli general reload conf` below. nm_np_state=$(nm_wan_state_netplan) if [ "$nm_np_state" = missing ]; then install_rendered ap/nm-uplink-pending.conf "$NM_PENDING_CONF" elif [ "$nm_np_state" = matches ]; then rm -f "$NM_PENDING_CONF" fi if ! nm_pkg_installed; then # Installed only now: the package starts NM, which manages Wi-Fi/gsm devices by # default, so van-ap-unmanaged.conf (AP radios, LAN ports) must already be in place. apt-get update -qq || true DEBIAN_FRONTEND=noninteractive apt-get install -y network-manager \ || { echo "deploy: apt-get install network-manager failed" >&2; exit 1; } fi systemctl enable --now NetworkManager ModemManager >/dev/null 2>&1 \ || warn "failed to enable/start NetworkManager/ModemManager (check: systemctl status NetworkManager ModemManager)" nmcli general reload conf 2>/dev/null || true for nm_if in "$WIFI_5G_IFACE" "$WIFI_2G_IFACE" "$LAN_IFACE" ${ONBOARD_LAN_IFACE:-}; do [ -e "/sys/class/net/$nm_if" ] || continue nm_state=$(nmcli -t -f DEVICE,STATE device 2>/dev/null | awk -F: -v d="$nm_if" '$1 == d { print $2 }') || true [ "$nm_state" = unmanaged ] \ || warn "NetworkManager manages $nm_if (state: ${nm_state:-unknown}) — AP/LAN ports must be unmanaged (see ap/van-ap-unmanaged.conf)" done if [ -n "${CELLULAR_APN:-}" ]; then nm_keyfile="/etc/NetworkManager/system-connections/${CELLULAR_CONN_NAME}.nmconnection" if cellular_profile_exists; then echo " cellular connection ${CELLULAR_CONN_NAME} exists — left unchanged" elif [ -z "${CELLULAR_UUID:-}" ]; then warn "CELLULAR_UUID is empty — not creating the cellular keyfile $nm_keyfile" else install_rendered ap/cellular.nmconnection "$nm_keyfile" 0600 nmcli connection reload 2>/dev/null || true echo " created cellular connection ${CELLULAR_CONN_NAME} (apn ${CELLULAR_APN}) — no modem needed" fi fi if [ "$nm_np_state" != matches ]; then warn "WAN netplan is $nm_np_state (not provisioned/up to date): run ./deploy.sh --apply-network (backs up /etc/netplan and rolls back automatically if the uplink does not come back)" fi fi echo "== apply ==" sysctl --system >/dev/null systemctl daemon-reload # Re-exec PID1 so the system.conf.d watchdog drop-in takes effect (daemon-reload # alone does NOT re-arm RuntimeWatchdogSec). Safe online. systemctl daemon-reexec # Safe online: picks up 99-van-router-dns.conf immediately. NM's # ignore-auto-dns only takes effect on a connection's next activation though — # an already-up WAN keeps its currently-applied DNS until it reconnects (or # reboot), deliberately not forced here (reconnecting a WAN flaps it). systemctl restart systemd-resolved systemctl restart mosquitto # networkd here owns only the AP radios + bridge + wired LAN port (no real uplink); # its wait-online would just stall network-online.target. NM-wait-online covers WANs. systemctl mask systemd-networkd-wait-online.service >/dev/null 2>&1 || true systemctl unmask hostapd >/dev/null 2>&1 || true # Disable system dnsmasq.service to avoid port 53 conflicts — van-ap-dnsmasq owns AP DHCP systemctl disable dnsmasq.service >/dev/null 2>&1 || true set_units systemctl enable "${ENABLE_UNITS[@]}" >/dev/null 2>&1 || true # bluetooth: host BlueZ serves the onboard hci0 to the HA container over D-Bus systemctl enable --now bluetooth >/dev/null 2>&1 || true if [ "$ENABLE_HA" = 1 ]; then systemctl start homeassistant || warn "homeassistant failed to start (podman/quadlet — check journalctl -u homeassistant)" systemctl start esphome || warn "esphome failed to start (podman/quadlet — check journalctl -u esphome)" fi if [ "$ENABLE_FRIGATE" = 1 ]; then systemctl start frigate || warn "frigate failed to start (podman/quadlet — check journalctl -u frigate)" fi systemctl restart van-thermal if [ "$ENABLE_NVME_WATCH" = 1 ]; then systemctl restart van-nvme-watch fi systemctl restart van-sms-watch systemctl restart van-gps-owntracks if [ "$ENABLE_LI3" = 1 ]; then systemctl restart van-li3-battery fi systemctl restart van-router-mqtt # Pick up unmanaged-devices changes so NM releases/keeps the right interfaces. nmcli general reload 2>/dev/null || systemctl reload NetworkManager 2>/dev/null || true # restart in dependency order: bridge + members first, then hostapd enslaves the # radios, then dnsmasq binds br0, then NAT/failover systemctl restart systemd-networkd for u in van-ap-dnsmasq nftables van-failover van-wlan-watchdog; do systemctl restart "$u" || warn "$u failed to restart (journalctl -xeu $u)" done # An AP radio may be absent; the start then fails but Restart=always keeps # retrying and claims it the moment it enumerates (post_check reports absent radios). systemctl restart hostapd hostapd-2g || true # AP watchdogs last, after hostapd is back up (they only ever restart a wedged hostapd) systemctl restart van-ap-watchdog van-ap-watchdog-2g networkctl reload 2>/dev/null || true post_check write_warnings if [ ${#WARNINGS[@]} -gt 0 ]; then echo echo "== warnings ==" for w in "${WARNINGS[@]}"; do echo " -> $w"; done fi echo echo "Deployed host $(hostname -s) with $(readlink -f "$VANLINK_CONF")." echo "Verify further:" echo " cat /run/van-failover/state.json" echo " cat /run/van-thermal/state.json" echo " journalctl -u van-router-mqtt -n 20" echo " journalctl -u van-sms-watch -n 20" if [ "$ENABLE_NVME_WATCH" = 1 ]; then echo " cat /run/van-nvme-watch/state.json" fi if [ "$ENABLE_LI3" = 1 ]; then echo " journalctl -u van-li3-battery -n 20" fi echo " ./deploy.sh --post-check # re-run the checks above"