vanlink/deploy.sh

#!/usr/bin/env bash
# Deploy vanlink configs/scripts to their system locations. One checkout is
# shared by every host; host-specific values come from /etc/vanlink.conf
# (symlink to hosts/<hostname>.conf — see hosts/vanlink.conf.example).
# Usage:  cd ~/vanlink && sudo ./deploy.sh
#         ./deploy.sh --check [conf]       # validate conf + render all templates, no install
#         ./deploy.sh --diff [conf]        # what would change in /etc
#         ./deploy.sh --post-check [conf]  # read-only live checks (units, br0, LAN, NAT, dnsmasq, radios, uplink); exit 1 on any warning
#                                    # POSTCHECK_WAIT=<s>: seconds retried probes may wait in total (default 15; 0 = single attempt)
#         ./deploy.sh --classify-unit <ActiveState> <Result> <ExecMainStatus>  # prints ok|finished|wait|warn (used by post_check; for tests)
#         ./deploy.sh --classify-radio     # reads `iw dev <if> info` text on stdin, prints beaconing|silent (no ssid line = silent; used by post_check; for tests)
#         ./deploy.sh --render-wan-netplan # rendered /etc/netplan/50-van-wan.yaml (ethernet part + wifis stanza when WIFI_WAN_IFACE is set)
#         ./deploy.sh --wan-plan [conf]    # what the NetworkManager WAN provisioning would do (read-only, no root)
#         ./deploy.sh render <file>        # preview a templated file on stdout
#         ./deploy.sh --redact             # filter stdin, masking secret values (used by --diff; for tests)
# Idempotent. Netplan (wlan0 + Starlink NIC = NM-managed WANs; eth0 is a LAN
# port on br0) is NOT deployed here — reference copy in ap/50-van-wan.yaml,
# applied once manually (apply flaps uplinks). We do check it for drift below.
set -euo pipefail
cd "$(dirname "$(readlink -f "$0")")"

# Host-specific values (interface names, addressing, feature flags) live in
# /etc/vanlink.conf — a symlink to hosts/<hostname>.conf in this repo. Config
# files below carry @TOKEN@ placeholders substituted from those variables via
# render()/install_rendered() — edit hosts/<hostname>.conf, not the configs.
VANLINK_CONF=${VANLINK_CONF:-/etc/vanlink.conf}

# Must be defined (non-empty) in the host file; deploy aborts naming any missing.
REQUIRED_VARS=(
    WIFI_5G_IFACE WIFI_2G_IFACE LAN_IFACE STARLINK_IFACE
    GATEWAY_IP GATEWAY_PREFIX GATEWAY_NETWORK DHCP_RANGE_START DHCP_RANGE_END
    DNS_RESOLVERS MODEM_USB_VENDOR AP_SSID_5G AP_SSID_2G AP_5G_RADIO
    ENABLE_COCKPIT ENABLE_LI3 ENABLE_NVME_WATCH ENABLE_HA ENABLE_FRIGATE
    ENABLE_NM_WAN
)
# May be empty or absent; NVME_* are required only when ENABLE_NVME_WATCH=1.
OPTIONAL_VARS=(
    WIFI_WAN_IFACE CELLULAR_CONN_NAME CELLULAR_APN
    ONBOARD_LAN_IFACE NVME_DEVICE NVME_FS_DEVICE
    USB_BACKUP_ROOT_PART USB_BACKUP_MOUNT SD_BACKUP_UUID
)
# Computed in load_conf; usable as @TOKEN@ like the rest.
DERIVED_VARS=(ONBOARD_LAN_UNMANAGED CELLULAR_UUID)
TOKEN_VARS=("${REQUIRED_VARS[@]}" "${OPTIONAL_VARS[@]}" "${DERIVED_VARS[@]}")
TOKEN_RE='@[A-Z][A-Z0-9_]*@'

die() { echo "deploy: $*" >&2; exit 1; }

load_conf() {  # load_conf <file> — source and validate a host config
    local f=$1 v flag missing=()
    [ -e "$f" ] || die "$f not found (missing or dangling symlink). Create it for this host with:
  sudo ln -sfn \"$PWD/hosts/$(hostname -s).conf\" /etc/vanlink.conf"
    for v in "${TOKEN_VARS[@]}"; do unset "$v"; done
    # shellcheck disable=SC1090
    . "$f"
    for v in "${REQUIRED_VARS[@]}"; do
        [ -n "${!v:-}" ] || missing+=("$v")
    done
    [ ${#missing[@]} -eq 0 ] || die "$f is missing required variable(s): ${missing[*]}"
    for flag in ENABLE_COCKPIT ENABLE_LI3 ENABLE_NVME_WATCH ENABLE_HA ENABLE_FRIGATE ENABLE_NM_WAN; do
        case ${!flag} in
            0|1) ;;
            *) die "$f: $flag must be 0 or 1 (got '${!flag}')" ;;
        esac
    done
    if [ "$ENABLE_NVME_WATCH" = 1 ]; then
        { [ -n "${NVME_DEVICE:-}" ] && [ -n "${NVME_FS_DEVICE:-}" ]; } \
            || die "$f: ENABLE_NVME_WATCH=1 needs NVME_DEVICE and NVME_FS_DEVICE"
    fi
    ONBOARD_LAN_UNMANAGED=${ONBOARD_LAN_IFACE:+;interface-name:$ONBOARD_LAN_IFACE}
    # Stable per connection name; pure bash + sha1sum (coreutils), no uuid-runtime dependency.
    local h
    h=$(printf 'vanlink-%s' "${CELLULAR_CONN_NAME:-}" | sha1sum | cut -c1-32)
    CELLULAR_UUID="${h:0:8}-${h:8:4}-5${h:13:3}-8${h:17:3}-${h:20:12}"
}

render() {  # render <file> -> stdout, with @TOKEN@ placeholders substituted
    local args=() v
    for v in "${TOKEN_VARS[@]}"; do
        args+=(-e "s|@${v}@|${!v-}|g")
    done
    sed "${args[@]}" "$1"
}

# The netplan is two templates: ap/50-van-wan.yaml (ethernet/Starlink) and
# ap/50-van-wan-wifi.yaml (the wifis: stanza, appended verbatim only for hosts
# with a Wi-Fi uplink, so a host without one gets no empty/invalid wifis: block).
render_wan_netplan() {  # ap/50-van-wan.yaml, plus the wifis stanza when the host has a Wi-Fi uplink
    render ap/50-van-wan.yaml
    if [ -n "${WIFI_WAN_IFACE:-}" ]; then
        render ap/50-van-wan-wifi.yaml
    fi
}

nm_wan_state_netplan() {  # missing | differs | matches (the installed WAN netplan vs the rendered one)
    if [ ! -f /etc/netplan/50-van-wan.yaml ]; then echo missing
    elif diff -q <(render_wan_netplan) /etc/netplan/50-van-wan.yaml >/dev/null 2>&1; then echo matches
    else echo differs; fi
}

check_rendered() {  # check_rendered <src> <rendered-file> — fail on a leftover @TOKEN@
    local left
    left=$(grep -oE "$TOKEN_RE" "$2" | sort -u | tr '\n' ' ') || true
    [ -z "$left" ] && return 0
    echo "deploy: $1: unrendered token(s) after render: $left" >&2
    return 1
}

install_rendered() {  # install_rendered <src> <dst> [mode]
    local tmp
    tmp=$(mktemp)
    render "$1" > "$tmp"
    check_rendered "$1" "$tmp" || { rm -f "$tmp"; exit 1; }
    install -D -m"${3:-0644}" "$tmp" "$2"
    rm -f "$tmp"
}

redact_secrets() {  # stdin -> stdout; mask values of secret-looking keys, keep key + diff markers
    sed -E \
        -e 's/((pass(word|phrase)|secret|token)[A-Za-z_]*[[:space:]]*=).*/\1<redacted>/I' \
        -e 's/("[A-Za-z_]*(pass(word|phrase)|secret|token|user)[A-Za-z_]*"[[:space:]]*:[[:space:]]*)"[^"]*"/\1"<redacted>"/I'
}

template_files() {  # every repo file that carries an @TOKEN@ placeholder
    grep -rlE "$TOKEN_RE" --exclude-dir=.git --exclude-dir=__pycache__ --exclude-dir=.superpowers \
        --exclude-dir=docs --exclude-dir=tests --exclude-dir=hosts \
        --exclude='*.bak' --exclude='*.md' --exclude=deploy.sh . | sed 's|^\./||' | sort
}

# Collected during a deploy and persisted to WARNINGS_FILE so the Cockpit vanrouter page
# can surface deploy-time issues (missing deps, unedited example configs, drift)
# without someone having to remember to scroll back through deploy output.
WARNINGS=()
WARNINGS_FILE=/var/lib/vanlink/deploy-warnings.json
warn() {  # warn <message...> — prints "   -> <message>" (as before) and records it
    echo "   -> $*"
    WARNINGS+=("$*")
}

# The units a deploy enables (and post_check expects active). Needs the host
# config loaded. The `if` guards sit in column 0 on purpose: tests/check-templates.sh
# check 6 requires flag-gated unit names to appear only inside such guards.
set_units() {
ENABLE_UNITS=(regdomain.service van-br0-setup.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-modem-watch van-sms-watch van-gps-owntracks van-router-mqtt)
EXTRA_CHECK_UNITS=()   # started by deploy but not in the `enable` list
if [ "$ENABLE_NVME_WATCH" = 1 ]; then
    ENABLE_UNITS+=(van-nvme-watch)
fi
if [ "$ENABLE_LI3" = 1 ]; then
    ENABLE_UNITS+=(van-li3-battery)
fi
if [ "$ENABLE_NM_WAN" = 1 ]; then
    ENABLE_UNITS+=(NetworkManager ModemManager)
fi
if [ "$ENABLE_HA" = 1 ]; then
    EXTRA_CHECK_UNITS+=(homeassistant esphome)
fi
if [ "$ENABLE_FRIGATE" = 1 ]; then
    EXTRA_CHECK_UNITS+=(frigate)
fi
}

classify_unit() {  # classify_unit <ActiveState> <Result> <ExecMainStatus> -> ok | finished | wait | warn
    case "$1" in
        active) echo ok ;;
        activating|reloading) echo wait ;;
        inactive) if [ "${2:-}" = success ] && [ "${3:-}" = 0 ]; then echo finished; else echo warn; fi ;;
        *) echo warn ;;
    esac
}

classify_radio() {  # stdin: `iw dev <if> info` text -> beaconing (has an ssid line) | silent
    local line
    while IFS= read -r line || [ -n "$line" ]; do
        line=${line#"${line%%[![:space:]]*}"}
        case "$line" in "ssid "*) echo beaconing; return 0 ;; esac
    done
    echo silent
}

post_check() {  # read-only live checks; every probe is guarded, warns instead of aborting
    set_units
    echo "== post-deploy check =="
    local u pending=() still=() n f out
    local -A radio=([hostapd]=$WIFI_5G_IFACE [hostapd-2g]=$WIFI_2G_IFACE)
    # One shared deadline: probes are retried ~1/s until it passes (0 = single attempt),
    # since a restart just before may not have settled yet.
    local deadline=$((SECONDS + ${POSTCHECK_WAIT:-15}))
    retry() { until "$@"; do [ "$SECONDS" -lt "$deadline" ] || return 1; sleep 1; done; }
    # a. units (AP units are judged under (f))
    for u in "${ENABLE_UNITS[@]}" "${EXTRA_CHECK_UNITS[@]}"; do
        if [ -n "${radio[$u]:-}" ] && ! ip link show dev "${radio[$u]}" >/dev/null 2>&1; then continue; fi
        pending+=("$u")
    done
    still=("${pending[@]}")
    local -A verdict=() detail=()
    local k v as res st ls
    while :; do
        local next=()
        for u in "${still[@]}"; do
            as= res= st= ls=
            while IFS='=' read -r k v; do
                case "$k" in LoadState) ls=$v ;; ActiveState) as=$v ;; Result) res=$v ;; ExecMainStatus) st=$v ;; esac
            done < <(systemctl show -p LoadState,ActiveState,Result,ExecMainStatus "$u" 2>/dev/null || true)
            verdict[$u]=$(classify_unit "$as" "$res" "$st")
            [ "$ls" = loaded ] || { verdict[$u]=warn; as="${as:-?}(LoadState=${ls:-?})"; }
            detail[$u]="ActiveState=${as:-?} Result=${res:-?}"
            case "${verdict[$u]}" in ok|finished) ;; *) next+=("$u") ;; esac
        done
        still=("${next[@]}")
        [ ${#still[@]} -eq 0 ] && break
        [ "$SECONDS" -lt "$deadline" ] || break
        sleep 1
    done
    for u in "${pending[@]}"; do
        case "${verdict[$u]:-warn}" in
            ok) echo "   ok    unit $u active" ;;
            finished) echo "   ok    unit $u finished (exit 0)" ;;
            *) warn "post-deploy: unit $u is not active (${detail[$u]:-state unknown})" ;;
        esac
    done
    # b. bridge address
    has_br0_addr() { local o; o=$(ip -4 -br addr show dev br0 2>/dev/null) || true; [[ $o == *"$GATEWAY_IP/$GATEWAY_PREFIX"* ]]; }
    if retry has_br0_addr; then echo "   ok    br0 has $GATEWAY_IP/$GATEWAY_PREFIX"
    else warn "post-deploy: br0 does not have $GATEWAY_IP/$GATEWAY_PREFIX"; fi
    # c. LAN ports picked up our .network files (not netplan's catch-all)
    uses_file() { local o; o=$(networkctl status "$1" 2>/dev/null) || true; [[ $o == *"/etc/systemd/network/$2"* ]]; }
    for f in "$LAN_IFACE:05-van-lan.network" "${ONBOARD_LAN_IFACE:-}:06-van-lan-eth0.network"; do
        n=${f%%:*}; [ -n "$n" ] || continue
        if ! ip link show dev "$n" >/dev/null 2>&1; then warn "post-deploy: LAN port $n is absent"
        elif retry uses_file "$n" "${f#*:}"; then echo "   ok    $n uses ${f#*:}"
        else warn "post-deploy: $n is not using /etc/systemd/network/${f#*:} (another .network file shadows it?)"; fi
    done
    # d. NAT
    has_nat() { local o; o=$(nft list ruleset 2>/dev/null) || true
        [[ $o == *"ip saddr $GATEWAY_NETWORK"*masquerade* ]]; }
    if retry has_nat; then echo "   ok    NAT masquerade for $GATEWAY_NETWORK"
    elif [ "$(id -u)" != 0 ]; then warn "post-deploy: no masquerade rule for ip saddr $GATEWAY_NETWORK found (nft list ruleset needs root to read)"
    else warn "post-deploy: NAT rule for $GATEWAY_NETWORK not found"; fi
    # e. DHCP
    has_dhcp() { local o; o=$(ss -ulnH 2>/dev/null | awk '{print $4}') || true; [[ $'\n'$o == *:67$'\n'* || $o == *:67 ]]; }
    if retry has_dhcp; then echo "   ok    dnsmasq bound to UDP 67"
    else warn "post-deploy: nothing bound to UDP port 67 (dnsmasq DHCP)"; fi
    # f. AP radios (present but no ssid = hostapd is not beaconing)
    radio_beaconing() { out=$(iw dev "$1" info 2>/dev/null) || return 1
        [ "$(classify_radio <<<"$out")" = beaconing ]; }
    for n in "$WIFI_5G_IFACE" "$WIFI_2G_IFACE"; do
        u=hostapd; [ "$n" = "$WIFI_5G_IFACE" ] || u=hostapd-2g
        out=
        if retry radio_beaconing "$n"; then
            echo "   ok    radio $n: $(echo "$out" | grep -E 'ssid|channel' | sed 's/^[[:space:]]*//' | tr '\n' ';')"
        elif [ -z "$out" ]; then warn "post-deploy: radio $n not present (its hostapd keeps retrying until it appears)"
        else warn "post-deploy: radio $n is present but not beaconing (no ssid) — check: journalctl -u $u -n 30"; fi
    done
    # g. uplink
    out=$(ip route show default 2>/dev/null) || true
    if [ -z "$out" ]; then warn "post-deploy: no default route"
    elif ping -c1 -W2 1.1.1.1 >/dev/null 2>&1; then echo "   ok    uplink: default route, 1.1.1.1 reachable"
    else warn "post-deploy: default route present but 1.1.1.1 does not answer ping"; fi
}

case "${1:-}" in
    --templates)
        template_files
        exit 0 ;;
    --check)    # ./deploy.sh --check [conf] — validate conf, render every template, install nothing
        load_conf "${2:-$VANLINK_CONF}"
        rc=0
        tmp=$(mktemp)
        for f in $(template_files); do
            render "$f" > "$tmp"
            check_rendered "$f" "$tmp" || rc=1
        done
        rm -f "$tmp"
        [ "$rc" -eq 0 ] && echo "check OK: $(template_files | wc -l) templates render cleanly"
        exit "$rc" ;;
    --diff)     # ./deploy.sh --diff [conf] — what would change in /etc, without installing
        load_conf "${2:-$VANLINK_CONF}"
        grep -E '^[[:space:]]*install_rendered +[^" ]+ +/' deploy.sh | while read -r _ src dst _; do
            if [ ! -e "$dst" ]; then
                echo "NEW      $dst"
            elif ! diff -q <(render "$src") "$dst" >/dev/null; then
                echo "CHANGED  $dst"
                diff -u "$dst" <(render "$src") | redact_secrets | sed 's/^/    /' || true
            fi
        done
        # Also lists plain `install` files and `rm -f` targets — these sections may be
        # flag-gated in the real run, but --diff reports them regardless of the flag.
        grep -E '^[[:space:]]*install -D -m[0-9]+ +[^" ]+ +/' deploy.sh | while read -r _ _ mode src dst _; do
            # seed-if-absent secret files (from *.example, or 0600): never diff or print
            case "$src" in *.example) continue ;; esac
            [ "$mode" = -m0600 ] && continue
            if [ ! -e "$dst" ]; then
                echo "NEW      $dst"
            elif ! cmp -s "$src" "$dst"; then
                echo "CHANGED  $dst"
                diff -u "$dst" "$src" | redact_secrets | sed 's/^/    /' || true
            fi
        done
        grep -E '^[[:space:]]*rm -f +/' deploy.sh | while read -r _ _ path _; do
            [ -e "$path" ] && echo "REMOVE   $path"
        done || true
        exit 0 ;;
    --redact)   # ./deploy.sh --redact — stdin through redact_secrets (no config, no root)
        redact_secrets
        exit 0 ;;
    --render-wan-netplan) load_conf "$VANLINK_CONF"; render_wan_netplan; exit 0 ;;
    --wan-plan)  # ./deploy.sh --wan-plan [conf] — what the NM WAN provisioning would do; changes nothing
        load_conf "${2:-$VANLINK_CONF}"
        echo "wan-plan: ENABLE_NM_WAN=$ENABLE_NM_WAN"
        if [ "$ENABLE_NM_WAN" = 1 ]; then
            if [ -n "${WIFI_WAN_IFACE:-}" ]; then echo "  wifi WAN: $WIFI_WAN_IFACE"; else echo "  wifi WAN: none (wifis stanza omitted)"; fi
            if dpkg -s network-manager >/dev/null 2>&1; then echo "  packages: network-manager installed"; else echo "  packages: network-manager would install"; fi
            echo "  netplan: /etc/netplan/50-van-wan.yaml $(nm_wan_state_netplan)"
            if [ -f /etc/netplan/10-dhcp-all-interfaces.yaml ]; then echo "  armbian: /etc/netplan/10-dhcp-all-interfaces.yaml present -> would move aside"; else echo "  armbian: /etc/netplan/10-dhcp-all-interfaces.yaml absent"; fi
            if [ -z "${CELLULAR_APN:-}" ]; then
                echo "  cellular profile: skipped (no CELLULAR_APN)"
            elif [ -e "/etc/NetworkManager/system-connections/${CELLULAR_CONN_NAME}.nmconnection" ]; then
                echo "  cellular profile: ${CELLULAR_CONN_NAME} apn=${CELLULAR_APN} exists"
            else
                echo "  cellular profile: ${CELLULAR_CONN_NAME} apn=${CELLULAR_APN} would create"
            fi
        fi
        exit 0 ;;
    --classify-unit)   # ./deploy.sh --classify-unit <ActiveState> <Result> <ExecMainStatus>
        classify_unit "${2:-}" "${3:-}" "${4:-}"
        exit 0 ;;
    --classify-radio)   # ./deploy.sh --classify-radio < `iw dev <if> info` text
        classify_radio
        exit 0 ;;
    --post-check)   # ./deploy.sh --post-check [conf] — live checks, no root, no install; exit 1 on any warning
        load_conf "${2:-$VANLINK_CONF}"
        post_check
        if [ ${#WARNINGS[@]} -gt 0 ]; then
            echo
            echo "== warnings =="
            for w in "${WARNINGS[@]}"; do echo "   -> $w"; done
            exit 1
        fi
        exit 0 ;;
    render)
        [ -n "${2:-}" ] || { echo "Usage: $0 render <file>"; exit 1; }
        load_conf "$VANLINK_CONF"
        render "$2"
        exit 0 ;;
esac

[ "$(id -u)" = 0 ] || { echo "Run with sudo (writes to /etc, /usr)."; exit 1; }
load_conf "$VANLINK_CONF"
echo "== host config: $(readlink -f "$VANLINK_CONF") =="

# Check for essential package dependencies
echo "== dependency check =="
REQUIRED_PACKAGES=(
    hostapd
    iw
    nftables
    dnsmasq
    systemd-resolved
    python3-gps
    python3-bleak
    modemmanager
    usb-modeswitch
    gpsd
    gpsd-clients
    rsync
    mosquitto
    python3-paho-mqtt
    mobile-broadband-provider-info
    zerotier-one
)
if [ "$ENABLE_HA" = 1 ]; then
    REQUIRED_PACKAGES+=(podman)
fi
OPTIONAL_PACKAGES=(
)
missing_pkgs=()
for pkg in "${REQUIRED_PACKAGES[@]}"; do
    dpkg -s "$pkg" >/dev/null 2>&1 || missing_pkgs+=("$pkg")
done
if [ ${#missing_pkgs[@]} -gt 0 ]; then
    echo "   installing missing required packages: ${missing_pkgs[*]}"
    apt-get update -qq || echo "   -> apt-get update failed (continuing with the cached lists)"
    DEBIAN_FRONTEND=noninteractive apt-get install -y "${missing_pkgs[@]}" \
        || { echo "deploy: apt-get install failed for: ${missing_pkgs[*]}" >&2; exit 1; }
fi

# Check optional packages
for pkg in "${OPTIONAL_PACKAGES[@]}"; do
    if ! dpkg -s "$pkg" >/dev/null 2>&1; then
        warn "optional package not installed: $pkg (apt install $pkg)"
    fi
done
write_warnings() {
    install -d -m0755 "$(dirname "$WARNINGS_FILE")"
    {
        printf '{\n  "deployed": "%s",\n  "warnings": [' "$(date -Iseconds)"
        local first=1 w e
        for w in "${WARNINGS[@]}"; do
            [ "$first" = 1 ] || printf ','
            first=0
            e=${w//\\/\\\\}; e=${e//\"/\\\"}
            printf '\n    "%s"' "$e"
        done
        printf '\n  ]\n}\n'
    } > "$WARNINGS_FILE"
}

# A host file with uncommitted edits means this deploy uses unreviewed values.
conf_real=$(readlink -f "$VANLINK_CONF")
if git -c safe.directory='*' -C "$(dirname "$conf_real")" status --porcelain -- "$conf_real" 2>/dev/null | grep -q .; then
    warn "$conf_real has uncommitted changes — commit it so the host config stays reviewable"
fi

if [ "$ENABLE_NM_WAN" != 1 ]; then
    echo "== netplan drift check =="
    # Netplan is never installed by this script (applying it flaps the uplinks —
    # see note below), so it's easy to edit ap/50-van-wan.yaml and forget the
    # manual `netplan apply` step. Warn loudly rather than silently drifting.
    if [ ! -f /etc/netplan/50-van-wan.yaml ]; then
        warn "netplan: /etc/netplan/50-van-wan.yaml is missing — repo config was never deployed. Run: sudo ./deploy.sh --render-wan-netplan | sudo tee /etc/netplan/50-van-wan.yaml && sudo netplan generate && sudo netplan apply"
    elif ! diff -q <(render_wan_netplan) /etc/netplan/50-van-wan.yaml >/dev/null 2>&1; then
        warn "netplan: /etc/netplan/50-van-wan.yaml differs from ap/50-van-wan.yaml (rendered). Deploy manually: sudo ./deploy.sh --render-wan-netplan | sudo tee /etc/netplan/50-van-wan.yaml && sudo netplan generate && sudo netplan apply"
        diff -u /etc/netplan/50-van-wan.yaml <(render_wan_netplan) || true
    fi
fi

echo "== access point =="
install_rendered ap/hostapd.conf            /etc/hostapd/hostapd.conf
install -D -m0644 ap/hostapd-restart.conf   /etc/systemd/system/hostapd.service.d/restart.conf
install -D -m0644 ap/hostapd-regdomain.conf /etc/systemd/system/hostapd.service.d/regdomain.conf
install -D -m0644 ap/default-hostapd        /etc/default/hostapd
install_rendered ap/hostapd-2g.conf         /etc/hostapd/hostapd-2g.conf
install_rendered ap/hostapd-2g.service      /etc/systemd/system/hostapd-2g.service
install -D -m0644 ap/hostapd-regdomain.conf /etc/systemd/system/hostapd-2g.service.d/regdomain.conf
install_rendered ap/11-van-ap-2g.network    /etc/systemd/network/11-van-ap-2g.network
install -D -m0644 ap/van-ap-watchdog-2g.service /etc/systemd/system/van-ap-watchdog-2g.service
install -D -m0755 ap/van-ap-watchdog        /usr/local/sbin/van-ap-watchdog
install -D -m0644 ap/van-ap-watchdog.service /etc/systemd/system/van-ap-watchdog.service
install_rendered ap/van-ap-dnsmasq.conf     /etc/van-ap/dnsmasq.conf
install -D -m0644 ap/van-ap-dnsmasq.service /etc/systemd/system/van-ap-dnsmasq.service
install_rendered ap/10-van-ap.network       /etc/systemd/network/10-van-ap.network
install -D -m0644 ap/20-van-br0.netdev      /etc/systemd/network/20-van-br0.netdev
install_rendered ap/21-van-br0.network      /etc/systemd/network/21-van-br0.network
# Named 05- (not 22-) so it sorts before Armbian's netplan-generated
# /run/systemd/network/10-netplan-all-eth-interfaces.network, which matches
# every e* port with DHCP and otherwise wins over this file (first match by
# filename), leaving the LAN port un-bridged.
install_rendered ap/22-van-lan.network      /etc/systemd/network/05-van-lan.network
rm -f /etc/systemd/network/22-van-lan.network
if [ -n "${ONBOARD_LAN_IFACE:-}" ]; then
    # 06- sorts before netplan's 10-* catch-all, like 05-van-lan above
    install_rendered ap/23-van-lan-eth0.network /etc/systemd/network/06-van-lan-eth0.network
    rm -f /etc/systemd/network/23-van-lan-eth0.network
else
    rm -f /etc/systemd/network/23-van-lan-eth0.network /etc/systemd/network/06-van-lan-eth0.network
fi
install_rendered ap/van-ap-unmanaged.conf   /etc/NetworkManager/conf.d/van-ap-unmanaged.conf
# Mask dracut's initramfs-generated catch-all (/run/systemd/network/
# zzzz-dracut-default.network, regenerated every boot): it matches every
# unconfigured link, so networkd co-managed the NM-owned WANs — a second
# DHCPv4 client on wlan0 and IPv6 that worked only by accident. NM owns WAN
# IPv6 now (see ap/50-van-wan.yaml). /etc overrides /run; /dev/null masks.
ln -sfn /dev/null /etc/systemd/network/zzzz-dracut-default.network
install_rendered ap/nftables.conf           /etc/nftables.conf
install -D -m0644 ap/regdomain.service      /etc/systemd/system/regdomain.service
install_rendered ap/van-br0-setup.service   /etc/systemd/system/van-br0-setup.service
install -D -m0644 ap/80-regdomain.rules     /etc/udev/rules.d/80-regdomain.rules
install -D -m0644 ap/rtl8188gu-modeswitch.conf /etc/usb-modeswitch.d/0bda:1a2b
install -D -m0644 ap/rtw89.conf             /etc/modprobe.d/rtw89.conf
install -D -m0644 ap/99-van-router.conf     /etc/sysctl.d/99-van-router.conf

echo "== dns =="
# Disable systemd-resolved DNS listener — dnsmasq handles DNS+DHCP on bridge.
# Even localhost-only listening blocks dnsmasq from binding to port 53.
install -D -m0644 ap/systemd-resolved-disable.conf /etc/systemd/system/systemd-resolved.service.d/disable.conf
# Fixed upstream resolvers (/etc/vanlink.conf's DNS_RESOLVERS), never a WAN's own
# DHCP/RA-provided DNS — see the files themselves for the full rationale.
# (NM's ipv4/ipv6.ignore-auto-dns can't be set as a config-file connection
# default — NM rejects it there — so 60-van-wan-dns enforces this directly
# against resolved instead, on every WAN connect/lease event.)
# .local (mDNS) is handled separately, scoped to br0 (see 21-van-br0.network
# above). ZeroTier-managed DNS (zt.wrede.pvt) is a separate, additive path.
install_rendered ap/99-van-router-dns.conf  /etc/systemd/resolved.conf.d/99-van-router-dns.conf
install -D -m0755 failover/60-van-wan-dns   /etc/NetworkManager/dispatcher.d/60-van-wan-dns
# ZeroTier-managed DNS for zt.wrede.pvt (see README's "ZeroTier managed DNS").
# allowDNS on the network + the search-domain drop-in are repo-tracked so a
# reimage doesn't need the manual `zerotier-cli set ... allowDNS=1` step
# remembered by hand; the manager binary itself is a hand-installed .deb
# (not in apt) — see https://github.com/zerotier/zerotier-systemd-manager/releases.
install -D -m0600 dns/zt-network.local.conf /var/lib/zerotier-one/networks.d/d3ecf5726d041b2a.local.conf
install -D -m0644 dns/zt-search.conf        /etc/systemd/network/99-ztuga7c2kh.network.d/search.conf
dpkg -s zerotier-systemd-manager >/dev/null 2>&1 \
    || warn "zerotier-systemd-manager not installed — zt.wrede.pvt won't resolve. Install the arm64 .deb from https://github.com/zerotier/zerotier-systemd-manager/releases"

echo "== failover =="
install -D -m0755 failover/van-failover         /usr/local/sbin/van-failover
install_rendered failover/config.json           /etc/van-failover/config.json
install -D -m0644 failover/van-failover.service /etc/systemd/system/van-failover.service
install -D -m0755 failover/50-disable-eee       /etc/NetworkManager/dispatcher.d/50-disable-eee
install -D -m0644 failover/99-van-arp.conf      /etc/sysctl.d/99-van-arp.conf
# Backstop for wlan0's post-boot NM no-secrets wedge (see the script's docstring).
install -D -m0755 failover/van-wlan-watchdog         /usr/local/sbin/van-wlan-watchdog
install_rendered failover/van-wlan-watchdog.service /etc/systemd/system/van-wlan-watchdog.service

echo "== cellular modem (GSM/LTE) =="
# NM's gsm.auto-config APN lookup needs this apt-only carrier database; without
# it even the right APN can't be auto-detected, and MVNOs (e.g. Koodo, which
# isn't listed under its own name — only under host network "Telus Mobility")
# often aren't in it anyway, so the modem's NM connection profile may still
# need an explicit gsm.apn set by hand regardless.
dpkg -s mobile-broadband-provider-info >/dev/null 2>&1 \
    || warn "mobile-broadband-provider-info missing (apt install mobile-broadband-provider-info) — GSM APN auto-config will fail"
# Backstop for the modem sometimes never enumerating at boot (see the
# script's docstring) — detection + Pushover alert only. No automated
# recovery: a hub power-cycle was proven not to fix this (only a genuine
# physical unplug/replug does), so the modem is now on the Pi's native USB
# port rather than through the hub. This just pages if it ever recurs.
install -D -m0755 failover/van-modem-watch       /usr/local/sbin/van-modem-watch
install_rendered failover/van-modem-watch.service /etc/systemd/system/van-modem-watch.service
# SMS archive + Pushover relay: polls ModemManager for inbound SMS, appends
# each to /var/log/van-sms.jsonl, pages via Pushover, then deletes it from
# the modem's own flash storage (small, fills up silently otherwise). MMS
# can't be fetched here (see script docstring) — notification-only paging.
install -D -m0755 modem/van-sms-watch         /usr/local/sbin/van-sms-watch
install -D -m0644 modem/van-sms-watch.service /etc/systemd/system/van-sms-watch.service
# CLI to send an SMS ad hoc (needs sudo — ModemManager's Messaging actions
# are PolicyKit-gated). Not a service, just a tool: sudo van-sms-send <number> <text>
install -D -m0755 modem/van-sms-send /usr/local/sbin/van-sms-send

if [ "$ENABLE_COCKPIT" = 1 ]; then
    echo "== cockpit plugin =="
    install -d /usr/share/cockpit/vanrouter
    for f in cockpit/vanrouter/*; do
        b=$(basename "$f")
        if [ "$b" = "vanrouter.js" ]; then
            install_rendered "$f" "/usr/share/cockpit/vanrouter/$b"
        else
            install -m0644 "$f" "/usr/share/cockpit/vanrouter/$b"
        fi
    done
    # Bridge fd headroom (Python bridge frees spawn pipes only at GC; 1024 is too tight)
    install -D -m0644 cockpit/cockpit-session-nofile.conf /etc/systemd/system/cockpit-session@.service.d/nofile.conf
fi

# The Starlink card queries the dish's gRPC API; grpcurl isn't packaged in apt.
command -v grpcurl >/dev/null 2>&1 \
    || warn "grpcurl missing (Starlink card will say so): install linux_arm64 binary from github.com/fullstorydev/grpcurl/releases"

echo "== router status (MQTT -> Home Assistant, replacing the Cockpit plugin above) =="
# Same status/actions as the cockpit plugin, published via MQTT discovery
# instead of a web UI of its own (Cockpit's login was unreliable and it was a
# resource hog on this Pi). No secrets in config.json (the local Mosquitto
# broker currently allows anonymous connections) so it's rendered + installed
# every deploy, same as failover/config.json.
install -D -m0755 router-status/van-router-mqtt        /usr/local/sbin/van-router-mqtt
install_rendered router-status/config.json              /etc/van-router-mqtt/config.json
install -D -m0644 router-status/van-router-mqtt.service /etc/systemd/system/van-router-mqtt.service
python3 -c 'import paho.mqtt' 2>/dev/null \
    || warn "python3-paho-mqtt missing (apt install python3-paho-mqtt) — van-router-mqtt won't start"
# MQTT broker must listen on all interfaces (not just localhost) for clients on br0
install -D -m0644 router-status/mosquitto-listener.conf /etc/mosquitto/conf.d/van-listener.conf
# Restart limit drop-ins: prevent infinite restart loops that trigger hardware watchdog
install -D -m0644 router-status/van-gps-owntracks-restart-limit.conf /etc/systemd/system/van-gps-owntracks.service.d/restart-limit.conf

echo "== gps (cellular modem GNSS -> gpsd) =="
if dpkg -s gpsd >/dev/null 2>&1; then
    install -D -m0644 gps/77-modem-gps.rules /etc/udev/rules.d/77-modem-gps.rules
    install -D -m0644 gps/gpsd.default       /etc/default/gpsd
    udevadm control --reload
else
    warn "gpsd not installed (apt install gpsd gpsd-clients) — skipping GPS setup"
fi
# OwnTracks publisher: gpsd fix -> MQTT (broker + creds live only on the
# system, 0600 — same pattern as pushover.json).
install -D -m0755 gps/van-gps-owntracks         /usr/local/sbin/van-gps-owntracks
install -D -m0644 gps/van-gps-owntracks.service /etc/systemd/system/van-gps-owntracks.service
if [ ! -f /etc/van-gps/config.json ]; then
    install -D -m0600 gps/config.json.example /etc/van-gps/config.json
    warn "seeded /etc/van-gps/config.json (EDIT IT: add MQTT username + password)"
fi
python3 -c 'import gps' 2>/dev/null \
    || warn "python3-gps missing (apt install python3-gps) — van-gps-owntracks won't start"
python3 -c 'import paho.mqtt' 2>/dev/null \
    || warn "python3-paho-mqtt missing (apt install python3-paho-mqtt) — van-gps-owntracks won't start"

if [ "$ENABLE_LI3" = 1 ]; then
    echo "== li3 (RV house battery BMS -> MQTT/HA) =="
    # NOT the laptop's own AC/battery monitor — that's power/van-battery (different
    # hardware, different concern). This is the Lithionics Li3 12V LiFePO4 house
    # battery, over its BLE HM-10 UART module (no pairing) -> Home Assistant MQTT
    # discovery. Broker + creds live only on the system (0600, seeded from
    # li3/config.json.example — same pattern as gps/pushover).
    install -D -m0755 li3/van-li3-battery         /usr/local/sbin/van-li3-battery
    install -D -m0644 li3/van-li3-battery.service /etc/systemd/system/van-li3-battery.service
    install -D -m0644 li3/van-li3-battery-restart-limit.conf /etc/systemd/system/van-li3-battery.service.d/restart-limit.conf
    if [ ! -f /etc/van-li3/config.json ]; then
        install -D -m0600 li3/config.json.example /etc/van-li3/config.json
        warn "seeded /etc/van-li3/config.json (EDIT IT: add MQTT username + password)"
    fi
    python3 -c 'import bleak' 2>/dev/null \
        || warn "python3-bleak missing (apt install python3-bleak) — van-li3-battery won't start"
fi

echo "== thermal monitor =="
install -D -m0755 power/van-thermal         /usr/local/sbin/van-thermal
install -D -m0644 power/thermal-config.json /etc/van-thermal/config.json
install -D -m0644 power/van-thermal.service /etc/systemd/system/van-thermal.service
# Pushover secrets live only on the system (0600), never in the repo. Path kept
# under /etc/van-battery/ for parity with wayback's van-thermal default.
if [ ! -f /etc/van-battery/pushover.json ]; then
    install -D -m0600 power/pushover.json.example /etc/van-battery/pushover.json
    warn "seeded /etc/van-battery/pushover.json (EDIT IT: add Pushover token + user)"
fi

if [ "$ENABLE_NVME_WATCH" = 1 ]; then
    echo "== nvme watchdog =="
    install -D -m0755 power/van-nvme-watch         /usr/local/sbin/van-nvme-watch
    install_rendered power/nvme-watch-config.json  /etc/van-nvme-watch/config.json
    install -D -m0644 power/van-nvme-watch.service /etc/systemd/system/van-nvme-watch.service
fi

echo "== home assistant =="
# Native HA (Podman Quadlet, replaced the ha_van VM). daemon-reload below
# regenerates homeassistant.service; started (not restarted) at the end so a
# deploy never bounces HA — after editing the .container, restart it manually.
if [ "$ENABLE_HA" = 1 ]; then
    install -D -m0644 ha/homeassistant.container /etc/containers/systemd/homeassistant.container
    install -d -m0755 /srv/homeassistant
    # ESPHome dashboard (sibling container, same rationale as HA above — no
    # Supervisor/add-on store here).
    install -D -m0644 ha/esphome.container /etc/containers/systemd/esphome.container
    install -d -m0755 /srv/esphome
fi
# Frigate NVR (sibling container, same rationale as HA/ESPHome above).
if [ "$ENABLE_FRIGATE" = 1 ]; then
    install -D -m0644 ha/frigate.container /etc/containers/systemd/frigate.container
    install -d -m0755 /srv/frigate/config /srv/frigate/storage
fi

echo "== hardware watchdog =="
install -D -m0644 power/10-vanlink-watchdog.conf /etc/systemd/system.conf.d/10-vanlink-watchdog.conf

if [ "$ENABLE_NM_WAN" = 1 ]; then
    echo "== networkmanager WANs =="
    if ! dpkg -s network-manager >/dev/null 2>&1; then
        # Installed only now: the package starts NM, which manages Wi-Fi/gsm devices by
        # default, so van-ap-unmanaged.conf (AP radios, LAN ports) must already be in place.
        apt-get update -qq || true
        DEBIAN_FRONTEND=noninteractive apt-get install -y network-manager \
            || { echo "deploy: apt-get install network-manager failed" >&2; exit 1; }
    fi
    systemctl enable --now NetworkManager ModemManager >/dev/null 2>&1 || true
    nmcli general reload conf 2>/dev/null || true
    for nm_if in "$WIFI_5G_IFACE" "$WIFI_2G_IFACE" "$LAN_IFACE" ${ONBOARD_LAN_IFACE:-}; do
        [ -e "/sys/class/net/$nm_if" ] || continue
        nm_state=$(nmcli -t -f DEVICE,STATE device 2>/dev/null | awk -F: -v d="$nm_if" '$1 == d { print $2 }') || true
        [ "$nm_state" = unmanaged ] \
            || warn "NetworkManager manages $nm_if (state: ${nm_state:-unknown}) — AP/LAN ports must be unmanaged (see ap/van-ap-unmanaged.conf)"
    done
    if [ -n "${CELLULAR_APN:-}" ]; then
        nm_keyfile="/etc/NetworkManager/system-connections/${CELLULAR_CONN_NAME}.nmconnection"
        if [ -e "$nm_keyfile" ]; then
            echo "   cellular connection ${CELLULAR_CONN_NAME} exists — left unchanged"
        elif [ -z "${CELLULAR_UUID:-}" ]; then
            warn "CELLULAR_UUID is empty — not creating the cellular keyfile $nm_keyfile"
        else
            install_rendered ap/cellular.nmconnection "$nm_keyfile" 0600
            nmcli connection reload 2>/dev/null || true
            echo "   created cellular connection ${CELLULAR_CONN_NAME} (apn ${CELLULAR_APN}) — no modem needed"
        fi
    fi
    if [ "$(nm_wan_state_netplan)" != matches ]; then
        warn "WAN netplan is not provisioned/up to date: run ./deploy.sh --apply-network (backs up /etc/netplan and rolls back automatically if the uplink does not come back)"
    fi
fi

echo "== apply =="
sysctl --system >/dev/null
systemctl daemon-reload
# Re-exec PID1 so the system.conf.d watchdog drop-in takes effect (daemon-reload
# alone does NOT re-arm RuntimeWatchdogSec). Safe online.
systemctl daemon-reexec
# Safe online: picks up 99-van-router-dns.conf immediately. NM's
# ignore-auto-dns only takes effect on a connection's next activation though —
# an already-up WAN keeps its currently-applied DNS until it reconnects (or
# reboot), deliberately not forced here (reconnecting a WAN flaps it).
systemctl restart systemd-resolved
systemctl restart mosquitto
# networkd here owns only the AP radios + bridge + wired LAN port (no real uplink);
# its wait-online would just stall network-online.target. NM-wait-online covers WANs.
systemctl mask systemd-networkd-wait-online.service >/dev/null 2>&1 || true
systemctl unmask hostapd >/dev/null 2>&1 || true
# Disable system dnsmasq.service to avoid port 53 conflicts — van-ap-dnsmasq owns AP DHCP
systemctl disable dnsmasq.service >/dev/null 2>&1 || true
set_units
systemctl enable "${ENABLE_UNITS[@]}" >/dev/null 2>&1 || true
# bluetooth: host BlueZ serves the onboard hci0 to the HA container over D-Bus
systemctl enable --now bluetooth >/dev/null 2>&1 || true
if [ "$ENABLE_HA" = 1 ]; then
    systemctl start homeassistant || warn "homeassistant failed to start (podman/quadlet — check journalctl -u homeassistant)"
    systemctl start esphome || warn "esphome failed to start (podman/quadlet — check journalctl -u esphome)"
fi
if [ "$ENABLE_FRIGATE" = 1 ]; then
    systemctl start frigate || warn "frigate failed to start (podman/quadlet — check journalctl -u frigate)"
fi
systemctl restart van-thermal
if [ "$ENABLE_NVME_WATCH" = 1 ]; then
    systemctl restart van-nvme-watch
fi
systemctl restart van-sms-watch
systemctl restart van-gps-owntracks
if [ "$ENABLE_LI3" = 1 ]; then
    systemctl restart van-li3-battery
fi
systemctl restart van-router-mqtt
# Pick up unmanaged-devices changes so NM releases/keeps the right interfaces.
nmcli general reload 2>/dev/null || systemctl reload NetworkManager 2>/dev/null || true
# restart in dependency order: bridge + members first, then hostapd enslaves the
# radios, then dnsmasq binds br0, then NAT/failover
systemctl restart systemd-networkd
for u in van-ap-dnsmasq nftables van-failover van-wlan-watchdog; do
    systemctl restart "$u" || warn "$u failed to restart (journalctl -xeu $u)"
done
# An AP radio may be absent; the start then fails but Restart=always keeps
# retrying and claims it the moment it enumerates (post_check reports absent radios).
systemctl restart hostapd hostapd-2g || true
# AP watchdogs last, after hostapd is back up (they only ever restart a wedged hostapd)
systemctl restart van-ap-watchdog van-ap-watchdog-2g
networkctl reload 2>/dev/null || true

post_check
write_warnings

if [ ${#WARNINGS[@]} -gt 0 ]; then
    echo
    echo "== warnings =="
    for w in "${WARNINGS[@]}"; do echo "   -> $w"; done
fi

echo
echo "Deployed host $(hostname -s) with $(readlink -f "$VANLINK_CONF")."
echo "Verify further:"
echo "  cat /run/van-failover/state.json"
echo "  cat /run/van-thermal/state.json"
echo "  journalctl -u van-router-mqtt -n 20"
echo "  journalctl -u van-sms-watch -n 20"
if [ "$ENABLE_NVME_WATCH" = 1 ]; then
    echo "  cat /run/van-nvme-watch/state.json"
fi
if [ "$ENABLE_LI3" = 1 ]; then
    echo "  journalctl -u van-li3-battery -n 20"
fi
echo "  ./deploy.sh --post-check     # re-run the checks above"