Andreas Wrede
e7a82126f2
Tails journalctl -kf for the "nvme nvmeN: I/O tag ... timeout, reset controller" signature that crashed and corrupted the root fs on 2026-08-02 (and recurred 2026-08-04, that time self-healing). Watches a short grace window to tell a clean self-heal from an escalation (repeated timeout or a following ext4 error) before paging via the existing Pushover credentials, with a live SMART/superblock snapshot in the alert body.
262 lines
15 KiB
Bash
Executable File
262 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Deploy vanlink configs/scripts (Pi 4 "wan" port) to their system locations.
|
|
# Usage: cd ~/vanlink && sudo ./deploy.sh
|
|
# sudo ./deploy.sh render <file> # preview a templated file on stdout
|
|
# Idempotent. Netplan (wlan0 + Starlink NIC = NM-managed WANs; eth0 is a LAN
|
|
# port on br0) is NOT deployed here — reference copy in ap/50-van-wan.yaml,
|
|
# applied once manually (apply flaps uplinks). We do check it for drift below.
|
|
set -euo pipefail
|
|
cd "$(dirname "$(readlink -f "$0")")"
|
|
|
|
# deploy.conf holds hardware-instance identifiers (interface names, USB vendor
|
|
# IDs) that change when a dongle/adapter gets physically swapped. Config files
|
|
# below carry @TOKEN@ placeholders substituted from these variables via
|
|
# render()/install_rendered() — edit deploy.conf, not the individual configs.
|
|
set -a
|
|
source ./deploy.conf
|
|
set +a
|
|
|
|
render() { # render <file> -> stdout, with @TOKEN@ placeholders substituted
|
|
sed -e "s|@WIFI_5G_IFACE@|$WIFI_5G_IFACE|g" \
|
|
-e "s|@WIFI_2G_IFACE@|$WIFI_2G_IFACE|g" \
|
|
-e "s|@LAN_USB_IFACE@|$LAN_USB_IFACE|g" \
|
|
-e "s|@STARLINK_IFACE@|$STARLINK_IFACE|g" \
|
|
-e "s|@MODEM_USB_VENDOR@|$MODEM_USB_VENDOR|g" \
|
|
-e "s|@DNS_RESOLVERS@|$DNS_RESOLVERS|g" \
|
|
"$1"
|
|
}
|
|
install_rendered() { # install_rendered <src> <dst> [mode]
|
|
local tmp
|
|
tmp=$(mktemp)
|
|
render "$1" > "$tmp"
|
|
install -D -m"${3:-0644}" "$tmp" "$2"
|
|
rm -f "$tmp"
|
|
}
|
|
|
|
if [ "${1:-}" = "render" ]; then
|
|
[ -n "${2:-}" ] || { echo "Usage: $0 render <file>"; exit 1; }
|
|
render "$2"
|
|
exit 0
|
|
fi
|
|
|
|
[ "$(id -u)" = 0 ] || { echo "Run with sudo (writes to /etc, /usr)."; exit 1; }
|
|
|
|
# Collected below and persisted to WARNINGS_FILE so the Cockpit vanrouter page
|
|
# can surface deploy-time issues (missing deps, unedited example configs, drift)
|
|
# without someone having to remember to scroll back through deploy output.
|
|
WARNINGS=()
|
|
WARNINGS_FILE=/var/lib/vanlink/deploy-warnings.json
|
|
warn() { # warn <message...> — prints " -> <message>" (as before) and records it
|
|
echo " -> $*"
|
|
WARNINGS+=("$*")
|
|
}
|
|
write_warnings() {
|
|
install -d -m0755 "$(dirname "$WARNINGS_FILE")"
|
|
{
|
|
printf '{\n "deployed": "%s",\n "warnings": [' "$(date -Iseconds)"
|
|
local first=1 w e
|
|
for w in "${WARNINGS[@]}"; do
|
|
[ "$first" = 1 ] || printf ','
|
|
first=0
|
|
e=${w//\\/\\\\}; e=${e//\"/\\\"}
|
|
printf '\n "%s"' "$e"
|
|
done
|
|
printf '\n ]\n}\n'
|
|
} > "$WARNINGS_FILE"
|
|
}
|
|
|
|
echo "== netplan drift check =="
|
|
# Netplan is never installed by this script (applying it flaps the uplinks —
|
|
# see note below), so it's easy to edit ap/50-van-wan.yaml and forget the
|
|
# manual `netplan apply` step. Warn loudly rather than silently drifting.
|
|
if [ ! -f /etc/netplan/50-van-wan.yaml ]; then
|
|
warn "netplan: /etc/netplan/50-van-wan.yaml is missing — repo config was never deployed. Run: sudo ./deploy.sh render ap/50-van-wan.yaml | sudo tee /etc/netplan/50-van-wan.yaml && sudo netplan generate && sudo netplan apply"
|
|
elif ! diff -q <(render ap/50-van-wan.yaml) /etc/netplan/50-van-wan.yaml >/dev/null 2>&1; then
|
|
warn "netplan: /etc/netplan/50-van-wan.yaml differs from ap/50-van-wan.yaml (rendered). Deploy manually: sudo ./deploy.sh render ap/50-van-wan.yaml | sudo tee /etc/netplan/50-van-wan.yaml && sudo netplan generate && sudo netplan apply"
|
|
diff -u /etc/netplan/50-van-wan.yaml <(render ap/50-van-wan.yaml) || true
|
|
fi
|
|
|
|
echo "== access point =="
|
|
install_rendered ap/hostapd.conf /etc/hostapd/hostapd.conf
|
|
install -D -m0644 ap/hostapd-restart.conf /etc/systemd/system/hostapd.service.d/restart.conf
|
|
install -D -m0644 ap/default-hostapd /etc/default/hostapd
|
|
install_rendered ap/hostapd-2g.conf /etc/hostapd/hostapd-2g.conf
|
|
install_rendered ap/hostapd-2g.service /etc/systemd/system/hostapd-2g.service
|
|
install_rendered ap/11-van-ap-2g.network /etc/systemd/network/11-van-ap-2g.network
|
|
install -D -m0644 ap/van-ap-watchdog-2g.service /etc/systemd/system/van-ap-watchdog-2g.service
|
|
install -D -m0755 ap/van-ap-watchdog /usr/local/sbin/van-ap-watchdog
|
|
install -D -m0644 ap/van-ap-watchdog.service /etc/systemd/system/van-ap-watchdog.service
|
|
install -D -m0644 ap/van-ap-dnsmasq.conf /etc/van-ap/dnsmasq.conf
|
|
install -D -m0644 ap/van-ap-dnsmasq.service /etc/systemd/system/van-ap-dnsmasq.service
|
|
install_rendered ap/10-van-ap.network /etc/systemd/network/10-van-ap.network
|
|
install -D -m0644 ap/20-van-br0.netdev /etc/systemd/network/20-van-br0.netdev
|
|
install -D -m0644 ap/21-van-br0.network /etc/systemd/network/21-van-br0.network
|
|
install_rendered ap/22-van-lan.network /etc/systemd/network/22-van-lan.network
|
|
install -D -m0644 ap/23-van-lan-eth0.network /etc/systemd/network/23-van-lan-eth0.network
|
|
install_rendered ap/van-ap-unmanaged.conf /etc/NetworkManager/conf.d/van-ap-unmanaged.conf
|
|
# Mask dracut's initramfs-generated catch-all (/run/systemd/network/
|
|
# zzzz-dracut-default.network, regenerated every boot): it matches every
|
|
# unconfigured link, so networkd co-managed the NM-owned WANs — a second
|
|
# DHCPv4 client on wlan0 and IPv6 that worked only by accident. NM owns WAN
|
|
# IPv6 now (see ap/50-van-wan.yaml). /etc overrides /run; /dev/null masks.
|
|
ln -sfn /dev/null /etc/systemd/network/zzzz-dracut-default.network
|
|
install -D -m0644 ap/nftables.conf /etc/nftables.conf
|
|
install -D -m0644 ap/regdomain.service /etc/systemd/system/regdomain.service
|
|
install -D -m0644 ap/rtw89.conf /etc/modprobe.d/rtw89.conf
|
|
install -D -m0644 ap/99-van-router.conf /etc/sysctl.d/99-van-router.conf
|
|
|
|
echo "== dns =="
|
|
# Fixed upstream resolvers (deploy.conf's DNS_RESOLVERS), never a WAN's own
|
|
# DHCP/RA-provided DNS — see the files themselves for the full rationale.
|
|
# (NM's ipv4/ipv6.ignore-auto-dns can't be set as a config-file connection
|
|
# default — NM rejects it there — so 60-van-wan-dns enforces this directly
|
|
# against resolved instead, on every WAN connect/lease event.)
|
|
# .local (mDNS) is handled separately, scoped to br0 (see 21-van-br0.network
|
|
# above). ZeroTier-managed DNS (zt.wrede.pvt) is a separate, additive path.
|
|
install_rendered ap/99-van-router-dns.conf /etc/systemd/resolved.conf.d/99-van-router-dns.conf
|
|
install -D -m0755 failover/60-van-wan-dns /etc/NetworkManager/dispatcher.d/60-van-wan-dns
|
|
# ZeroTier-managed DNS for zt.wrede.pvt (see README's "ZeroTier managed DNS").
|
|
# allowDNS on the network + the search-domain drop-in are repo-tracked so a
|
|
# reimage doesn't need the manual `zerotier-cli set ... allowDNS=1` step
|
|
# remembered by hand; the manager binary itself is a hand-installed .deb
|
|
# (not in apt) — see https://github.com/zerotier/zerotier-systemd-manager/releases.
|
|
install -D -m0600 dns/zt-network.local.conf /var/lib/zerotier-one/networks.d/d3ecf5726d041b2a.local.conf
|
|
install -D -m0644 dns/zt-search.conf /etc/systemd/network/99-ztuga7c2kh.network.d/search.conf
|
|
dpkg -s zerotier-systemd-manager >/dev/null 2>&1 \
|
|
|| warn "zerotier-systemd-manager not installed — zt.wrede.pvt won't resolve. Install the arm64 .deb from https://github.com/zerotier/zerotier-systemd-manager/releases"
|
|
|
|
echo "== failover =="
|
|
install -D -m0755 failover/van-failover /usr/local/sbin/van-failover
|
|
install_rendered failover/config.json /etc/van-failover/config.json
|
|
install -D -m0644 failover/van-failover.service /etc/systemd/system/van-failover.service
|
|
install -D -m0755 failover/50-disable-eee /etc/NetworkManager/dispatcher.d/50-disable-eee
|
|
install -D -m0644 failover/99-van-arp.conf /etc/sysctl.d/99-van-arp.conf
|
|
# Backstop for wlan0's post-boot NM no-secrets wedge (see the script's docstring).
|
|
install -D -m0755 failover/van-wlan-watchdog /usr/local/sbin/van-wlan-watchdog
|
|
install -D -m0644 failover/van-wlan-watchdog.service /etc/systemd/system/van-wlan-watchdog.service
|
|
|
|
echo "== cellular modem (GSM/LTE) =="
|
|
# NM's gsm.auto-config APN lookup needs this apt-only carrier database; without
|
|
# it even the right APN can't be auto-detected, and MVNOs (e.g. Koodo, which
|
|
# isn't listed under its own name — only under host network "Telus Mobility")
|
|
# often aren't in it anyway, so the modem's NM connection profile may still
|
|
# need an explicit gsm.apn set by hand regardless.
|
|
dpkg -s mobile-broadband-provider-info >/dev/null 2>&1 \
|
|
|| warn "mobile-broadband-provider-info missing (apt install mobile-broadband-provider-info) — GSM APN auto-config will fail"
|
|
|
|
echo "== cockpit plugin =="
|
|
install -d /usr/share/cockpit/vanrouter
|
|
for f in cockpit/vanrouter/*; do
|
|
b=$(basename "$f")
|
|
if [ "$b" = "vanrouter.js" ]; then
|
|
install_rendered "$f" "/usr/share/cockpit/vanrouter/$b"
|
|
else
|
|
install -m0644 "$f" "/usr/share/cockpit/vanrouter/$b"
|
|
fi
|
|
done
|
|
# Bridge fd headroom (Python bridge frees spawn pipes only at GC; 1024 is too tight)
|
|
install -D -m0644 cockpit/cockpit-session-nofile.conf /etc/systemd/system/cockpit-session@.service.d/nofile.conf
|
|
# The Starlink card queries the dish's gRPC API; grpcurl isn't packaged in apt.
|
|
command -v grpcurl >/dev/null 2>&1 \
|
|
|| warn "grpcurl missing (Starlink card will say so): install linux_arm64 binary from github.com/fullstorydev/grpcurl/releases"
|
|
|
|
echo "== gps (cellular modem GNSS -> gpsd) =="
|
|
if dpkg -s gpsd >/dev/null 2>&1; then
|
|
install -D -m0644 gps/77-modem-gps.rules /etc/udev/rules.d/77-modem-gps.rules
|
|
install -D -m0644 gps/gpsd.default /etc/default/gpsd
|
|
udevadm control --reload
|
|
else
|
|
warn "gpsd not installed (apt install gpsd gpsd-clients) — skipping GPS setup"
|
|
fi
|
|
# OwnTracks publisher: gpsd fix -> MQTT (broker + creds live only on the
|
|
# system, 0600 — same pattern as pushover.json).
|
|
install -D -m0755 gps/van-gps-owntracks /usr/local/sbin/van-gps-owntracks
|
|
install -D -m0644 gps/van-gps-owntracks.service /etc/systemd/system/van-gps-owntracks.service
|
|
if [ ! -f /etc/van-gps/config.json ]; then
|
|
install -D -m0600 gps/config.json.example /etc/van-gps/config.json
|
|
warn "seeded /etc/van-gps/config.json (EDIT IT: add MQTT username + password)"
|
|
fi
|
|
python3 -c 'import gps' 2>/dev/null \
|
|
|| warn "python3-gps missing (apt install python3-gps) — van-gps-owntracks won't start"
|
|
python3 -c 'import paho.mqtt' 2>/dev/null \
|
|
|| warn "python3-paho-mqtt missing (apt install python3-paho-mqtt) — van-gps-owntracks won't start"
|
|
|
|
echo "== thermal monitor =="
|
|
install -D -m0755 power/van-thermal /usr/local/sbin/van-thermal
|
|
install -D -m0644 power/thermal-config.json /etc/van-thermal/config.json
|
|
install -D -m0644 power/van-thermal.service /etc/systemd/system/van-thermal.service
|
|
# Pushover secrets live only on the system (0600), never in the repo. Path kept
|
|
# under /etc/van-battery/ for parity with wayback's van-thermal default.
|
|
if [ ! -f /etc/van-battery/pushover.json ]; then
|
|
install -D -m0600 power/pushover.json.example /etc/van-battery/pushover.json
|
|
warn "seeded /etc/van-battery/pushover.json (EDIT IT: add Pushover token + user)"
|
|
fi
|
|
|
|
echo "== nvme watchdog =="
|
|
install -D -m0755 power/van-nvme-watch /usr/local/sbin/van-nvme-watch
|
|
install -D -m0644 power/nvme-watch-config.json /etc/van-nvme-watch/config.json
|
|
install -D -m0644 power/van-nvme-watch.service /etc/systemd/system/van-nvme-watch.service
|
|
|
|
echo "== home assistant =="
|
|
# Native HA (Podman Quadlet, replaced the ha_van VM). daemon-reload below
|
|
# regenerates homeassistant.service; started (not restarted) at the end so a
|
|
# deploy never bounces HA — after editing the .container, restart it manually.
|
|
install -D -m0644 ha/homeassistant.container /etc/containers/systemd/homeassistant.container
|
|
install -d -m0755 /srv/homeassistant
|
|
# ESPHome dashboard (sibling container, same rationale as HA above — no
|
|
# Supervisor/add-on store here).
|
|
install -D -m0644 ha/esphome.container /etc/containers/systemd/esphome.container
|
|
install -d -m0755 /srv/esphome
|
|
|
|
echo "== hardware watchdog =="
|
|
install -D -m0644 power/10-vanlink-watchdog.conf /etc/systemd/system.conf.d/10-vanlink-watchdog.conf
|
|
|
|
echo "== apply =="
|
|
sysctl --system >/dev/null
|
|
systemctl daemon-reload
|
|
# Re-exec PID1 so the system.conf.d watchdog drop-in takes effect (daemon-reload
|
|
# alone does NOT re-arm RuntimeWatchdogSec). Safe online.
|
|
systemctl daemon-reexec
|
|
# Safe online: picks up 99-van-router-dns.conf immediately. NM's
|
|
# ignore-auto-dns only takes effect on a connection's next activation though —
|
|
# an already-up WAN keeps its currently-applied DNS until it reconnects (or
|
|
# reboot), deliberately not forced here (reconnecting a WAN flaps it).
|
|
systemctl restart systemd-resolved
|
|
# networkd here owns only the AP radios + bridge + wired LAN port (no real uplink);
|
|
# its wait-online would just stall network-online.target. NM-wait-online covers WANs.
|
|
systemctl mask systemd-networkd-wait-online.service >/dev/null 2>&1 || true
|
|
systemctl unmask hostapd >/dev/null 2>&1 || true
|
|
systemctl enable regdomain.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-nvme-watch van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-gps-owntracks >/dev/null 2>&1 || true
|
|
# bluetooth: host BlueZ serves the onboard hci0 to the HA container over D-Bus
|
|
systemctl enable --now bluetooth >/dev/null 2>&1 || true
|
|
systemctl start homeassistant || warn "homeassistant failed to start (podman/quadlet — check journalctl -u homeassistant)"
|
|
systemctl start esphome || warn "esphome failed to start (podman/quadlet — check journalctl -u esphome)"
|
|
systemctl restart van-thermal
|
|
systemctl restart van-nvme-watch
|
|
systemctl restart van-gps-owntracks
|
|
# Pick up unmanaged-devices changes so NM releases/keeps the right interfaces.
|
|
nmcli general reload 2>/dev/null || systemctl reload NetworkManager 2>/dev/null || true
|
|
# restart in dependency order: bridge + members first, then hostapd enslaves the
|
|
# radios, then dnsmasq binds br0, then NAT/failover
|
|
systemctl restart systemd-networkd
|
|
systemctl restart van-ap-dnsmasq nftables van-failover van-wlan-watchdog
|
|
# The AP radios live on the USB hub and may be absent; the start then fails but
|
|
# Restart=always keeps retrying and claims them the moment they enumerate.
|
|
systemctl restart hostapd hostapd-2g \
|
|
|| echo " -> hostapd(-2g) waiting for AP radios (USB hub not plugged in)"
|
|
# AP watchdogs last, after hostapd is back up (they only ever restart a wedged hostapd)
|
|
systemctl restart van-ap-watchdog van-ap-watchdog-2g
|
|
networkctl reload 2>/dev/null || true
|
|
|
|
write_warnings
|
|
|
|
echo
|
|
echo "Deployed. NOTE: until the USB hub (AP radios + LAN/Starlink adapters) is"
|
|
echo "plugged in, hostapd/hostapd-2g just retry every 5s — that is by design."
|
|
echo "Verify (with hub present):"
|
|
echo " iw dev $WIFI_5G_IFACE info | grep -E 'ssid|channel|width'"
|
|
echo " iw dev $WIFI_2G_IFACE info | grep -E 'ssid|channel|width'"
|
|
echo " cat /run/van-failover/state.json"
|
|
echo " cat /run/van-thermal/state.json"
|
|
echo " cat /run/van-nvme-watch/state.json"
|