diff --git a/deploy.sh b/deploy.sh index 8c805af..6afa33d 100755 --- a/deploy.sh +++ b/deploy.sh @@ -143,10 +143,13 @@ echo "== cellular modem (GSM/LTE) ==" # need an explicit gsm.apn set by hand regardless. dpkg -s mobile-broadband-provider-info >/dev/null 2>&1 \ || warn "mobile-broadband-provider-info missing (apt install mobile-broadband-provider-info) — GSM APN auto-config will fail" -# Backstop for the modem sometimes never enumerating at boot on the new -# powered hub (see the script's docstring). -install -D -m0755 failover/van-modem-usb-kick /usr/local/sbin/van-modem-usb-kick -install_rendered failover/van-modem-usb-kick.service /etc/systemd/system/van-modem-usb-kick.service +# Backstop for the modem sometimes never enumerating at boot (see the +# script's docstring) — detection + Pushover alert only. No automated +# recovery: a hub power-cycle was proven not to fix this (only a genuine +# physical unplug/replug does), so the modem is now on the Pi's native USB +# port rather than through the hub. This just pages if it ever recurs. +install -D -m0755 failover/van-modem-watch /usr/local/sbin/van-modem-watch +install_rendered failover/van-modem-watch.service /etc/systemd/system/van-modem-watch.service echo "== cockpit plugin ==" install -d /usr/share/cockpit/vanrouter @@ -230,7 +233,7 @@ systemctl restart systemd-resolved # its wait-online would just stall network-online.target. NM-wait-online covers WANs. systemctl mask systemd-networkd-wait-online.service >/dev/null 2>&1 || true systemctl unmask hostapd >/dev/null 2>&1 || true -systemctl enable regdomain.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-nvme-watch van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-modem-usb-kick van-gps-owntracks >/dev/null 2>&1 || true +systemctl enable regdomain.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-nvme-watch van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-modem-watch van-gps-owntracks >/dev/null 2>&1 || true # bluetooth: host BlueZ serves the onboard hci0 to the HA container over D-Bus systemctl enable --now bluetooth >/dev/null 2>&1 || true systemctl start homeassistant || warn "homeassistant failed to start (podman/quadlet — check journalctl -u homeassistant)" diff --git a/failover/van-modem-watch b/failover/van-modem-watch new file mode 100755 index 0000000..4bac0ac --- /dev/null +++ b/failover/van-modem-watch @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""van-modem-watch — pages if the EC25 modem never enumerates at boot. + +Detection only, deliberately no recovery attempt. On 2026-08-04 the modem +(then on a shared powered USB hub) sometimes failed to enumerate on a cold +boot. Investigation ruled out a boot-timing race and a bad cable/port: a +genuine hub-commanded VBUS power-cycle (confirmed via kernel disconnect/ +reconnect events, held off for a full 10s) did not recover it, even with the +rest of the system already up and stable — only a real physical unplug/ +replug of the connector ever did. That means no software action from this +host can fix it once it happens; the modem was moved off the hub onto the +Pi's native USB port as the actual fix (a direct port doesn't reproduce the +failure). This just watches for a recurrence and pages, since if it comes +back the only real fix is someone physically reseating the connector. + +Pushover credentials shared with van-battery/van-thermal/van-nvme-watch +(/etc/van-battery/pushover.json, 0600). Publishes /run/van-modem-watch/ +state.json (same convention as the other watchdogs). Stdlib only. + +Usage: van-modem-watch [modem-usb-vendor] +Defaults to 2c7c (Quectel) — see deploy.conf. +""" + +import json +import socket +import sys +import time +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +MODEM_VENDOR = sys.argv[1] if len(sys.argv) > 1 else "2c7c" +INTERVAL = 20 # seconds between checks +BUDGET_S = 6 * INTERVAL # ~2 minutes of retries after boot, then give up quietly +USB_DEVICES = Path("/sys/bus/usb/devices") +STATE_DIR = Path("/run/van-modem-watch") +STATE_PATH = STATE_DIR / "state.json" +CREDENTIALS_PATH = Path("/etc/van-battery/pushover.json") +HOST = socket.gethostname() +PLACEHOLDERS = {"", "REPLACE_ME", "your-token-here", "your-user-key-here"} + + +def log(msg, level="info"): + # systemd journal severity prefixes (sd-daemon), same convention as van-ap-watchdog. + pri = {"info": "<6>", "warn": "<4>", "crit": "<2>"}.get(level, "<6>") + print(pri + msg, flush=True) + + +def modem_present(vendor): + for f in USB_DEVICES.glob("*/idVendor"): + try: + if f.read_text().strip() == vendor: + return True + except OSError: + continue + return False + + +def load_creds(): + try: + c = json.loads(CREDENTIALS_PATH.read_text()) + token, user = str(c.get("token", "")).strip(), str(c.get("user", "")).strip() + if token in PLACEHOLDERS or user in PLACEHOLDERS: + return None + return token, user + except FileNotFoundError: + return None + except Exception as e: + log(f"credentials {CREDENTIALS_PATH} unreadable ({e})", "warn") + return None + + +def pushover(title, message, attempts=3, retry_delay=15): + creds = load_creds() + if not creds: + log(f"pushover skipped (no credentials): {title} — {message}", "warn") + return False + token, user = creds + data = urllib.parse.urlencode({ + "token": token, "user": user, "title": title, "message": message, + }).encode() + req = urllib.request.Request("https://api.pushover.net/1/messages.json", data=data) + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(req, timeout=10) as resp: + ok = resp.status == 200 + if not ok: + log(f"pushover HTTP {resp.status}", "warn") + return ok + except Exception as e: + last = attempt == attempts + log(f"pushover send failed ({attempt}/{attempts}): {e}", "warn") + if not last: + time.sleep(retry_delay) + return False + + +def write_state(payload): + STATE_DIR.mkdir(parents=True, exist_ok=True) + tmp = STATE_PATH.with_suffix(".tmp") + tmp.write_text(json.dumps(payload)) + tmp.replace(STATE_PATH) + + +def main(): + log(f"van-modem-watch up: watching for USB vendor {MODEM_VENDOR} for up to " + f"{BUDGET_S}s after boot") + elapsed = 0 + while elapsed < BUDGET_S: + if modem_present(MODEM_VENDOR): + log("modem present — exiting") + write_state({ + "updated": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "present": True, + }) + return + time.sleep(INTERVAL) + elapsed += INTERVAL + + message = (f"modem (USB vendor {MODEM_VENDOR}) not seen {BUDGET_S}s after boot — " + f"needs a physical unplug/replug, no software fix works for this") + log(message, "crit") + write_state({ + "updated": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "present": False, + }) + pushover(f"🚨 {HOST}: cellular modem absent after boot", message) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(0) diff --git a/failover/van-modem-watch.service b/failover/van-modem-watch.service new file mode 100644 index 0000000..d211dee --- /dev/null +++ b/failover/van-modem-watch.service @@ -0,0 +1,11 @@ +[Unit] +Description=EC25 modem boot-presence watchdog — pages if it never enumerated (no auto-recovery, see script docstring) +After=systemd-udev-settle.service +Wants=systemd-udev-settle.service + +[Service] +Type=simple +ExecStart=/usr/local/sbin/van-modem-watch @MODEM_USB_VENDOR@ + +[Install] +WantedBy=multi-user.target