initial commit

failover/50-disable-eee

@@ -0,0 +1,8 @@
+#!/bin/sh
+# Disable EEE on USB ethernet (r8152): its idle power-save parks the backup WAN link
+# and breaks van-failover health probes. See wayback-campervan-ap memory.
+iface="$1"; action="$2"
+[ "$action" = "up" ] || exit 0
+case "$iface" in
+  en*|eth*) /usr/sbin/ethtool --set-eee "$iface" eee off >/dev/null 2>&1 || true ;;
+esac

failover/99-van-arp.conf

@@ -0,0 +1,4 @@
+# Multi-WAN on possibly-shared subnets: answer ARP only for the incoming iface,
+# announce the best local source — avoids ARP flux between eth/wifi WANs.
+net.ipv4.conf.all.arp_ignore=1
+net.ipv4.conf.all.arp_announce=2

failover/config.json

@@ -0,0 +1,16 @@
+{
+  "probe_interval": 4,
+  "probe_timeout": 3,
+  "fail_threshold": 3,
+  "ok_threshold": 2,
+  "probe_urls": [
+    "http://connectivity-check.ubuntu.com/",
+    "http://www.gstatic.com/generate_204",
+    "http://cp.cloudflare.com/"
+  ],
+  "wans": [
+    { "name": "wifi",     "device": "wlp1s0",          "metric": 100 },
+    { "name": "starlink", "device": "enxd8ec5eeb3512", "metric": 200 },
+    { "name": "cellular", "connection": "Koodo",       "metric": 300 }
+  ]
+}

failover/van-failover

@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+"""van-failover — multi-WAN health-probe + metric-based failover for the campervan router.
+
+Each WAN gets a base route-metric defining priority (lower = preferred). The daemon
+probes each WAN's *real* internet reachability independently (HTTP 204 bound to the
+interface, so it catches "link up but no internet" AND captive portals), and demotes
+a failed WAN by raising its NM route-metric so the kernel routes via the next-best
+healthy WAN. Auto fail-back on recovery, with hysteresis. Writes /run/van-failover/state.json
+for the Cockpit dashboard. NAT (nftables masquerade oifname != AP) already follows
+whatever the active default route is, so nothing else is needed.
+"""
+import json
+import os
+import subprocess
+import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+CONFIG = "/etc/van-failover/config.json"
+STATE = "/run/van-failover/state.json"
+PREFER = "/run/van-failover/prefer"  # optional: a device name the Cockpit UI asks us to prefer
+PENALTY = 10000  # added to a WAN's metric while it is unhealthy
+PREFER_METRIC = 50  # base metric given to the user-preferred WAN (below every config metric)
+
+
+def sh(args, timeout=10):
+    try:
+        return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+    except (subprocess.TimeoutExpired, OSError):
+        return None
+
+
+def active_connections():
+    """List of {name, device, type} for currently-active NM connections."""
+    r = sh(["nmcli", "-t", "-f", "NAME,DEVICE,TYPE", "connection", "show", "--active"])
+    out = []
+    if r and r.returncode == 0:
+        for line in r.stdout.splitlines():
+            # nmcli -t escapes ':' inside fields as '\:'; split on unescaped ':'
+            parts = _split_nmcli(line)
+            if len(parts) >= 3:
+                out.append({"name": parts[0], "device": parts[1], "type": parts[2]})
+    return out
+
+
+def _split_nmcli(line):
+    parts, cur, esc = [], "", False
+    for ch in line:
+        if esc:
+            cur += ch
+            esc = False
+        elif ch == "\\":
+            esc = True
+        elif ch == ":":
+            parts.append(cur)
+            cur = ""
+        else:
+            cur += ch
+    parts.append(cur)
+    return parts
+
+
+def resolve(wan, actives):
+    """Return (device, connection) for a WAN, using whichever identifier is configured."""
+    if wan.get("device"):
+        dev = wan["device"]
+        conn = next((a["name"] for a in actives if a["device"] == dev), None)
+        return dev, conn
+    conn = wan.get("connection")
+    dev = next((a["device"] for a in actives if a["name"] == conn), None)
+    return dev, conn
+
+
+def probe_one(dev, url, timeout):
+    """True if URL returns HTTP 204 out the bound device (if! forces SO_BINDTODEVICE)."""
+    r = sh(["curl", "-s", "-m", str(timeout), "--interface", "if!" + dev,
+            "-o", "/dev/null", "-w", "%{http_code}", url], timeout=timeout + 2)
+    return bool(r and r.stdout.strip() == "204")
+
+
+def default_routes():
+    """dict device -> metric for current default routes."""
+    r = sh(["ip", "-j", "route", "show", "default"])
+    out = {}
+    if r and r.stdout.strip():
+        try:
+            for rt in json.loads(r.stdout):
+                if rt.get("dev"):
+                    out[rt["dev"]] = rt.get("metric", 0)
+        except json.JSONDecodeError:
+            pass
+    return out
+
+
+def routes_on(dev):
+    """List of {metric, gw} for default routes currently on a device."""
+    r = sh(["ip", "-j", "route", "show", "default", "dev", dev])
+    out = []
+    if r and r.stdout.strip():
+        try:
+            for rt in json.loads(r.stdout):
+                out.append({"metric": rt.get("metric", 0), "gw": rt.get("gateway")})
+        except json.JSONDecodeError:
+            pass
+    return out
+
+
+def enforce_route(dev, metric):
+    """Ensure exactly one default route on dev at the desired metric, via `ip route`.
+    NEVER use `nmcli device reapply` — it resets r8152 USB-ethernet carriers and causes
+    a failover flap. This is a pure routing change (carrier-safe, verified)."""
+    if not dev:
+        return
+    rts = routes_on(dev)
+    if any(r["metric"] == metric for r in rts):
+        # desired metric already present; just prune any stale others
+        for r in rts:
+            if r["metric"] != metric and r["gw"]:
+                sh(["ip", "route", "del", "default", "via", r["gw"], "dev", dev, "metric", str(r["metric"])])
+        return
+    # Prefer the gw from an existing default route; fall back to NM's known gateway so we can
+    # also *restore* a route that went missing while the carrier is still up (not just rebase one).
+    gw = next((r["gw"] for r in rts if r["gw"]), None) or device_gateway(dev)
+    if not gw:
+        return  # no gateway known (carrier down); profile metric still set
+    r = sh(["ip", "route", "add", "default", "via", gw, "dev", dev, "metric", str(metric), "proto", "static"])
+    if not (r and r.returncode == 0):
+        # Add failed — most likely another dev transiently holds this exact metric during a
+        # preference swap. Leave the existing route intact and retry next loop; do NOT prune,
+        # or we'd strand this dev with no default route at all.
+        return
+    for old in rts:
+        if old["metric"] != metric and old["gw"]:
+            sh(["ip", "route", "del", "default", "via", old["gw"], "dev", dev, "metric", str(old["metric"])])
+
+
+def device_gateway(dev):
+    """NM's gateway for a device — available even when its default route is missing."""
+    r = sh(["nmcli", "-g", "IP4.GATEWAY", "device", "show", dev])
+    if r and r.returncode == 0:
+        return r.stdout.strip() or None
+    return None
+
+
+def set_profile_metric(conn, metric):
+    """Update the NM profile's route-metric (no reapply) so NM re-assertions stay consistent."""
+    if conn:
+        sh(["nmcli", "connection", "modify", conn, "ipv4.route-metric", str(metric)])
+
+
+def read_prefer():
+    """Device name the UI wants preferred, or None. Ephemeral (cleared on reboot)."""
+    try:
+        with open(PREFER) as f:
+            return f.read().strip() or None
+    except OSError:
+        return None
+
+
+def write_state(state):
+    os.makedirs(os.path.dirname(STATE), exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(STATE))
+    with os.fdopen(fd, "w") as f:
+        json.dump(state, f, indent=2)
+    os.chmod(tmp, 0o644)
+    os.replace(tmp, STATE)
+
+
+def main():
+    with open(CONFIG) as f:
+        cfg = json.load(f)
+    interval = cfg.get("probe_interval", 5)
+    fail_th = cfg.get("fail_threshold", 3)
+    ok_th = cfg.get("ok_threshold", 2)
+    ptimeout = cfg.get("probe_timeout", 4)
+    urls = cfg["probe_urls"]
+    wans = cfg["wans"]
+
+    # Optimistic start: assume up so healthy WANs immediately get their base metric.
+    rt = {w["name"]: {"up": True, "ok": ok_th, "fail": 0, "applied": None} for w in wans}
+    print(f"van-failover started: {[w['name'] for w in wans]}", flush=True)
+
+    pool = ThreadPoolExecutor(max_workers=max(4, len(wans) * len(urls)))
+    while True:
+        actives = active_connections()
+        resolved = {w["name"]: resolve(w, actives) for w in wans}
+        prefer_dev = read_prefer()
+        # A WAN is "present" (probeable/manageable) only with both an active device and connection.
+        present = {w["name"]: bool(resolved[w["name"]][0]) and bool(resolved[w["name"]][1]) for w in wans}
+
+        # Probe every (present WAN x url) pair concurrently; a WAN is healthy if ANY url
+        # returns 204. Concurrency bounds a failed WAN to ~one timeout, not N serial timeouts.
+        health = {w["name"]: (False if present[w["name"]] else None) for w in wans}
+        tasks = [(w["name"], pool.submit(probe_one, resolved[w["name"]][0], url, ptimeout))
+                 for w in wans if present[w["name"]] for url in urls]
+        for name, fut in tasks:
+            if fut.result():
+                health[name] = True
+
+        for w in wans:
+            name = w["name"]
+            dev, conn = resolved[name]
+            # The preferred WAN gets the lowest base so it wins while healthy; the
+            # health PENALTY still applies on top, so failover/fail-back is unchanged.
+            base = PREFER_METRIC if prefer_dev and dev == prefer_dev else w["metric"]
+            s = rt[name]
+            if not present[name]:
+                # Absent (e.g. modem unplugged): don't probe/penalize; just keep the
+                # profile's base metric so it lands at the right priority when it connects.
+                if conn and s["applied"] != base:
+                    set_profile_metric(conn, base)
+                    s["applied"] = base
+                continue
+            if health[name]:
+                s["ok"] += 1
+                s["fail"] = 0
+                if s["ok"] >= ok_th:
+                    s["up"] = True
+            else:
+                s["fail"] += 1
+                s["ok"] = 0
+                if s["fail"] >= fail_th:
+                    s["up"] = False
+            desired = base if s["up"] else base + PENALTY
+            # Enforce the live route every loop (carrier-safe, corrects any NM drift);
+            # update the NM profile only on an actual state change.
+            enforce_route(dev, desired)
+            if s["applied"] != desired:
+                print(f"{name}: {'UP' if s['up'] else 'DOWN'} -> metric {desired}", flush=True)
+                set_profile_metric(conn, desired)
+                s["applied"] = desired
+
+        routes = default_routes()
+        active_dev = min(routes, key=routes.get) if routes else None
+        report = []
+        for w in wans:
+            name = w["name"]
+            dev, conn = resolved[name]
+            report.append({
+                "name": name, "priority": w["metric"], "device": dev, "connection": conn,
+                "present": present[name], "up": rt[name]["up"] if present[name] else False,
+                "healthy": health[name], "preferred": dev is not None and dev == prefer_dev,
+                "route_metric": routes.get(dev), "active": dev is not None and dev == active_dev,
+            })
+        write_state({"updated": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+                     "active_device": active_dev, "wans": report})
+        time.sleep(interval)
+
+
+if __name__ == "__main__":
+    main()

failover/van-failover.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Multi-WAN health-probe failover for the campervan router
+After=NetworkManager.service network-online.target
+Wants=NetworkManager.service
+
+[Service]
+Type=simple
+ExecStart=/usr/local/sbin/van-failover
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target