#!/usr/bin/env python3 """van-failover — multi-WAN health-probe + metric-based failover for the campervan router. Each WAN gets a base route-metric defining priority (lower = preferred). The daemon probes each WAN's *real* internet reachability independently (HTTP 204 bound to the interface, so it catches "link up but no internet" AND captive portals), and demotes a failed WAN by raising its NM route-metric so the kernel routes via the next-best healthy WAN. Auto fail-back on recovery, with hysteresis. Writes /run/van-failover/state.json for the Cockpit dashboard. NAT (nftables masquerade oifname != AP) already follows whatever the active default route is, so nothing else is needed. """ import json import os import subprocess import tempfile import time from concurrent.futures import ThreadPoolExecutor CONFIG = "/etc/van-failover/config.json" STATE = "/run/van-failover/state.json" PREFER = "/run/van-failover/prefer" # optional: a device name the Cockpit UI asks us to prefer PENALTY = 10000 # added to a WAN's metric while it is unhealthy PREFER_METRIC = 50 # base metric given to the user-preferred WAN (below every config metric) def sh(args, timeout=10): try: return subprocess.run(args, capture_output=True, text=True, timeout=timeout) except (subprocess.TimeoutExpired, OSError): return None def active_connections(): """List of {name, device, type} for currently-active NM connections.""" r = sh(["nmcli", "-t", "-f", "NAME,DEVICE,TYPE", "connection", "show", "--active"]) out = [] if r and r.returncode == 0: for line in r.stdout.splitlines(): # nmcli -t escapes ':' inside fields as '\:'; split on unescaped ':' parts = _split_nmcli(line) if len(parts) >= 3: out.append({"name": parts[0], "device": parts[1], "type": parts[2]}) return out def _split_nmcli(line): parts, cur, esc = [], "", False for ch in line: if esc: cur += ch esc = False elif ch == "\\": esc = True elif ch == ":": parts.append(cur) cur = "" else: cur += ch parts.append(cur) return parts def resolve(wan, actives): """Return (device, connection) for a WAN, using whichever identifier is configured.""" if wan.get("device"): dev = wan["device"] conn = next((a["name"] for a in actives if a["device"] == dev), None) return dev, conn conn = wan.get("connection") dev = next((a["device"] for a in actives if a["name"] == conn), None) return dev, conn def probe_one(dev, url, timeout): """True if URL returns HTTP 204 out the bound device (if! forces SO_BINDTODEVICE).""" r = sh(["curl", "-s", "-m", str(timeout), "--interface", "if!" + dev, "-o", "/dev/null", "-w", "%{http_code}", url], timeout=timeout + 2) return bool(r and r.stdout.strip() == "204") def default_routes(): """dict device -> metric for current default routes.""" r = sh(["ip", "-j", "route", "show", "default"]) out = {} if r and r.stdout.strip(): try: for rt in json.loads(r.stdout): if rt.get("dev"): out[rt["dev"]] = rt.get("metric", 0) except json.JSONDecodeError: pass return out def routes_on(dev): """List of {metric, gw} for default routes currently on a device.""" r = sh(["ip", "-j", "route", "show", "default", "dev", dev]) out = [] if r and r.stdout.strip(): try: for rt in json.loads(r.stdout): out.append({"metric": rt.get("metric", 0), "gw": rt.get("gateway")}) except json.JSONDecodeError: pass return out def enforce_route(dev, metric): """Ensure exactly one default route on dev at the desired metric, via `ip route`. NEVER use `nmcli device reapply` — it resets r8152 USB-ethernet carriers and causes a failover flap. This is a pure routing change (carrier-safe, verified).""" if not dev: return rts = routes_on(dev) if any(r["metric"] == metric for r in rts): # desired metric already present; just prune any stale others for r in rts: if r["metric"] != metric and r["gw"]: sh(["ip", "route", "del", "default", "via", r["gw"], "dev", dev, "metric", str(r["metric"])]) return # Prefer the gw from an existing default route; fall back to NM's known gateway so we can # also *restore* a route that went missing while the carrier is still up (not just rebase one). gw = next((r["gw"] for r in rts if r["gw"]), None) or device_gateway(dev) if not gw: return # no gateway known (carrier down); profile metric still set r = sh(["ip", "route", "add", "default", "via", gw, "dev", dev, "metric", str(metric), "proto", "static"]) if not (r and r.returncode == 0): # Add failed — most likely another dev transiently holds this exact metric during a # preference swap. Leave the existing route intact and retry next loop; do NOT prune, # or we'd strand this dev with no default route at all. return for old in rts: if old["metric"] != metric and old["gw"]: sh(["ip", "route", "del", "default", "via", old["gw"], "dev", dev, "metric", str(old["metric"])]) def device_gateway(dev): """NM's gateway for a device — available even when its default route is missing.""" r = sh(["nmcli", "-g", "IP4.GATEWAY", "device", "show", dev]) if r and r.returncode == 0: return r.stdout.strip() or None return None def set_profile_metric(conn, metric): """Update the NM profile's route-metric (no reapply) so NM re-assertions stay consistent.""" if conn: sh(["nmcli", "connection", "modify", conn, "ipv4.route-metric", str(metric)]) def read_prefer(): """Device name the UI wants preferred, or None. Ephemeral (cleared on reboot).""" try: with open(PREFER) as f: return f.read().strip() or None except OSError: return None def write_state(state): os.makedirs(os.path.dirname(STATE), exist_ok=True) fd, tmp = tempfile.mkstemp(dir=os.path.dirname(STATE)) with os.fdopen(fd, "w") as f: json.dump(state, f, indent=2) os.chmod(tmp, 0o644) os.replace(tmp, STATE) def main(): with open(CONFIG) as f: cfg = json.load(f) interval = cfg.get("probe_interval", 5) fail_th = cfg.get("fail_threshold", 3) ok_th = cfg.get("ok_threshold", 2) ptimeout = cfg.get("probe_timeout", 4) urls = cfg["probe_urls"] wans = cfg["wans"] # Optimistic start: assume up so healthy WANs immediately get their base metric. rt = {w["name"]: {"up": True, "ok": ok_th, "fail": 0, "applied": None} for w in wans} print(f"van-failover started: {[w['name'] for w in wans]}", flush=True) pool = ThreadPoolExecutor(max_workers=max(4, len(wans) * len(urls))) while True: actives = active_connections() resolved = {w["name"]: resolve(w, actives) for w in wans} prefer_dev = read_prefer() # A WAN is "present" (probeable/manageable) only with both an active device and connection. present = {w["name"]: bool(resolved[w["name"]][0]) and bool(resolved[w["name"]][1]) for w in wans} # Probe every (present WAN x url) pair concurrently; a WAN is healthy if ANY url # returns 204. Concurrency bounds a failed WAN to ~one timeout, not N serial timeouts. health = {w["name"]: (False if present[w["name"]] else None) for w in wans} tasks = [(w["name"], pool.submit(probe_one, resolved[w["name"]][0], url, ptimeout)) for w in wans if present[w["name"]] for url in urls] for name, fut in tasks: if fut.result(): health[name] = True for w in wans: name = w["name"] dev, conn = resolved[name] # The preferred WAN gets the lowest base so it wins while healthy; the # health PENALTY still applies on top, so failover/fail-back is unchanged. base = PREFER_METRIC if prefer_dev and dev == prefer_dev else w["metric"] s = rt[name] if not present[name]: # Absent (e.g. modem unplugged): don't probe/penalize; just keep the # profile's base metric so it lands at the right priority when it connects. if conn and s["applied"] != base: set_profile_metric(conn, base) s["applied"] = base continue if health[name]: s["ok"] += 1 s["fail"] = 0 if s["ok"] >= ok_th: s["up"] = True else: s["fail"] += 1 s["ok"] = 0 if s["fail"] >= fail_th: s["up"] = False desired = base if s["up"] else base + PENALTY # Enforce the live route every loop (carrier-safe, corrects any NM drift); # update the NM profile only on an actual state change. enforce_route(dev, desired) if s["applied"] != desired: print(f"{name}: {'UP' if s['up'] else 'DOWN'} -> metric {desired}", flush=True) set_profile_metric(conn, desired) s["applied"] = desired routes = default_routes() active_dev = min(routes, key=routes.get) if routes else None report = [] for w in wans: name = w["name"] dev, conn = resolved[name] report.append({ "name": name, "priority": w["metric"], "device": dev, "connection": conn, "present": present[name], "up": rt[name]["up"] if present[name] else False, "healthy": health[name], "preferred": dev is not None and dev == prefer_dev, "route_metric": routes.get(dev), "active": dev is not None and dev == active_dev, }) write_state({"updated": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "active_device": active_dev, "wans": report}) time.sleep(interval) if __name__ == "__main__": main()