Andreas WredeandClaude Fable 5
fe9e6d3b30
NM reports a gsm connection's device as the control port (cdc-wdm0), but IP + routes live on the wwan netdev (wwu1u2i4 on the Quectel EC25-AF), so probes and route enforcement silently saw nothing (cellular stuck "down", no metric). Map connection-keyed WANs through GENERAL.IP-IFACE, and let the gateway fallback query the connection since the netdev isn't an NM device. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
267 lines
11 KiB
Python
Executable File
267 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""van-failover — multi-WAN health-probe + metric-based failover for the campervan router.
|
|
|
|
Each WAN gets a base route-metric defining priority (lower = preferred). The daemon
|
|
probes each WAN's *real* internet reachability independently (HTTP 204 bound to the
|
|
interface, so it catches "link up but no internet" AND captive portals), and demotes
|
|
a failed WAN by raising its NM route-metric so the kernel routes via the next-best
|
|
healthy WAN. Auto fail-back on recovery, with hysteresis. Writes /run/van-failover/state.json
|
|
for the Cockpit dashboard. NAT (nftables masquerade oifname != AP) already follows
|
|
whatever the active default route is, so nothing else is needed.
|
|
"""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
CONFIG = "/etc/van-failover/config.json"
|
|
STATE = "/run/van-failover/state.json"
|
|
PREFER = "/run/van-failover/prefer" # optional: a device name the Cockpit UI asks us to prefer
|
|
PENALTY = 10000 # added to a WAN's metric while it is unhealthy
|
|
PREFER_METRIC = 50 # base metric given to the user-preferred WAN (below every config metric)
|
|
|
|
|
|
def sh(args, timeout=10):
|
|
try:
|
|
return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return None
|
|
|
|
|
|
def active_connections():
|
|
"""List of {name, device, type} for currently-active NM connections."""
|
|
r = sh(["nmcli", "-t", "-f", "NAME,DEVICE,TYPE", "connection", "show", "--active"])
|
|
out = []
|
|
if r and r.returncode == 0:
|
|
for line in r.stdout.splitlines():
|
|
# nmcli -t escapes ':' inside fields as '\:'; split on unescaped ':'
|
|
parts = _split_nmcli(line)
|
|
if len(parts) >= 3:
|
|
out.append({"name": parts[0], "device": parts[1], "type": parts[2]})
|
|
return out
|
|
|
|
|
|
def _split_nmcli(line):
|
|
parts, cur, esc = [], "", False
|
|
for ch in line:
|
|
if esc:
|
|
cur += ch
|
|
esc = False
|
|
elif ch == "\\":
|
|
esc = True
|
|
elif ch == ":":
|
|
parts.append(cur)
|
|
cur = ""
|
|
else:
|
|
cur += ch
|
|
parts.append(cur)
|
|
return parts
|
|
|
|
|
|
def resolve(wan, actives):
|
|
"""Return (device, connection) for a WAN, using whichever identifier is configured."""
|
|
if wan.get("device"):
|
|
dev = wan["device"]
|
|
conn = next((a["name"] for a in actives if a["device"] == dev), None)
|
|
return dev, conn
|
|
conn = wan.get("connection")
|
|
dev = next((a["device"] for a in actives if a["name"] == conn), None)
|
|
return ip_iface(dev), conn
|
|
|
|
|
|
def ip_iface(dev):
|
|
"""The routed netdev for an NM device. For MBIM/QMI modems NM's device is the
|
|
control port (cdc-wdm0) while IP/routes live on the wwan netdev — probing and
|
|
`ip route` must use the latter."""
|
|
if not dev:
|
|
return dev
|
|
r = sh(["nmcli", "-g", "GENERAL.IP-IFACE", "device", "show", dev])
|
|
if r and r.returncode == 0 and r.stdout.strip():
|
|
return r.stdout.strip()
|
|
return dev
|
|
|
|
|
|
def probe_one(dev, url, timeout):
|
|
"""True if URL returns HTTP 204 out the bound device (if! forces SO_BINDTODEVICE)."""
|
|
r = sh(["curl", "-s", "-m", str(timeout), "--interface", "if!" + dev,
|
|
"-o", "/dev/null", "-w", "%{http_code}", url], timeout=timeout + 2)
|
|
return bool(r and r.stdout.strip() == "204")
|
|
|
|
|
|
def default_routes():
|
|
"""dict device -> metric for current default routes."""
|
|
r = sh(["ip", "-j", "route", "show", "default"])
|
|
out = {}
|
|
if r and r.stdout.strip():
|
|
try:
|
|
for rt in json.loads(r.stdout):
|
|
if rt.get("dev"):
|
|
out[rt["dev"]] = rt.get("metric", 0)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def routes_on(dev):
|
|
"""List of {metric, gw} for default routes currently on a device."""
|
|
r = sh(["ip", "-j", "route", "show", "default", "dev", dev])
|
|
out = []
|
|
if r and r.stdout.strip():
|
|
try:
|
|
for rt in json.loads(r.stdout):
|
|
out.append({"metric": rt.get("metric", 0), "gw": rt.get("gateway")})
|
|
except json.JSONDecodeError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def enforce_route(dev, metric, conn=None):
|
|
"""Ensure exactly one default route on dev at the desired metric, via `ip route`.
|
|
NEVER use `nmcli device reapply` — it resets r8152 USB-ethernet carriers and causes
|
|
a failover flap. This is a pure routing change (carrier-safe, verified)."""
|
|
if not dev:
|
|
return
|
|
rts = routes_on(dev)
|
|
if any(r["metric"] == metric for r in rts):
|
|
# desired metric already present; just prune any stale others
|
|
for r in rts:
|
|
if r["metric"] != metric and r["gw"]:
|
|
sh(["ip", "route", "del", "default", "via", r["gw"], "dev", dev, "metric", str(r["metric"])])
|
|
return
|
|
# Prefer the gw from an existing default route; fall back to NM's known gateway so we can
|
|
# also *restore* a route that went missing while the carrier is still up (not just rebase one).
|
|
gw = next((r["gw"] for r in rts if r["gw"]), None) or nm_gateway(dev, conn)
|
|
if not gw:
|
|
return # no gateway known (carrier down); profile metric still set
|
|
r = sh(["ip", "route", "add", "default", "via", gw, "dev", dev, "metric", str(metric), "proto", "static"])
|
|
if not (r and r.returncode == 0):
|
|
# Add failed — most likely another dev transiently holds this exact metric during a
|
|
# preference swap. Leave the existing route intact and retry next loop; do NOT prune,
|
|
# or we'd strand this dev with no default route at all.
|
|
return
|
|
for old in rts:
|
|
if old["metric"] != metric and old["gw"]:
|
|
sh(["ip", "route", "del", "default", "via", old["gw"], "dev", dev, "metric", str(old["metric"])])
|
|
|
|
|
|
def nm_gateway(dev, conn):
|
|
"""NM's gateway — available even when the default route is missing. Try the device,
|
|
then the connection (a wwan netdev is not an NM device, but its connection is active)."""
|
|
for kind, name in (("device", dev), ("connection", conn)):
|
|
if name:
|
|
r = sh(["nmcli", "-g", "IP4.GATEWAY", kind, "show", name])
|
|
if r and r.returncode == 0 and r.stdout.strip():
|
|
return r.stdout.strip()
|
|
return None
|
|
|
|
|
|
def set_profile_metric(conn, metric):
|
|
"""Update the NM profile's route-metric (no reapply) so NM re-assertions stay consistent."""
|
|
if conn:
|
|
sh(["nmcli", "connection", "modify", conn, "ipv4.route-metric", str(metric)])
|
|
|
|
|
|
def read_prefer():
|
|
"""Device name the UI wants preferred, or None. Ephemeral (cleared on reboot)."""
|
|
try:
|
|
with open(PREFER) as f:
|
|
return f.read().strip() or None
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def write_state(state):
|
|
os.makedirs(os.path.dirname(STATE), exist_ok=True)
|
|
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(STATE))
|
|
with os.fdopen(fd, "w") as f:
|
|
json.dump(state, f, indent=2)
|
|
os.chmod(tmp, 0o644)
|
|
os.replace(tmp, STATE)
|
|
|
|
|
|
def main():
|
|
with open(CONFIG) as f:
|
|
cfg = json.load(f)
|
|
interval = cfg.get("probe_interval", 5)
|
|
fail_th = cfg.get("fail_threshold", 3)
|
|
ok_th = cfg.get("ok_threshold", 2)
|
|
ptimeout = cfg.get("probe_timeout", 4)
|
|
urls = cfg["probe_urls"]
|
|
wans = cfg["wans"]
|
|
|
|
# Optimistic start: assume up so healthy WANs immediately get their base metric.
|
|
rt = {w["name"]: {"up": True, "ok": ok_th, "fail": 0, "applied": None} for w in wans}
|
|
print(f"van-failover started: {[w['name'] for w in wans]}", flush=True)
|
|
|
|
pool = ThreadPoolExecutor(max_workers=max(4, len(wans) * len(urls)))
|
|
while True:
|
|
actives = active_connections()
|
|
resolved = {w["name"]: resolve(w, actives) for w in wans}
|
|
prefer_dev = read_prefer()
|
|
# A WAN is "present" (probeable/manageable) only with both an active device and connection.
|
|
present = {w["name"]: bool(resolved[w["name"]][0]) and bool(resolved[w["name"]][1]) for w in wans}
|
|
|
|
# Probe every (present WAN x url) pair concurrently; a WAN is healthy if ANY url
|
|
# returns 204. Concurrency bounds a failed WAN to ~one timeout, not N serial timeouts.
|
|
health = {w["name"]: (False if present[w["name"]] else None) for w in wans}
|
|
tasks = [(w["name"], pool.submit(probe_one, resolved[w["name"]][0], url, ptimeout))
|
|
for w in wans if present[w["name"]] for url in urls]
|
|
for name, fut in tasks:
|
|
if fut.result():
|
|
health[name] = True
|
|
|
|
for w in wans:
|
|
name = w["name"]
|
|
dev, conn = resolved[name]
|
|
# The preferred WAN gets the lowest base so it wins while healthy; the
|
|
# health PENALTY still applies on top, so failover/fail-back is unchanged.
|
|
base = PREFER_METRIC if prefer_dev and dev == prefer_dev else w["metric"]
|
|
s = rt[name]
|
|
if not present[name]:
|
|
# Absent (e.g. modem unplugged): don't probe/penalize; just keep the
|
|
# profile's base metric so it lands at the right priority when it connects.
|
|
if conn and s["applied"] != base:
|
|
set_profile_metric(conn, base)
|
|
s["applied"] = base
|
|
continue
|
|
if health[name]:
|
|
s["ok"] += 1
|
|
s["fail"] = 0
|
|
if s["ok"] >= ok_th:
|
|
s["up"] = True
|
|
else:
|
|
s["fail"] += 1
|
|
s["ok"] = 0
|
|
if s["fail"] >= fail_th:
|
|
s["up"] = False
|
|
desired = base if s["up"] else base + PENALTY
|
|
# Enforce the live route every loop (carrier-safe, corrects any NM drift);
|
|
# update the NM profile only on an actual state change.
|
|
enforce_route(dev, desired, conn)
|
|
if s["applied"] != desired:
|
|
print(f"{name}: {'UP' if s['up'] else 'DOWN'} -> metric {desired}", flush=True)
|
|
set_profile_metric(conn, desired)
|
|
s["applied"] = desired
|
|
|
|
routes = default_routes()
|
|
active_dev = min(routes, key=routes.get) if routes else None
|
|
report = []
|
|
for w in wans:
|
|
name = w["name"]
|
|
dev, conn = resolved[name]
|
|
report.append({
|
|
"name": name, "priority": w["metric"], "device": dev, "connection": conn,
|
|
"present": present[name], "up": rt[name]["up"] if present[name] else False,
|
|
"healthy": health[name], "preferred": dev is not None and dev == prefer_dev,
|
|
"route_metric": routes.get(dev), "active": dev is not None and dev == active_dev,
|
|
})
|
|
write_state({"updated": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
|
"active_device": active_dev, "wans": report})
|
|
time.sleep(interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|