Andreas WredeandClaude Sonnet 5
6d5184e2e7
van-failover is now the sole owner of default-route selection: netplan/NM profiles get ipv4.never-default so NM's own DHCP/lease renewals stop reinstalling competing default routes (was racing van-failover's enforce_route on Starlink's 16s lease). set_profile_metric() also pins never-default on NM-managed connections it doesn't own the netplan source for (e.g. cellular), and nm_gateway() falls back to the raw DHCP4 lease's `routers` option since NM stops populating IP4.GATEWAY once never-default is set. Probing moved from one shared round to independent per-WAN retry scheduling, so a flaky WAN retries on its own clock instead of dragging healthy WANs into extra probes or throttling a failing one to the slow steady-state cadence. config.json: probe_interval 4->60, fail/ok_threshold 3/2->2/1, single probe URL — verified live across this reboot (cellular took ~2min after NM reported "activated" to actually pass traffic; van-failover correctly withheld/ deprioritized the default route until then, then installed it automatically). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
312 lines
14 KiB
Python
Executable File
312 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""van-failover — multi-WAN health-probe + metric-based failover for the campervan router.
|
|
|
|
Each WAN gets a base route-metric defining priority (lower = preferred). The daemon
|
|
probes each WAN's *real* internet reachability independently (HTTP 204 bound to the
|
|
interface, so it catches "link up but no internet" AND captive portals), and demotes
|
|
a failed WAN by raising its NM route-metric so the kernel routes via the next-best
|
|
healthy WAN. Auto fail-back on recovery, with hysteresis. Writes /run/van-failover/state.json
|
|
for the Cockpit dashboard. NAT (nftables masquerade oifname != AP) already follows
|
|
whatever the active default route is, so nothing else is needed.
|
|
"""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
CONFIG = "/etc/van-failover/config.json"
|
|
STATE = "/run/van-failover/state.json"
|
|
PREFER = "/run/van-failover/prefer" # optional: a device name the Cockpit UI asks us to prefer
|
|
PENALTY = 10000 # added to a WAN's metric while it is unhealthy
|
|
PREFER_METRIC = 50 # base metric given to the user-preferred WAN (below every config metric)
|
|
|
|
|
|
def sh(args, timeout=10):
|
|
try:
|
|
return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return None
|
|
|
|
|
|
def active_connections():
|
|
"""List of {name, device, type} for currently-active NM connections."""
|
|
r = sh(["nmcli", "-t", "-f", "NAME,DEVICE,TYPE", "connection", "show", "--active"])
|
|
out = []
|
|
if r and r.returncode == 0:
|
|
for line in r.stdout.splitlines():
|
|
# nmcli -t escapes ':' inside fields as '\:'; split on unescaped ':'
|
|
parts = _split_nmcli(line)
|
|
if len(parts) >= 3:
|
|
out.append({"name": parts[0], "device": parts[1], "type": parts[2]})
|
|
return out
|
|
|
|
|
|
def _split_nmcli(line):
|
|
parts, cur, esc = [], "", False
|
|
for ch in line:
|
|
if esc:
|
|
cur += ch
|
|
esc = False
|
|
elif ch == "\\":
|
|
esc = True
|
|
elif ch == ":":
|
|
parts.append(cur)
|
|
cur = ""
|
|
else:
|
|
cur += ch
|
|
parts.append(cur)
|
|
return parts
|
|
|
|
|
|
def resolve(wan, actives):
|
|
"""Return (device, connection) for a WAN, using whichever identifier is configured."""
|
|
if wan.get("device"):
|
|
dev = wan["device"]
|
|
conn = next((a["name"] for a in actives if a["device"] == dev), None)
|
|
return dev, conn
|
|
conn = wan.get("connection")
|
|
dev = next((a["device"] for a in actives if a["name"] == conn), None)
|
|
return ip_iface(dev), conn
|
|
|
|
|
|
def ip_iface(dev):
|
|
"""The routed netdev for an NM device. For MBIM/QMI modems NM's device is the
|
|
control port (cdc-wdm0) while IP/routes live on the wwan netdev — probing and
|
|
`ip route` must use the latter."""
|
|
if not dev:
|
|
return dev
|
|
r = sh(["nmcli", "-g", "GENERAL.IP-IFACE", "device", "show", dev])
|
|
if r and r.returncode == 0 and r.stdout.strip():
|
|
return r.stdout.strip()
|
|
return dev
|
|
|
|
|
|
def probe_one(dev, url, timeout):
|
|
"""True if URL returns HTTP 204 out the bound device (if! forces SO_BINDTODEVICE)."""
|
|
r = sh(["curl", "-s", "-m", str(timeout), "--interface", "if!" + dev,
|
|
"-o", "/dev/null", "-w", "%{http_code}", url], timeout=timeout + 2)
|
|
return bool(r and r.stdout.strip() == "204")
|
|
|
|
|
|
def default_routes():
|
|
"""dict device -> metric for current default routes."""
|
|
r = sh(["ip", "-j", "route", "show", "default"])
|
|
out = {}
|
|
if r and r.stdout.strip():
|
|
try:
|
|
for rt in json.loads(r.stdout):
|
|
if rt.get("dev"):
|
|
out[rt["dev"]] = rt.get("metric", 0)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def routes_on(dev):
|
|
"""List of {metric, gw} for default routes currently on a device."""
|
|
r = sh(["ip", "-j", "route", "show", "default", "dev", dev])
|
|
out = []
|
|
if r and r.stdout.strip():
|
|
try:
|
|
for rt in json.loads(r.stdout):
|
|
out.append({"metric": rt.get("metric", 0), "gw": rt.get("gateway")})
|
|
except json.JSONDecodeError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def enforce_route(dev, metric, conn=None):
|
|
"""Ensure exactly one default route on dev at the desired metric, via `ip route`.
|
|
NEVER use `nmcli device reapply` — it resets r8152 USB-ethernet carriers and causes
|
|
a failover flap. This is a pure routing change (carrier-safe, verified)."""
|
|
if not dev:
|
|
return
|
|
rts = routes_on(dev)
|
|
if any(r["metric"] == metric for r in rts):
|
|
# desired metric already present; just prune any stale others
|
|
for r in rts:
|
|
if r["metric"] != metric and r["gw"]:
|
|
sh(["ip", "route", "del", "default", "via", r["gw"], "dev", dev, "metric", str(r["metric"])])
|
|
return
|
|
# Prefer the gw from an existing default route; fall back to NM's known gateway so we can
|
|
# also *restore* a route that went missing while the carrier is still up (not just rebase one).
|
|
gw = next((r["gw"] for r in rts if r["gw"]), None) or nm_gateway(dev, conn)
|
|
if not gw:
|
|
return # no gateway known (carrier down); profile metric still set
|
|
r = sh(["ip", "route", "add", "default", "via", gw, "dev", dev, "metric", str(metric), "proto", "static"])
|
|
if not (r and r.returncode == 0):
|
|
# Add failed — most likely another dev transiently holds this exact metric during a
|
|
# preference swap. Leave the existing route intact and retry next loop; do NOT prune,
|
|
# or we'd strand this dev with no default route at all.
|
|
return
|
|
for old in rts:
|
|
if old["metric"] != metric and old["gw"]:
|
|
sh(["ip", "route", "del", "default", "via", old["gw"], "dev", dev, "metric", str(old["metric"])])
|
|
|
|
|
|
def nm_gateway(dev, conn):
|
|
"""NM's gateway — available even when the default route is missing. Try the device,
|
|
then the connection (a wwan netdev is not an NM device, but its connection is active).
|
|
Falls back to the raw DHCP4 lease's `routers` option: with ipv4.never-default set (so NM
|
|
never installs its own competing default route — see set_profile_metric), NM also stops
|
|
populating IP4.GATEWAY, even though the DHCP-negotiated router is still known underneath."""
|
|
for kind, name in (("device", dev), ("connection", conn)):
|
|
if name:
|
|
r = sh(["nmcli", "-g", "IP4.GATEWAY", kind, "show", name])
|
|
if r and r.returncode == 0 and r.stdout.strip():
|
|
return r.stdout.strip()
|
|
if dev:
|
|
r = sh(["nmcli", "-g", "DHCP4.OPTION", "device", "show", dev])
|
|
if r and r.returncode == 0:
|
|
for opt in r.stdout.split(" | "):
|
|
key, _, val = opt.strip().partition(" = ")
|
|
if key == "routers" and val:
|
|
return val.split()[0]
|
|
return None
|
|
|
|
|
|
def set_profile_metric(conn, metric):
|
|
"""Update the NM profile's route-metric (no reapply) so NM re-assertions stay consistent.
|
|
Also pins never-default=yes: without it, NM's own DHCP client reinstalls its own default
|
|
route (its device-type default metric, e.g. 100 for ethernet) on every lease renewal,
|
|
racing the `ip route`-managed one enforce_route() maintains (observed every ~8s on a
|
|
Starlink dongle with a 16s DHCP lease). Like the metric change, this only takes effect on
|
|
this connection's *next* activation, not the currently-active one (a live `nmcli modify`
|
|
doesn't retroactively change an already-active connection's installed routes, and we
|
|
deliberately never `nmcli device reapply` — see enforce_route) — belt-and-suspenders for
|
|
connections van-failover doesn't own the netplan source for (e.g. cellular)."""
|
|
if conn:
|
|
sh(["nmcli", "connection", "modify", conn,
|
|
"ipv4.route-metric", str(metric), "ipv4.never-default", "yes"])
|
|
|
|
|
|
def read_prefer():
|
|
"""Device name the UI wants preferred, or None. Ephemeral (cleared on reboot)."""
|
|
try:
|
|
with open(PREFER) as f:
|
|
return f.read().strip() or None
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def write_state(state):
|
|
os.makedirs(os.path.dirname(STATE), exist_ok=True)
|
|
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(STATE))
|
|
with os.fdopen(fd, "w") as f:
|
|
json.dump(state, f, indent=2)
|
|
os.chmod(tmp, 0o644)
|
|
os.replace(tmp, STATE)
|
|
|
|
|
|
def main():
|
|
with open(CONFIG) as f:
|
|
cfg = json.load(f)
|
|
interval = cfg.get("probe_interval", 5)
|
|
fail_th = cfg.get("fail_threshold", 3)
|
|
ok_th = cfg.get("ok_threshold", 2)
|
|
ptimeout = cfg.get("probe_timeout", 4)
|
|
urls = cfg["probe_urls"]
|
|
wans = cfg["wans"]
|
|
|
|
# Optimistic start: assume up so healthy WANs immediately get their base metric.
|
|
rt = {w["name"]: {"up": True, "ok": ok_th, "fail": 0, "applied": None, "healthy": None} for w in wans}
|
|
# Per-WAN independent probe clock: on the road every link can be flaky independently, so a
|
|
# struggling WAN retries on its own schedule instead of a shared round dragging healthy WANs
|
|
# into extra probes (metered cellular data) or throttling a failing one down to the slow
|
|
# steady-state cadence. 0.0 (epoch) means "due immediately" — probe everything on startup.
|
|
next_due = {w["name"]: 0.0 for w in wans}
|
|
print(f"van-failover started: {[w['name'] for w in wans]}", flush=True)
|
|
|
|
pool = ThreadPoolExecutor(max_workers=max(4, len(wans) * len(urls)))
|
|
while True:
|
|
now = time.monotonic()
|
|
actives = active_connections()
|
|
resolved = {w["name"]: resolve(w, actives) for w in wans}
|
|
prefer_dev = read_prefer()
|
|
# A WAN is "present" (probeable/manageable) only with both an active device and connection.
|
|
present = {w["name"]: bool(resolved[w["name"]][0]) and bool(resolved[w["name"]][1]) for w in wans}
|
|
due = [w for w in wans if present[w["name"]] and now >= next_due[w["name"]]]
|
|
|
|
# Probe only WANs that are due, one url-fan-out per WAN, concurrently. A WAN is healthy
|
|
# if ANY url returns 204. `finish` tracks each WAN's own completion time (not one shared
|
|
# round time) — needed below to tell an instant failure (DNS/route error, no time spent
|
|
# waiting) apart from a real probe_timeout-bound one.
|
|
health = {w["name"]: False for w in due}
|
|
finish = {}
|
|
tasks = [(w["name"], pool.submit(probe_one, resolved[w["name"]][0], url, ptimeout))
|
|
for w in due for url in urls]
|
|
for name, fut in tasks:
|
|
ok = fut.result()
|
|
finish[name] = max(finish.get(name, now), time.monotonic())
|
|
if ok:
|
|
health[name] = True
|
|
|
|
for w in wans:
|
|
name = w["name"]
|
|
dev, conn = resolved[name]
|
|
# The preferred WAN gets the lowest base so it wins while healthy; the
|
|
# health PENALTY still applies on top, so failover/fail-back is unchanged.
|
|
base = PREFER_METRIC if prefer_dev and dev == prefer_dev else w["metric"]
|
|
s = rt[name]
|
|
if not present[name]:
|
|
# Absent (e.g. modem unplugged): don't probe/penalize; just keep the profile's
|
|
# base metric so it lands at the right priority when it connects, and probe it
|
|
# right away once it reappears.
|
|
next_due[name] = 0.0
|
|
if conn and s["applied"] != base:
|
|
set_profile_metric(conn, base)
|
|
s["applied"] = base
|
|
continue
|
|
if name in finish:
|
|
s["healthy"] = health[name]
|
|
if health[name]:
|
|
s["ok"] += 1
|
|
s["fail"] = 0
|
|
if s["ok"] >= ok_th:
|
|
s["up"] = True
|
|
next_due[name] = now + interval
|
|
else:
|
|
s["fail"] += 1
|
|
s["ok"] = 0
|
|
if s["fail"] >= fail_th:
|
|
s["up"] = False
|
|
# Retry as soon as we've spent a full probe_timeout since this attempt
|
|
# started — immediately if the failure itself ate the whole timeout (a real
|
|
# timeout), otherwise topped up with a short wait. That's the early-out for
|
|
# instant failures (DNS/route errors that return in milliseconds): without
|
|
# the top-up, a permanently-unreachable WAN would retry in a tight loop.
|
|
next_due[name] = max(finish[name], now + ptimeout)
|
|
desired = base if s["up"] else base + PENALTY
|
|
# Enforce the live route every loop (carrier-safe, corrects any NM drift);
|
|
# update the NM profile only on an actual state change.
|
|
enforce_route(dev, desired, conn)
|
|
if s["applied"] != desired:
|
|
print(f"{name}: {'UP' if s['up'] else 'DOWN'} -> metric {desired}", flush=True)
|
|
set_profile_metric(conn, desired)
|
|
s["applied"] = desired
|
|
|
|
routes = default_routes()
|
|
active_dev = min(routes, key=routes.get) if routes else None
|
|
report = []
|
|
for w in wans:
|
|
name = w["name"]
|
|
dev, conn = resolved[name]
|
|
report.append({
|
|
"name": name, "priority": w["metric"], "device": dev, "connection": conn,
|
|
"present": present[name], "up": rt[name]["up"] if present[name] else False,
|
|
"healthy": rt[name]["healthy"] if present[name] else None,
|
|
"preferred": dev is not None and dev == prefer_dev,
|
|
"route_metric": routes.get(dev), "active": dev is not None and dev == active_dev,
|
|
})
|
|
write_state({"updated": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
|
"active_device": active_dev, "wans": report})
|
|
|
|
upcoming = [next_due[w["name"]] for w in wans if present[w["name"]]]
|
|
time.sleep(max(0.0, min(upcoming) - time.monotonic()) if upcoming else interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|