initial commit
This commit is contained in:
Andreas Wrede
Executable
+251
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""van-failover — multi-WAN health-probe + metric-based failover for the campervan router.
|
||||
|
||||
Each WAN gets a base route-metric defining priority (lower = preferred). The daemon
|
||||
probes each WAN's *real* internet reachability independently (HTTP 204 bound to the
|
||||
interface, so it catches "link up but no internet" AND captive portals), and demotes
|
||||
a failed WAN by raising its NM route-metric so the kernel routes via the next-best
|
||||
healthy WAN. Auto fail-back on recovery, with hysteresis. Writes /run/van-failover/state.json
|
||||
for the Cockpit dashboard. NAT (nftables masquerade oifname != AP) already follows
|
||||
whatever the active default route is, so nothing else is needed.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
CONFIG = "/etc/van-failover/config.json"
|
||||
STATE = "/run/van-failover/state.json"
|
||||
PREFER = "/run/van-failover/prefer" # optional: a device name the Cockpit UI asks us to prefer
|
||||
PENALTY = 10000 # added to a WAN's metric while it is unhealthy
|
||||
PREFER_METRIC = 50 # base metric given to the user-preferred WAN (below every config metric)
|
||||
|
||||
|
||||
def sh(args, timeout=10):
|
||||
try:
|
||||
return subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def active_connections():
|
||||
"""List of {name, device, type} for currently-active NM connections."""
|
||||
r = sh(["nmcli", "-t", "-f", "NAME,DEVICE,TYPE", "connection", "show", "--active"])
|
||||
out = []
|
||||
if r and r.returncode == 0:
|
||||
for line in r.stdout.splitlines():
|
||||
# nmcli -t escapes ':' inside fields as '\:'; split on unescaped ':'
|
||||
parts = _split_nmcli(line)
|
||||
if len(parts) >= 3:
|
||||
out.append({"name": parts[0], "device": parts[1], "type": parts[2]})
|
||||
return out
|
||||
|
||||
|
||||
def _split_nmcli(line):
|
||||
parts, cur, esc = [], "", False
|
||||
for ch in line:
|
||||
if esc:
|
||||
cur += ch
|
||||
esc = False
|
||||
elif ch == "\\":
|
||||
esc = True
|
||||
elif ch == ":":
|
||||
parts.append(cur)
|
||||
cur = ""
|
||||
else:
|
||||
cur += ch
|
||||
parts.append(cur)
|
||||
return parts
|
||||
|
||||
|
||||
def resolve(wan, actives):
|
||||
"""Return (device, connection) for a WAN, using whichever identifier is configured."""
|
||||
if wan.get("device"):
|
||||
dev = wan["device"]
|
||||
conn = next((a["name"] for a in actives if a["device"] == dev), None)
|
||||
return dev, conn
|
||||
conn = wan.get("connection")
|
||||
dev = next((a["device"] for a in actives if a["name"] == conn), None)
|
||||
return dev, conn
|
||||
|
||||
|
||||
def probe_one(dev, url, timeout):
|
||||
"""True if URL returns HTTP 204 out the bound device (if! forces SO_BINDTODEVICE)."""
|
||||
r = sh(["curl", "-s", "-m", str(timeout), "--interface", "if!" + dev,
|
||||
"-o", "/dev/null", "-w", "%{http_code}", url], timeout=timeout + 2)
|
||||
return bool(r and r.stdout.strip() == "204")
|
||||
|
||||
|
||||
def default_routes():
|
||||
"""dict device -> metric for current default routes."""
|
||||
r = sh(["ip", "-j", "route", "show", "default"])
|
||||
out = {}
|
||||
if r and r.stdout.strip():
|
||||
try:
|
||||
for rt in json.loads(r.stdout):
|
||||
if rt.get("dev"):
|
||||
out[rt["dev"]] = rt.get("metric", 0)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def routes_on(dev):
|
||||
"""List of {metric, gw} for default routes currently on a device."""
|
||||
r = sh(["ip", "-j", "route", "show", "default", "dev", dev])
|
||||
out = []
|
||||
if r and r.stdout.strip():
|
||||
try:
|
||||
for rt in json.loads(r.stdout):
|
||||
out.append({"metric": rt.get("metric", 0), "gw": rt.get("gateway")})
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def enforce_route(dev, metric):
|
||||
"""Ensure exactly one default route on dev at the desired metric, via `ip route`.
|
||||
NEVER use `nmcli device reapply` — it resets r8152 USB-ethernet carriers and causes
|
||||
a failover flap. This is a pure routing change (carrier-safe, verified)."""
|
||||
if not dev:
|
||||
return
|
||||
rts = routes_on(dev)
|
||||
if any(r["metric"] == metric for r in rts):
|
||||
# desired metric already present; just prune any stale others
|
||||
for r in rts:
|
||||
if r["metric"] != metric and r["gw"]:
|
||||
sh(["ip", "route", "del", "default", "via", r["gw"], "dev", dev, "metric", str(r["metric"])])
|
||||
return
|
||||
# Prefer the gw from an existing default route; fall back to NM's known gateway so we can
|
||||
# also *restore* a route that went missing while the carrier is still up (not just rebase one).
|
||||
gw = next((r["gw"] for r in rts if r["gw"]), None) or device_gateway(dev)
|
||||
if not gw:
|
||||
return # no gateway known (carrier down); profile metric still set
|
||||
r = sh(["ip", "route", "add", "default", "via", gw, "dev", dev, "metric", str(metric), "proto", "static"])
|
||||
if not (r and r.returncode == 0):
|
||||
# Add failed — most likely another dev transiently holds this exact metric during a
|
||||
# preference swap. Leave the existing route intact and retry next loop; do NOT prune,
|
||||
# or we'd strand this dev with no default route at all.
|
||||
return
|
||||
for old in rts:
|
||||
if old["metric"] != metric and old["gw"]:
|
||||
sh(["ip", "route", "del", "default", "via", old["gw"], "dev", dev, "metric", str(old["metric"])])
|
||||
|
||||
|
||||
def device_gateway(dev):
|
||||
"""NM's gateway for a device — available even when its default route is missing."""
|
||||
r = sh(["nmcli", "-g", "IP4.GATEWAY", "device", "show", dev])
|
||||
if r and r.returncode == 0:
|
||||
return r.stdout.strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def set_profile_metric(conn, metric):
|
||||
"""Update the NM profile's route-metric (no reapply) so NM re-assertions stay consistent."""
|
||||
if conn:
|
||||
sh(["nmcli", "connection", "modify", conn, "ipv4.route-metric", str(metric)])
|
||||
|
||||
|
||||
def read_prefer():
|
||||
"""Device name the UI wants preferred, or None. Ephemeral (cleared on reboot)."""
|
||||
try:
|
||||
with open(PREFER) as f:
|
||||
return f.read().strip() or None
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def write_state(state):
|
||||
os.makedirs(os.path.dirname(STATE), exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=os.path.dirname(STATE))
|
||||
with os.fdopen(fd, "w") as f:
|
||||
json.dump(state, f, indent=2)
|
||||
os.chmod(tmp, 0o644)
|
||||
os.replace(tmp, STATE)
|
||||
|
||||
|
||||
def main():
|
||||
with open(CONFIG) as f:
|
||||
cfg = json.load(f)
|
||||
interval = cfg.get("probe_interval", 5)
|
||||
fail_th = cfg.get("fail_threshold", 3)
|
||||
ok_th = cfg.get("ok_threshold", 2)
|
||||
ptimeout = cfg.get("probe_timeout", 4)
|
||||
urls = cfg["probe_urls"]
|
||||
wans = cfg["wans"]
|
||||
|
||||
# Optimistic start: assume up so healthy WANs immediately get their base metric.
|
||||
rt = {w["name"]: {"up": True, "ok": ok_th, "fail": 0, "applied": None} for w in wans}
|
||||
print(f"van-failover started: {[w['name'] for w in wans]}", flush=True)
|
||||
|
||||
pool = ThreadPoolExecutor(max_workers=max(4, len(wans) * len(urls)))
|
||||
while True:
|
||||
actives = active_connections()
|
||||
resolved = {w["name"]: resolve(w, actives) for w in wans}
|
||||
prefer_dev = read_prefer()
|
||||
# A WAN is "present" (probeable/manageable) only with both an active device and connection.
|
||||
present = {w["name"]: bool(resolved[w["name"]][0]) and bool(resolved[w["name"]][1]) for w in wans}
|
||||
|
||||
# Probe every (present WAN x url) pair concurrently; a WAN is healthy if ANY url
|
||||
# returns 204. Concurrency bounds a failed WAN to ~one timeout, not N serial timeouts.
|
||||
health = {w["name"]: (False if present[w["name"]] else None) for w in wans}
|
||||
tasks = [(w["name"], pool.submit(probe_one, resolved[w["name"]][0], url, ptimeout))
|
||||
for w in wans if present[w["name"]] for url in urls]
|
||||
for name, fut in tasks:
|
||||
if fut.result():
|
||||
health[name] = True
|
||||
|
||||
for w in wans:
|
||||
name = w["name"]
|
||||
dev, conn = resolved[name]
|
||||
# The preferred WAN gets the lowest base so it wins while healthy; the
|
||||
# health PENALTY still applies on top, so failover/fail-back is unchanged.
|
||||
base = PREFER_METRIC if prefer_dev and dev == prefer_dev else w["metric"]
|
||||
s = rt[name]
|
||||
if not present[name]:
|
||||
# Absent (e.g. modem unplugged): don't probe/penalize; just keep the
|
||||
# profile's base metric so it lands at the right priority when it connects.
|
||||
if conn and s["applied"] != base:
|
||||
set_profile_metric(conn, base)
|
||||
s["applied"] = base
|
||||
continue
|
||||
if health[name]:
|
||||
s["ok"] += 1
|
||||
s["fail"] = 0
|
||||
if s["ok"] >= ok_th:
|
||||
s["up"] = True
|
||||
else:
|
||||
s["fail"] += 1
|
||||
s["ok"] = 0
|
||||
if s["fail"] >= fail_th:
|
||||
s["up"] = False
|
||||
desired = base if s["up"] else base + PENALTY
|
||||
# Enforce the live route every loop (carrier-safe, corrects any NM drift);
|
||||
# update the NM profile only on an actual state change.
|
||||
enforce_route(dev, desired)
|
||||
if s["applied"] != desired:
|
||||
print(f"{name}: {'UP' if s['up'] else 'DOWN'} -> metric {desired}", flush=True)
|
||||
set_profile_metric(conn, desired)
|
||||
s["applied"] = desired
|
||||
|
||||
routes = default_routes()
|
||||
active_dev = min(routes, key=routes.get) if routes else None
|
||||
report = []
|
||||
for w in wans:
|
||||
name = w["name"]
|
||||
dev, conn = resolved[name]
|
||||
report.append({
|
||||
"name": name, "priority": w["metric"], "device": dev, "connection": conn,
|
||||
"present": present[name], "up": rt[name]["up"] if present[name] else False,
|
||||
"healthy": health[name], "preferred": dev is not None and dev == prefer_dev,
|
||||
"route_metric": routes.get(dev), "active": dev is not None and dev == active_dev,
|
||||
})
|
||||
write_state({"updated": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"active_device": active_dev, "wans": report})
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user