From 6d5184e2e78cd18f5c9c94f3949cf40654c95870 Mon Sep 17 00:00:00 2001 From: Andreas Wrede Date: Thu, 6 Aug 2026 14:22:46 -0400 Subject: [PATCH] failover: ipv4.never-default + per-WAN probe scheduling; tune probe cadence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit van-failover is now the sole owner of default-route selection: netplan/NM profiles get ipv4.never-default so NM's own DHCP/lease renewals stop reinstalling competing default routes (was racing van-failover's enforce_route on Starlink's 16s lease). set_profile_metric() also pins never-default on NM-managed connections it doesn't own the netplan source for (e.g. cellular), and nm_gateway() falls back to the raw DHCP4 lease's `routers` option since NM stops populating IP4.GATEWAY once never-default is set. Probing moved from one shared round to independent per-WAN retry scheduling, so a flaky WAN retries on its own clock instead of dragging healthy WANs into extra probes or throttling a failing one to the slow steady-state cadence. config.json: probe_interval 4->60, fail/ok_threshold 3/2->2/1, single probe URL — verified live across this reboot (cellular took ~2min after NM reported "activated" to actually pass traffic; van-failover correctly withheld/ deprioritized the default route until then, then installed it automatically). Co-Authored-By: Claude Sonnet 5 --- ap/50-van-wan.yaml | 17 ++++++++ failover/config.json | 10 ++--- failover/van-failover | 91 ++++++++++++++++++++++++++++++++----------- 3 files changed, 89 insertions(+), 29 deletions(-) diff --git a/ap/50-van-wan.yaml b/ap/50-van-wan.yaml index d6b5a55..3972000 100644 --- a/ap/50-van-wan.yaml +++ b/ap/50-van-wan.yaml @@ -21,6 +21,18 @@ network: routes: - to: 192.168.100.1/32 scope: link + # never-default: van-failover is the sole owner of the default route (it manages it + # directly via `ip route`, deliberately never `nmcli device reapply`d — that resets + # r8152 USB-ethernet carriers). Without this, NM's own DHCP client reinstalls its own + # default route (metric 100, NM's ethernet default) on every lease renewal — observed + # every ~8s here (dish hands out a 16s lease) — which van-failover's loop prunes within + # ~1s, but during that window it can tie wifi's own healthy base metric (also 100). + # Setting this at connection-creation time (vs. a live `nmcli modify`, which does NOT + # take effect on an already-active connection without a reactivation) closes the race + # for good. + networkmanager: + passthrough: + ipv4.never-default: "true" wifis: wlan0: renderer: NetworkManager @@ -34,6 +46,11 @@ network: # and DNS see one stable source address per uplink. dhcp6: true ipv6-privacy: false + # never-default: see the Starlink stanza above — same reasoning, applies to every + # van-failover-managed WAN. + networkmanager: + passthrough: + ipv4.never-default: "true" access-points: "Wapana": auth: diff --git a/failover/config.json b/failover/config.json index 1f1b2c5..42ff062 100644 --- a/failover/config.json +++ b/failover/config.json @@ -1,12 +1,10 @@ { - "probe_interval": 4, + "probe_interval": 60, "probe_timeout": 3, - "fail_threshold": 3, - "ok_threshold": 2, + "fail_threshold": 2, + "ok_threshold": 1, "probe_urls": [ - "http://connectivity-check.ubuntu.com/", - "http://www.gstatic.com/generate_204", - "http://cp.cloudflare.com/" + "http://connectivity-check.ubuntu.com/" ], "wans": [ { "name": "wifi", "device": "wlan0", "metric": 100 }, diff --git a/failover/van-failover b/failover/van-failover index ae76668..4127ecc 100755 --- a/failover/van-failover +++ b/failover/van-failover @@ -148,19 +148,38 @@ def enforce_route(dev, metric, conn=None): def nm_gateway(dev, conn): """NM's gateway — available even when the default route is missing. Try the device, - then the connection (a wwan netdev is not an NM device, but its connection is active).""" + then the connection (a wwan netdev is not an NM device, but its connection is active). + Falls back to the raw DHCP4 lease's `routers` option: with ipv4.never-default set (so NM + never installs its own competing default route — see set_profile_metric), NM also stops + populating IP4.GATEWAY, even though the DHCP-negotiated router is still known underneath.""" for kind, name in (("device", dev), ("connection", conn)): if name: r = sh(["nmcli", "-g", "IP4.GATEWAY", kind, "show", name]) if r and r.returncode == 0 and r.stdout.strip(): return r.stdout.strip() + if dev: + r = sh(["nmcli", "-g", "DHCP4.OPTION", "device", "show", dev]) + if r and r.returncode == 0: + for opt in r.stdout.split(" | "): + key, _, val = opt.strip().partition(" = ") + if key == "routers" and val: + return val.split()[0] return None def set_profile_metric(conn, metric): - """Update the NM profile's route-metric (no reapply) so NM re-assertions stay consistent.""" + """Update the NM profile's route-metric (no reapply) so NM re-assertions stay consistent. + Also pins never-default=yes: without it, NM's own DHCP client reinstalls its own default + route (its device-type default metric, e.g. 100 for ethernet) on every lease renewal, + racing the `ip route`-managed one enforce_route() maintains (observed every ~8s on a + Starlink dongle with a 16s DHCP lease). Like the metric change, this only takes effect on + this connection's *next* activation, not the currently-active one (a live `nmcli modify` + doesn't retroactively change an already-active connection's installed routes, and we + deliberately never `nmcli device reapply` — see enforce_route) — belt-and-suspenders for + connections van-failover doesn't own the netplan source for (e.g. cellular).""" if conn: - sh(["nmcli", "connection", "modify", conn, "ipv4.route-metric", str(metric)]) + sh(["nmcli", "connection", "modify", conn, + "ipv4.route-metric", str(metric), "ipv4.never-default", "yes"]) def read_prefer(): @@ -192,24 +211,36 @@ def main(): wans = cfg["wans"] # Optimistic start: assume up so healthy WANs immediately get their base metric. - rt = {w["name"]: {"up": True, "ok": ok_th, "fail": 0, "applied": None} for w in wans} + rt = {w["name"]: {"up": True, "ok": ok_th, "fail": 0, "applied": None, "healthy": None} for w in wans} + # Per-WAN independent probe clock: on the road every link can be flaky independently, so a + # struggling WAN retries on its own schedule instead of a shared round dragging healthy WANs + # into extra probes (metered cellular data) or throttling a failing one down to the slow + # steady-state cadence. 0.0 (epoch) means "due immediately" — probe everything on startup. + next_due = {w["name"]: 0.0 for w in wans} print(f"van-failover started: {[w['name'] for w in wans]}", flush=True) pool = ThreadPoolExecutor(max_workers=max(4, len(wans) * len(urls))) while True: + now = time.monotonic() actives = active_connections() resolved = {w["name"]: resolve(w, actives) for w in wans} prefer_dev = read_prefer() # A WAN is "present" (probeable/manageable) only with both an active device and connection. present = {w["name"]: bool(resolved[w["name"]][0]) and bool(resolved[w["name"]][1]) for w in wans} + due = [w for w in wans if present[w["name"]] and now >= next_due[w["name"]]] - # Probe every (present WAN x url) pair concurrently; a WAN is healthy if ANY url - # returns 204. Concurrency bounds a failed WAN to ~one timeout, not N serial timeouts. - health = {w["name"]: (False if present[w["name"]] else None) for w in wans} + # Probe only WANs that are due, one url-fan-out per WAN, concurrently. A WAN is healthy + # if ANY url returns 204. `finish` tracks each WAN's own completion time (not one shared + # round time) — needed below to tell an instant failure (DNS/route error, no time spent + # waiting) apart from a real probe_timeout-bound one. + health = {w["name"]: False for w in due} + finish = {} tasks = [(w["name"], pool.submit(probe_one, resolved[w["name"]][0], url, ptimeout)) - for w in wans if present[w["name"]] for url in urls] + for w in due for url in urls] for name, fut in tasks: - if fut.result(): + ok = fut.result() + finish[name] = max(finish.get(name, now), time.monotonic()) + if ok: health[name] = True for w in wans: @@ -220,22 +251,33 @@ def main(): base = PREFER_METRIC if prefer_dev and dev == prefer_dev else w["metric"] s = rt[name] if not present[name]: - # Absent (e.g. modem unplugged): don't probe/penalize; just keep the - # profile's base metric so it lands at the right priority when it connects. + # Absent (e.g. modem unplugged): don't probe/penalize; just keep the profile's + # base metric so it lands at the right priority when it connects, and probe it + # right away once it reappears. + next_due[name] = 0.0 if conn and s["applied"] != base: set_profile_metric(conn, base) s["applied"] = base continue - if health[name]: - s["ok"] += 1 - s["fail"] = 0 - if s["ok"] >= ok_th: - s["up"] = True - else: - s["fail"] += 1 - s["ok"] = 0 - if s["fail"] >= fail_th: - s["up"] = False + if name in finish: + s["healthy"] = health[name] + if health[name]: + s["ok"] += 1 + s["fail"] = 0 + if s["ok"] >= ok_th: + s["up"] = True + next_due[name] = now + interval + else: + s["fail"] += 1 + s["ok"] = 0 + if s["fail"] >= fail_th: + s["up"] = False + # Retry as soon as we've spent a full probe_timeout since this attempt + # started — immediately if the failure itself ate the whole timeout (a real + # timeout), otherwise topped up with a short wait. That's the early-out for + # instant failures (DNS/route errors that return in milliseconds): without + # the top-up, a permanently-unreachable WAN would retry in a tight loop. + next_due[name] = max(finish[name], now + ptimeout) desired = base if s["up"] else base + PENALTY # Enforce the live route every loop (carrier-safe, corrects any NM drift); # update the NM profile only on an actual state change. @@ -254,12 +296,15 @@ def main(): report.append({ "name": name, "priority": w["metric"], "device": dev, "connection": conn, "present": present[name], "up": rt[name]["up"] if present[name] else False, - "healthy": health[name], "preferred": dev is not None and dev == prefer_dev, + "healthy": rt[name]["healthy"] if present[name] else None, + "preferred": dev is not None and dev == prefer_dev, "route_metric": routes.get(dev), "active": dev is not None and dev == active_dev, }) write_state({"updated": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "active_device": active_dev, "wans": report}) - time.sleep(interval) + + upcoming = [next_due[w["name"]] for w in wans if present[w["name"]]] + time.sleep(max(0.0, min(upcoming) - time.monotonic()) if upcoming else interval) if __name__ == "__main__":