From 95355374f8d11f8c56704ecec590e8ac067296ab Mon Sep 17 00:00:00 2001 From: Andreas Wrede Date: Mon, 21 Sep 2026 09:50:43 -0400 Subject: [PATCH] fix: never park the last present family as absent Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Uc9V1v7iXruubpoLiSE24M --- hbd/client/main.py | 2 +- hbd/server/hbdclass.py | 8 +++++++- hbd/server/udp.py | 8 +++++++- tests/test_host_status.py | 26 ++++++++++++++++++++++++++ tests/test_udp_presence.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 3 deletions(-) diff --git a/hbd/client/main.py b/hbd/client/main.py index e883cb4..011e286 100644 --- a/hbd/client/main.py +++ b/hbd/client/main.py @@ -626,7 +626,6 @@ async def async_main(args, config): loop.add_signal_handler(signal.SIGHUP, _sighup) # Start async tasks - # Heartbeat senders (one per connection) # Per-protocol internet presence probe (shared with the network_monitor plugin) net_cfg = dict((config.get("plugins") or {}).get("network_monitor") or config.get("network_monitor") or {}) @@ -636,6 +635,7 @@ async def async_main(args, config): set_shared(connectivity) active_tasks.append(asyncio.create_task(connectivity.run(shutdown_event))) + # Heartbeat senders (one per connection) for conn in connections: task = asyncio.create_task(heartbeat_sender(conn, interval, connectivity)) active_tasks.append(task) diff --git a/hbd/server/hbdclass.py b/hbd/server/hbdclass.py index 715008b..0211c4f 100644 --- a/hbd/server/hbdclass.py +++ b/hbd/server/hbdclass.py @@ -485,12 +485,18 @@ class Host: """Host-level status: down, overdue, simplex, up or unknown. overdue: every present family is overdue. simplex: both present, one up, one overdue - (same severity as up). Families the client reports absent are not considered. + (same severity as up). Families the client reports absent are not considered, + unless none is present: then every connection not parked as ABSENT is considered. """ conns = self.connections if any(c.state == Connection.DOWN for c in conns.values()): return "down" present = [c for afam, c in conns.items() if self.family_present(afam)] + if not present: + # The client reports no protocol present (e.g. its internet probe fails on every + # family) yet may still be heard: monitor whatever is not parked as absent, so a + # host that then dies is "overdue", not silently "unknown". + present = [c for c in conns.values() if c.state != Connection.ABSENT] up = sum(c.state == Connection.UP for c in present) overdue = sum(c.state == Connection.OVERDUE for c in present) if up and overdue: diff --git a/hbd/server/udp.py b/hbd/server/udp.py index b7abf24..e6034a4 100644 --- a/hbd/server/udp.py +++ b/hbd/server/udp.py @@ -249,7 +249,11 @@ def _make_timer_callbacks(uname, host, ctx): return now = time.time() afam = connection.afam - if not host.flag_present(afam): + # Park only while some other family is still present: the last monitored family + # must never be parked silently, or a host whose client reports no internet on any + # protocol would die without an alert. + if not host.flag_present(afam) and any( + af != afam and host.family_present(af) for af in host.connections): # The client says this protocol has no connectivity: not an outage. connection.newstate(cls.ABSENT, now) connection.cancel_overdue_timer() @@ -258,6 +262,8 @@ def _make_timer_callbacks(uname, host, ctx): msg_to_websockets("host", host.stateinfo()) return + # If any connection is DOWN (clean shutdown reported) Host.status() is "down", so a + # later overdue of the other family raises no separate event/alert; the shutdown did. prev = host.status() connection.newstate(cls.OVERDUE, now, cfg.get("grace", 2)) new = host.status() diff --git a/tests/test_host_status.py b/tests/test_host_status.py index 35d11cf..dafda5d 100644 --- a/tests/test_host_status.py +++ b/tests/test_host_status.py @@ -92,3 +92,29 @@ def test_presence_survives_pickle_and_old_pickles_get_default(): del h2.__dict__["presence"] # simulate a pre-feature pickle h2.fixup() assert h2.presence == {} + + +def _all_absent(name, s4, s6=None): + h = _host(name) + _conn(h, "IPv4", s4) + h.set_presence("IPv4", False, time.time()) + if s6 is not None: + _conn(h, "IPv6", s6) + h.set_presence("IPv6", False, time.time()) + return h + + +def test_all_absent_single_family_overdue_is_overdue(): + assert _all_absent("abs-single", C.OVERDUE).status() == "overdue" + + +def test_all_absent_dual_stack_one_absent_one_overdue_is_overdue(): + assert _all_absent("abs-dual-od", C.ABSENT, C.OVERDUE).status() == "overdue" + + +def test_all_absent_dual_stack_v6_up_is_up(): + assert _all_absent("abs-dual-up", C.ABSENT, C.UP).status() == "up" + + +def test_all_absent_all_unknown_is_unknown(): + assert _all_absent("abs-unknown", C.UNKNOWN, C.UNKNOWN).status() == "unknown" diff --git a/tests/test_udp_presence.py b/tests/test_udp_presence.py index 060dc86..d200b65 100644 --- a/tests/test_udp_presence.py +++ b/tests/test_udp_presence.py @@ -217,3 +217,34 @@ def test_v4_only_host_overdue_is_single_critical(monkeypatch): assert "connectivity.IPv4" in h.alert_states asyncio.run(on_overdue(h.connections["IPv4"])) assert len([e for e in events if e[0] == "CRITICAL"]) == 1 + + +def test_all_families_absent_v4_only_host_still_alerts_when_it_dies(monkeypatch): + events = _events(monkeypatch) + ctx = _ctx() + _fresh("pres-allabsent") + _send("pres-allabsent", "127.0.0.1", ctx, p4=0) + h = hbdclass.Host.hosts["pres-allabsent"] + on_overdue, _ = udp._make_timer_callbacks("pres-allabsent", h, ctx) + asyncio.run(on_overdue(h.connections["IPv4"])) + assert [e for e in events if e[0] == "CRITICAL"] == [("CRITICAL", "IPv4 overdue")] + assert h.connections["IPv4"].state == C.OVERDUE + assert h.status() == "overdue" + assert "connectivity.IPv4" in h.alert_states + + +def test_last_present_family_is_not_parked_when_other_is_absent(monkeypatch): + events = _events(monkeypatch) + ctx = _ctx() + h = _dual_stack("pres-lastfam", ctx) + _send("pres-lastfam", "127.0.0.1", ctx, p4=0, p6=0) + events.clear() + on_overdue, _ = udp._make_timer_callbacks("pres-lastfam", h, ctx) + asyncio.run(on_overdue(h.connections["IPv4"])) + assert h.connections["IPv4"].state == C.ABSENT + assert not any(level == "CRITICAL" for level, _ in events) + assert h.status() == "up" + asyncio.run(on_overdue(h.connections["IPv6"])) + assert [e for e in events if e[0] == "CRITICAL"] == [("CRITICAL", "IPv6 overdue")] + assert h.connections["IPv6"].state == C.OVERDUE + assert h.status() == "overdue"