fix: never park the last present family as absent

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uc9V1v7iXruubpoLiSE24M
This commit is contained in:
2026-09-21 09:50:43 -04:00
co-authored by Claude Sonnet 5
parent 8122c8d7dc
commit 95355374f8
5 changed files with 72 additions and 3 deletions
+1 -1
View File
@@ -626,7 +626,6 @@ async def async_main(args, config):
loop.add_signal_handler(signal.SIGHUP, _sighup) loop.add_signal_handler(signal.SIGHUP, _sighup)
# Start async tasks # Start async tasks
# Heartbeat senders (one per connection)
# Per-protocol internet presence probe (shared with the network_monitor plugin) # Per-protocol internet presence probe (shared with the network_monitor plugin)
net_cfg = dict((config.get("plugins") or {}).get("network_monitor") net_cfg = dict((config.get("plugins") or {}).get("network_monitor")
or config.get("network_monitor") or {}) or config.get("network_monitor") or {})
@@ -636,6 +635,7 @@ async def async_main(args, config):
set_shared(connectivity) set_shared(connectivity)
active_tasks.append(asyncio.create_task(connectivity.run(shutdown_event))) active_tasks.append(asyncio.create_task(connectivity.run(shutdown_event)))
# Heartbeat senders (one per connection)
for conn in connections: for conn in connections:
task = asyncio.create_task(heartbeat_sender(conn, interval, connectivity)) task = asyncio.create_task(heartbeat_sender(conn, interval, connectivity))
active_tasks.append(task) active_tasks.append(task)
+7 -1
View File
@@ -485,12 +485,18 @@ class Host:
"""Host-level status: down, overdue, simplex, up or unknown. """Host-level status: down, overdue, simplex, up or unknown.
overdue: every present family is overdue. simplex: both present, one up, one overdue overdue: every present family is overdue. simplex: both present, one up, one overdue
(same severity as up). Families the client reports absent are not considered. (same severity as up). Families the client reports absent are not considered,
unless none is present: then every connection not parked as ABSENT is considered.
""" """
conns = self.connections conns = self.connections
if any(c.state == Connection.DOWN for c in conns.values()): if any(c.state == Connection.DOWN for c in conns.values()):
return "down" return "down"
present = [c for afam, c in conns.items() if self.family_present(afam)] present = [c for afam, c in conns.items() if self.family_present(afam)]
if not present:
# The client reports no protocol present (e.g. its internet probe fails on every
# family) yet may still be heard: monitor whatever is not parked as absent, so a
# host that then dies is "overdue", not silently "unknown".
present = [c for c in conns.values() if c.state != Connection.ABSENT]
up = sum(c.state == Connection.UP for c in present) up = sum(c.state == Connection.UP for c in present)
overdue = sum(c.state == Connection.OVERDUE for c in present) overdue = sum(c.state == Connection.OVERDUE for c in present)
if up and overdue: if up and overdue:
+7 -1
View File
@@ -249,7 +249,11 @@ def _make_timer_callbacks(uname, host, ctx):
return return
now = time.time() now = time.time()
afam = connection.afam afam = connection.afam
if not host.flag_present(afam): # Park only while some other family is still present: the last monitored family
# must never be parked silently, or a host whose client reports no internet on any
# protocol would die without an alert.
if not host.flag_present(afam) and any(
af != afam and host.family_present(af) for af in host.connections):
# The client says this protocol has no connectivity: not an outage. # The client says this protocol has no connectivity: not an outage.
connection.newstate(cls.ABSENT, now) connection.newstate(cls.ABSENT, now)
connection.cancel_overdue_timer() connection.cancel_overdue_timer()
@@ -258,6 +262,8 @@ def _make_timer_callbacks(uname, host, ctx):
msg_to_websockets("host", host.stateinfo()) msg_to_websockets("host", host.stateinfo())
return return
# If any connection is DOWN (clean shutdown reported) Host.status() is "down", so a
# later overdue of the other family raises no separate event/alert; the shutdown did.
prev = host.status() prev = host.status()
connection.newstate(cls.OVERDUE, now, cfg.get("grace", 2)) connection.newstate(cls.OVERDUE, now, cfg.get("grace", 2))
new = host.status() new = host.status()
+26
View File
@@ -92,3 +92,29 @@ def test_presence_survives_pickle_and_old_pickles_get_default():
del h2.__dict__["presence"] # simulate a pre-feature pickle del h2.__dict__["presence"] # simulate a pre-feature pickle
h2.fixup() h2.fixup()
assert h2.presence == {} assert h2.presence == {}
def _all_absent(name, s4, s6=None):
h = _host(name)
_conn(h, "IPv4", s4)
h.set_presence("IPv4", False, time.time())
if s6 is not None:
_conn(h, "IPv6", s6)
h.set_presence("IPv6", False, time.time())
return h
def test_all_absent_single_family_overdue_is_overdue():
assert _all_absent("abs-single", C.OVERDUE).status() == "overdue"
def test_all_absent_dual_stack_one_absent_one_overdue_is_overdue():
assert _all_absent("abs-dual-od", C.ABSENT, C.OVERDUE).status() == "overdue"
def test_all_absent_dual_stack_v6_up_is_up():
assert _all_absent("abs-dual-up", C.ABSENT, C.UP).status() == "up"
def test_all_absent_all_unknown_is_unknown():
assert _all_absent("abs-unknown", C.UNKNOWN, C.UNKNOWN).status() == "unknown"
+31
View File
@@ -217,3 +217,34 @@ def test_v4_only_host_overdue_is_single_critical(monkeypatch):
assert "connectivity.IPv4" in h.alert_states assert "connectivity.IPv4" in h.alert_states
asyncio.run(on_overdue(h.connections["IPv4"])) asyncio.run(on_overdue(h.connections["IPv4"]))
assert len([e for e in events if e[0] == "CRITICAL"]) == 1 assert len([e for e in events if e[0] == "CRITICAL"]) == 1
def test_all_families_absent_v4_only_host_still_alerts_when_it_dies(monkeypatch):
events = _events(monkeypatch)
ctx = _ctx()
_fresh("pres-allabsent")
_send("pres-allabsent", "127.0.0.1", ctx, p4=0)
h = hbdclass.Host.hosts["pres-allabsent"]
on_overdue, _ = udp._make_timer_callbacks("pres-allabsent", h, ctx)
asyncio.run(on_overdue(h.connections["IPv4"]))
assert [e for e in events if e[0] == "CRITICAL"] == [("CRITICAL", "IPv4 overdue")]
assert h.connections["IPv4"].state == C.OVERDUE
assert h.status() == "overdue"
assert "connectivity.IPv4" in h.alert_states
def test_last_present_family_is_not_parked_when_other_is_absent(monkeypatch):
events = _events(monkeypatch)
ctx = _ctx()
h = _dual_stack("pres-lastfam", ctx)
_send("pres-lastfam", "127.0.0.1", ctx, p4=0, p6=0)
events.clear()
on_overdue, _ = udp._make_timer_callbacks("pres-lastfam", h, ctx)
asyncio.run(on_overdue(h.connections["IPv4"]))
assert h.connections["IPv4"].state == C.ABSENT
assert not any(level == "CRITICAL" for level, _ in events)
assert h.status() == "up"
asyncio.run(on_overdue(h.connections["IPv6"]))
assert [e for e in events if e[0] == "CRITICAL"] == [("CRITICAL", "IPv6 overdue")]
assert h.connections["IPv6"].state == C.OVERDUE
assert h.status() == "overdue"