fix: never park the last present family as absent
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uc9V1v7iXruubpoLiSE24M
This commit is contained in:
+1
-1
@@ -626,7 +626,6 @@ async def async_main(args, config):
|
||||
loop.add_signal_handler(signal.SIGHUP, _sighup)
|
||||
|
||||
# Start async tasks
|
||||
# Heartbeat senders (one per connection)
|
||||
# Per-protocol internet presence probe (shared with the network_monitor plugin)
|
||||
net_cfg = dict((config.get("plugins") or {}).get("network_monitor")
|
||||
or config.get("network_monitor") or {})
|
||||
@@ -636,6 +635,7 @@ async def async_main(args, config):
|
||||
set_shared(connectivity)
|
||||
active_tasks.append(asyncio.create_task(connectivity.run(shutdown_event)))
|
||||
|
||||
# Heartbeat senders (one per connection)
|
||||
for conn in connections:
|
||||
task = asyncio.create_task(heartbeat_sender(conn, interval, connectivity))
|
||||
active_tasks.append(task)
|
||||
|
||||
@@ -485,12 +485,18 @@ class Host:
|
||||
"""Host-level status: down, overdue, simplex, up or unknown.
|
||||
|
||||
overdue: every present family is overdue. simplex: both present, one up, one overdue
|
||||
(same severity as up). Families the client reports absent are not considered.
|
||||
(same severity as up). Families the client reports absent are not considered,
|
||||
unless none is present: then every connection not parked as ABSENT is considered.
|
||||
"""
|
||||
conns = self.connections
|
||||
if any(c.state == Connection.DOWN for c in conns.values()):
|
||||
return "down"
|
||||
present = [c for afam, c in conns.items() if self.family_present(afam)]
|
||||
if not present:
|
||||
# The client reports no protocol present (e.g. its internet probe fails on every
|
||||
# family) yet may still be heard: monitor whatever is not parked as absent, so a
|
||||
# host that then dies is "overdue", not silently "unknown".
|
||||
present = [c for c in conns.values() if c.state != Connection.ABSENT]
|
||||
up = sum(c.state == Connection.UP for c in present)
|
||||
overdue = sum(c.state == Connection.OVERDUE for c in present)
|
||||
if up and overdue:
|
||||
|
||||
+7
-1
@@ -249,7 +249,11 @@ def _make_timer_callbacks(uname, host, ctx):
|
||||
return
|
||||
now = time.time()
|
||||
afam = connection.afam
|
||||
if not host.flag_present(afam):
|
||||
# Park only while some other family is still present: the last monitored family
|
||||
# must never be parked silently, or a host whose client reports no internet on any
|
||||
# protocol would die without an alert.
|
||||
if not host.flag_present(afam) and any(
|
||||
af != afam and host.family_present(af) for af in host.connections):
|
||||
# The client says this protocol has no connectivity: not an outage.
|
||||
connection.newstate(cls.ABSENT, now)
|
||||
connection.cancel_overdue_timer()
|
||||
@@ -258,6 +262,8 @@ def _make_timer_callbacks(uname, host, ctx):
|
||||
msg_to_websockets("host", host.stateinfo())
|
||||
return
|
||||
|
||||
# If any connection is DOWN (clean shutdown reported) Host.status() is "down", so a
|
||||
# later overdue of the other family raises no separate event/alert; the shutdown did.
|
||||
prev = host.status()
|
||||
connection.newstate(cls.OVERDUE, now, cfg.get("grace", 2))
|
||||
new = host.status()
|
||||
|
||||
@@ -92,3 +92,29 @@ def test_presence_survives_pickle_and_old_pickles_get_default():
|
||||
del h2.__dict__["presence"] # simulate a pre-feature pickle
|
||||
h2.fixup()
|
||||
assert h2.presence == {}
|
||||
|
||||
|
||||
def _all_absent(name, s4, s6=None):
|
||||
h = _host(name)
|
||||
_conn(h, "IPv4", s4)
|
||||
h.set_presence("IPv4", False, time.time())
|
||||
if s6 is not None:
|
||||
_conn(h, "IPv6", s6)
|
||||
h.set_presence("IPv6", False, time.time())
|
||||
return h
|
||||
|
||||
|
||||
def test_all_absent_single_family_overdue_is_overdue():
|
||||
assert _all_absent("abs-single", C.OVERDUE).status() == "overdue"
|
||||
|
||||
|
||||
def test_all_absent_dual_stack_one_absent_one_overdue_is_overdue():
|
||||
assert _all_absent("abs-dual-od", C.ABSENT, C.OVERDUE).status() == "overdue"
|
||||
|
||||
|
||||
def test_all_absent_dual_stack_v6_up_is_up():
|
||||
assert _all_absent("abs-dual-up", C.ABSENT, C.UP).status() == "up"
|
||||
|
||||
|
||||
def test_all_absent_all_unknown_is_unknown():
|
||||
assert _all_absent("abs-unknown", C.UNKNOWN, C.UNKNOWN).status() == "unknown"
|
||||
|
||||
@@ -217,3 +217,34 @@ def test_v4_only_host_overdue_is_single_critical(monkeypatch):
|
||||
assert "connectivity.IPv4" in h.alert_states
|
||||
asyncio.run(on_overdue(h.connections["IPv4"]))
|
||||
assert len([e for e in events if e[0] == "CRITICAL"]) == 1
|
||||
|
||||
|
||||
def test_all_families_absent_v4_only_host_still_alerts_when_it_dies(monkeypatch):
|
||||
events = _events(monkeypatch)
|
||||
ctx = _ctx()
|
||||
_fresh("pres-allabsent")
|
||||
_send("pres-allabsent", "127.0.0.1", ctx, p4=0)
|
||||
h = hbdclass.Host.hosts["pres-allabsent"]
|
||||
on_overdue, _ = udp._make_timer_callbacks("pres-allabsent", h, ctx)
|
||||
asyncio.run(on_overdue(h.connections["IPv4"]))
|
||||
assert [e for e in events if e[0] == "CRITICAL"] == [("CRITICAL", "IPv4 overdue")]
|
||||
assert h.connections["IPv4"].state == C.OVERDUE
|
||||
assert h.status() == "overdue"
|
||||
assert "connectivity.IPv4" in h.alert_states
|
||||
|
||||
|
||||
def test_last_present_family_is_not_parked_when_other_is_absent(monkeypatch):
|
||||
events = _events(monkeypatch)
|
||||
ctx = _ctx()
|
||||
h = _dual_stack("pres-lastfam", ctx)
|
||||
_send("pres-lastfam", "127.0.0.1", ctx, p4=0, p6=0)
|
||||
events.clear()
|
||||
on_overdue, _ = udp._make_timer_callbacks("pres-lastfam", h, ctx)
|
||||
asyncio.run(on_overdue(h.connections["IPv4"]))
|
||||
assert h.connections["IPv4"].state == C.ABSENT
|
||||
assert not any(level == "CRITICAL" for level, _ in events)
|
||||
assert h.status() == "up"
|
||||
asyncio.run(on_overdue(h.connections["IPv6"]))
|
||||
assert [e for e in events if e[0] == "CRITICAL"] == [("CRITICAL", "IPv6 overdue")]
|
||||
assert h.connections["IPv6"].state == C.OVERDUE
|
||||
assert h.status() == "overdue"
|
||||
|
||||
Reference in New Issue
Block a user