fix: clear flap state when a host is dropped

flap._state persists at module level keyed by (host, service) and only
clears via a RECOVER-triggered quiet window. A host dropped mid-flap with
no RECOVER ever received leaves ok_since permanently None, so the
flapping flag could never clear on its own.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-07 08:35:10 -04:00
co-authored by Claude Sonnet 5
parent 136b10239f
commit a7b698e6b9
3 changed files with 33 additions and 0 deletions
+11
View File
@@ -100,6 +100,17 @@ def observe(host: str, service: str, level: str) -> str:
return SUPPRESS return SUPPRESS
def clear_host(host: str) -> None:
"""Discard all flap state for *host* (called when a host is dropped).
A dropped host may be mid-flap with no RECOVER ever received, in which
case ``ok_since`` stays ``None`` and ``_sweep`` can never clear it on its
own — the state would otherwise persist forever.
"""
for key in [k for k in _state if k[0] == host]:
del _state[key]
def flapping_services(host: str) -> list: def flapping_services(host: str) -> list:
"""Return the services of *host* that are currently flapping. """Return the services of *host* that are currently flapping.
+2
View File
@@ -14,6 +14,7 @@ import logging
from aiohttp import web from aiohttp import web
import jinja2 import jinja2
from . import data from . import data
from . import flap as flap_mod
from . import notify as notify_mod from . import notify as notify_mod
from . import settings as settings_mod from . import settings as settings_mod
from . import users as users_mod from . import users as users_mod
@@ -439,6 +440,7 @@ async def start(
return web.json_response({"error": "Forbidden"}, status=403) return web.json_response({"error": "Forbidden"}, status=403)
eventlog(uname, "INFO", "dropped") eventlog(uname, "INFO", "dropped")
del hbdclass.Host.hosts[uname] del hbdclass.Host.hosts[uname]
flap_mod.clear_host(uname)
return web.Response(text="Done") return web.Response(text="Done")
async def register(request): async def register(request):
+20
View File
@@ -98,6 +98,26 @@ def test_alert_during_the_quiet_window_keeps_it_flapping():
assert flap.flapping_services("h1") == ["cpu"] assert flap.flapping_services("h1") == ["cpu"]
# --- host removal ------------------------------------------------------------
def test_clear_host_drops_flapping_state_even_without_a_recover():
# A host dropped mid-flap (no RECOVER ever received) must not stay
# flapping forever: nothing will ever set ok_since for it again.
alerts(4)
assert flap.flapping_services("h1") == ["cpu"]
flap.clear_host("h1")
assert flap.flapping_services("h1") == []
assert flap.observe("h1", "cpu", "CRITICAL") == flap.PASS
def test_clear_host_only_affects_the_named_host():
alerts(4, host="h1")
alerts(4, host="h2")
flap.clear_host("h1")
assert flap.flapping_services("h1") == []
assert flap.flapping_services("h2") == ["cpu"]
# --- keying ----------------------------------------------------------------- # --- keying -----------------------------------------------------------------
def test_services_and_hosts_are_tracked_independently(): def test_services_and_hosts_are_tracked_independently():