fix: clear flap state when a host is dropped
flap._state persists at module level keyed by (host, service) and only clears via a RECOVER-triggered quiet window. A host dropped mid-flap with no RECOVER ever received leaves ok_since permanently None, so the flapping flag could never clear on its own. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -100,6 +100,17 @@ def observe(host: str, service: str, level: str) -> str:
|
|||||||
return SUPPRESS
|
return SUPPRESS
|
||||||
|
|
||||||
|
|
||||||
|
def clear_host(host: str) -> None:
|
||||||
|
"""Discard all flap state for *host* (called when a host is dropped).
|
||||||
|
|
||||||
|
A dropped host may be mid-flap with no RECOVER ever received, in which
|
||||||
|
case ``ok_since`` stays ``None`` and ``_sweep`` can never clear it on its
|
||||||
|
own — the state would otherwise persist forever.
|
||||||
|
"""
|
||||||
|
for key in [k for k in _state if k[0] == host]:
|
||||||
|
del _state[key]
|
||||||
|
|
||||||
|
|
||||||
def flapping_services(host: str) -> list:
|
def flapping_services(host: str) -> list:
|
||||||
"""Return the services of *host* that are currently flapping.
|
"""Return the services of *host* that are currently flapping.
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import logging
|
|||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
import jinja2
|
import jinja2
|
||||||
from . import data
|
from . import data
|
||||||
|
from . import flap as flap_mod
|
||||||
from . import notify as notify_mod
|
from . import notify as notify_mod
|
||||||
from . import settings as settings_mod
|
from . import settings as settings_mod
|
||||||
from . import users as users_mod
|
from . import users as users_mod
|
||||||
@@ -439,6 +440,7 @@ async def start(
|
|||||||
return web.json_response({"error": "Forbidden"}, status=403)
|
return web.json_response({"error": "Forbidden"}, status=403)
|
||||||
eventlog(uname, "INFO", "dropped")
|
eventlog(uname, "INFO", "dropped")
|
||||||
del hbdclass.Host.hosts[uname]
|
del hbdclass.Host.hosts[uname]
|
||||||
|
flap_mod.clear_host(uname)
|
||||||
return web.Response(text="Done")
|
return web.Response(text="Done")
|
||||||
|
|
||||||
async def register(request):
|
async def register(request):
|
||||||
|
|||||||
@@ -98,6 +98,26 @@ def test_alert_during_the_quiet_window_keeps_it_flapping():
|
|||||||
assert flap.flapping_services("h1") == ["cpu"]
|
assert flap.flapping_services("h1") == ["cpu"]
|
||||||
|
|
||||||
|
|
||||||
|
# --- host removal ------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_clear_host_drops_flapping_state_even_without_a_recover():
|
||||||
|
# A host dropped mid-flap (no RECOVER ever received) must not stay
|
||||||
|
# flapping forever: nothing will ever set ok_since for it again.
|
||||||
|
alerts(4)
|
||||||
|
assert flap.flapping_services("h1") == ["cpu"]
|
||||||
|
flap.clear_host("h1")
|
||||||
|
assert flap.flapping_services("h1") == []
|
||||||
|
assert flap.observe("h1", "cpu", "CRITICAL") == flap.PASS
|
||||||
|
|
||||||
|
|
||||||
|
def test_clear_host_only_affects_the_named_host():
|
||||||
|
alerts(4, host="h1")
|
||||||
|
alerts(4, host="h2")
|
||||||
|
flap.clear_host("h1")
|
||||||
|
assert flap.flapping_services("h1") == []
|
||||||
|
assert flap.flapping_services("h2") == ["cpu"]
|
||||||
|
|
||||||
|
|
||||||
# --- keying -----------------------------------------------------------------
|
# --- keying -----------------------------------------------------------------
|
||||||
|
|
||||||
def test_services_and_hosts_are_tracked_independently():
|
def test_services_and_hosts_are_tracked_independently():
|
||||||
|
|||||||
Reference in New Issue
Block a user