andreasandClaude Sonnet 5
a7b698e6b9
flap._state persists at module level keyed by (host, service) and only clears via a RECOVER-triggered quiet window. A host dropped mid-flap with no RECOVER ever received leaves ok_since permanently None, so the flapping flag could never clear on its own. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
131 lines
4.2 KiB
Python
131 lines
4.2 KiB
Python
"""Flap detection: silence checks that toggle faster than they are useful.
|
|
|
|
A ``(host, service)`` pair is *flapping* once it exceeds ``flap_count``
|
|
WARNING/CRITICAL notifications within ``flap_interval`` minutes. The
|
|
notification that trips the threshold is delivered with ``FLAP_MARKER``
|
|
appended; every notification for that pair afterwards is dropped. The state
|
|
clears silently ``flap_interval`` minutes after a RECOVER, provided no further
|
|
WARNING/CRITICAL arrived in the meantime.
|
|
|
|
Only outbound notifications are affected — ``notify.eventlog`` keeps recording
|
|
every event, so the journal and ``/log`` retain the full history of the flap.
|
|
|
|
State lives here at module level rather than on ``Host`` so it is never
|
|
pickled: a restart starts every check with a clean slate.
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FLAP_MARKER = "Now flapping!! No more messages!"
|
|
|
|
# Actions returned by observe()
|
|
PASS = "pass" # deliver unchanged
|
|
TRIP = "trip" # deliver with FLAP_MARKER appended
|
|
SUPPRESS = "suppress" # drop
|
|
|
|
_count = 0 # flap_count: alerts within the window needed to trip
|
|
_window = 0.0 # flap_interval in seconds
|
|
|
|
# {(host, service): {"events": [ts, ...], "flapping": bool, "ok_since": float|None}}
|
|
_state: dict = {}
|
|
|
|
|
|
def setup(cfg) -> None:
|
|
"""Read flap_count / flap_interval from *cfg* (also called on reload)."""
|
|
global _count, _window
|
|
_count = int(cfg.get("flap_count", 0) or 0)
|
|
_window = float(cfg.get("flap_interval", 0) or 0) * 60.0
|
|
|
|
|
|
def _enabled() -> bool:
|
|
return _count > 0 and _window > 0
|
|
|
|
|
|
def _label(key) -> str:
|
|
host, service = key
|
|
return f"{host}/{service}" if service else host
|
|
|
|
|
|
def _sweep(st: dict, now: float) -> bool:
|
|
"""Clear the flapping flag once the post-RECOVER quiet window has elapsed."""
|
|
if st["flapping"] and st["ok_since"] is not None and now - st["ok_since"] >= _window:
|
|
st["flapping"] = False
|
|
st["ok_since"] = None
|
|
st["events"].clear()
|
|
return True
|
|
return False
|
|
|
|
|
|
def observe(host: str, service: str, level: str) -> str:
|
|
"""Record a notification for *host*/*service* and return what to do with it.
|
|
|
|
Returns PASS, TRIP or SUPPRESS.
|
|
"""
|
|
if not _enabled():
|
|
return PASS
|
|
|
|
now = time.time()
|
|
key = (host or "", service or "")
|
|
level = (level or "").upper()
|
|
st = _state.get(key)
|
|
if st is not None and _sweep(st, now):
|
|
logger.info("flapping cleared for %s", _label(key))
|
|
|
|
if level in ("WARNING", "CRITICAL"):
|
|
if st is None:
|
|
st = _state[key] = {"events": [], "flapping": False, "ok_since": None}
|
|
# An alert inside the quiet window means it never settled.
|
|
st["ok_since"] = None
|
|
st["events"] = [t for t in st["events"] if now - t < _window]
|
|
st["events"].append(now)
|
|
if st["flapping"]:
|
|
return SUPPRESS
|
|
if len(st["events"]) > _count:
|
|
st["flapping"] = True
|
|
logger.info(
|
|
"flapping detected for %s (%d alerts in %.0f min)",
|
|
_label(key), len(st["events"]), _window / 60,
|
|
)
|
|
return TRIP
|
|
return PASS
|
|
|
|
if st is None or not st["flapping"]:
|
|
return PASS
|
|
if level == "RECOVER":
|
|
# Starts the quiet window; the state ends silently when it elapses.
|
|
st["ok_since"] = now
|
|
return SUPPRESS
|
|
|
|
|
|
def clear_host(host: str) -> None:
|
|
"""Discard all flap state for *host* (called when a host is dropped).
|
|
|
|
A dropped host may be mid-flap with no RECOVER ever received, in which
|
|
case ``ok_since`` stays ``None`` and ``_sweep`` can never clear it on its
|
|
own — the state would otherwise persist forever.
|
|
"""
|
|
for key in [k for k in _state if k[0] == host]:
|
|
del _state[key]
|
|
|
|
|
|
def flapping_services(host: str) -> list:
|
|
"""Return the services of *host* that are currently flapping.
|
|
|
|
An empty string in the result means the host itself (connectivity, boot,
|
|
shutdown) rather than a named service.
|
|
"""
|
|
if not _enabled():
|
|
return []
|
|
now = time.time()
|
|
flapping = []
|
|
for (h, service), st in _state.items():
|
|
if h != host:
|
|
continue
|
|
_sweep(st, now)
|
|
if st["flapping"]:
|
|
flapping.append(service)
|
|
return sorted(flapping)
|