Andreas WredeandClaude Opus 4.8
4414967bdc
A (host, service) pair is flapping once it exceeds flap_count warning or critical notifications within flap_interval minutes. The notification that trips the state carries "Now flapping!! No more messages!" and every later one is dropped, including RECOVER. The state ends silently flap_interval minutes after a RECOVER, provided no further alert arrived meanwhile. Hooked into notify.send_notification, the single choke point for channel delivery, so only outbound notifications are suppressed — eventlog keeps recording, leaving the journal and /log with the full history of the flap. Threshold alerts key on their metric path, so a flapping disk check cannot silence a CPU alert; connectivity, boot and shutdown events key on the host itself. State lives at module level in flap.py and is never pickled. Flapping pairs surface in Host.stateinfo() and render as an amber badge on the live dashboard. Config: flap_count (5), flap_interval (10 minutes), 0 in either disables; both editable on the settings page. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
120 lines
3.8 KiB
Python
120 lines
3.8 KiB
Python
"""Flap detection: silence checks that toggle faster than they are useful.
|
|
|
|
A ``(host, service)`` pair is *flapping* once it exceeds ``flap_count``
|
|
WARNING/CRITICAL notifications within ``flap_interval`` minutes. The
|
|
notification that trips the threshold is delivered with ``FLAP_MARKER``
|
|
appended; every notification for that pair afterwards is dropped. The state
|
|
clears silently ``flap_interval`` minutes after a RECOVER, provided no further
|
|
WARNING/CRITICAL arrived in the meantime.
|
|
|
|
Only outbound notifications are affected — ``notify.eventlog`` keeps recording
|
|
every event, so the journal and ``/log`` retain the full history of the flap.
|
|
|
|
State lives here at module level rather than on ``Host`` so it is never
|
|
pickled: a restart starts every check with a clean slate.
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FLAP_MARKER = "Now flapping!! No more messages!"
|
|
|
|
# Actions returned by observe()
|
|
PASS = "pass" # deliver unchanged
|
|
TRIP = "trip" # deliver with FLAP_MARKER appended
|
|
SUPPRESS = "suppress" # drop
|
|
|
|
_count = 0 # flap_count: alerts within the window needed to trip
|
|
_window = 0.0 # flap_interval in seconds
|
|
|
|
# {(host, service): {"events": [ts, ...], "flapping": bool, "ok_since": float|None}}
|
|
_state: dict = {}
|
|
|
|
|
|
def setup(cfg) -> None:
|
|
"""Read flap_count / flap_interval from *cfg* (also called on reload)."""
|
|
global _count, _window
|
|
_count = int(cfg.get("flap_count", 0) or 0)
|
|
_window = float(cfg.get("flap_interval", 0) or 0) * 60.0
|
|
|
|
|
|
def _enabled() -> bool:
|
|
return _count > 0 and _window > 0
|
|
|
|
|
|
def _label(key) -> str:
|
|
host, service = key
|
|
return f"{host}/{service}" if service else host
|
|
|
|
|
|
def _sweep(st: dict, now: float) -> bool:
|
|
"""Clear the flapping flag once the post-RECOVER quiet window has elapsed."""
|
|
if st["flapping"] and st["ok_since"] is not None and now - st["ok_since"] >= _window:
|
|
st["flapping"] = False
|
|
st["ok_since"] = None
|
|
st["events"].clear()
|
|
return True
|
|
return False
|
|
|
|
|
|
def observe(host: str, service: str, level: str) -> str:
|
|
"""Record a notification for *host*/*service* and return what to do with it.
|
|
|
|
Returns PASS, TRIP or SUPPRESS.
|
|
"""
|
|
if not _enabled():
|
|
return PASS
|
|
|
|
now = time.time()
|
|
key = (host or "", service or "")
|
|
level = (level or "").upper()
|
|
st = _state.get(key)
|
|
if st is not None and _sweep(st, now):
|
|
logger.info("flapping cleared for %s", _label(key))
|
|
|
|
if level in ("WARNING", "CRITICAL"):
|
|
if st is None:
|
|
st = _state[key] = {"events": [], "flapping": False, "ok_since": None}
|
|
# An alert inside the quiet window means it never settled.
|
|
st["ok_since"] = None
|
|
st["events"] = [t for t in st["events"] if now - t < _window]
|
|
st["events"].append(now)
|
|
if st["flapping"]:
|
|
return SUPPRESS
|
|
if len(st["events"]) > _count:
|
|
st["flapping"] = True
|
|
logger.info(
|
|
"flapping detected for %s (%d alerts in %.0f min)",
|
|
_label(key), len(st["events"]), _window / 60,
|
|
)
|
|
return TRIP
|
|
return PASS
|
|
|
|
if st is None or not st["flapping"]:
|
|
return PASS
|
|
if level == "RECOVER":
|
|
# Starts the quiet window; the state ends silently when it elapses.
|
|
st["ok_since"] = now
|
|
return SUPPRESS
|
|
|
|
|
|
def flapping_services(host: str) -> list:
|
|
"""Return the services of *host* that are currently flapping.
|
|
|
|
An empty string in the result means the host itself (connectivity, boot,
|
|
shutdown) rather than a named service.
|
|
"""
|
|
if not _enabled():
|
|
return []
|
|
now = time.time()
|
|
flapping = []
|
|
for (h, service), st in _state.items():
|
|
if h != host:
|
|
continue
|
|
_sweep(st, now)
|
|
if st["flapping"]:
|
|
flapping.append(service)
|
|
return sorted(flapping)
|