"""Flap detection: silence checks that toggle faster than they are useful. A ``(host, service)`` pair is *flapping* once it exceeds ``flap_count`` WARNING/CRITICAL notifications within ``flap_interval`` minutes. The notification that trips the threshold is delivered with ``FLAP_MARKER`` appended; every notification for that pair afterwards is dropped. The state clears silently ``flap_interval`` minutes after a RECOVER, provided no further WARNING/CRITICAL arrived in the meantime. Only outbound notifications are affected — ``notify.eventlog`` keeps recording every event, so the journal and ``/log`` retain the full history of the flap. State lives here at module level rather than on ``Host`` so it is never pickled: a restart starts every check with a clean slate. """ import logging import time logger = logging.getLogger(__name__) FLAP_MARKER = "Now flapping!! No more messages!" # Actions returned by observe() PASS = "pass" # deliver unchanged TRIP = "trip" # deliver with FLAP_MARKER appended SUPPRESS = "suppress" # drop _count = 0 # flap_count: alerts within the window needed to trip _window = 0.0 # flap_interval in seconds # {(host, service): {"events": [ts, ...], "flapping": bool, "ok_since": float|None}} _state: dict = {} def setup(cfg) -> None: """Read flap_count / flap_interval from *cfg* (also called on reload).""" global _count, _window _count = int(cfg.get("flap_count", 0) or 0) _window = float(cfg.get("flap_interval", 0) or 0) * 60.0 def _enabled() -> bool: return _count > 0 and _window > 0 def _label(key) -> str: host, service = key return f"{host}/{service}" if service else host def _sweep(st: dict, now: float) -> bool: """Clear the flapping flag once the post-RECOVER quiet window has elapsed.""" if st["flapping"] and st["ok_since"] is not None and now - st["ok_since"] >= _window: st["flapping"] = False st["ok_since"] = None st["events"].clear() return True return False def observe(host: str, service: str, level: str) -> str: """Record a notification for *host*/*service* and return what to do with it. Returns PASS, TRIP or SUPPRESS. """ if not _enabled(): return PASS now = time.time() key = (host or "", service or "") level = (level or "").upper() st = _state.get(key) if st is not None and _sweep(st, now): logger.info("flapping cleared for %s", _label(key)) if level in ("WARNING", "CRITICAL"): if st is None: st = _state[key] = {"events": [], "flapping": False, "ok_since": None} # An alert inside the quiet window means it never settled. st["ok_since"] = None st["events"] = [t for t in st["events"] if now - t < _window] st["events"].append(now) if st["flapping"]: return SUPPRESS if len(st["events"]) > _count: st["flapping"] = True logger.info( "flapping detected for %s (%d alerts in %.0f min)", _label(key), len(st["events"]), _window / 60, ) return TRIP return PASS if st is None or not st["flapping"]: return PASS if level == "RECOVER": # Starts the quiet window; the state ends silently when it elapses. st["ok_since"] = now return SUPPRESS def clear_host(host: str) -> None: """Discard all flap state for *host* (called when a host is dropped). A dropped host may be mid-flap with no RECOVER ever received, in which case ``ok_since`` stays ``None`` and ``_sweep`` can never clear it on its own — the state would otherwise persist forever. """ for key in [k for k in _state if k[0] == host]: del _state[key] def flapping_services(host: str) -> list: """Return the services of *host* that are currently flapping. An empty string in the result means the host itself (connectivity, boot, shutdown) rather than a named service. """ if not _enabled(): return [] now = time.time() flapping = [] for (h, service), st in _state.items(): if h != host: continue _sweep(st, now) if st["flapping"]: flapping.append(service) return sorted(flapping)