feat: flapping detection suppresses notification storms
A (host, service) pair is flapping once it exceeds flap_count warning or critical notifications within flap_interval minutes. The notification that trips the state carries "Now flapping!! No more messages!" and every later one is dropped, including RECOVER. The state ends silently flap_interval minutes after a RECOVER, provided no further alert arrived meanwhile. Hooked into notify.send_notification, the single choke point for channel delivery, so only outbound notifications are suppressed — eventlog keeps recording, leaving the journal and /log with the full history of the flap. Threshold alerts key on their metric path, so a flapping disk check cannot silence a CPU alert; connectivity, boot and shutdown events key on the host itself. State lives at module level in flap.py and is never pickled. Flapping pairs surface in Host.stateinfo() and render as an amber badge on the live dashboard. Config: flap_count (5), flap_interval (10 minutes), 0 in either disables; both editable on the settings page. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Andreas Wrede
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
e3b0e5041f
commit
4414967bdc
@@ -30,6 +30,10 @@ SERVER_DEFAULTS = {
|
||||
"grace": 2, # Grace period (extra seconds before notifying after a missed heartbeat)
|
||||
"threshold_renotify_interval": 3600, # Seconds between threshold re-notifications
|
||||
|
||||
# Flap detection (0 in either key disables it)
|
||||
"flap_count": 5, # Warning/critical notifications within flap_interval that trip flapping
|
||||
"flap_interval": 10, # Minutes: the counting window, and the quiet window after an OK
|
||||
|
||||
# User management
|
||||
"users": {}, # username -> {full_name, avatar, password, admin, notification_channels}
|
||||
"default_owner": None, # Username that owns hosts with no explicit owner
|
||||
|
||||
@@ -19,6 +19,7 @@ def _make_yaml() -> YAML:
|
||||
_SERVER_KEYS = [
|
||||
"hbd_port", "hbd_host", "ws_port", "wss_port", "hb_port",
|
||||
"interval", "grace", "base_url", "threshold_renotify_interval",
|
||||
"flap_count", "flap_interval",
|
||||
"logfile", "pidfile", "pickfile", "journal_enabled", "journal_dir",
|
||||
"journal_max_size", "journal_max_backups", "default_owner",
|
||||
"default_threshold_config",
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Flap detection: silence checks that toggle faster than they are useful.
|
||||
|
||||
A ``(host, service)`` pair is *flapping* once it exceeds ``flap_count``
|
||||
WARNING/CRITICAL notifications within ``flap_interval`` minutes. The
|
||||
notification that trips the threshold is delivered with ``FLAP_MARKER``
|
||||
appended; every notification for that pair afterwards is dropped. The state
|
||||
clears silently ``flap_interval`` minutes after a RECOVER, provided no further
|
||||
WARNING/CRITICAL arrived in the meantime.
|
||||
|
||||
Only outbound notifications are affected — ``notify.eventlog`` keeps recording
|
||||
every event, so the journal and ``/log`` retain the full history of the flap.
|
||||
|
||||
State lives here at module level rather than on ``Host`` so it is never
|
||||
pickled: a restart starts every check with a clean slate.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FLAP_MARKER = "Now flapping!! No more messages!"
|
||||
|
||||
# Actions returned by observe()
|
||||
PASS = "pass" # deliver unchanged
|
||||
TRIP = "trip" # deliver with FLAP_MARKER appended
|
||||
SUPPRESS = "suppress" # drop
|
||||
|
||||
_count = 0 # flap_count: alerts within the window needed to trip
|
||||
_window = 0.0 # flap_interval in seconds
|
||||
|
||||
# {(host, service): {"events": [ts, ...], "flapping": bool, "ok_since": float|None}}
|
||||
_state: dict = {}
|
||||
|
||||
|
||||
def setup(cfg) -> None:
|
||||
"""Read flap_count / flap_interval from *cfg* (also called on reload)."""
|
||||
global _count, _window
|
||||
_count = int(cfg.get("flap_count", 0) or 0)
|
||||
_window = float(cfg.get("flap_interval", 0) or 0) * 60.0
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
return _count > 0 and _window > 0
|
||||
|
||||
|
||||
def _label(key) -> str:
|
||||
host, service = key
|
||||
return f"{host}/{service}" if service else host
|
||||
|
||||
|
||||
def _sweep(st: dict, now: float) -> bool:
|
||||
"""Clear the flapping flag once the post-RECOVER quiet window has elapsed."""
|
||||
if st["flapping"] and st["ok_since"] is not None and now - st["ok_since"] >= _window:
|
||||
st["flapping"] = False
|
||||
st["ok_since"] = None
|
||||
st["events"].clear()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def observe(host: str, service: str, level: str) -> str:
|
||||
"""Record a notification for *host*/*service* and return what to do with it.
|
||||
|
||||
Returns PASS, TRIP or SUPPRESS.
|
||||
"""
|
||||
if not _enabled():
|
||||
return PASS
|
||||
|
||||
now = time.time()
|
||||
key = (host or "", service or "")
|
||||
level = (level or "").upper()
|
||||
st = _state.get(key)
|
||||
if st is not None and _sweep(st, now):
|
||||
logger.info("flapping cleared for %s", _label(key))
|
||||
|
||||
if level in ("WARNING", "CRITICAL"):
|
||||
if st is None:
|
||||
st = _state[key] = {"events": [], "flapping": False, "ok_since": None}
|
||||
# An alert inside the quiet window means it never settled.
|
||||
st["ok_since"] = None
|
||||
st["events"] = [t for t in st["events"] if now - t < _window]
|
||||
st["events"].append(now)
|
||||
if st["flapping"]:
|
||||
return SUPPRESS
|
||||
if len(st["events"]) > _count:
|
||||
st["flapping"] = True
|
||||
logger.info(
|
||||
"flapping detected for %s (%d alerts in %.0f min)",
|
||||
_label(key), len(st["events"]), _window / 60,
|
||||
)
|
||||
return TRIP
|
||||
return PASS
|
||||
|
||||
if st is None or not st["flapping"]:
|
||||
return PASS
|
||||
if level == "RECOVER":
|
||||
# Starts the quiet window; the state ends silently when it elapses.
|
||||
st["ok_since"] = now
|
||||
return SUPPRESS
|
||||
|
||||
|
||||
def flapping_services(host: str) -> list:
|
||||
"""Return the services of *host* that are currently flapping.
|
||||
|
||||
An empty string in the result means the host itself (connectivity, boot,
|
||||
shutdown) rather than a named service.
|
||||
"""
|
||||
if not _enabled():
|
||||
return []
|
||||
now = time.time()
|
||||
flapping = []
|
||||
for (h, service), st in _state.items():
|
||||
if h != host:
|
||||
continue
|
||||
_sweep(st, now)
|
||||
if st["flapping"]:
|
||||
flapping.append(service)
|
||||
return sorted(flapping)
|
||||
@@ -428,6 +428,10 @@ class Host:
|
||||
ddict["alert_critical_unacked"] = critical_unacked
|
||||
ddict["alert_critical_acked"] = critical_acked
|
||||
|
||||
# Flap detection state (module-level in flap.py, never pickled)
|
||||
from . import flap
|
||||
ddict["flapping"] = flap.flapping_services(self.name)
|
||||
|
||||
# User access
|
||||
ddict["owner"] = getattr(self, "owner", None)
|
||||
ddict["managers"] = list(getattr(self, "managers", []))
|
||||
|
||||
@@ -25,6 +25,7 @@ from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from . import data
|
||||
from . import flap as flap_mod
|
||||
from . import ws as ws_mod
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -63,6 +64,7 @@ class Notification:
|
||||
body: str # detail message
|
||||
level: str # RECOVER | WARNING | CRITICAL | INFO
|
||||
url: str = "" # link to plugin metrics page
|
||||
service: str = "" # flap-detection key within the host ("" = the host itself)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -73,6 +75,7 @@ def setup(cfg: dict, loop: Optional[asyncio.AbstractEventLoop] = None):
|
||||
"""Initialize notifier from configuration dict."""
|
||||
global _config, _loop
|
||||
_config = dict(cfg)
|
||||
flap_mod.setup(_config)
|
||||
if loop is not None:
|
||||
_loop = loop
|
||||
|
||||
@@ -81,6 +84,7 @@ def reload_config(cfg: dict):
|
||||
"""Reload notification configuration on SIGHUP."""
|
||||
global _config
|
||||
_config = dict(cfg)
|
||||
flap_mod.setup(_config)
|
||||
logger.info("Notification configuration reloaded")
|
||||
|
||||
|
||||
@@ -439,11 +443,22 @@ async def send_notification(host_name: str, notif: Notification) -> dict:
|
||||
notification_channels, and dispatches. Silently does nothing if
|
||||
no users are configured.
|
||||
|
||||
Flap detection runs first: once *host_name*/*notif.service* is flapping the
|
||||
notification is dropped, and the one that trips the state carries
|
||||
``flap.FLAP_MARKER``.
|
||||
|
||||
Returns a dict of {channel_name: bool} results.
|
||||
"""
|
||||
from . import users as users_mod
|
||||
from . import hbdclass
|
||||
|
||||
action = flap_mod.observe(host_name, notif.service, notif.level)
|
||||
if action == flap_mod.SUPPRESS:
|
||||
logger.debug("flapping: suppressed %s notification for %s", notif.level, host_name)
|
||||
return {}
|
||||
if action == flap_mod.TRIP:
|
||||
notif.body = f"{notif.body} {flap_mod.FLAP_MARKER}"
|
||||
|
||||
if not users_mod.users_enabled():
|
||||
return {}
|
||||
|
||||
|
||||
@@ -386,6 +386,12 @@ def get_settings_sections(config: dict, threshold_checker=None, user=None) -> li
|
||||
"Extra seconds to wait after a missed heartbeat before sending notifications.", editable=True),
|
||||
field("threshold_renotify_interval", "Re-notify interval", "duration",
|
||||
"How often to re-send notifications for ongoing threshold alerts.", editable=True),
|
||||
field("flap_count", "Flap count", "number",
|
||||
"Warning/critical notifications within the flap interval that mark a "
|
||||
"service as flapping and silence it. 0 disables flap detection.", editable=True),
|
||||
field("flap_interval", "Flap interval", "number",
|
||||
"Minutes: the window flap count is measured over, and how long a service "
|
||||
"must stay OK before flapping ends.", editable=True),
|
||||
field("autosave_interval", "Autosave interval", "duration",
|
||||
"How often the server saves its state to disk."),
|
||||
field("base_url", "Base URL", "text",
|
||||
|
||||
@@ -45,6 +45,7 @@
|
||||
.dot.down { background: var(--st-crit); box-shadow: 0 0 0 3px var(--st-crit-soft); }
|
||||
.dot.overdue { background: transparent; border: 2px solid var(--st-warn); }
|
||||
.stale { color: var(--st-faint); font-size: 10px; border: 1px solid var(--st-line); border-radius: 4px; padding: 1px 4px; font-family: var(--st-mono); white-space: nowrap; }
|
||||
.flap { color: var(--st-warn); font-size: 10px; border: 1px solid var(--st-warn); border-radius: 4px; padding: 1px 4px; font-family: var(--st-mono); white-space: nowrap; }
|
||||
.chip.outline { background: transparent; border: 1px solid var(--st-line); }
|
||||
|
||||
.lat { font-family: var(--st-mono); font-size: 12px; color: var(--st-muted); font-variant-numeric: tabular-nums; }
|
||||
@@ -247,11 +248,17 @@
|
||||
staleTag = '<span class="stale" title="agent v' + escHtml(String(data.hbc_version)) + ' ≠ server v' + HBD_VERSION + '">v' + escHtml(String(data.hbc_version)) + '</span>';
|
||||
}
|
||||
var dotCls = w ? (w.state === 'up' ? sev : (w.state === 'down' ? 'down' : 'overdue')) : 'off';
|
||||
var flapTag = '';
|
||||
var flapping = data.flapping || [];
|
||||
if (flapping.length) {
|
||||
var svcs = flapping.map(function (s) { return s || 'host'; }).join(', ');
|
||||
flapTag = '<span class="flap" title="notifications suppressed: ' + escHtml(svcs) + '">flapping</span>';
|
||||
}
|
||||
|
||||
row.innerHTML =
|
||||
'<span class="host' + (data.watched ? ' watched' : '') + '">'
|
||||
+ '<span class="dot ' + dotCls + '"></span>'
|
||||
+ '<a href="/plugins#' + encodeURIComponent(name) + '">' + escHtml(name) + '</a>' + staleTag + '</span>'
|
||||
+ '<a href="/plugins#' + encodeURIComponent(name) + '">' + escHtml(name) + '</a>' + staleTag + flapTag + '</span>'
|
||||
+ '<span class="tip" tabindex="0">' + statusChip(data) + tipCard(data) + '</span>'
|
||||
+ '<span class="lat num">' + lat + '</span>'
|
||||
+ '<span>' + alertChips(data) + '</span>'
|
||||
|
||||
@@ -1251,6 +1251,7 @@ class ThresholdChecker:
|
||||
title=title,
|
||||
body=body,
|
||||
level=lvl,
|
||||
service=short_path,
|
||||
),
|
||||
))
|
||||
|
||||
@@ -1534,6 +1535,7 @@ class ThresholdChecker:
|
||||
title=f"[REMINDER/{alert_state.level.name}] {host_name} {short_path}",
|
||||
body=body,
|
||||
level=alert_state.level.name,
|
||||
service=short_path,
|
||||
),
|
||||
))
|
||||
logger.info("Re-notification sent: %s", message)
|
||||
|
||||
Reference in New Issue
Block a user