Files
heartbeat/hbd/server/config.py
T
Andreas WredeandClaude Opus 4.8 4414967bdc feat: flapping detection suppresses notification storms
A (host, service) pair is flapping once it exceeds flap_count warning or
critical notifications within flap_interval minutes. The notification that
trips the state carries "Now flapping!! No more messages!" and every later
one is dropped, including RECOVER. The state ends silently flap_interval
minutes after a RECOVER, provided no further alert arrived meanwhile.

Hooked into notify.send_notification, the single choke point for channel
delivery, so only outbound notifications are suppressed — eventlog keeps
recording, leaving the journal and /log with the full history of the flap.

Threshold alerts key on their metric path, so a flapping disk check cannot
silence a CPU alert; connectivity, boot and shutdown events key on the host
itself. State lives at module level in flap.py and is never pickled.

Flapping pairs surface in Host.stateinfo() and render as an amber badge on
the live dashboard. Config: flap_count (5), flap_interval (10 minutes),
0 in either disables; both editable on the settings page.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-23 09:53:46 -07:00

337 lines
11 KiB
Python

"""Configuration loader and defaults for hbd (HeartBeat Daemon/Server)."""
import asyncio
import logging
import os
try:
import yaml
except Exception:
yaml = None
SERVER_DEFAULTS = {
# Network settings
"hb_port": 50003, # Port to listen for heartbeats
"hbd_port": 50004, # HTTP API port
"hbd_host": "", # Bind address (empty = all interfaces)
# Persistence
"pickfile": os.path.join(os.path.expanduser("~"), ".hb.pick"), # File to store host state between restarts
"pidfile": os.path.join(os.path.expanduser("~"), ".hb.pid"), # PID file for stop/restart/reload
# Logging
"logfile": os.path.join(os.path.expanduser("~"), ".hb.log"),
# Notification channels
"notification_channels": {}, # Named channels with type and credentials
"base_url": "", # Base URL for notification links (e.g. https://hbd.example.com)
# Monitoring settings
"interval": 20, # Expected heartbeat interval (for server checks)
"grace": 2, # Grace period (extra seconds before notifying after a missed heartbeat)
"threshold_renotify_interval": 3600, # Seconds between threshold re-notifications
# Flap detection (0 in either key disables it)
"flap_count": 5, # Warning/critical notifications within flap_interval that trip flapping
"flap_interval": 10, # Minutes: the counting window, and the quiet window after an OK
# User management
"users": {}, # username -> {full_name, avatar, password, admin, notification_channels}
"default_owner": None, # Username that owns hosts with no explicit owner
# OAuth2 providers
"oauth": {}, # oauth.gitea.{url,client_id,client_secret}
# Host management
"hosts": {}, # Unified host definitions
"dyndomains": ["example.org"], # Domains to update via nsupdate when a host with dyndns: true is updated
# DNS updates
"nsupdate_bin": "/usr/bin/nsupdate", # Path to nsupdate binary
# WebSocket settings
"ws_port": 50005,
"wss_port": None,
"cert_path": "/usr/local/etc/ssl/",
"wss_pem": "fullchain.pem",
"wss_key": "privkey.pem",
# Message journal configuration
"journal_enabled": True,
"journal_dir": "/var/log/heartbeat",
"journal_file": "messages.journal",
"journal_max_size": 100 * 1024 * 1024, # 100MB
"journal_max_backups": 10,
# Runtime flags
"foreground": False,
"verbose": False,
"debug": 0,
# Plugin/threshold configs (for clients reporting to this server)
"plugins": {},
"thresholds": {},
}
THRESHOLD_DEFAULTS = {
'thresholds': {
'cpu_monitor': {
'cpu_percent': {
'warning': 80.0,
'critical': 90.0
}
},
'memory_monitor': {
'memory_percent': {
'warning': 85.0,
'critical': 95.0
},
'swap_percent': {
'warning': 40.0,
'critical': 75.0
}
},
'disk_monitor': {
'partitions': {
'/': {
'percent': {
'warning': 85.0,
'critical': 90.0
}
}
}
},
'rtt': {
'warning': 200,
'critical': 250.0,
'count': 3 # Optional: number of consecutive breaches before alerting
},
'nagios_runner': {
'status_code': {
'display': '{check_name} {output}',
'operator': "nagios"
}
},
'zfs_monitor': {
'pools': {
'*': {
'status': {
'warning': 1,
'critical': 2,
'operator': '>=',
'hysteresis': 0.0,
'grace': 0,
'display': 'ZFS pool {pool_name} is {health}'
},
'capacity': {
'warning': 80.0,
'critical': 90.0,
}
}
}
},
}
}
def load_config(path=None):
"""Load configuration from a YAML file and merge with server defaults.
If YAML is not available or the file does not exist, defaults are returned.
Args:
path: Path to YAML config file (default: ~/.hb.yaml)
Returns:
Dictionary with configuration
"""
cfg = SERVER_DEFAULTS.copy()
if not path:
# default path (~/.hb.yaml)
path = os.path.join(os.path.expanduser("~"), ".hb.yaml")
if os.path.exists(path):
if yaml:
with open(path) as fh:
data = yaml.safe_load(fh)
# Merge YAML data with defaults
# Keep all keys from YAML to support plugin configs and future extensions
for k, v in data.items():
cfg[k] = v
else:
# yaml not installed: do not attempt to parse; user must ensure defaults
pass
return cfg
class ReloadableConfig:
"""Thread-safe/async-safe configuration wrapper that supports runtime reloading.
This class wraps the configuration dictionary and provides:
- Thread-safe config reloading via SIGHUP
- Backward-compatible dict-like access
- Async lock to prevent concurrent reloads
"""
def __init__(self, initial_config, config_path=None):
"""Initialize with initial configuration.
Args:
initial_config: Initial configuration dictionary
config_path: Path to config file for reloading (optional)
"""
self._config = initial_config
self._config_path = config_path
self._lock = asyncio.Lock()
self._logger = logging.getLogger(__name__)
async def reload(self, config_path=None):
"""Reload configuration from file.
Args:
config_path: Path to config file (uses stored path if not provided)
Returns:
New configuration dictionary
Raises:
Exception if reload fails (keeps existing config)
"""
path = config_path or self._config_path
if not path:
raise ValueError("No config path specified for reload")
async with self._lock:
try:
# Load new config
new_config = load_config(path)
# Store old config for rollback if needed
old_config = self._config
# Update config
self._config = new_config
self._logger.info(f"Configuration reloaded from {path}")
return new_config
except Exception as e:
self._logger.error(f"Failed to reload config from {path}: {e}", exc_info=True)
# Keep existing config on error
raise
def get(self, key, default=None):
"""Get a config value (dict-compatible)."""
return self._config.get(key, default)
def __getitem__(self, key):
"""Get a config value via subscript (dict-compatible)."""
return self._config[key]
def __contains__(self, key):
"""Check if key exists (dict-compatible)."""
return key in self._config
def keys(self):
"""Return config keys (dict-compatible)."""
return self._config.keys()
def items(self):
"""Return config items (dict-compatible)."""
return self._config.items()
def values(self):
"""Return config values (dict-compatible)."""
return self._config.values()
@property
def config(self):
"""Get the underlying config dict (for components that need full dict)."""
return self._config
def get_watchhosts(config):
"""Extract watched hostnames from config (hosts with watch: true).
Returns:
# List of hostnames to watch
"""
watchhosts = []
hosts_config = config.get("hosts", {})
if isinstance(hosts_config, dict):
for host_name, host_attrs in hosts_config.items():
if isinstance(host_attrs, dict) and host_attrs.get("watch", True):
watchhosts.append(host_name)
return watchhosts
def get_dyndnshosts(config):
"""Return hostnames that have a dyndns setting in the hosts section."""
hosts_config = config.get("hosts", {})
if not isinstance(hosts_config, dict):
return []
return [
name for name, attrs in hosts_config.items()
if isinstance(attrs, dict) and attrs.get("dyndns")
]
def get_host_config(config, hostname):
"""Get configuration for a specific host from the hosts section.
Returns:
Dictionary with host attributes or empty dict
"""
hosts_config = config.get("hosts", {})
if isinstance(hosts_config, dict) and hostname in hosts_config:
val = hosts_config[hostname]
return val if isinstance(val, dict) else {}
return {}
# ---------------------------------------------------------------------------
# User / host-access helpers
# ---------------------------------------------------------------------------
def get_default_owner(config) -> str | None:
"""Return the configured default_owner username, or the first admin user, or None."""
explicit = config.get("default_owner")
if explicit:
return explicit
# Fall back to first admin user found in config
users_cfg = config.get("users", {})
if isinstance(users_cfg, dict):
for username, attrs in users_cfg.items():
if isinstance(attrs, dict) and attrs.get("admin", False):
return username
return None
def get_host_access(config, hostname) -> dict:
"""Return the access dict for *hostname*: owner, managers, monitors.
Falls back to default_owner for hosts without an explicit owner.
Returns:
{
"owner": str | None,
"managers": list[str],
"monitors": list[str],
}
"""
host_cfg = get_host_config(config, hostname)
owner = host_cfg.get("owner") # or get_default_owner(config)
managers = host_cfg.get("managers", [])
if isinstance(managers, str):
managers = [managers]
monitors = host_cfg.get("monitors", [])
if isinstance(monitors, str):
monitors = [monitors]
return {
"owner": owner,
"managers": list(managers),
"monitors": list(monitors),
}