andreas/heartbeat
Public Access
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 54 Wiki Activity

style: clear whitespace, unused-import and blank-line lint

Browse Source 
Strip trailing whitespace, drop unused imports/variables/global statements,
split multi-imports, fix E3xx/E1xx spacing and E712 comparisons. No behaviour
change; only E501 long lines remain in flake8.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
andreas
2026-10-28 04:59:34 -04:00
co-authored by Claude Sonnet 5.5
parent 85c6a4eebe
commit 748349a7b9
35 changed files with 705 additions and 719 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hbd/server/__init__.py
+1 -1
View File
@@ -1,3 +1,3 @@
"""HeartBeat Daemon (hbd) - Server/daemon component."""
from hbd import __version__
from hbd import __version__ # noqa: F401
hbd/server/cli.py
+10 -4
View File
@@ -120,7 +120,7 @@ def cmd_passwd(args):
def cmd_notify(args):
"""Send a test message via a single notification channel."""
from .config import load_config
from .notify import Notification, _dispatch_to_channel, setup
from .notify import Notification, setup
config = load_config(args.configfile)
setup(config)
@@ -193,7 +193,9 @@ def _read_pid(configfile) -> int | None:
def cmd_stop(args):
import os, signal as _signal, time
import os
import signal as _signal
import time
pid = _read_pid(args.configfile)
if pid is None:
sys.exit(1)
@@ -212,7 +214,8 @@ def cmd_stop(args):
def cmd_reload(args):
import os, signal as _signal
import os
import signal as _signal
pid = _read_pid(args.configfile)
if pid is None:
sys.exit(1)
@@ -222,7 +225,10 @@ def cmd_reload(args):
def cmd_restart(args):
import os, signal as _signal, time, subprocess
import os
import signal as _signal
import time
import subprocess
pid = _read_pid(args.configfile)
if pid is not None:
print(f"Stopping hbd (pid {pid})...")
hbd/server/config.py
+33 -36
View File
@@ -43,30 +43,30 @@ SERVER_DEFAULTS = {
# Host management
"hosts": {}, # Unified host definitions
"dyndomains": ["example.org"], # Domains to update via nsupdate when a host with dyndns: true is updated
"dyndomains": ["example.org"], # Domains to update via nsupdate when a host with dyndns: true is updated
# DNS updates
"nsupdate_bin": "/usr/bin/nsupdate", # Path to nsupdate binary
"nsupdate_bin": "/usr/bin/nsupdate", # Path to nsupdate binary
# WebSocket settings
"ws_port": 50005,
"ws_port": 50005,
"wss_port": None,
"cert_path": "/usr/local/etc/ssl/",
"wss_pem": "fullchain.pem",
"wss_key": "privkey.pem",
# Message journal configuration
"journal_enabled": True,
"journal_dir": "/var/log/heartbeat",
"journal_file": "messages.journal",
"journal_max_size": 100 * 1024 * 1024, # 100MB
"journal_max_backups": 10,
# Runtime flags
"foreground": False,
"verbose": False,
"debug": 0,
# Plugin/threshold configs (for clients reporting to this server)
"plugins": {},
"thresholds": {},
@@ -76,11 +76,11 @@ THRESHOLD_DEFAULTS = {
'thresholds': {
'cpu_monitor': {
'cpu_percent': {
'warning': 80.0,
'warning': 80.0,
'critical': 90.0
}
},
'memory_monitor': {
'memory_monitor': {
'memory_percent': {
'warning': 85.0,
'critical': 95.0
@@ -90,7 +90,7 @@ THRESHOLD_DEFAULTS = {
'critical': 75.0
}
},
'disk_monitor': {
'disk_monitor': {
'partitions': {
'/': {
'percent': {
@@ -100,18 +100,18 @@ THRESHOLD_DEFAULTS = {
}
}
},
'rtt': {
'rtt': {
'warning': 200,
'critical': 250.0,
'count': 3 # Optional: number of consecutive breaches before alerting
},
'nagios_runner': {
'nagios_runner': {
'status_code': {
'display': '{check_name} {output}',
'operator': "nagios"
}
},
'zfs_monitor': {
'zfs_monitor': {
'pools': {
'*': {
'status': {
@@ -137,10 +137,10 @@ def load_config(path=None):
"""Load configuration from a YAML file and merge with server defaults.
If YAML is not available or the file does not exist, defaults are returned.
Args:
path: Path to YAML config file (default: ~/.hb.yaml)
Returns:
Dictionary with configuration
"""
@@ -165,16 +165,16 @@ def load_config(path=None):
class ReloadableConfig:
"""Thread-safe/async-safe configuration wrapper that supports runtime reloading.
This class wraps the configuration dictionary and provides:
- Thread-safe config reloading via SIGHUP
- Backward-compatible dict-like access
- Async lock to prevent concurrent reloads
"""
def __init__(self, initial_config, config_path=None):
"""Initialize with initial configuration.
Args:
initial_config: Initial configuration dictionary
config_path: Path to config file for reloading (optional)
@@ -183,65 +183,62 @@ class ReloadableConfig:
self._config_path = config_path
self._lock = asyncio.Lock()
self._logger = logging.getLogger(__name__)
async def reload(self, config_path=None):
"""Reload configuration from file.
Args:
config_path: Path to config file (uses stored path if not provided)
Returns:
New configuration dictionary
Raises:
Exception if reload fails (keeps existing config)
"""
path = config_path or self._config_path
if not path:
raise ValueError("No config path specified for reload")
async with self._lock:
try:
# Load new config
new_config = load_config(path)
# Store old config for rollback if needed
old_config = self._config
# Update config
self._config = new_config
self._logger.info(f"Configuration reloaded from {path}")
return new_config
except Exception as e:
self._logger.error(f"Failed to reload config from {path}: {e}", exc_info=True)
# Keep existing config on error
raise
def get(self, key, default=None):
"""Get a config value (dict-compatible)."""
return self._config.get(key, default)
def __getitem__(self, key):
"""Get a config value via subscript (dict-compatible)."""
return self._config[key]
def __contains__(self, key):
"""Check if key exists (dict-compatible)."""
return key in self._config
def keys(self):
"""Return config keys (dict-compatible)."""
return self._config.keys()
def items(self):
"""Return config items (dict-compatible)."""
return self._config.items()
def values(self):
"""Return config values (dict-compatible)."""
return self._config.values()
@property
def config(self):
"""Get the underlying config dict (for components that need full dict)."""
@@ -319,7 +316,7 @@ def get_host_access(config, hostname) -> dict:
"""
host_cfg = get_host_config(config, hostname)
owner = host_cfg.get("owner") # or get_default_owner(config)
owner = host_cfg.get("owner") # or get_default_owner(config)
managers = host_cfg.get("managers", [])
if isinstance(managers, str):
hbd/server/configio.py
+1
View File
@@ -15,6 +15,7 @@ def _make_yaml() -> YAML:
y.preserve_quotes = True
return y
# Top-level keys managed by the 'server' logical section
_SERVER_KEYS = [
"hbd_port", "hbd_host", "ws_port", "wss_port", "hb_port",
hbd/server/data.py
+5 -3
View File
@@ -1,12 +1,14 @@
msgs = [] # in-memory list of recent messages for new websocket clients; also logged to file via notify.eventlog
class Data:
def __init__(self, config):
self.config = config
self.data = {}
def update(self, new_data):
self.data.update(new_data)
def get(self, key, default=None):
return self.data.get(key, default)
return self.data.get(key, default)
hbd/server/hbdclass.py
+34 -34
View File
@@ -43,7 +43,7 @@ class Connection:
self.statetime = self.lastbeat
self.deltastatetime = "computed"
self.state = Connection.UNKNOWN
# Timer-based reachability monitoring
self.overdue_timer = None
self.overdue_callback = None
@@ -196,39 +196,39 @@ class Connection:
if self.host.isDynDns():
Host.dnsQ.put((self.host.name, self.addr))
return r
def reset_overdue_timer(self, timeout_seconds, callback):
"""Reset the overdue timer for this connection.
Cancels any existing timer and sets a new one that will mark
the connection as overdue if no heartbeat arrives before timeout.
Args:
timeout_seconds: Seconds before marking as overdue
callback: Async function to call when timer expires
"""
import asyncio
# Cancel existing timer if any
if self.overdue_timer and not self.overdue_timer.cancelled():
self.overdue_timer.cancel()
# Store parameters for later reference
self.timeout_duration = timeout_seconds
self.overdue_callback = callback
# Create new timer
async def timer_expired():
await callback(self)
try:
loop = asyncio.get_event_loop()
self.overdue_timer = loop.call_later(timeout_seconds,
lambda: asyncio.create_task(timer_expired()))
self.overdue_timer = loop.call_later(timeout_seconds,
lambda: asyncio.create_task(timer_expired()))
except RuntimeError:
# No event loop running yet
pass
def cancel_overdue_timer(self):
"""Cancel the overdue timer if it exists and clear all timer references."""
if self.overdue_timer:
@@ -241,37 +241,37 @@ class Connection:
self.overdue_timer = None
self.overdue_callback = None
self.timeout_duration = None
def get_avg_rtt(self):
"""Get average RTT from recent samples."""
valid_rtts = [r for r in self.rtts if r > 0]
if valid_rtts:
return sum(valid_rtts) / len(valid_rtts)
return 0
def get_current_rtt(self):
"""Get most recent RTT value."""
return self.rtts[-1] if self.rtts else 0
def check_rtt_threshold(self, warning_threshold=None, critical_threshold=None):
"""Check if RTT exceeds thresholds.
Args:
warning_threshold: RTT in ms for warning level
critical_threshold: RTT in ms for critical level
Returns:
Tuple of (level, rtt_value) where level is None, 'WARNING', or 'CRITICAL'
"""
rtt = self.get_current_rtt()
if rtt <= 0:
return (None, rtt)
if critical_threshold and rtt > critical_threshold:
return ('CRITICAL', rtt)
elif warning_threshold and rtt > warning_threshold:
return ('WARNING', rtt)
return (None, rtt)
@@ -328,7 +328,7 @@ class Host:
d["dyn"] = str(self.dyn)
d["num"] = self.num
d["status"] = self.status()
# Add alert counts (split by acknowledged status)
warning_unacked = 0
warning_acked = 0
@@ -347,12 +347,12 @@ class Host:
critical_acked += 1
else:
critical_unacked += 1
d["alert_warning_unacked"] = warning_unacked
d["alert_warning_acked"] = warning_acked
d["alert_critical_unacked"] = critical_unacked
d["alert_critical_acked"] = critical_acked
for c in ["IPv4", "IPv6"]:
if c in self.connections:
cs = self.connections[c].statedict()
@@ -409,7 +409,7 @@ class Host:
ddict[d] = cl
else:
ddict[d] = self.__dict__[d]
# Add alert counts (computed from alert_states)
warning_unacked = 0
warning_acked = 0
@@ -428,7 +428,7 @@ class Host:
critical_acked += 1
else:
critical_unacked += 1
ddict["alert_warning_unacked"] = warning_unacked
ddict["alert_warning_acked"] = warning_acked
ddict["alert_critical_unacked"] = critical_unacked
@@ -554,7 +554,7 @@ class Host:
def add_plugin_data(self, plugin_name, data, timestamp=None):
"""Store plugin data with timestamp.
Args:
plugin_name: Name of the plugin (e.g., "cpu_monitor")
data: Dict of plugin data
@@ -562,24 +562,24 @@ class Host:
"""
if timestamp is None:
timestamp = time.time()
if plugin_name not in self.plugin_data:
self.plugin_data[plugin_name] = []
# Add new data
self.plugin_data[plugin_name].append((timestamp, data))
# Enforce retention limit (keep last N samples)
if len(self.plugin_data[plugin_name]) > self.plugin_retention:
self.plugin_data[plugin_name] = self.plugin_data[plugin_name][-self.plugin_retention:]
def get_plugin_data(self, plugin_name, limit=None):
"""Retrieve plugin data for a specific plugin.
Args:
plugin_name: Name of the plugin
limit: Optional limit on number of recent samples to return
Returns:
List of (timestamp, data) tuples, most recent last
"""
@@ -587,19 +587,19 @@ class Host:
if limit and len(data) > limit:
return data[-limit:]
return data
def get_latest_plugin_data(self, plugin_name):
"""Get the most recent plugin data for a plugin.
Args:
plugin_name: Name of the plugin
Returns:
(timestamp, data) tuple or None if no data
"""
data = self.plugin_data.get(plugin_name, [])
return data[-1] if data else None
def get_all_plugin_data(self):
"""Get all plugin data for this host.
hbd/server/http.py
+22 -22
View File
@@ -109,6 +109,7 @@ def _insert_threshold_metric(thresholds: dict, metric_path: str, values: dict) -
else:
thresholds[plugin].setdefault(intermediate, {})[leaf] = cfg
def _render_template(html_str: str, **context) -> str:
tmpl = jinja2.Template(html_str)
return tmpl.render(**context)
@@ -563,7 +564,7 @@ async def start(
if err:
return err
hostname = request.match_info.get("hostname")
if hostname not in hbdclass.Host.hosts:
return web.json_response({"error": f"Host '{hostname}' not found"}, status=404)
@@ -582,7 +583,7 @@ async def start(
"data": data,
"sample_count": len(samples),
}
return web.json_response({
"hostname": hostname,
"plugins": plugins_summary,
@@ -602,23 +603,23 @@ async def start(
host = hbdclass.Host.hosts[hostname]
if not _can_view_host(user, host):
return web.json_response({"error": "Forbidden"}, status=403)
# Get limit from query parameter
limit = request.rel_url.query.get("limit", "10")
try:
limit = int(limit)
except ValueError:
limit = 10
# Get plugin data
samples = host.get_plugin_data(plugin_name, limit=limit)
if not samples:
return web.json_response(
{"error": f"No data for plugin '{plugin_name}' on host '{hostname}'"},
status=404
)
# Format samples
formatted_samples = [
{
@@ -627,7 +628,7 @@ async def start(
}
for ts, data in samples
]
return web.json_response({
"hostname": hostname,
"plugin": plugin_name,
@@ -648,17 +649,17 @@ async def start(
host = hbdclass.Host.hosts[hostname]
if not _can_view_host(user, host):
return web.json_response({"error": "Forbidden"}, status=403)
# Get alert states
alerts = []
for metric_path, alert_state in host.alert_states.items():
alerts.append(alert_state.to_dict())
# Get summary if threshold_checker available
summary = {"ok": 0, "warning": 0, "critical": 0, "unknown": 0}
if threshold_checker:
summary = threshold_checker.get_alert_summary(host.alert_states)
return web.json_response({
"hostname": hostname,
"alerts": alerts,
@@ -684,31 +685,31 @@ async def start(
state for state in host.alert_states.values()
if state.level != AlertLevel.OK
]
for alert in active_alerts:
alert_dict = alert.to_dict()
alert_dict["hostname"] = hostname
all_alerts.append(alert_dict)
# Sort by level (critical first) then by hostname
level_order = {"CRITICAL": 0, "WARNING": 1, "UNKNOWN": 2, "OK": 3}
all_alerts.sort(
key=lambda a: (level_order.get(a["level"], 99), a["hostname"], a["metric_path"])
)
# Get summary counts
summary = {"critical": 0, "warning": 0, "unknown": 0, "total": len(all_alerts)}
for alert in all_alerts:
level = alert["level"].lower()
if level in summary:
summary[level] += 1
return web.json_response({
"alerts": all_alerts,
"summary": summary,
"host_count": len(hbdclass.Host.hosts),
})
async def api_acknowledge_alert(request):
"""Acknowledge an alert to stop reminder notifications."""
user, err = _require_auth(request)
@@ -721,22 +722,22 @@ async def start(
{"error": "Invalid JSON in request body"},
status=400
)
hostname = data.get("hostname")
metric_path = data.get("metric_path")
if not hostname or not metric_path:
return web.json_response(
{"error": "Missing required fields: hostname and metric_path"},
status=400
)
if hostname not in hbdclass.Host.hosts:
return web.json_response(
{"error": f"Host '{hostname}' not found"},
status=404
)
host = hbdclass.Host.hosts[hostname]
if not _can_view_host(user, host):
return web.json_response({"error": "Forbidden"}, status=403)
@@ -746,10 +747,10 @@ async def start(
{"error": f"Alert '{metric_path}' not found for host '{hostname}'"},
status=404
)
alert_state = host.alert_states[metric_path]
alert_state.acknowledge()
return web.json_response({
"success": True,
"hostname": hostname,
@@ -1132,7 +1133,6 @@ async def start(
elif is_mon:
monitored.append(hostname)
# Resolve notification channel configs for display
notif_channels = []
if current_user:
hbd/server/journal.py
+52 -52
View File
@@ -19,14 +19,14 @@ logger = logging.getLogger(__name__)
class MessageJournal:
"""
Journal logger for heartbeat messages with size-based rotation.
Features:
- Logs all received messages in JSON format
- Automatic rotation when file size exceeds threshold
- Keeps configurable number of rotated logs
- Thread-safe and async-safe operation
- Configurable log directory and file naming
Configuration:
journal_dir: Directory for journal files (default: /var/log/heartbeat/)
journal_file: Base filename (default: messages.journal)
@@ -34,69 +34,69 @@ class MessageJournal:
max_backups: Number of backup files to keep (default: 10)
enabled: Enable/disable journaling (default: True)
"""
def __init__(self, config: Optional[Dict[str, Any]] = None):
"""
Initialize the message journal.
Args:
config: Configuration dictionary with journal settings
"""
self.config = config or {}
# Configuration options
self.journal_dir = Path(self.config.get('journal_dir', '/var/log/heartbeat'))
self.journal_file = self.config.get('journal_file', 'messages.journal')
self.max_size = self.config.get('journal_max_size', 100 * 1024 * 1024) # 100MB default
self.max_backups = self.config.get('journal_max_backups', 10)
self.enabled = self.config.get('journal_enabled', True)
# Runtime state
self._file_handle = None
self._current_size = 0
self._lock = asyncio.Lock()
self._initialized = False
# Full path to current journal file
self.journal_path = self.journal_dir / self.journal_file
async def initialize(self) -> bool:
"""
Initialize the journal.
Creates journal directory if needed and opens the journal file.
Returns:
True if initialization successful, False otherwise
"""
if not self.enabled:
logger.info("Message journal disabled in configuration")
return True
try:
# Create journal directory if it doesn't exist
self.journal_dir.mkdir(parents=True, exist_ok=True)
# Open journal file in append mode
self._file_handle = open(self.journal_path, 'a', encoding='utf-8')
# Get current file size
try:
self._current_size = os.path.getsize(self.journal_path)
except OSError:
self._current_size = 0
self._initialized = True
logger.info(f"Message journal initialized: {self.journal_path} "
f"(current size: {self._current_size:,} bytes, "
f"max: {self.max_size:,} bytes)")
f"(current size: {self._current_size:,} bytes, "
f"max: {self.max_size:,} bytes)")
return True
except Exception as e:
logger.error(f"Failed to initialize message journal: {e}")
self.enabled = False
return False
async def log_message(
self,
msg: Dict[str, Any],
@@ -105,7 +105,7 @@ class MessageJournal:
):
"""
Log a received message to the journal.
Args:
msg: Parsed message dictionary
addr: Source address (ip, port) tuple
@@ -113,19 +113,19 @@ class MessageJournal:
"""
if not self.enabled or not self._initialized:
return
# Skip HTB (heartbeat) messages - too verbose
msg_id = msg.get('ID', '')
if msg_id == 'HTB':
return
async with self._lock:
try:
# Prepare journal entry
if timestamp is None:
import time
timestamp = time.time()
entry = {
'timestamp': timestamp,
'datetime': datetime.fromtimestamp(timestamp).isoformat(),
@@ -133,21 +133,21 @@ class MessageJournal:
'source_port': addr[1] if isinstance(addr, (tuple, list)) and len(addr) > 1 else None,
'message': msg
}
# Serialize to JSON (one line per entry)
json_line = json.dumps(entry, separators=(',', ':')) + '\n'
json_bytes = json_line.encode('utf-8')
# Check if rotation is needed
if self._current_size + len(json_bytes) > self.max_size:
await self._rotate()
# Write to journal
if self._file_handle:
self._file_handle.write(json_line)
self._file_handle.flush() # Ensure data is written
self._current_size += len(json_bytes)
logger.debug(f"Logged message from {addr[0]}: {msg.get('ID', 'UNKNOWN')}")
except Exception as e:
@@ -181,7 +181,7 @@ class MessageJournal:
async def _rotate(self):
"""
Rotate the journal file.
Renames current file with timestamp, opens new file, and removes
old backups exceeding max_backups limit.
"""
@@ -190,25 +190,25 @@ class MessageJournal:
if self._file_handle:
self._file_handle.close()
self._file_handle = None
# Generate backup filename with timestamp
timestamp_str = datetime.now().strftime('%Y%m%d-%H%M%S')
backup_name = f"{self.journal_file}.{timestamp_str}"
backup_path = self.journal_dir / backup_name
# Rename current file to backup
if self.journal_path.exists():
self.journal_path.rename(backup_path)
logger.info(f"Rotated journal: {backup_path} "
f"(size: {self._current_size:,} bytes)")
f"(size: {self._current_size:,} bytes)")
# Open new journal file
self._file_handle = open(self.journal_path, 'a', encoding='utf-8')
self._current_size = 0
# Clean up old backups
await self._cleanup_old_backups()
except Exception as e:
logger.error(f"Error rotating journal: {e}")
# Try to reopen the file even if rotation failed
@@ -217,18 +217,18 @@ class MessageJournal:
except Exception as e2:
logger.error(f"Failed to reopen journal after rotation error: {e2}")
self.enabled = False
async def _cleanup_old_backups(self):
"""
Remove old backup files exceeding max_backups limit.
Keeps only the most recent backups based on filename (which includes timestamp).
"""
try:
# Find all backup files
backup_pattern = f"{self.journal_file}.*"
backup_files = sorted(self.journal_dir.glob(backup_pattern))
# Remove oldest backups if we have too many
if len(backup_files) > self.max_backups:
files_to_remove = backup_files[:len(backup_files) - self.max_backups]
@@ -238,10 +238,10 @@ class MessageJournal:
logger.info(f"Removed old backup: {backup_file.name}")
except Exception as e:
logger.warning(f"Failed to remove old backup {backup_file}: {e}")
except Exception as e:
logger.error(f"Error cleaning up old backups: {e}")
async def log_threshold_event(
self,
host_name: str,
@@ -253,7 +253,7 @@ class MessageJournal:
):
"""
Log a threshold state change event.
Args:
host_name: Name of the host
metric_path: Full metric path (e.g., "cpu_monitor.cpu_percent")
@@ -264,11 +264,11 @@ class MessageJournal:
"""
if not self.enabled or not self._initialized:
return
try:
if timestamp is None:
timestamp = __import__('time').time()
event = {
'timestamp': timestamp,
'iso_time': datetime.fromtimestamp(timestamp).isoformat(),
@@ -279,30 +279,30 @@ class MessageJournal:
'new_level': new_level,
'value': value,
}
async with self._lock:
if not self._file_handle:
return
# Check if rotation is needed
if self._current_size >= self.max_size:
await self._rotate()
# Write event
line = json.dumps(event) + '\n'
self._file_handle.write(line)
self._file_handle.flush()
# Update size
self._current_size += len(line.encode('utf-8'))
except Exception as e:
logger.error(f"Error logging threshold event: {e}")
async def close(self):
"""
Close the journal and release resources.
Should be called during shutdown.
"""
async with self._lock:
@@ -315,11 +315,11 @@ class MessageJournal:
finally:
self._file_handle = None
self._initialized = False
def get_stats(self) -> Dict[str, Any]:
"""
Get journal statistics.
Returns:
Dictionary with journal stats
"""
@@ -423,10 +423,10 @@ _journal_instance: Optional[MessageJournal] = None
def get_journal(config: Optional[Dict[str, Any]] = None) -> MessageJournal:
"""
Get or create the global journal instance.
Args:
config: Configuration dictionary (only used on first call)
Returns:
MessageJournal instance
"""
hbd/server/main.py
+19 -20
View File
@@ -5,8 +5,6 @@ import logging
import socket
import time
import signal
import sys
import ssl
from . import __version__
from . import udp
@@ -23,6 +21,7 @@ eventlog = notify_mod.eventlog
# shared runtime collections and helpers
def save_state(config, hbdclass):
"""Save current state to pickle file. Safe to call at any time."""
import pickle
@@ -68,21 +67,21 @@ def cleanup_function(config, hbdclass):
async def reload_configuration(config_obj, config_path, components):
"""Reload configuration and update all components.
Args:
config_obj: ReloadableConfig instance
config_path: Path to config file
components: Dict with threshold_checker and other components
Returns:
True if reload succeeded, False otherwise
"""
try:
logger.info("Starting configuration reload...")
# Reload config file
new_config = await config_obj.reload(config_path)
# Update notify module
notify_mod.reload_config(new_config)
@@ -103,23 +102,23 @@ async def reload_configuration(config_obj, config_path, components):
if 'threshold_checker' in components:
components['threshold_checker'].reload(new_config)
components['threshold_checker'].purge_stale_alerts(hbdclass)
# Note: Changes to the following require restart:
# - hb_port, hbd_port, ws_port (already bound)
# - SSL certificates (already loaded)
# - pickfile (already opened)
# - journal settings (journal already initialized)
# These are reloadable and effective immediately:
# - notification_channels
# - threshold_configs
# - hosts (watchhosts, dyndns, notification_channels)
# - grace period (used on next heartbeat)
# - debug/verbose flags (used on next message)
logger.info("Configuration reload completed successfully")
return True
except Exception as e:
logger.error("=" * 60)
logger.error(f"Failed to reload configuration: {e}", exc_info=True)
@@ -142,7 +141,7 @@ async def _run_async(config, config_path=None):
sig_name = signal.Signals(signum).name if hasattr(signal, "Signals") else signum
logger.info(f"Received {sig_name}, initiating shutdown...")
loop.call_soon_threadsafe(shutdown_event.set)
def reload_handler(signum, frame):
sig_name = signal.Signals(signum).name if hasattr(signal, "Signals") else signum
logger.info(f"Received {sig_name}, initiating config reload...")
@@ -181,7 +180,7 @@ async def _run_async(config, config_path=None):
journal=msg_journal,
)
logger.info("Threshold checker initialized")
# Components dict for reload orchestration
components = {
'threshold_checker': threshold_checker,
@@ -324,7 +323,7 @@ async def _run_async(config, config_path=None):
],
return_when=asyncio.FIRST_COMPLETED
)
# Check which event was triggered
if shutdown_event.is_set():
logger.info("Shutdown signal received, stopping services...")
@@ -332,24 +331,24 @@ async def _run_async(config, config_path=None):
for task in pending:
task.cancel()
break
if reload_event.is_set():
# Clear the event for next reload
reload_event.clear()
# Cancel pending wait tasks
for task in pending:
task.cancel()
# Perform reload if config_path is available
if config_path:
await reload_configuration(config, config_path, components)
else:
logger.warning("Cannot reload: no config path available")
# Continue main loop
continue
except Exception as e:
logger.exception("Error in main loop: %s", e)
finally:
@@ -381,7 +380,7 @@ async def _run_async(config, config_path=None):
logger.warning("Timeout waiting for tasks to cancel")
except Exception as e:
logger.debug("Exception during task cancellation: %s", e)
# Close message journal
try:
await msg_journal.close()
@@ -487,7 +486,7 @@ def run(config, config_path=None):
"""Start the hbd service (blocking).
Manually manages the event loop to ensure clean shutdown.
Args:
config: Configuration dictionary
config_path: Path to config file (for reload support)
hbd/server/monitor.py
+2 -5
View File
@@ -1,15 +1,13 @@
"""Monitor helper for heartbeat daemon.
This module provides monitoring tasks for the heartbeat daemon.
The primary reachability monitoring is now event-driven (timers set/reset
The primary reachability monitoring is now event-driven (timers set/reset
on HTB arrival in udp.py) rather than periodic polling.
This module can be extended for additional monitoring tasks.
"""
from __future__ import annotations
import asyncio
import time
from . import notify as notify_mod
DROPOVERDUE = 7 * 24 * 3600
@@ -18,11 +16,10 @@ eventlog = notify_mod.eventlog
async def cleanup_connections(hbdclass):
"""Clean up connection timers on shutdown.
Cancels all active overdue timers to prevent callbacks after shutdown.
"""
for hostname, host in list(hbdclass.Host.hosts.items()):
for conn_type, conn in host.connections.items():
if hasattr(conn, 'cancel_overdue_timer'):
conn.cancel_overdue_timer()
hbd/server/notify.py
+2 -6
View File
@@ -21,7 +21,7 @@ import smtplib
import subprocess
import time
import sys
from dataclasses import dataclass, field
from dataclasses import dataclass
from typing import Optional
from . import data
@@ -49,6 +49,7 @@ logf = None
_LEVEL_ORDER = {"RECOVER": 0, "INFO": 0, "WARNING": 1, "CRITICAL": 2}
def _level_value(level: str) -> int:
return _LEVEL_ORDER.get(level.upper(), 0)
@@ -103,7 +104,6 @@ def initlog(logfile):
def closelog():
global logf
if logf and logf != sys.stderr:
try:
logf.close()
@@ -331,8 +331,6 @@ async def _send_sms_voipms_async(channel_cfg: dict, notif: Notification) -> bool
return False
async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
"""Send a Matrix message using matrix-nio."""
try:
@@ -341,7 +339,6 @@ async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
logger.error("matrix-nio not installed; pip install matrix-nio")
return False
from nio import AsyncClient
homeserver = channel_cfg.get("homeserver", "")
access_token = channel_cfg.get("access_token", "")
room_id = channel_cfg.get("room_id", "")
@@ -359,7 +356,6 @@ async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
client = AsyncClient(homeserver)
client.access_token = access_token
try:
from nio import RoomSendResponse
content = {
"msgtype": "m.text",
"body": text,
hbd/server/settings.py
+1 -1
View File
@@ -420,7 +420,7 @@ def get_settings_sections(config: dict, threshold_checker=None, user=None) -> li
"fields": [
field("journal_enabled", "Enabled", "boolean",
"Turn journalling on or off.", editable=True),
field("journal_dir", "Journal directory","path",
field("journal_dir", "Journal directory", "path",
"Directory where journal files are written.", editable=True),
field("journal_file", "Journal filename", "text",
"Base filename for the journal (rotated copies get a numeric suffix)."),
hbd/server/threshold.py
+110 -107
View File
@@ -13,13 +13,14 @@ import asyncio
import logging
import time
from enum import Enum
from typing import Dict, List, Any, Optional, Tuple, Callable
from typing import Dict, List, Any, Optional, Tuple
from . import notify as notify_mod
from .config import THRESHOLD_DEFAULTS
logger = logging.getLogger(__name__)
eventlog = notify_mod.eventlog
class AlertLevel(Enum):
"""Alert severity levels."""
OK = 0
@@ -64,30 +65,30 @@ class AlertState:
self.acknowledged_at = None # Timestamp when acknowledged
self.consecutive_count = 0 # Consecutive exceedances while still OK (for count gating)
self.pending_since: Optional[float] = None # non-None while waiting out grace period before notifying
def update(
self,
level: AlertLevel,
self,
level: AlertLevel,
value: Any,
threshold_value: Optional[float] = None,
operator: Optional[str] = None
) -> bool:
"""
Update alert state.
Args:
level: New alert level
value: Current metric value
threshold_value: The threshold value that was exceeded (if applicable)
operator: The comparison operator (>, <, >=, etc.)
Returns:
True if state changed (notification needed), False otherwise
"""
now = time.time()
self.last_check = now
self.last_value = value
# Update threshold info when alert is active
if level != AlertLevel.OK:
self.threshold_value = threshold_value
@@ -96,7 +97,7 @@ class AlertState:
# Clear threshold info when returning to OK
self.threshold_value = None
self.operator = None
# Check if state changed
if level != self.level:
logger.info(
@@ -116,13 +117,13 @@ class AlertState:
self.acknowledged = False
self.acknowledged_at = None
return True
return False
def to_dict(self) -> dict:
"""Convert alert state to dictionary for serialization."""
import math
# Helper to sanitize numeric values for JSON (handle inf/nan)
def sanitize_value(val):
if isinstance(val, float):
@@ -131,7 +132,7 @@ class AlertState:
if math.isnan(val):
return None
return val
result = {
"metric_path": self.metric_path,
"level": self.level.name,
@@ -141,11 +142,11 @@ class AlertState:
"notification_count": self.notification_count,
"acknowledged": self.acknowledged,
}
# Include acknowledgment timestamp if acknowledged
if self.acknowledged_at is not None:
result["acknowledged_at"] = self.acknowledged_at
# Include threshold info if available
if self.threshold_value is not None:
result["threshold_value"] = sanitize_value(self.threshold_value)
@@ -164,7 +165,7 @@ class AlertState:
result["recovery_threshold"] = round(self.threshold_value + ha, 4)
return result
def __setstate__(self, state):
"""Restore from pickle, backfilling fields added after the pickle was written."""
self.__dict__.update(state)
@@ -182,6 +183,7 @@ class AlertState:
def __str__(self):
return self.to_dict().__str__()
class ThresholdConfig:
"""Configuration for a single threshold check."""
@@ -218,7 +220,7 @@ class ThresholdConfig:
self.display = display
self.count = max(1, int(count))
self.grace = float(grace) if grace is not None else None
# Parse operator
try:
self.operator = ComparisonOperator(operator)
@@ -229,7 +231,7 @@ class ThresholdConfig:
metric_path
)
self.operator = ComparisonOperator.GT
def evaluate(self, value: float) -> AlertLevel:
"""
Evaluate a value against this threshold.
@@ -271,7 +273,7 @@ class ThresholdConfig:
return AlertLevel.WARNING
return AlertLevel.OK
def evaluate_with_hysteresis(
self,
value: float,
@@ -279,11 +281,11 @@ class ThresholdConfig:
) -> AlertLevel:
"""
Evaluate with hysteresis to prevent flapping.
Args:
value: Current metric value
current_level: Current alert level
Returns:
New alert level considering hysteresis
"""
@@ -296,7 +298,7 @@ class ThresholdConfig:
# If no hysteresis, return new level
if self.hysteresis == 0.0:
return new_level
# If improving (going to a lower severity), apply hysteresis
if new_level.value < current_level.value:
# For recovery, value must be better by hysteresis amount
@@ -306,10 +308,10 @@ class ThresholdConfig:
threshold = self.warning
else:
return new_level
# Calculate hysteresis threshold
hysteresis_amount = abs(threshold * self.hysteresis)
if self.operator in [ComparisonOperator.GT, ComparisonOperator.GTE]:
# For "greater than" thresholds, value must go below by hysteresis
recovery_threshold = threshold - hysteresis_amount
@@ -322,9 +324,9 @@ class ThresholdConfig:
if value <= recovery_threshold:
# Not enough improvement, keep current level
return current_level
return new_level
def _compare(self, value: float, threshold: float) -> bool:
"""Perform comparison based on operator."""
if self.operator == ComparisonOperator.GT:
@@ -344,7 +346,7 @@ class ThresholdConfig:
class ThresholdChecker:
"""Main threshold checking and alerting system."""
def __init__(
self,
config: Dict[str, Any],
@@ -353,7 +355,7 @@ class ThresholdChecker:
):
"""
Initialize threshold checker.
Args:
config: Threshold configuration dictionary from YAML
renotify_interval: Seconds between repeat notifications (default: 1 hour)
@@ -373,14 +375,14 @@ class ThresholdChecker:
# Default config name to use when no mapping exists
self.default_config = "default"
self.renotify_interval = renotify_interval
self.grace_seconds: float = float(config.get("grace", 2))
self.journal = journal
# Parse configuration
self._parse_config(config)
total_thresholds = sum(len(cfg) for cfg in self.threshold_configs.values())
if total_thresholds == 0 and len(self.thresholds) > 0:
# Backward compatibility: using single threshold set
@@ -392,18 +394,18 @@ class ThresholdChecker:
len(self.threshold_configs),
total_thresholds
)
def reload(self, config: Dict[str, Any]):
"""Reload threshold configuration from new config dict.
This clears all existing thresholds and re-parses from the new configuration.
Alert states are preserved to maintain hysteresis across reloads.
Args:
config: New configuration dictionary
"""
logger.info("Reloading threshold configuration...")
# Clear old configuration
self.threshold_configs.clear()
self.threshold_raw_configs.clear()
@@ -413,13 +415,13 @@ class ThresholdChecker:
# Parse new configuration
self._parse_config(config)
total_thresholds = sum(len(cfg) for cfg in self.threshold_configs.values())
if total_thresholds == 0 and len(self.thresholds) > 0:
total_thresholds = len(self.thresholds)
logger.info("Threshold configuration reloaded: %d total thresholds", total_thresholds)
def _parse_config(self, config: Dict[str, Any]):
"""Parse threshold configuration from YAML structure.
@@ -449,7 +451,7 @@ class ThresholdChecker:
self._parse_legacy_config(config)
else:
logger.info("No thresholds configured")
def _parse_multi_config(self, config: Dict[str, Any]):
"""Parse multiple named threshold configurations."""
threshold_configs = config.get("threshold_configs", {})
@@ -545,34 +547,34 @@ class ThresholdChecker:
if isinstance(legacy_mapping, dict):
for host_name, value in legacy_mapping.items():
self.host_config_mapping[host_name] = _normalise(value)
# Set default config (first one alphabetically or explicitly set)
self.default_config = config.get("default_threshold_config", "default")
if self.default_config not in self.threshold_configs and self.threshold_configs:
# Use first available config as default
self.default_config = sorted(self.threshold_configs.keys())[0]
logger.info("Using '%s' as default threshold config", self.default_config)
logger.info(
"Loaded %d threshold configurations with %d host mappings",
len(self.threshold_configs),
len(self.host_config_mapping)
)
def _parse_legacy_config(self, config: Dict[str, Any]):
"""Parse legacy single threshold configuration for backward compatibility."""
if not config or "thresholds" not in config:
logger.info("No thresholds configured")
return
thresholds_config = config["thresholds"]
for plugin_name, plugin_thresholds in thresholds_config.items():
if not isinstance(plugin_thresholds, dict):
continue
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=self.thresholds)
def _parse_plugin_thresholds(
self,
plugin_name: str,
@@ -580,7 +582,7 @@ class ThresholdChecker:
target_dict: Optional[Dict[str, ThresholdConfig]] = None
):
"""Parse thresholds for a specific plugin.
Args:
plugin_name: Name of the plugin
thresholds: Threshold configuration dictionary
@@ -588,7 +590,7 @@ class ThresholdChecker:
"""
if target_dict is None:
target_dict = self.thresholds
# Special handling for RTT thresholds (per-host)
if plugin_name == "rtt":
self._parse_rtt_thresholds(thresholds, target_dict)
@@ -606,7 +608,7 @@ class ThresholdChecker:
for metric_name, threshold_config in thresholds.items():
if not isinstance(threshold_config, dict):
continue
# Handle nested metrics (e.g., partitions./.percent or pools.*.status)
if metric_name == "partitions":
self._parse_partition_thresholds(plugin_name, threshold_config, target_dict)
@@ -614,9 +616,9 @@ class ThresholdChecker:
if metric_name == "pools":
self._parse_pool_thresholds(plugin_name, threshold_config, target_dict)
continue
metric_path = f"{plugin_name}.{metric_name}"
# Extract threshold values
warning = threshold_config.get("warning")
critical = threshold_config.get("critical")
@@ -643,7 +645,7 @@ class ThresholdChecker:
display=display,
grace=grace,
)
target_dict[metric_path] = threshold
logger.debug(
"Registered threshold for %s: warn=%s, crit=%s, op=%s",
@@ -652,7 +654,7 @@ class ThresholdChecker:
critical,
operator
)
def _parse_partition_thresholds(
self,
plugin_name: str,
@@ -660,7 +662,7 @@ class ThresholdChecker:
target_dict: Optional[Dict[str, ThresholdConfig]] = None
):
"""Parse partition-specific thresholds for disk monitoring.
Args:
plugin_name: Name of the plugin
partitions: Partition threshold configuration
@@ -668,18 +670,18 @@ class ThresholdChecker:
"""
if target_dict is None:
target_dict = self.thresholds
for partition, metrics in partitions.items():
if not isinstance(metrics, dict):
continue
for metric_name, threshold_config in metrics.items():
if not isinstance(threshold_config, dict):
continue
# Create metric path like "disk_monitor./dev/sda1.percent"
metric_path = f"{plugin_name}.{partition}.{metric_name}"
warning = threshold_config.get("warning")
critical = threshold_config.get("critical")
operator = threshold_config.get("operator", ">")
@@ -700,7 +702,7 @@ class ThresholdChecker:
display=display,
grace=grace,
)
target_dict[metric_path] = threshold
def _parse_pool_thresholds(
@@ -761,26 +763,26 @@ class ThresholdChecker:
target_dict: Optional[Dict[str, ThresholdConfig]] = None
):
"""Parse RTT thresholds (network latency thresholds).
RTT thresholds are configured as:
thresholds:
rtt:
warning: 100.0 # ms
critical: 500.0 # ms
Args:
rtt_thresholds: RTT threshold configuration
target_dict: Dictionary to store parsed thresholds
"""
if target_dict is None:
target_dict = self.thresholds
if not isinstance(rtt_thresholds, dict):
return
# Metric path is simply "rtt" (not per-host)
metric_path = "rtt"
warning = rtt_thresholds.get("warning")
critical = rtt_thresholds.get("critical")
operator = rtt_thresholds.get("operator", ">")
@@ -813,7 +815,7 @@ class ThresholdChecker:
critical,
count,
)
def get_thresholds_for_host(self, host_name: str) -> Dict[str, ThresholdConfig]:
"""Get the effective threshold configuration for a host.
@@ -864,7 +866,7 @@ class ThresholdChecker:
else:
result.update(raw)
return result
def check_value(
self,
host_name: str,
@@ -874,30 +876,30 @@ class ThresholdChecker:
) -> Optional[Tuple[AlertLevel, AlertLevel]]:
"""
Check a single value against configured threshold.
Args:
host_name: Name of the host
metric_path: Full metric path (e.g., "rtt.hostname")
value: The metric value to check
alert_states: Host's alert_states dictionary
Returns:
Tuple of (old_level, new_level) if state changed, None otherwise
"""
# Get host-specific thresholds
thresholds = self.get_thresholds_for_host(host_name)
if metric_path not in thresholds:
return None
threshold = thresholds[metric_path]
# Get or create alert state
if metric_path not in alert_states:
alert_states[metric_path] = AlertState(metric_path)
alert_state = alert_states[metric_path]
# Evaluate threshold with hysteresis
new_level = threshold.evaluate_with_hysteresis(
value,
@@ -946,6 +948,7 @@ class ThresholdChecker:
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, None)
return None
def _find_threshold(
self, thresholds: Dict[str, "ThresholdConfig"], metric_path: str
) -> Tuple[Optional["ThresholdConfig"], Optional[str]]:
@@ -985,21 +988,21 @@ class ThresholdChecker:
) -> list:
"""
Check plugin data against configured thresholds.
Args:
host_name: Name of the host
plugin_name: Name of the plugin
data: Plugin data dictionary
alert_states: Host's alert_states dictionary
Returns:
List of (metric_path, old_level, new_level, value) tuples for state changes
"""
state_changes = []
# Get host-specific thresholds
thresholds = self.get_thresholds_for_host(host_name)
# Check flat metrics
for metric_name, value in data.items():
metric_path = f"{plugin_name}.{metric_name}"
@@ -1045,9 +1048,9 @@ class ThresholdChecker:
alert_states,
state_changes
)
return state_changes
def _check_nested_metrics(
self,
host_name: str,
@@ -1059,7 +1062,7 @@ class ThresholdChecker:
"""Check nested metrics like partition-specific thresholds."""
# Get host-specific thresholds
thresholds = self.get_thresholds_for_host(host_name)
# ZFS pool health checks
if plugin_name == "zfs_monitor" and "pools" in data:
pools = data["pools"]
@@ -1103,29 +1106,29 @@ class ThresholdChecker:
partitions = data["partitions"]
if not isinstance(partitions, dict):
return
for partition, metrics in partitions.items():
if not isinstance(metrics, dict):
continue
for metric_name, value in metrics.items():
metric_path = f"{plugin_name}.{partition}.{metric_name}"
if metric_path not in thresholds:
continue
threshold = thresholds[metric_path]
if metric_path not in alert_states:
alert_states[metric_path] = AlertState(metric_path)
alert_state = alert_states[metric_path]
new_level = threshold.evaluate_with_hysteresis(
value,
alert_state.level
)
# Determine which threshold was exceeded
threshold_value = None
if new_level == AlertLevel.CRITICAL and threshold.critical is not None:
@@ -1141,7 +1144,7 @@ class ThresholdChecker:
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, data)
elif new_level != AlertLevel.OK:
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, data)
def _trigger_notification(
self,
host_name: str,
@@ -1155,7 +1158,7 @@ class ThresholdChecker:
metric_name: Optional[str] = None,
):
"""Trigger a notification for an alert state change.
Args:
host_name: Name of the host
metric_path: Full metric path
@@ -1171,7 +1174,7 @@ class ThresholdChecker:
threshold_value = threshold.critical
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
threshold_value = threshold.warning
# Format operator symbol
op_symbol = threshold.operator.value
@@ -1223,7 +1226,7 @@ class ThresholdChecker:
formatted_threshold_msg = _fmt() if has_display and new_level != AlertLevel.OK else None
return lvl, message, formatted_threshold_msg
def _send_notification(
self,
host_name: str,
@@ -1254,7 +1257,7 @@ class ThresholdChecker:
service=short_path,
),
))
# Log to journal
if self.journal is not None:
try:
@@ -1270,7 +1273,7 @@ class ThresholdChecker:
logger.debug(f"Failed to log threshold event to journal: {e}")
# Log to eventlog as well
eventlog(host_name, lvl, message, service="threshold")
def _format_display(
self,
display_format: str,
@@ -1330,7 +1333,7 @@ class ThresholdChecker:
status = plugin_data.get(f"{check_name}_status")
if status is not None:
format_context['status'] = status
try:
# Format the display string
return display_format.format(**format_context)
@@ -1349,7 +1352,7 @@ class ThresholdChecker:
e
)
return f"(threshold: {op_symbol} {threshold_value})"
def _apply_grace(
self,
host_name: str,
@@ -1472,7 +1475,7 @@ class ThresholdChecker:
metric_name: Optional[str] = None,
):
"""Check if we should send a repeat notification.
Args:
host_name: Name of the host
alert_state: Current alert state
@@ -1487,16 +1490,16 @@ class ThresholdChecker:
# Skip reminders if alert has been acknowledged
if alert_state.acknowledged:
return
now = time.time()
# Check if we should re-notify
if alert_state.last_notification is None:
# First notification already sent during state change
alert_state.last_notification = now
alert_state.notification_count = 1
return
if (now - alert_state.last_notification) >= self.renotify_interval:
# Determine which threshold is active
threshold_value = None
@@ -1504,7 +1507,7 @@ class ThresholdChecker:
threshold_value = threshold.critical
elif alert_state.level == AlertLevel.WARNING and threshold.warning is not None:
threshold_value = threshold.warning
# Format operator symbol
op_symbol = threshold.operator.value
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
@@ -1541,7 +1544,7 @@ class ThresholdChecker:
logger.info("Re-notification sent: %s", message)
alert_state.last_notification = now
alert_state.notification_count += 1
def purge_stale_alerts(self, hbdclass) -> None:
"""Remove alert states that have no matching threshold configuration.
@@ -1578,10 +1581,10 @@ class ThresholdChecker:
def get_active_alerts(self, alert_states: Dict[str, AlertState]) -> list:
"""
Get all currently active (non-OK) alerts.
Args:
alert_states: Host's alert_states dictionary
Returns:
List of AlertState objects that are not OK
"""
@@ -1589,19 +1592,19 @@ class ThresholdChecker:
state for state in alert_states.values()
if state.level != AlertLevel.OK
]
def get_alert_summary(self, alert_states: Dict[str, AlertState]) -> Dict[str, int]:
"""
Get summary counts of alert levels.
Args:
alert_states: Host's alert_states dictionary
Returns:
Dictionary with counts: {"ok": N, "warning": N, "critical": N}
"""
summary = {"ok": 0, "warning": 0, "critical": 0, "unknown": 0}
for state in alert_states.values():
if state.level == AlertLevel.OK:
summary["ok"] += 1
@@ -1611,5 +1614,5 @@ class ThresholdChecker:
summary["critical"] += 1
elif state.level == AlertLevel.UNKNOWN:
summary["unknown"] += 1
return summary
hbd/server/udp.py
+11 -11
View File
@@ -23,14 +23,14 @@ _STALE_INTERVAL_MULTIPLIER = 3
# SO_TIMESTAMP: kernel attaches a struct timeval to each received datagram.
# Supported on Linux, FreeBSD, and macOS. The constant is not exposed by
# Python's socket module on all platforms
# Python's socket module on all platforms
platform = platform_system()
if platform == "Darwin":
_SO_TIMESTAMP = 1024 # SO_TIMESTAMP on macOS (not in Python's socket module)
elif platform == "Linux":
_SO_TIMESTAMP = 29 # Linux value (not in older Python versions)
elif platform == "FreeBSD":
_SO_TIMESTAMP = 32 # FreeBSD value (not in older Python versions)
_SO_TIMESTAMP = 32 # FreeBSD value (not in older Python versions)
else:
logger.warning("SO_TIMESTAMP may not be supported on this platform (%s)", platform)
_SO_TIMESTAMP = None
@@ -71,6 +71,7 @@ class RecvmsgTransport:
Exposes the same sendto() / close() interface as asyncio's DatagramTransport
so the rest of the code does not need to know which path is in use.
"""
def __init__(self, loop, sock):
self._loop = loop
self._sock = sock
@@ -410,7 +411,7 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
if not msg:
return
now = ctx.get("recv_ts") or time.time()
# Log message to journal
msg_journal = ctx.get("msg_journal")
if msg_journal:
@@ -421,7 +422,7 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
loop.create_task(msg_journal.log_message(msg, addr, now))
except Exception as e:
logger.debug(f"Failed to log message to journal: {e}")
# Only named HTB/PLG datagrams come from clients; anything else (e.g. an internet scanner's
# SIP probe) is journaled above but must not sign on a host.
if msg.get("ID") not in ("HTB", "PLG") or not msg.get("name"):
@@ -432,7 +433,6 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
hbdcls = ctx.get("hbdclass")
msg_to_websockets = ctx.get("msg_to_websockets")
DEBUG = ctx.get("DEBUG", 0)
verbose = ctx.get("verbose", False)
# normalize addr (ip, port)
ip = addr[0] if isinstance(addr, (list, tuple)) else addr
@@ -457,7 +457,7 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
else:
host = hbdcls.Host.hosts[uname]
newh = False
cid = msg.get("id", 0)
try:
rtt = float(msg.get("rtt"))
@@ -523,14 +523,14 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
# If os_info reports an owner and none is configured server-side, apply it
if plugin_name == "os_info":
config_owner = config_mod.get_host_access(cfg, uname).get("owner")
config_owner = config_mod.get_host_access(cfg, uname).get("owner")
default_owner = config_mod.get_default_owner(cfg)
inferred_owner = config_owner or plugin_data.get("owner") or default_owner
host.owner = inferred_owner
logger.info(f"owner for {uname} is {host.owner}")
if DEBUG > 1:
print(f"Stored plugin data for {uname}: {plugin_name}")
# Check thresholds if checker is available
threshold_checker = ctx.get("threshold_checker")
if threshold_checker:
@@ -545,7 +545,7 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
print(f"Threshold state changes for {uname}: {state_changes}")
except Exception as e:
logger.error(f"Error checking thresholds for {uname}.{plugin_name}: {e}")
# Notify websockets of plugin update
if msg_to_websockets:
try:
@@ -671,13 +671,13 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
timeout_seconds = interval + grace
on_overdue, _ = _make_timer_callbacks(uname, host, ctx)
conn.reset_overdue_timer(timeout_seconds, on_overdue)
# Check RTT thresholds using the threshold checker
threshold_checker = ctx.get("threshold_checker")
if threshold_checker and rtt and rtt > 0:
# Metric path for RTT is simply "rtt"
metric_path = "rtt"
# Check against configured thresholds (handles alerts, notifications, etc.)
threshold_checker.check_value(
host_name=uname,