andreas/heartbeat
Public Access
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 54 Wiki Activity

style: clear whitespace, unused-import and blank-line lint

Browse Source 
Strip trailing whitespace, drop unused imports/variables/global statements,
split multi-imports, fix E3xx/E1xx spacing and E712 comparisons. No behaviour
change; only E501 long lines remain in flake8.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
andreas
2026-10-28 04:59:34 -04:00
co-authored by Claude Sonnet 5.5
parent 85c6a4eebe
commit 748349a7b9
35 changed files with 705 additions and 719 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hbd/server/threshold.py
+110 -107
View File
@@ -13,13 +13,14 @@ import asyncio
import logging
import time
from enum import Enum
from typing import Dict, List, Any, Optional, Tuple, Callable
from typing import Dict, List, Any, Optional, Tuple
from . import notify as notify_mod
from .config import THRESHOLD_DEFAULTS
logger = logging.getLogger(__name__)
eventlog = notify_mod.eventlog
class AlertLevel(Enum):
"""Alert severity levels."""
OK = 0
@@ -64,30 +65,30 @@ class AlertState:
self.acknowledged_at = None # Timestamp when acknowledged
self.consecutive_count = 0 # Consecutive exceedances while still OK (for count gating)
self.pending_since: Optional[float] = None # non-None while waiting out grace period before notifying
def update(
self,
level: AlertLevel,
self,
level: AlertLevel,
value: Any,
threshold_value: Optional[float] = None,
operator: Optional[str] = None
) -> bool:
"""
Update alert state.
Args:
level: New alert level
value: Current metric value
threshold_value: The threshold value that was exceeded (if applicable)
operator: The comparison operator (>, <, >=, etc.)
Returns:
True if state changed (notification needed), False otherwise
"""
now = time.time()
self.last_check = now
self.last_value = value
# Update threshold info when alert is active
if level != AlertLevel.OK:
self.threshold_value = threshold_value
@@ -96,7 +97,7 @@ class AlertState:
# Clear threshold info when returning to OK
self.threshold_value = None
self.operator = None
# Check if state changed
if level != self.level:
logger.info(
@@ -116,13 +117,13 @@ class AlertState:
self.acknowledged = False
self.acknowledged_at = None
return True
return False
def to_dict(self) -> dict:
"""Convert alert state to dictionary for serialization."""
import math
# Helper to sanitize numeric values for JSON (handle inf/nan)
def sanitize_value(val):
if isinstance(val, float):
@@ -131,7 +132,7 @@ class AlertState:
if math.isnan(val):
return None
return val
result = {
"metric_path": self.metric_path,
"level": self.level.name,
@@ -141,11 +142,11 @@ class AlertState:
"notification_count": self.notification_count,
"acknowledged": self.acknowledged,
}
# Include acknowledgment timestamp if acknowledged
if self.acknowledged_at is not None:
result["acknowledged_at"] = self.acknowledged_at
# Include threshold info if available
if self.threshold_value is not None:
result["threshold_value"] = sanitize_value(self.threshold_value)
@@ -164,7 +165,7 @@ class AlertState:
result["recovery_threshold"] = round(self.threshold_value + ha, 4)
return result
def __setstate__(self, state):
"""Restore from pickle, backfilling fields added after the pickle was written."""
self.__dict__.update(state)
@@ -182,6 +183,7 @@ class AlertState:
def __str__(self):
return self.to_dict().__str__()
class ThresholdConfig:
"""Configuration for a single threshold check."""
@@ -218,7 +220,7 @@ class ThresholdConfig:
self.display = display
self.count = max(1, int(count))
self.grace = float(grace) if grace is not None else None
# Parse operator
try:
self.operator = ComparisonOperator(operator)
@@ -229,7 +231,7 @@ class ThresholdConfig:
metric_path
)
self.operator = ComparisonOperator.GT
def evaluate(self, value: float) -> AlertLevel:
"""
Evaluate a value against this threshold.
@@ -271,7 +273,7 @@ class ThresholdConfig:
return AlertLevel.WARNING
return AlertLevel.OK
def evaluate_with_hysteresis(
self,
value: float,
@@ -279,11 +281,11 @@ class ThresholdConfig:
) -> AlertLevel:
"""
Evaluate with hysteresis to prevent flapping.
Args:
value: Current metric value
current_level: Current alert level
Returns:
New alert level considering hysteresis
"""
@@ -296,7 +298,7 @@ class ThresholdConfig:
# If no hysteresis, return new level
if self.hysteresis == 0.0:
return new_level
# If improving (going to a lower severity), apply hysteresis
if new_level.value < current_level.value:
# For recovery, value must be better by hysteresis amount
@@ -306,10 +308,10 @@ class ThresholdConfig:
threshold = self.warning
else:
return new_level
# Calculate hysteresis threshold
hysteresis_amount = abs(threshold * self.hysteresis)
if self.operator in [ComparisonOperator.GT, ComparisonOperator.GTE]:
# For "greater than" thresholds, value must go below by hysteresis
recovery_threshold = threshold - hysteresis_amount
@@ -322,9 +324,9 @@ class ThresholdConfig:
if value <= recovery_threshold:
# Not enough improvement, keep current level
return current_level
return new_level
def _compare(self, value: float, threshold: float) -> bool:
"""Perform comparison based on operator."""
if self.operator == ComparisonOperator.GT:
@@ -344,7 +346,7 @@ class ThresholdConfig:
class ThresholdChecker:
"""Main threshold checking and alerting system."""
def __init__(
self,
config: Dict[str, Any],
@@ -353,7 +355,7 @@ class ThresholdChecker:
):
"""
Initialize threshold checker.
Args:
config: Threshold configuration dictionary from YAML
renotify_interval: Seconds between repeat notifications (default: 1 hour)
@@ -373,14 +375,14 @@ class ThresholdChecker:
# Default config name to use when no mapping exists
self.default_config = "default"
self.renotify_interval = renotify_interval
self.grace_seconds: float = float(config.get("grace", 2))
self.journal = journal
# Parse configuration
self._parse_config(config)
total_thresholds = sum(len(cfg) for cfg in self.threshold_configs.values())
if total_thresholds == 0 and len(self.thresholds) > 0:
# Backward compatibility: using single threshold set
@@ -392,18 +394,18 @@ class ThresholdChecker:
len(self.threshold_configs),
total_thresholds
)
def reload(self, config: Dict[str, Any]):
"""Reload threshold configuration from new config dict.
This clears all existing thresholds and re-parses from the new configuration.
Alert states are preserved to maintain hysteresis across reloads.
Args:
config: New configuration dictionary
"""
logger.info("Reloading threshold configuration...")
# Clear old configuration
self.threshold_configs.clear()
self.threshold_raw_configs.clear()
@@ -413,13 +415,13 @@ class ThresholdChecker:
# Parse new configuration
self._parse_config(config)
total_thresholds = sum(len(cfg) for cfg in self.threshold_configs.values())
if total_thresholds == 0 and len(self.thresholds) > 0:
total_thresholds = len(self.thresholds)
logger.info("Threshold configuration reloaded: %d total thresholds", total_thresholds)
def _parse_config(self, config: Dict[str, Any]):
"""Parse threshold configuration from YAML structure.
@@ -449,7 +451,7 @@ class ThresholdChecker:
self._parse_legacy_config(config)
else:
logger.info("No thresholds configured")
def _parse_multi_config(self, config: Dict[str, Any]):
"""Parse multiple named threshold configurations."""
threshold_configs = config.get("threshold_configs", {})
@@ -545,34 +547,34 @@ class ThresholdChecker:
if isinstance(legacy_mapping, dict):
for host_name, value in legacy_mapping.items():
self.host_config_mapping[host_name] = _normalise(value)
# Set default config (first one alphabetically or explicitly set)
self.default_config = config.get("default_threshold_config", "default")
if self.default_config not in self.threshold_configs and self.threshold_configs:
# Use first available config as default
self.default_config = sorted(self.threshold_configs.keys())[0]
logger.info("Using '%s' as default threshold config", self.default_config)
logger.info(
"Loaded %d threshold configurations with %d host mappings",
len(self.threshold_configs),
len(self.host_config_mapping)
)
def _parse_legacy_config(self, config: Dict[str, Any]):
"""Parse legacy single threshold configuration for backward compatibility."""
if not config or "thresholds" not in config:
logger.info("No thresholds configured")
return
thresholds_config = config["thresholds"]
for plugin_name, plugin_thresholds in thresholds_config.items():
if not isinstance(plugin_thresholds, dict):
continue
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=self.thresholds)
def _parse_plugin_thresholds(
self,
plugin_name: str,
@@ -580,7 +582,7 @@ class ThresholdChecker:
target_dict: Optional[Dict[str, ThresholdConfig]] = None
):
"""Parse thresholds for a specific plugin.
Args:
plugin_name: Name of the plugin
thresholds: Threshold configuration dictionary
@@ -588,7 +590,7 @@ class ThresholdChecker:
"""
if target_dict is None:
target_dict = self.thresholds
# Special handling for RTT thresholds (per-host)
if plugin_name == "rtt":
self._parse_rtt_thresholds(thresholds, target_dict)
@@ -606,7 +608,7 @@ class ThresholdChecker:
for metric_name, threshold_config in thresholds.items():
if not isinstance(threshold_config, dict):
continue
# Handle nested metrics (e.g., partitions./.percent or pools.*.status)
if metric_name == "partitions":
self._parse_partition_thresholds(plugin_name, threshold_config, target_dict)
@@ -614,9 +616,9 @@ class ThresholdChecker:
if metric_name == "pools":
self._parse_pool_thresholds(plugin_name, threshold_config, target_dict)
continue
metric_path = f"{plugin_name}.{metric_name}"
# Extract threshold values
warning = threshold_config.get("warning")
critical = threshold_config.get("critical")
@@ -643,7 +645,7 @@ class ThresholdChecker:
display=display,
grace=grace,
)
target_dict[metric_path] = threshold
logger.debug(
"Registered threshold for %s: warn=%s, crit=%s, op=%s",
@@ -652,7 +654,7 @@ class ThresholdChecker:
critical,
operator
)
def _parse_partition_thresholds(
self,
plugin_name: str,
@@ -660,7 +662,7 @@ class ThresholdChecker:
target_dict: Optional[Dict[str, ThresholdConfig]] = None
):
"""Parse partition-specific thresholds for disk monitoring.
Args:
plugin_name: Name of the plugin
partitions: Partition threshold configuration
@@ -668,18 +670,18 @@ class ThresholdChecker:
"""
if target_dict is None:
target_dict = self.thresholds
for partition, metrics in partitions.items():
if not isinstance(metrics, dict):
continue
for metric_name, threshold_config in metrics.items():
if not isinstance(threshold_config, dict):
continue
# Create metric path like "disk_monitor./dev/sda1.percent"
metric_path = f"{plugin_name}.{partition}.{metric_name}"
warning = threshold_config.get("warning")
critical = threshold_config.get("critical")
operator = threshold_config.get("operator", ">")
@@ -700,7 +702,7 @@ class ThresholdChecker:
display=display,
grace=grace,
)
target_dict[metric_path] = threshold
def _parse_pool_thresholds(
@@ -761,26 +763,26 @@ class ThresholdChecker:
target_dict: Optional[Dict[str, ThresholdConfig]] = None
):
"""Parse RTT thresholds (network latency thresholds).
RTT thresholds are configured as:
thresholds:
rtt:
warning: 100.0 # ms
critical: 500.0 # ms
Args:
rtt_thresholds: RTT threshold configuration
target_dict: Dictionary to store parsed thresholds
"""
if target_dict is None:
target_dict = self.thresholds
if not isinstance(rtt_thresholds, dict):
return
# Metric path is simply "rtt" (not per-host)
metric_path = "rtt"
warning = rtt_thresholds.get("warning")
critical = rtt_thresholds.get("critical")
operator = rtt_thresholds.get("operator", ">")
@@ -813,7 +815,7 @@ class ThresholdChecker:
critical,
count,
)
def get_thresholds_for_host(self, host_name: str) -> Dict[str, ThresholdConfig]:
"""Get the effective threshold configuration for a host.
@@ -864,7 +866,7 @@ class ThresholdChecker:
else:
result.update(raw)
return result
def check_value(
self,
host_name: str,
@@ -874,30 +876,30 @@ class ThresholdChecker:
) -> Optional[Tuple[AlertLevel, AlertLevel]]:
"""
Check a single value against configured threshold.
Args:
host_name: Name of the host
metric_path: Full metric path (e.g., "rtt.hostname")
value: The metric value to check
alert_states: Host's alert_states dictionary
Returns:
Tuple of (old_level, new_level) if state changed, None otherwise
"""
# Get host-specific thresholds
thresholds = self.get_thresholds_for_host(host_name)
if metric_path not in thresholds:
return None
threshold = thresholds[metric_path]
# Get or create alert state
if metric_path not in alert_states:
alert_states[metric_path] = AlertState(metric_path)
alert_state = alert_states[metric_path]
# Evaluate threshold with hysteresis
new_level = threshold.evaluate_with_hysteresis(
value,
@@ -946,6 +948,7 @@ class ThresholdChecker:
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, None)
return None
def _find_threshold(
self, thresholds: Dict[str, "ThresholdConfig"], metric_path: str
) -> Tuple[Optional["ThresholdConfig"], Optional[str]]:
@@ -985,21 +988,21 @@ class ThresholdChecker:
) -> list:
"""
Check plugin data against configured thresholds.
Args:
host_name: Name of the host
plugin_name: Name of the plugin
data: Plugin data dictionary
alert_states: Host's alert_states dictionary
Returns:
List of (metric_path, old_level, new_level, value) tuples for state changes
"""
state_changes = []
# Get host-specific thresholds
thresholds = self.get_thresholds_for_host(host_name)
# Check flat metrics
for metric_name, value in data.items():
metric_path = f"{plugin_name}.{metric_name}"
@@ -1045,9 +1048,9 @@ class ThresholdChecker:
alert_states,
state_changes
)
return state_changes
def _check_nested_metrics(
self,
host_name: str,
@@ -1059,7 +1062,7 @@ class ThresholdChecker:
"""Check nested metrics like partition-specific thresholds."""
# Get host-specific thresholds
thresholds = self.get_thresholds_for_host(host_name)
# ZFS pool health checks
if plugin_name == "zfs_monitor" and "pools" in data:
pools = data["pools"]
@@ -1103,29 +1106,29 @@ class ThresholdChecker:
partitions = data["partitions"]
if not isinstance(partitions, dict):
return
for partition, metrics in partitions.items():
if not isinstance(metrics, dict):
continue
for metric_name, value in metrics.items():
metric_path = f"{plugin_name}.{partition}.{metric_name}"
if metric_path not in thresholds:
continue
threshold = thresholds[metric_path]
if metric_path not in alert_states:
alert_states[metric_path] = AlertState(metric_path)
alert_state = alert_states[metric_path]
new_level = threshold.evaluate_with_hysteresis(
value,
alert_state.level
)
# Determine which threshold was exceeded
threshold_value = None
if new_level == AlertLevel.CRITICAL and threshold.critical is not None:
@@ -1141,7 +1144,7 @@ class ThresholdChecker:
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, data)
elif new_level != AlertLevel.OK:
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, data)
def _trigger_notification(
self,
host_name: str,
@@ -1155,7 +1158,7 @@ class ThresholdChecker:
metric_name: Optional[str] = None,
):
"""Trigger a notification for an alert state change.
Args:
host_name: Name of the host
metric_path: Full metric path
@@ -1171,7 +1174,7 @@ class ThresholdChecker:
threshold_value = threshold.critical
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
threshold_value = threshold.warning
# Format operator symbol
op_symbol = threshold.operator.value
@@ -1223,7 +1226,7 @@ class ThresholdChecker:
formatted_threshold_msg = _fmt() if has_display and new_level != AlertLevel.OK else None
return lvl, message, formatted_threshold_msg
def _send_notification(
self,
host_name: str,
@@ -1254,7 +1257,7 @@ class ThresholdChecker:
service=short_path,
),
))
# Log to journal
if self.journal is not None:
try:
@@ -1270,7 +1273,7 @@ class ThresholdChecker:
logger.debug(f"Failed to log threshold event to journal: {e}")
# Log to eventlog as well
eventlog(host_name, lvl, message, service="threshold")
def _format_display(
self,
display_format: str,
@@ -1330,7 +1333,7 @@ class ThresholdChecker:
status = plugin_data.get(f"{check_name}_status")
if status is not None:
format_context['status'] = status
try:
# Format the display string
return display_format.format(**format_context)
@@ -1349,7 +1352,7 @@ class ThresholdChecker:
e
)
return f"(threshold: {op_symbol} {threshold_value})"
def _apply_grace(
self,
host_name: str,
@@ -1472,7 +1475,7 @@ class ThresholdChecker:
metric_name: Optional[str] = None,
):
"""Check if we should send a repeat notification.
Args:
host_name: Name of the host
alert_state: Current alert state
@@ -1487,16 +1490,16 @@ class ThresholdChecker:
# Skip reminders if alert has been acknowledged
if alert_state.acknowledged:
return
now = time.time()
# Check if we should re-notify
if alert_state.last_notification is None:
# First notification already sent during state change
alert_state.last_notification = now
alert_state.notification_count = 1
return
if (now - alert_state.last_notification) >= self.renotify_interval:
# Determine which threshold is active
threshold_value = None
@@ -1504,7 +1507,7 @@ class ThresholdChecker:
threshold_value = threshold.critical
elif alert_state.level == AlertLevel.WARNING and threshold.warning is not None:
threshold_value = threshold.warning
# Format operator symbol
op_symbol = threshold.operator.value
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
@@ -1541,7 +1544,7 @@ class ThresholdChecker:
logger.info("Re-notification sent: %s", message)
alert_state.last_notification = now
alert_state.notification_count += 1
def purge_stale_alerts(self, hbdclass) -> None:
"""Remove alert states that have no matching threshold configuration.
@@ -1578,10 +1581,10 @@ class ThresholdChecker:
def get_active_alerts(self, alert_states: Dict[str, AlertState]) -> list:
"""
Get all currently active (non-OK) alerts.
Args:
alert_states: Host's alert_states dictionary
Returns:
List of AlertState objects that are not OK
"""
@@ -1589,19 +1592,19 @@ class ThresholdChecker:
state for state in alert_states.values()
if state.level != AlertLevel.OK
]
def get_alert_summary(self, alert_states: Dict[str, AlertState]) -> Dict[str, int]:
"""
Get summary counts of alert levels.
Args:
alert_states: Host's alert_states dictionary
Returns:
Dictionary with counts: {"ok": N, "warning": N, "critical": N}
"""
summary = {"ok": 0, "warning": 0, "critical": 0, "unknown": 0}
for state in alert_states.values():
if state.level == AlertLevel.OK:
summary["ok"] += 1
@@ -1611,5 +1614,5 @@ class ThresholdChecker:
summary["critical"] += 1
elif state.level == AlertLevel.UNKNOWN:
summary["unknown"] += 1
return summary