Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
13f9916693 | ||
|
|
80b830fb07 | ||
|
|
6790ff53a8 | ||
|
|
445ff6b474 | ||
|
|
748349a7b9 | ||
|
|
85c6a4eebe | ||
|
|
952adb68f8 | ||
|
|
3e2d343fe7 | ||
|
|
885d5dc937 |
@@ -2,6 +2,22 @@
|
|||||||
|
|
||||||
All notable changes to this project are documented here, organized by release.
|
All notable changes to this project are documented here, organized by release.
|
||||||
|
|
||||||
|
## [5.4.7]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- clear all mypy errors
|
||||||
|
- honour the legacy thresholds: config; repair the stale test suite
|
||||||
|
- ignore datagrams that are not named HTB/PLG messages
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.4.6]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- re-resolve the server hostnames at the DNS TTL
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## [5.4.5]
|
## [5.4.5]
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ hbd passwd <username>
|
|||||||
|
|
||||||
Tests live in `tests/` (pytest, imported as a package). A second directory `test/` contains only TLS fixtures (`.pem` files), not test code.
|
Tests live in `tests/` (pytest, imported as a package). A second directory `test/` contains only TLS fixtures (`.pem` files), not test code.
|
||||||
|
|
||||||
Line length: 111 characters (`black`, `flake8`, both configured).
|
Line length: `black` and `flake8` both use 165 characters (set in `pyproject.toml`, `tox.ini` and `.flake8`).
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ A lightweight UDP-based host monitoring system. Monitored hosts run a client (`h
|
|||||||
└────────────────────┘ └────────────────────────────┘
|
└────────────────────┘ └────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
**Package:** `hbd` v5.4.5
|
**Package:** `hbd` v5.4.7
|
||||||
**Python:** 3.11+
|
**Python:** 3.11+
|
||||||
|
|
||||||
### Subpackages
|
### Subpackages
|
||||||
@@ -242,6 +242,7 @@ plugins:
|
|||||||
### Connection behaviour
|
### Connection behaviour
|
||||||
|
|
||||||
- The client sends heartbeats over UDP to each server address resolved from the hostname (IPv4 and IPv6).
|
- The client sends heartbeats over UDP to each server address resolved from the hostname (IPv4 and IPv6).
|
||||||
|
- The server hostname is re-resolved every TTL seconds (the smallest A/AAAA TTL, at least 30s; 300s when the TTL cannot be determined, e.g. an IP literal), so an address change is followed without restarting the client. A failed lookup keeps the current addresses.
|
||||||
- If a connection fails to open at startup, IPv6 connections are dropped after 3 consecutive failures. IPv4 connections retry indefinitely.
|
- If a connection fails to open at startup, IPv6 connections are dropped after 3 consecutive failures. IPv4 connections retry indefinitely.
|
||||||
- In daemon mode (`-d`), all log output goes to syslog (`LOG_DAEMON` facility).
|
- In daemon mode (`-d`), all log output goes to syslog (`LOG_DAEMON` facility).
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -4,7 +4,7 @@ This package contains both the heartbeat client (hbc) and server (hbd) component
|
|||||||
organized into separate subpackages:
|
organized into separate subpackages:
|
||||||
|
|
||||||
- hbd.client: Client component with system monitoring plugins
|
- hbd.client: Client component with system monitoring plugins
|
||||||
- hbd.server: Server/daemon component with web UI and notifications
|
- hbd.server: Server/daemon component with web UI and notifications
|
||||||
- hbd.common: Shared utilities and protocol definitions
|
- hbd.common: Shared utilities and protocol definitions
|
||||||
|
|
||||||
Install options:
|
Install options:
|
||||||
@@ -14,4 +14,4 @@ Install options:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
__all__ = ["__version__"]
|
__all__ = ["__version__"]
|
||||||
__version__ = "5.4.5"
|
__version__ = "5.4.7"
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
"""HeartBeat Client (hbc) - System monitoring client."""
|
"""HeartBeat Client (hbc) - System monitoring client."""
|
||||||
|
|
||||||
from hbd import __version__
|
from hbd import __version__ # noqa: F401
|
||||||
|
|||||||
@@ -2,16 +2,16 @@
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import logging
|
from typing import Any, Dict
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import yaml
|
import yaml
|
||||||
except Exception:
|
except Exception:
|
||||||
yaml = None
|
yaml = None # type: ignore[assignment]
|
||||||
|
|
||||||
CLIENT_DEFAULTS = {
|
CLIENT_DEFAULTS: Dict[str, Any] = {
|
||||||
# Network settings
|
# Network settings
|
||||||
"hb_port": 50003, # Port where hbd servers listen
|
"hb_port": 50003, # Port where hbd servers listen
|
||||||
"interval": 10, # Heartbeat interval in seconds
|
"interval": 10, # Heartbeat interval in seconds
|
||||||
@@ -37,10 +37,10 @@ def load_config(path=None):
|
|||||||
"""Load configuration from a YAML file and merge with client defaults.
|
"""Load configuration from a YAML file and merge with client defaults.
|
||||||
|
|
||||||
If YAML is not available or the file does not exist, defaults are returned.
|
If YAML is not available or the file does not exist, defaults are returned.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
path: Path to YAML config file (default: ~/.hbc.yaml)
|
path: Path to YAML config file (default: ~/.hbc.yaml)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with configuration
|
Dictionary with configuration
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""Re-resolution of the server hostnames at the TTL of their A/AAAA records.
|
||||||
|
|
||||||
|
getaddrinfo() hides the TTL, so a minimal stub-resolver query (resolv.conf nameservers, search
|
||||||
|
list) is made only to learn it. The addresses themselves still come from getaddrinfo(). When
|
||||||
|
the TTL cannot be learned (IP literal, /etc/hosts name, no resolv.conf, resolver timeout)
|
||||||
|
DEFAULT_TTL is used.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
from typing import List, Optional, Tuple
|
||||||
|
|
||||||
|
logger = logging.getLogger("hbc.dns")
|
||||||
|
|
||||||
|
DEFAULT_TTL = 300 # used when the real TTL is unknown
|
||||||
|
MIN_TTL = 30 # a TTL of 0 must not turn into a resolution loop
|
||||||
|
FAILED_RETRY = 60 # re-try delay after a failed resolution
|
||||||
|
QUERY_TIMEOUT = 2.0
|
||||||
|
|
||||||
|
TYPE_A, TYPE_CNAME, TYPE_AAAA = 1, 5, 28
|
||||||
|
|
||||||
|
|
||||||
|
def parse_resolv_conf(path: str = "/etc/resolv.conf") -> Tuple[List[str], List[str]]:
|
||||||
|
"""Return (nameservers, search domains)."""
|
||||||
|
servers: List[str] = []
|
||||||
|
search: List[str] = []
|
||||||
|
try:
|
||||||
|
with open(path) as f:
|
||||||
|
for line in f:
|
||||||
|
parts = line.split()
|
||||||
|
if len(parts) < 2:
|
||||||
|
continue
|
||||||
|
if parts[0] == "nameserver":
|
||||||
|
servers.append(parts[1].split("%")[0])
|
||||||
|
elif parts[0] in ("search", "domain"):
|
||||||
|
search = parts[1:]
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return servers, search
|
||||||
|
|
||||||
|
|
||||||
|
def build_query(name: str, qtype: int, qid: int) -> bytes:
|
||||||
|
out = struct.pack("!HHHHHH", qid, 0x0100, 1, 0, 0, 0)
|
||||||
|
for label in name.rstrip(".").split("."):
|
||||||
|
out += bytes([len(label)]) + label.encode("idna")
|
||||||
|
return out + b"\x00" + struct.pack("!HH", qtype, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _skip_name(buf: bytes, off: int) -> int:
|
||||||
|
while True:
|
||||||
|
n = buf[off]
|
||||||
|
if n == 0:
|
||||||
|
return off + 1
|
||||||
|
if n & 0xC0:
|
||||||
|
return off + 2
|
||||||
|
off += n + 1
|
||||||
|
|
||||||
|
|
||||||
|
def min_answer_ttl(resp: bytes, qid: int, qtype: int) -> Optional[int]:
|
||||||
|
"""Smallest TTL among the answer RRs (CNAMEs included), or None if there is no qtype answer."""
|
||||||
|
try:
|
||||||
|
rid, flags, qd, an = struct.unpack("!HHHH", resp[:8])
|
||||||
|
if rid != qid or flags & 0x000F:
|
||||||
|
return None
|
||||||
|
off = 12
|
||||||
|
for _ in range(qd):
|
||||||
|
off = _skip_name(resp, off) + 4
|
||||||
|
ttls: List[int] = []
|
||||||
|
found = False
|
||||||
|
for _ in range(an):
|
||||||
|
off = _skip_name(resp, off)
|
||||||
|
rtype, _cls, ttl, rdlen = struct.unpack("!HHIH", resp[off:off + 10])
|
||||||
|
off += 10 + rdlen
|
||||||
|
if rtype in (qtype, TYPE_CNAME):
|
||||||
|
ttls.append(ttl)
|
||||||
|
found = found or rtype == qtype
|
||||||
|
return min(ttls) if found else None
|
||||||
|
except (struct.error, IndexError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _query_ttl(server: str, name: str, qtype: int) -> Optional[int]:
|
||||||
|
qid = struct.unpack("!H", os.urandom(2))[0]
|
||||||
|
try:
|
||||||
|
af = socket.AF_INET6 if ":" in server else socket.AF_INET
|
||||||
|
with socket.socket(af, socket.SOCK_DGRAM) as s:
|
||||||
|
s.settimeout(QUERY_TIMEOUT)
|
||||||
|
s.sendto(build_query(name, qtype, qid), (server, 53))
|
||||||
|
return min_answer_ttl(s.recv(4096), qid, qtype)
|
||||||
|
except (OSError, UnicodeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def lookup_ttl(host: str, af_filter: int = 0) -> Optional[int]:
|
||||||
|
"""Blocking. Smallest A/AAAA TTL for host, or None when it cannot be determined."""
|
||||||
|
try:
|
||||||
|
ipaddress.ip_address(host)
|
||||||
|
return None
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
servers, search = parse_resolv_conf()
|
||||||
|
if host.endswith("."):
|
||||||
|
names = [host]
|
||||||
|
elif "." in host:
|
||||||
|
names = [host] + [f"{host}.{d}" for d in search]
|
||||||
|
else:
|
||||||
|
names = [f"{host}.{d}" for d in search] + [host]
|
||||||
|
qtypes = [t for t, af in ((TYPE_A, socket.AF_INET), (TYPE_AAAA, socket.AF_INET6))
|
||||||
|
if af_filter in (0, af)]
|
||||||
|
for name in names:
|
||||||
|
ttls = []
|
||||||
|
for qtype in qtypes:
|
||||||
|
for server in servers:
|
||||||
|
ttl = _query_ttl(server, name, qtype)
|
||||||
|
if ttl is not None:
|
||||||
|
ttls.append(ttl)
|
||||||
|
break
|
||||||
|
if ttls:
|
||||||
|
return min(ttls)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_delay(ttls: List[Optional[int]]) -> int:
|
||||||
|
"""Seconds until the next resolution: the smallest known TTL, floored at MIN_TTL."""
|
||||||
|
known = [t for t in ttls if t is not None]
|
||||||
|
return max(min(known), MIN_TTL) if known else DEFAULT_TTL
|
||||||
|
|
||||||
|
|
||||||
|
def resolve(hosts: List[str], port: int, af_filter: int) -> Optional[List[Tuple[int, str]]]:
|
||||||
|
"""Blocking. Every (family, address) the hosts resolve to, deduplicated, or None if any lookup fails."""
|
||||||
|
found: List[Tuple[int, str]] = []
|
||||||
|
for host in hosts:
|
||||||
|
try:
|
||||||
|
infos = socket.getaddrinfo(host, port, af_filter, 0, socket.SOL_UDP)
|
||||||
|
except socket.gaierror as e:
|
||||||
|
logger.warning(f"Cannot resolve {host}: {e}")
|
||||||
|
return None
|
||||||
|
for info in infos:
|
||||||
|
entry = (info[0], str(info[4][0]))
|
||||||
|
if entry not in found:
|
||||||
|
found.append(entry)
|
||||||
|
return found or None
|
||||||
|
|
||||||
|
|
||||||
|
def retarget(connections: list, resolved: List[Tuple[int, str]]) -> Tuple[List[Tuple[int, str]], list]:
|
||||||
|
"""Point the connections at the resolved addresses, in place.
|
||||||
|
|
||||||
|
A connection whose address is gone is re-pointed at an address that is new, in list order, so
|
||||||
|
the first connection (the one plugin data uses) survives as long as anything resolves.
|
||||||
|
Returns (new addresses still needing a connection, connections to drop).
|
||||||
|
"""
|
||||||
|
current = {c.addr for c in connections}
|
||||||
|
wanted = {addr for _, addr in resolved}
|
||||||
|
new = [(af, addr) for af, addr in resolved if addr not in current]
|
||||||
|
gone = [c for c in connections if c.addr not in wanted]
|
||||||
|
paired = min(len(gone), len(new))
|
||||||
|
for conn, (af, addr) in zip(gone, new):
|
||||||
|
logger.info(f"Server address {conn.addr} -> {addr}")
|
||||||
|
conn.set_target(addr, af)
|
||||||
|
drop = gone[paired:]
|
||||||
|
if connections and connections[0] in drop:
|
||||||
|
# plugin data goes out on the first connection: hand it a surviving address and drop that one instead
|
||||||
|
keep = next(c for c in connections if c not in gone)
|
||||||
|
connections[0].set_target(keep.addr, keep.af)
|
||||||
|
drop.remove(connections[0])
|
||||||
|
drop.append(keep)
|
||||||
|
return new[paired:], drop
|
||||||
+114
-60
@@ -16,7 +16,7 @@ import sys
|
|||||||
import time
|
import time
|
||||||
from logging.handlers import SysLogHandler
|
from logging.handlers import SysLogHandler
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, List, Optional
|
from typing import List, Optional
|
||||||
|
|
||||||
# Import protocol and config
|
# Import protocol and config
|
||||||
from .config import load_config
|
from .config import load_config
|
||||||
@@ -25,6 +25,7 @@ from .. import __version__
|
|||||||
|
|
||||||
# Import plugin system
|
# Import plugin system
|
||||||
from .plugin import PluginRegistry, PluginLoader, InfoPlugin, MonitorPlugin
|
from .plugin import PluginRegistry, PluginLoader, InfoPlugin, MonitorPlugin
|
||||||
|
from . import dnsttl
|
||||||
from .connectivity import Connectivity, set_shared
|
from .connectivity import Connectivity, set_shared
|
||||||
|
|
||||||
# Constants
|
# Constants
|
||||||
@@ -35,6 +36,7 @@ MAXRECV = 32767
|
|||||||
# Global state
|
# Global state
|
||||||
running = True
|
running = True
|
||||||
dorestart = False
|
dorestart = False
|
||||||
|
send_shutdown = False
|
||||||
shutdown_event: Optional[asyncio.Event] = None
|
shutdown_event: Optional[asyncio.Event] = None
|
||||||
active_tasks: List[asyncio.Task] = []
|
active_tasks: List[asyncio.Task] = []
|
||||||
|
|
||||||
@@ -45,20 +47,20 @@ allow_remote_command = False
|
|||||||
|
|
||||||
class AsyncConnection:
|
class AsyncConnection:
|
||||||
"""Async UDP connection to a heartbeat server."""
|
"""Async UDP connection to a heartbeat server."""
|
||||||
|
|
||||||
def __init__(self, conn_id: int, addr: str, port: int, af: int, name: str):
|
def __init__(self, conn_id: int, addr: str, port: int, af: int, name: str):
|
||||||
self.conn_id = conn_id
|
self.conn_id = conn_id
|
||||||
self.addr = addr
|
self.addr = addr
|
||||||
self.port = port
|
self.port = port
|
||||||
self.af = af
|
self.af = af
|
||||||
self.name = name
|
self.name = name
|
||||||
|
|
||||||
self.ackcount = 0
|
self.ackcount = 0
|
||||||
self.lastack = 0.0
|
self.lastack = 0.0
|
||||||
self.send_count = 0
|
self.send_count = 0
|
||||||
self.lastsend = 0.0
|
self.lastsend = 0.0
|
||||||
self.rtts = [0.0]
|
self.rtts = [0.0]
|
||||||
|
|
||||||
self.transport: Optional[asyncio.DatagramTransport] = None
|
self.transport: Optional[asyncio.DatagramTransport] = None
|
||||||
self.protocol: Optional[asyncio.DatagramProtocol] = None
|
self.protocol: Optional[asyncio.DatagramProtocol] = None
|
||||||
self._dead = False
|
self._dead = False
|
||||||
@@ -88,17 +90,25 @@ class AsyncConnection:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Failed to open connection: {e}")
|
self.logger.error(f"Failed to open connection: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
"""Close the connection."""
|
"""Close the connection."""
|
||||||
if self.transport:
|
if self.transport:
|
||||||
self.transport.close()
|
self.transport.close()
|
||||||
self.transport = None
|
self.transport = None
|
||||||
self.protocol = None
|
self.protocol = None
|
||||||
|
|
||||||
|
def set_target(self, addr: str, af: int):
|
||||||
|
"""Re-point at a new server address; the socket is reopened on the next send if the family changed."""
|
||||||
|
if af != self.af:
|
||||||
|
self.close()
|
||||||
|
self.af = af
|
||||||
|
self.addr = addr
|
||||||
|
self.logger = logging.getLogger(f"hbc.conn.{addr}")
|
||||||
|
|
||||||
async def sendto(self, msg: dict, msg_id: str = "HTB"):
|
async def sendto(self, msg: dict, msg_id: str = "HTB"):
|
||||||
"""Send a message to the server.
|
"""Send a message to the server.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
msg: Message dictionary
|
msg: Message dictionary
|
||||||
msg_id: Message ID (HTB, PLG, etc.)
|
msg_id: Message ID (HTB, PLG, etc.)
|
||||||
@@ -112,36 +122,36 @@ class AsyncConnection:
|
|||||||
if not self.transport:
|
if not self.transport:
|
||||||
self.logger.error("Cannot send - no transport")
|
self.logger.error("Cannot send - no transport")
|
||||||
return
|
return
|
||||||
|
|
||||||
# Add standard fields
|
# Add standard fields
|
||||||
msg["name"] = shortname(self.name)
|
msg["name"] = shortname(self.name)
|
||||||
msg["id"] = self.conn_id
|
msg["id"] = self.conn_id
|
||||||
msg["time"] = time.time()
|
msg["time"] = time.time()
|
||||||
|
|
||||||
# Encode message
|
# Encode message
|
||||||
data = dicttos(msg_id, msg)
|
data = dicttos(msg_id, msg)
|
||||||
|
|
||||||
# Send
|
# Send
|
||||||
self.transport.sendto(data, (self.addr, self.port))
|
self.transport.sendto(data, (self.addr, self.port))
|
||||||
self.send_count += 1
|
self.send_count += 1
|
||||||
self.lastsend = time.time()
|
self.lastsend = time.time()
|
||||||
|
|
||||||
self.logger.debug(f"Sent {msg_id} message ({len(data)} bytes)")
|
self.logger.debug(f"Sent {msg_id} message ({len(data)} bytes)")
|
||||||
|
|
||||||
def handle_ack(self, msg: dict, now: float):
|
def handle_ack(self, msg: dict, now: float):
|
||||||
"""Handle ACK message from server.
|
"""Handle ACK message from server.
|
||||||
|
|
||||||
RTT is calculated as: (time ACK received) - (time HTB sent)
|
RTT is calculated as: (time ACK received) - (time HTB sent)
|
||||||
"""
|
"""
|
||||||
self.lastack = now
|
self.lastack = now
|
||||||
|
|
||||||
# Calculate RTT: time ACK received minus time HTB sent
|
# Calculate RTT: time ACK received minus time HTB sent
|
||||||
rtt = (now - self.lastsend) * 1000.0 # Convert to ms
|
rtt = (now - self.lastsend) * 1000.0 # Convert to ms
|
||||||
|
|
||||||
self.rtts.append(rtt)
|
self.rtts.append(rtt)
|
||||||
if len(self.rtts) > 10:
|
if len(self.rtts) > 10:
|
||||||
self.rtts.pop(0)
|
self.rtts.pop(0)
|
||||||
|
|
||||||
self.ackcount += 1
|
self.ackcount += 1
|
||||||
self.logger.debug(f"ACK received, RTT: {rtt:.1f}ms")
|
self.logger.debug(f"ACK received, RTT: {rtt:.1f}ms")
|
||||||
if msg.get("request_update"):
|
if msg.get("request_update"):
|
||||||
@@ -151,11 +161,11 @@ class AsyncConnection:
|
|||||||
|
|
||||||
class HeartbeatProtocol(asyncio.DatagramProtocol):
|
class HeartbeatProtocol(asyncio.DatagramProtocol):
|
||||||
"""Protocol handler for incoming UDP messages."""
|
"""Protocol handler for incoming UDP messages."""
|
||||||
|
|
||||||
def __init__(self, connection: AsyncConnection):
|
def __init__(self, connection: AsyncConnection):
|
||||||
self.connection = connection
|
self.connection = connection
|
||||||
self.logger = logging.getLogger("hbc.protocol")
|
self.logger = logging.getLogger("hbc.protocol")
|
||||||
|
|
||||||
def datagram_received(self, data: bytes, addr):
|
def datagram_received(self, data: bytes, addr):
|
||||||
"""Handle incoming datagram."""
|
"""Handle incoming datagram."""
|
||||||
try:
|
try:
|
||||||
@@ -163,10 +173,10 @@ class HeartbeatProtocol(asyncio.DatagramProtocol):
|
|||||||
if not msg:
|
if not msg:
|
||||||
self.logger.warning(f"Failed to parse message from {addr}")
|
self.logger.warning(f"Failed to parse message from {addr}")
|
||||||
return
|
return
|
||||||
|
|
||||||
now = time.time()
|
now = time.time()
|
||||||
msg_id = msg.get("ID")
|
msg_id = msg.get("ID")
|
||||||
|
|
||||||
if msg_id == "ACK":
|
if msg_id == "ACK":
|
||||||
self.connection.handle_ack(msg, now)
|
self.connection.handle_ack(msg, now)
|
||||||
elif msg_id == "CMD":
|
elif msg_id == "CMD":
|
||||||
@@ -177,10 +187,10 @@ class HeartbeatProtocol(asyncio.DatagramProtocol):
|
|||||||
asyncio.create_task(handle_update(self.connection, msg))
|
asyncio.create_task(handle_update(self.connection, msg))
|
||||||
else:
|
else:
|
||||||
self.logger.warning(f"Unknown message type: {msg_id}")
|
self.logger.warning(f"Unknown message type: {msg_id}")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error processing datagram: {e}", exc_info=True)
|
self.logger.error(f"Error processing datagram: {e}", exc_info=True)
|
||||||
|
|
||||||
def error_received(self, exc):
|
def error_received(self, exc):
|
||||||
"""Handle protocol errors — close transport so the heartbeat sender retries."""
|
"""Handle protocol errors — close transport so the heartbeat sender retries."""
|
||||||
self.logger.warning(f"Protocol error on {self.connection.addr}: {exc} — will retry")
|
self.logger.warning(f"Protocol error on {self.connection.addr}: {exc} — will retry")
|
||||||
@@ -221,7 +231,7 @@ async def handle_command(conn: AsyncConnection, msg: dict):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
result = str(e)
|
result = str(e)
|
||||||
status = "Error"
|
status = "Error"
|
||||||
|
|
||||||
# Send response
|
# Send response
|
||||||
response = {
|
response = {
|
||||||
"service": "command",
|
"service": "command",
|
||||||
@@ -437,14 +447,14 @@ async def plugin_collector_interval(
|
|||||||
interval: int
|
interval: int
|
||||||
):
|
):
|
||||||
"""Collect plugins on a specific interval.
|
"""Collect plugins on a specific interval.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
conn: Connection to send on
|
conn: Connection to send on
|
||||||
plugins: List of plugins to collect
|
plugins: List of plugins to collect
|
||||||
interval: Collection interval in seconds
|
interval: Collection interval in seconds
|
||||||
"""
|
"""
|
||||||
logger = logging.getLogger(f"hbc.plugins.{interval}s")
|
logger = logging.getLogger(f"hbc.plugins.{interval}s")
|
||||||
|
|
||||||
while running:
|
while running:
|
||||||
for plugin in plugins:
|
for plugin in plugins:
|
||||||
try:
|
try:
|
||||||
@@ -462,7 +472,7 @@ async def plugin_collector_interval(
|
|||||||
f"Error collecting {plugin.name}: {e}",
|
f"Error collecting {plugin.name}: {e}",
|
||||||
exc_info=True
|
exc_info=True
|
||||||
)
|
)
|
||||||
|
|
||||||
# Wait for next interval or shutdown event
|
# Wait for next interval or shutdown event
|
||||||
try:
|
try:
|
||||||
if shutdown_event:
|
if shutdown_event:
|
||||||
@@ -489,22 +499,64 @@ def stop():
|
|||||||
"""Stop the event loop."""
|
"""Stop the event loop."""
|
||||||
global running
|
global running
|
||||||
running = False
|
running = False
|
||||||
|
|
||||||
# Set shutdown event to wake up sleeping tasks
|
# Set shutdown event to wake up sleeping tasks
|
||||||
if shutdown_event:
|
if shutdown_event:
|
||||||
shutdown_event.set()
|
shutdown_event.set()
|
||||||
|
|
||||||
# Cancel all active tasks
|
# Cancel all active tasks
|
||||||
for task in active_tasks:
|
for task in active_tasks:
|
||||||
if not task.done():
|
if not task.done():
|
||||||
task.cancel()
|
task.cancel()
|
||||||
|
|
||||||
|
|
||||||
|
async def dns_refresher(connections: List[AsyncConnection], hosts: List[str], port: int, af_filter: int,
|
||||||
|
name: str, interval: int, connectivity: Optional[Connectivity], next_id: int):
|
||||||
|
"""Re-resolve the server hosts every TTL seconds and follow address changes.
|
||||||
|
|
||||||
|
A failed lookup leaves the connections untouched. Addresses that disappear are re-pointed at
|
||||||
|
new ones where possible; the rest are retired, and surplus new ones get a connection.
|
||||||
|
"""
|
||||||
|
logger = logging.getLogger("hbc.dns")
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
delay = await loop.run_in_executor(None, _ttl_delay, hosts, af_filter)
|
||||||
|
while running:
|
||||||
|
try:
|
||||||
|
if shutdown_event:
|
||||||
|
await asyncio.wait_for(shutdown_event.wait(), timeout=delay)
|
||||||
|
break
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
resolved = await loop.run_in_executor(None, dnsttl.resolve, hosts, port, af_filter)
|
||||||
|
if resolved is None:
|
||||||
|
delay = dnsttl.FAILED_RETRY
|
||||||
|
continue
|
||||||
|
new, gone = dnsttl.retarget(connections, resolved)
|
||||||
|
for conn in gone:
|
||||||
|
if len(connections) > 1:
|
||||||
|
logger.info(f"Dropping server address {conn.addr}")
|
||||||
|
conn._dead = True
|
||||||
|
conn.close()
|
||||||
|
connections.remove(conn)
|
||||||
|
for af, addr in new:
|
||||||
|
conn = AsyncConnection(next_id, addr, port, af, name)
|
||||||
|
next_id += 1
|
||||||
|
logger.info(f"Adding server address {addr}")
|
||||||
|
connections.append(conn)
|
||||||
|
active_tasks.append(asyncio.create_task(heartbeat_sender(conn, interval, connectivity)))
|
||||||
|
delay = await loop.run_in_executor(None, _ttl_delay, hosts, af_filter)
|
||||||
|
|
||||||
|
|
||||||
|
def _ttl_delay(hosts: List[str], af_filter: int) -> int:
|
||||||
|
return dnsttl.refresh_delay([dnsttl.lookup_ttl(h, af_filter) for h in hosts])
|
||||||
|
|
||||||
|
|
||||||
async def cleanup(connections: List[AsyncConnection]):
|
async def cleanup(connections: List[AsyncConnection]):
|
||||||
"""Cleanup connections on shutdown."""
|
"""Cleanup connections on shutdown."""
|
||||||
logger = logging.getLogger("hbc.cleanup")
|
logger = logging.getLogger("hbc.cleanup")
|
||||||
logger.info("Cleaning up connections")
|
logger.info("Cleaning up connections")
|
||||||
|
|
||||||
target = next((c for c in connections if c.transport), connections[0] if connections else None)
|
target = next((c for c in connections if c.transport), connections[0] if connections else None)
|
||||||
# A SIGHUP restart is not a host shutdown, so don't announce one.
|
# A SIGHUP restart is not a host shutdown, so don't announce one.
|
||||||
if target and send_shutdown and not dorestart:
|
if target and send_shutdown and not dorestart:
|
||||||
@@ -514,35 +566,35 @@ async def cleanup(connections: List[AsyncConnection]):
|
|||||||
logger.error(f"Error sending shutdown: {e}")
|
logger.error(f"Error sending shutdown: {e}")
|
||||||
for conn in connections:
|
for conn in connections:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
# Give messages time to send
|
# Give messages time to send
|
||||||
await asyncio.sleep(0.5)
|
await asyncio.sleep(0.5)
|
||||||
|
|
||||||
|
|
||||||
async def async_main(args, config):
|
async def async_main(args, config):
|
||||||
"""Async main function."""
|
"""Async main function."""
|
||||||
global running, shutdown_event, active_tasks, send_shutdown, allow_remote_command
|
global shutdown_event, active_tasks, send_shutdown, allow_remote_command
|
||||||
|
|
||||||
# Create shutdown event
|
# Create shutdown event
|
||||||
shutdown_event = asyncio.Event()
|
shutdown_event = asyncio.Event()
|
||||||
active_tasks = []
|
active_tasks = []
|
||||||
allow_remote_command = bool(config.get("allow_remote_command", False))
|
allow_remote_command = bool(config.get("allow_remote_command", False))
|
||||||
|
|
||||||
logger = logging.getLogger("hbc.main")
|
logger = logging.getLogger("hbc.main")
|
||||||
|
|
||||||
# Setup
|
# Setup
|
||||||
iam = socket.gethostname()
|
iam = socket.gethostname()
|
||||||
if args.name:
|
if args.name:
|
||||||
iam = args.name
|
iam = args.name
|
||||||
|
|
||||||
hb_hosts = args.hosts
|
hb_hosts = args.hosts
|
||||||
hb_port = config.get("hb_port", PORT)
|
hb_port = config.get("hb_port", PORT)
|
||||||
interval = config.get("interval", INTERVAL)
|
interval = config.get("interval", INTERVAL)
|
||||||
|
|
||||||
logger.info(f"hbc {__version__} on {iam} -> {hb_hosts} port={hb_port}, interval={interval}s")
|
logger.info(f"hbc {__version__} on {iam} -> {hb_hosts} port={hb_port}, interval={interval}s")
|
||||||
if allow_remote_command:
|
if allow_remote_command:
|
||||||
logger.warning("allow_remote_command is true — CMD packets from the server will be executed")
|
logger.warning("allow_remote_command is true — CMD packets from the server will be executed")
|
||||||
|
|
||||||
af_filter = (socket.AF_INET if getattr(args, "ipv4_only", False)
|
af_filter = (socket.AF_INET if getattr(args, "ipv4_only", False)
|
||||||
else socket.AF_INET6 if getattr(args, "ipv6_only", False)
|
else socket.AF_INET6 if getattr(args, "ipv6_only", False)
|
||||||
else 0)
|
else 0)
|
||||||
@@ -581,7 +633,7 @@ async def async_main(args, config):
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
logger.info(f"Created {len(connections)} connections")
|
logger.info(f"Created {len(connections)} connections")
|
||||||
|
|
||||||
# Send boot/message if requested
|
# Send boot/message if requested
|
||||||
send_shutdown = False
|
send_shutdown = False
|
||||||
if args.boot or args.message:
|
if args.boot or args.message:
|
||||||
@@ -592,27 +644,27 @@ async def async_main(args, config):
|
|||||||
if args.message:
|
if args.message:
|
||||||
boot_msg["service"] = "service"
|
boot_msg["service"] = "service"
|
||||||
boot_msg["msg"] = args.message
|
boot_msg["msg"] = args.message
|
||||||
|
|
||||||
boot_msg["acks"] = 0
|
boot_msg["acks"] = 0
|
||||||
target = next((c for c in connections if c.transport), connections[0])
|
target = next((c for c in connections if c.transport), connections[0])
|
||||||
await target.sendto(boot_msg)
|
await target.sendto(boot_msg)
|
||||||
|
|
||||||
if args.message and not args.daemon:
|
if args.message and not args.daemon:
|
||||||
# Message-only mode
|
# Message-only mode
|
||||||
await cleanup(connections)
|
await cleanup(connections)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
# Load plugins
|
# Load plugins
|
||||||
registry = PluginRegistry()
|
registry = PluginRegistry()
|
||||||
loader = PluginLoader(registry)
|
loader = PluginLoader(registry)
|
||||||
|
|
||||||
plugin_dir = Path(__file__).parent / "plugins"
|
plugin_dir = Path(__file__).parent / "plugins"
|
||||||
if plugin_dir.exists():
|
if plugin_dir.exists():
|
||||||
count = await loader.load_from_directory(plugin_dir, config)
|
count = await loader.load_from_directory(plugin_dir, config)
|
||||||
logger.info(f"Loaded {count} plugins")
|
logger.info(f"Loaded {count} plugins")
|
||||||
else:
|
else:
|
||||||
logger.warning(f"Plugin directory not found: {plugin_dir}")
|
logger.warning(f"Plugin directory not found: {plugin_dir}")
|
||||||
|
|
||||||
# Setup signal handlers
|
# Setup signal handlers
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||||
@@ -624,7 +676,7 @@ async def async_main(args, config):
|
|||||||
stop()
|
stop()
|
||||||
|
|
||||||
loop.add_signal_handler(signal.SIGHUP, _sighup)
|
loop.add_signal_handler(signal.SIGHUP, _sighup)
|
||||||
|
|
||||||
# Start async tasks
|
# Start async tasks
|
||||||
# Per-protocol internet presence probe (shared with the network_monitor plugin)
|
# Per-protocol internet presence probe (shared with the network_monitor plugin)
|
||||||
net_cfg = dict((config.get("plugins") or {}).get("network_monitor")
|
net_cfg = dict((config.get("plugins") or {}).get("network_monitor")
|
||||||
@@ -639,23 +691,26 @@ async def async_main(args, config):
|
|||||||
for conn in connections:
|
for conn in connections:
|
||||||
task = asyncio.create_task(heartbeat_sender(conn, interval, connectivity))
|
task = asyncio.create_task(heartbeat_sender(conn, interval, connectivity))
|
||||||
active_tasks.append(task)
|
active_tasks.append(task)
|
||||||
|
|
||||||
|
active_tasks.append(asyncio.create_task(
|
||||||
|
dns_refresher(connections, hb_hosts, hb_port, af_filter, iam, interval, connectivity, conn_id)))
|
||||||
|
|
||||||
# Plugin collector (uses all connections, but we'll use first one)
|
# Plugin collector (uses all connections, but we'll use first one)
|
||||||
if connections and registry.get_enabled():
|
if connections and registry.get_enabled():
|
||||||
task = asyncio.create_task(plugin_collector(connections[0], registry))
|
task = asyncio.create_task(plugin_collector(connections[0], registry))
|
||||||
active_tasks.append(task)
|
active_tasks.append(task)
|
||||||
|
|
||||||
# Wait for stop or tasks to complete
|
# Wait for stop or tasks to complete
|
||||||
try:
|
try:
|
||||||
await asyncio.gather(*active_tasks, return_exceptions=True)
|
await asyncio.gather(*active_tasks, return_exceptions=True)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
logger.info("Tasks cancelled")
|
logger.info("Tasks cancelled")
|
||||||
|
|
||||||
# Cleanup
|
# Cleanup
|
||||||
logger.info("Shutting down...")
|
logger.info("Shutting down...")
|
||||||
await cleanup(connections)
|
await cleanup(connections)
|
||||||
await loader.unload_all()
|
await loader.unload_all()
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
@@ -673,11 +728,11 @@ def daemonize(
|
|||||||
except OSError as e:
|
except OSError as e:
|
||||||
sys.stderr.write(f"fork #1 failed: {e}\n")
|
sys.stderr.write(f"fork #1 failed: {e}\n")
|
||||||
os._exit(1)
|
os._exit(1)
|
||||||
|
|
||||||
os.chdir(working_dir)
|
os.chdir(working_dir)
|
||||||
os.setsid()
|
os.setsid()
|
||||||
os.umask(0)
|
os.umask(0)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
pid = os.fork()
|
pid = os.fork()
|
||||||
if pid > 0:
|
if pid > 0:
|
||||||
@@ -685,14 +740,14 @@ def daemonize(
|
|||||||
except OSError as e:
|
except OSError as e:
|
||||||
sys.stderr.write(f"fork #2 failed: {e}\n")
|
sys.stderr.write(f"fork #2 failed: {e}\n")
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
si = open(stdin, "r")
|
si = open(stdin, "r")
|
||||||
so = open(stdout, "a+")
|
so = open(stdout, "a+")
|
||||||
se = open(stderr, "a+")
|
se = open(stderr, "a+")
|
||||||
|
|
||||||
os.dup2(si.fileno(), sys.stdin.fileno())
|
os.dup2(si.fileno(), sys.stdin.fileno())
|
||||||
os.dup2(so.fileno(), sys.stdout.fileno())
|
os.dup2(so.fileno(), sys.stdout.fileno())
|
||||||
os.dup2(se.fileno(), sys.stderr.fileno())
|
os.dup2(se.fileno(), sys.stderr.fileno())
|
||||||
@@ -784,18 +839,17 @@ def build_parser():
|
|||||||
|
|
||||||
def main(argv=None):
|
def main(argv=None):
|
||||||
"""Main entry point."""
|
"""Main entry point."""
|
||||||
global running, dorestart
|
|
||||||
|
|
||||||
parser = build_parser()
|
parser = build_parser()
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
# Setup logging
|
# Setup logging
|
||||||
log_level = logging.WARNING
|
log_level = logging.WARNING
|
||||||
if args.verbose:
|
if args.verbose:
|
||||||
log_level = logging.INFO
|
log_level = logging.INFO
|
||||||
if args.debug:
|
if args.debug:
|
||||||
log_level = logging.DEBUG
|
log_level = logging.DEBUG
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=log_level,
|
level=log_level,
|
||||||
format="%(asctime)s %(name)s %(levelname)s: %(message)s",
|
format="%(asctime)s %(name)s %(levelname)s: %(message)s",
|
||||||
@@ -804,14 +858,14 @@ def main(argv=None):
|
|||||||
|
|
||||||
# Load config
|
# Load config
|
||||||
config = load_config(args.configfile)
|
config = load_config(args.configfile)
|
||||||
|
|
||||||
# Daemonize if requested
|
# Daemonize if requested
|
||||||
if args.daemon:
|
if args.daemon:
|
||||||
logging.info("Daemonizing...")
|
logging.info("Daemonizing...")
|
||||||
daemonize()
|
daemonize()
|
||||||
_reconfigure_logging_for_daemon(log_level)
|
_reconfigure_logging_for_daemon(log_level)
|
||||||
logging.info(f"hbc starting, sending heartbeat to {', '.join(args.hosts)}")
|
logging.info(f"hbc starting, sending heartbeat to {', '.join(args.hosts)}")
|
||||||
|
|
||||||
# Run async main
|
# Run async main
|
||||||
try:
|
try:
|
||||||
exit_code = asyncio.run(async_main(args, config))
|
exit_code = asyncio.run(async_main(args, config))
|
||||||
@@ -821,7 +875,7 @@ def main(argv=None):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"Fatal error: {e}", exc_info=True)
|
logging.error(f"Fatal error: {e}", exc_info=True)
|
||||||
exit_code = 1
|
exit_code = 1
|
||||||
|
|
||||||
# Handle restart
|
# Handle restart
|
||||||
if dorestart:
|
if dorestart:
|
||||||
logging.info("Restarting...")
|
logging.info("Restarting...")
|
||||||
@@ -829,7 +883,7 @@ def main(argv=None):
|
|||||||
# a SIGHUP restart is not a host reboot.
|
# a SIGHUP restart is not a host reboot.
|
||||||
restart_argv = [sys.argv[0]] + [a for a in sys.argv[1:] if a not in ("-b", "--boot")]
|
restart_argv = [sys.argv[0]] + [a for a in sys.argv[1:] if a not in ("-b", "--boot")]
|
||||||
os.execv(restart_argv[0], restart_argv)
|
os.execv(restart_argv[0], restart_argv)
|
||||||
|
|
||||||
sys.exit(exit_code)
|
sys.exit(exit_code)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+79
-79
@@ -17,7 +17,7 @@ import logging
|
|||||||
import sys
|
import sys
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Type
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
|
||||||
class Plugin(ABC):
|
class Plugin(ABC):
|
||||||
@@ -31,13 +31,13 @@ class Plugin(ABC):
|
|||||||
enabled: Whether plugin is active (can be disabled via config)
|
enabled: Whether plugin is active (can be disabled via config)
|
||||||
skip_reason: Set by plugin before returning False from initialize(); causes loader to log INFO instead of WARNING.
|
skip_reason: Set by plugin before returning False from initialize(); causes loader to log INFO instead of WARNING.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name: str = ""
|
name: str = ""
|
||||||
version: str = "1.0.0"
|
version: str = "1.0.0"
|
||||||
description: str = ""
|
description: str = ""
|
||||||
interval: int = 0
|
interval: int = 0
|
||||||
enabled: bool = True
|
enabled: bool = True
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
"""Initialize plugin with optional configuration.
|
"""Initialize plugin with optional configuration.
|
||||||
|
|
||||||
@@ -48,51 +48,51 @@ class Plugin(ABC):
|
|||||||
self.logger = logging.getLogger(f"plugin.{self.name}")
|
self.logger = logging.getLogger(f"plugin.{self.name}")
|
||||||
self._initialized = False
|
self._initialized = False
|
||||||
self.skip_reason: Optional[str] = None
|
self.skip_reason: Optional[str] = None
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def initialize(self) -> bool:
|
async def initialize(self) -> bool:
|
||||||
"""Initialize plugin (load resources, check dependencies).
|
"""Initialize plugin (load resources, check dependencies).
|
||||||
|
|
||||||
Called once when plugin is loaded. Plugins should validate dependencies
|
Called once when plugin is loaded. Plugins should validate dependencies
|
||||||
(e.g., check if psutil is available) and prepare any resources.
|
(e.g., check if psutil is available) and prepare any resources.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if initialization succeeded, False otherwise
|
True if initialization succeeded, False otherwise
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def collect(self) -> Dict[str, Any]:
|
async def collect(self) -> Dict[str, Any]:
|
||||||
"""Collect data from the system.
|
"""Collect data from the system.
|
||||||
|
|
||||||
This is the main method called on each collection interval. Should return
|
This is the main method called on each collection interval. Should return
|
||||||
a dictionary of key-value pairs representing the collected data.
|
a dictionary of key-value pairs representing the collected data.
|
||||||
|
|
||||||
Keys should be strings (metric names). Values can be:
|
Keys should be strings (metric names). Values can be:
|
||||||
- Scalars: int, float, str, bool
|
- Scalars: int, float, str, bool
|
||||||
- Lists/dicts (will be serialized appropriately)
|
- Lists/dicts (will be serialized appropriately)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary of collected metrics, or empty dict on error
|
Dictionary of collected metrics, or empty dict on error
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def cleanup(self) -> None:
|
async def cleanup(self) -> None:
|
||||||
"""Cleanup plugin resources before shutdown.
|
"""Cleanup plugin resources before shutdown.
|
||||||
|
|
||||||
Called when plugin is being unloaded or on system shutdown.
|
Called when plugin is being unloaded or on system shutdown.
|
||||||
Override to release resources, close connections, etc.
|
Override to release resources, close connections, etc.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def validate_data(self, data: Dict[str, Any]) -> bool:
|
def validate_data(self, data: Dict[str, Any]) -> bool:
|
||||||
"""Validate collected data before sending to server.
|
"""Validate collected data before sending to server.
|
||||||
|
|
||||||
Override to implement custom validation logic.
|
Override to implement custom validation logic.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
data: Data returned from collect()
|
data: Data returned from collect()
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if data is valid, False otherwise
|
True if data is valid, False otherwise
|
||||||
"""
|
"""
|
||||||
@@ -101,46 +101,46 @@ class Plugin(ABC):
|
|||||||
|
|
||||||
class InfoPlugin(Plugin):
|
class InfoPlugin(Plugin):
|
||||||
"""Plugin for collecting static or rarely-changing information.
|
"""Plugin for collecting static or rarely-changing information.
|
||||||
|
|
||||||
InfoPlugins collect data that doesn't change frequently:
|
InfoPlugins collect data that doesn't change frequently:
|
||||||
- OS name and version
|
- OS name and version
|
||||||
- Hardware specifications (CPU model, RAM size)
|
- Hardware specifications (CPU model, RAM size)
|
||||||
- Network interface MAC addresses
|
- Network interface MAC addresses
|
||||||
|
|
||||||
Characteristics:
|
Characteristics:
|
||||||
- interval = 0 (collected once at startup by default)
|
- interval = 0 (collected once at startup by default)
|
||||||
- Can specify interval > 0 for periodic refresh (e.g., check for hardware changes)
|
- Can specify interval > 0 for periodic refresh (e.g., check for hardware changes)
|
||||||
- Data is cached and reused until next collection
|
- Data is cached and reused until next collection
|
||||||
"""
|
"""
|
||||||
|
|
||||||
interval: int = 0 # Collect once at startup
|
interval: int = 0 # Collect once at startup
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
super().__init__(config)
|
super().__init__(config)
|
||||||
self._cached_data: Optional[Dict[str, Any]] = None
|
self._cached_data: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
async def get_cached_data(self) -> Optional[Dict[str, Any]]:
|
async def get_cached_data(self) -> Optional[Dict[str, Any]]:
|
||||||
"""Get cached data if available (avoids re-collection).
|
"""Get cached data if available (avoids re-collection).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Cached data dict, or None if not yet collected
|
Cached data dict, or None if not yet collected
|
||||||
"""
|
"""
|
||||||
return self._cached_data
|
return self._cached_data
|
||||||
|
|
||||||
async def collect(self) -> Dict[str, Any]:
|
async def collect(self) -> Dict[str, Any]:
|
||||||
"""Collect and cache static information."""
|
"""Collect and cache static information."""
|
||||||
if self._cached_data is None:
|
if self._cached_data is None:
|
||||||
self._cached_data = await self._collect_info()
|
self._cached_data = await self._collect_info()
|
||||||
return self._cached_data
|
return self._cached_data
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def _collect_info(self) -> Dict[str, Any]:
|
async def _collect_info(self) -> Dict[str, Any]:
|
||||||
"""Internal method to perform actual data collection.
|
"""Internal method to perform actual data collection.
|
||||||
|
|
||||||
Override this method instead of collect() for InfoPlugins.
|
Override this method instead of collect() for InfoPlugins.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def invalidate_cache(self) -> None:
|
def invalidate_cache(self) -> None:
|
||||||
"""Force re-collection on next collect() call."""
|
"""Force re-collection on next collect() call."""
|
||||||
self._cached_data = None
|
self._cached_data = None
|
||||||
@@ -148,33 +148,33 @@ class InfoPlugin(Plugin):
|
|||||||
|
|
||||||
class MonitorPlugin(Plugin):
|
class MonitorPlugin(Plugin):
|
||||||
"""Plugin for collecting periodic monitoring data.
|
"""Plugin for collecting periodic monitoring data.
|
||||||
|
|
||||||
MonitorPlugins collect time-series metrics that change frequently:
|
MonitorPlugins collect time-series metrics that change frequently:
|
||||||
- CPU usage percentage
|
- CPU usage percentage
|
||||||
- Memory consumption
|
- Memory consumption
|
||||||
- Disk I/O statistics
|
- Disk I/O statistics
|
||||||
- Network traffic
|
- Network traffic
|
||||||
|
|
||||||
Characteristics:
|
Characteristics:
|
||||||
- interval > 0 (e.g., 30 seconds for CPU, 60 for disk)
|
- interval > 0 (e.g., 30 seconds for CPU, 60 for disk)
|
||||||
- Collected continuously on schedule
|
- Collected continuously on schedule
|
||||||
- Data includes timestamps for time-series tracking
|
- Data includes timestamps for time-series tracking
|
||||||
"""
|
"""
|
||||||
|
|
||||||
interval: int = 30 # Default: collect every 30 seconds
|
interval: int = 30 # Default: collect every 30 seconds
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
super().__init__(config)
|
super().__init__(config)
|
||||||
self._last_reading: Optional[Dict[str, Any]] = None
|
self._last_reading: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
def get_last_reading(self) -> Optional[Dict[str, Any]]:
|
def get_last_reading(self) -> Optional[Dict[str, Any]]:
|
||||||
"""Get the last collected reading.
|
"""Get the last collected reading.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Last reading dict with timestamp, or None if not yet collected
|
Last reading dict with timestamp, or None if not yet collected
|
||||||
"""
|
"""
|
||||||
return self._last_reading
|
return self._last_reading
|
||||||
|
|
||||||
async def collect(self) -> Dict[str, Any]:
|
async def collect(self) -> Dict[str, Any]:
|
||||||
"""Collect monitoring data and store as last reading."""
|
"""Collect monitoring data and store as last reading."""
|
||||||
data = await self._collect_metrics()
|
data = await self._collect_metrics()
|
||||||
@@ -184,11 +184,11 @@ class MonitorPlugin(Plugin):
|
|||||||
data['_timestamp'] = time.time()
|
data['_timestamp'] = time.time()
|
||||||
self._last_reading = data
|
self._last_reading = data
|
||||||
return data
|
return data
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||||
"""Internal method to perform actual metric collection.
|
"""Internal method to perform actual metric collection.
|
||||||
|
|
||||||
Override this method instead of collect() for MonitorPlugins.
|
Override this method instead of collect() for MonitorPlugins.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
@@ -196,38 +196,38 @@ class MonitorPlugin(Plugin):
|
|||||||
|
|
||||||
class PluginRegistry:
|
class PluginRegistry:
|
||||||
"""Registry for managing loaded plugins.
|
"""Registry for managing loaded plugins.
|
||||||
|
|
||||||
Maintains a collection of loaded plugins and provides methods to
|
Maintains a collection of loaded plugins and provides methods to
|
||||||
query plugins by name, type, or interval.
|
query plugins by name, type, or interval.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._plugins: Dict[str, Plugin] = {}
|
self._plugins: Dict[str, Plugin] = {}
|
||||||
self.logger = logging.getLogger("plugin.registry")
|
self.logger = logging.getLogger("plugin.registry")
|
||||||
|
|
||||||
def register(self, plugin: Plugin) -> bool:
|
def register(self, plugin: Plugin) -> bool:
|
||||||
"""Register a plugin instance.
|
"""Register a plugin instance.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
plugin: Plugin instance to register
|
plugin: Plugin instance to register
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if registered successfully, False if name conflict
|
True if registered successfully, False if name conflict
|
||||||
"""
|
"""
|
||||||
if plugin.name in self._plugins:
|
if plugin.name in self._plugins:
|
||||||
self.logger.error(f"Plugin '{plugin.name}' already registered")
|
self.logger.error(f"Plugin '{plugin.name}' already registered")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
self._plugins[plugin.name] = plugin
|
self._plugins[plugin.name] = plugin
|
||||||
self.logger.info(f"Registered plugin: {plugin.name} v{plugin.version}")
|
self.logger.info(f"Registered plugin: {plugin.name} v{plugin.version}")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def unregister(self, name: str) -> bool:
|
def unregister(self, name: str) -> bool:
|
||||||
"""Unregister a plugin by name.
|
"""Unregister a plugin by name.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
name: Plugin name to unregister
|
name: Plugin name to unregister
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if unregistered, False if not found
|
True if unregistered, False if not found
|
||||||
"""
|
"""
|
||||||
@@ -236,43 +236,43 @@ class PluginRegistry:
|
|||||||
self.logger.info(f"Unregistered plugin: {name}")
|
self.logger.info(f"Unregistered plugin: {name}")
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def get(self, name: str) -> Optional[Plugin]:
|
def get(self, name: str) -> Optional[Plugin]:
|
||||||
"""Get plugin by name.
|
"""Get plugin by name.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
name: Plugin name
|
name: Plugin name
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Plugin instance or None if not found
|
Plugin instance or None if not found
|
||||||
"""
|
"""
|
||||||
return self._plugins.get(name)
|
return self._plugins.get(name)
|
||||||
|
|
||||||
def get_all(self) -> List[Plugin]:
|
def get_all(self) -> List[Plugin]:
|
||||||
"""Get all registered plugins."""
|
"""Get all registered plugins."""
|
||||||
return list(self._plugins.values())
|
return list(self._plugins.values())
|
||||||
|
|
||||||
def get_enabled(self) -> List[Plugin]:
|
def get_enabled(self) -> List[Plugin]:
|
||||||
"""Get all enabled plugins."""
|
"""Get all enabled plugins."""
|
||||||
return [p for p in self._plugins.values() if p.enabled]
|
return [p for p in self._plugins.values() if p.enabled]
|
||||||
|
|
||||||
def get_by_type(self, plugin_type: Type[Plugin]) -> List[Plugin]:
|
def get_by_type(self, plugin_type: type) -> List[Plugin]:
|
||||||
"""Get all plugins of a specific type.
|
"""Get all plugins of a specific type.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
plugin_type: Plugin class (InfoPlugin or MonitorPlugin)
|
plugin_type: Plugin class (InfoPlugin or MonitorPlugin)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of plugins matching the type
|
List of plugins matching the type
|
||||||
"""
|
"""
|
||||||
return [p for p in self._plugins.values() if isinstance(p, plugin_type)]
|
return [p for p in self._plugins.values() if isinstance(p, plugin_type)]
|
||||||
|
|
||||||
def get_by_interval(self, interval: int) -> List[Plugin]:
|
def get_by_interval(self, interval: int) -> List[Plugin]:
|
||||||
"""Get all plugins with a specific collection interval.
|
"""Get all plugins with a specific collection interval.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
interval: Interval in seconds (0 for one-time collection)
|
interval: Interval in seconds (0 for one-time collection)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of plugins with matching interval
|
List of plugins with matching interval
|
||||||
"""
|
"""
|
||||||
@@ -281,51 +281,51 @@ class PluginRegistry:
|
|||||||
|
|
||||||
class PluginLoader:
|
class PluginLoader:
|
||||||
"""Load plugins from filesystem and instantiate them.
|
"""Load plugins from filesystem and instantiate them.
|
||||||
|
|
||||||
Scans plugin directories for Python modules containing Plugin subclasses,
|
Scans plugin directories for Python modules containing Plugin subclasses,
|
||||||
loads them dynamically, and registers them with the PluginRegistry.
|
loads them dynamically, and registers them with the PluginRegistry.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, registry: PluginRegistry):
|
def __init__(self, registry: PluginRegistry):
|
||||||
self.registry = registry
|
self.registry = registry
|
||||||
self.logger = logging.getLogger("plugin.loader")
|
self.logger = logging.getLogger("plugin.loader")
|
||||||
self._loaded_modules: Dict[str, Any] = {}
|
self._loaded_modules: Dict[str, Any] = {}
|
||||||
|
|
||||||
async def load_from_directory(
|
async def load_from_directory(
|
||||||
self,
|
self,
|
||||||
directory: Path,
|
directory: Path,
|
||||||
config: Optional[Dict[str, Any]] = None
|
config: Optional[Dict[str, Any]] = None
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Load all plugins from a directory.
|
"""Load all plugins from a directory.
|
||||||
|
|
||||||
Scans for .py files, imports them, finds Plugin subclasses,
|
Scans for .py files, imports them, finds Plugin subclasses,
|
||||||
instantiates them with config, initializes, and registers.
|
instantiates them with config, initializes, and registers.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
directory: Path to plugin directory
|
directory: Path to plugin directory
|
||||||
config: Configuration dict (may contain per-plugin config)
|
config: Configuration dict (may contain per-plugin config)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Number of plugins successfully loaded
|
Number of plugins successfully loaded
|
||||||
"""
|
"""
|
||||||
if not directory.exists() or not directory.is_dir():
|
if not directory.exists() or not directory.is_dir():
|
||||||
self.logger.warning(f"Plugin directory not found: {directory}")
|
self.logger.warning(f"Plugin directory not found: {directory}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
loaded_count = 0
|
loaded_count = 0
|
||||||
raw_config = config or {}
|
raw_config = config or {}
|
||||||
# Per-plugin config lives under the 'plugins' key or at top-level.
|
# Per-plugin config lives under the 'plugins' key or at top-level.
|
||||||
# CLIENT_DEFAULTS seeds "plugins": {} so the key always exists; check
|
# CLIENT_DEFAULTS seeds "plugins": {} so the key always exists; check
|
||||||
# both the subdict and top-level so that either layout in .hbc.yaml works.
|
# both the subdict and top-level so that either layout in .hbc.yaml works.
|
||||||
plugins_subconfig = raw_config.get("plugins", {})
|
plugins_subconfig = raw_config.get("plugins", {})
|
||||||
|
|
||||||
# Scan for Python files
|
# Scan for Python files
|
||||||
for plugin_file in directory.glob("*.py"):
|
for plugin_file in directory.glob("*.py"):
|
||||||
if plugin_file.name.startswith("_"):
|
if plugin_file.name.startswith("_"):
|
||||||
continue # Skip __init__.py and private modules
|
continue # Skip __init__.py and private modules
|
||||||
|
|
||||||
self.logger.debug(f"Processing plugin file: {plugin_file.name}")
|
self.logger.debug(f"Processing plugin file: {plugin_file.name}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Load module dynamically
|
# Load module dynamically
|
||||||
module_name = f"plugins.{plugin_file.stem}"
|
module_name = f"plugins.{plugin_file.stem}"
|
||||||
@@ -333,17 +333,17 @@ class PluginLoader:
|
|||||||
if not spec or not spec.loader:
|
if not spec or not spec.loader:
|
||||||
self.logger.warning(f"Could not create spec for {plugin_file}")
|
self.logger.warning(f"Could not create spec for {plugin_file}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
module = importlib.util.module_from_spec(spec)
|
||||||
sys.modules[module_name] = module
|
sys.modules[module_name] = module
|
||||||
spec.loader.exec_module(module)
|
spec.loader.exec_module(module)
|
||||||
self._loaded_modules[module_name] = module
|
self._loaded_modules[module_name] = module
|
||||||
|
|
||||||
self.logger.debug(f"Loaded module: {module_name}")
|
self.logger.debug(f"Loaded module: {module_name}")
|
||||||
|
|
||||||
# Track which plugin classes we've already processed to avoid duplicates
|
# Track which plugin classes we've already processed to avoid duplicates
|
||||||
processed_classes = set()
|
processed_classes = set()
|
||||||
|
|
||||||
# Find Plugin subclasses in module
|
# Find Plugin subclasses in module
|
||||||
for name, obj in inspect.getmembers(module, inspect.isclass):
|
for name, obj in inspect.getmembers(module, inspect.isclass):
|
||||||
# Skip base classes and non-Plugin classes
|
# Skip base classes and non-Plugin classes
|
||||||
@@ -353,15 +353,15 @@ class PluginLoader:
|
|||||||
if not issubclass(obj, Plugin):
|
if not issubclass(obj, Plugin):
|
||||||
self.logger.debug(f"Skipping non-Plugin class: {name}")
|
self.logger.debug(f"Skipping non-Plugin class: {name}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Skip if we've already processed this class (handles module-level aliases)
|
# Skip if we've already processed this class (handles module-level aliases)
|
||||||
if id(obj) in processed_classes:
|
if id(obj) in processed_classes:
|
||||||
self.logger.debug(f"Skipping duplicate reference to: {obj.__name__}")
|
self.logger.debug(f"Skipping duplicate reference to: {obj.__name__}")
|
||||||
continue
|
continue
|
||||||
processed_classes.add(id(obj))
|
processed_classes.add(id(obj))
|
||||||
|
|
||||||
self.logger.debug(f"Found plugin class: {name}")
|
self.logger.debug(f"Found plugin class: {name}")
|
||||||
|
|
||||||
# Instantiate plugin with config — check plugins subdict first,
|
# Instantiate plugin with config — check plugins subdict first,
|
||||||
# then top-level keys (e.g. nagios_runner: ... at root of config).
|
# then top-level keys (e.g. nagios_runner: ... at root of config).
|
||||||
plugin_instance_config = dict(plugins_subconfig.get(obj.name) or raw_config.get(obj.name) or {})
|
plugin_instance_config = dict(plugins_subconfig.get(obj.name) or raw_config.get(obj.name) or {})
|
||||||
@@ -369,7 +369,7 @@ class PluginLoader:
|
|||||||
if "owner" in raw_config and "owner" not in plugin_instance_config:
|
if "owner" in raw_config and "owner" not in plugin_instance_config:
|
||||||
plugin_instance_config["owner"] = raw_config["owner"]
|
plugin_instance_config["owner"] = raw_config["owner"]
|
||||||
plugin = obj(config=plugin_instance_config)
|
plugin = obj(config=plugin_instance_config)
|
||||||
|
|
||||||
# Initialize plugin
|
# Initialize plugin
|
||||||
try:
|
try:
|
||||||
initialized = await plugin.initialize()
|
initialized = await plugin.initialize()
|
||||||
@@ -389,7 +389,7 @@ class PluginLoader:
|
|||||||
exc_info=True
|
exc_info=True
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Register with registry
|
# Register with registry
|
||||||
if self.registry.register(plugin):
|
if self.registry.register(plugin):
|
||||||
loaded_count += 1
|
loaded_count += 1
|
||||||
@@ -397,15 +397,15 @@ class PluginLoader:
|
|||||||
f"Loaded plugin: {plugin.name} v{plugin.version} "
|
f"Loaded plugin: {plugin.name} v{plugin.version} "
|
||||||
f"(interval: {plugin.interval}s)"
|
f"(interval: {plugin.interval}s)"
|
||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(
|
self.logger.error(
|
||||||
f"Error loading plugin from {plugin_file}: {e}",
|
f"Error loading plugin from {plugin_file}: {e}",
|
||||||
exc_info=True
|
exc_info=True
|
||||||
)
|
)
|
||||||
|
|
||||||
return loaded_count
|
return loaded_count
|
||||||
|
|
||||||
async def unload_all(self) -> None:
|
async def unload_all(self) -> None:
|
||||||
"""Unload all plugins and cleanup resources."""
|
"""Unload all plugins and cleanup resources."""
|
||||||
for plugin in self.registry.get_all():
|
for plugin in self.registry.get_all():
|
||||||
@@ -417,7 +417,7 @@ class PluginLoader:
|
|||||||
exc_info=True
|
exc_info=True
|
||||||
)
|
)
|
||||||
self.registry.unregister(plugin.name)
|
self.registry.unregister(plugin.name)
|
||||||
|
|
||||||
# Remove loaded modules
|
# Remove loaded modules
|
||||||
for module_name in self._loaded_modules:
|
for module_name in self._loaded_modules:
|
||||||
if module_name in sys.modules:
|
if module_name in sys.modules:
|
||||||
|
|||||||
@@ -5,8 +5,6 @@ load average, and process counts.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
# Import from parent package
|
# Import from parent package
|
||||||
from hbd.client.plugin import MonitorPlugin
|
from hbd.client.plugin import MonitorPlugin
|
||||||
@@ -14,7 +12,7 @@ from hbd.client.plugin import MonitorPlugin
|
|||||||
|
|
||||||
class CPUMonitorPlugin(MonitorPlugin):
|
class CPUMonitorPlugin(MonitorPlugin):
|
||||||
"""Monitor CPU usage and load.
|
"""Monitor CPU usage and load.
|
||||||
|
|
||||||
Collects:
|
Collects:
|
||||||
- Overall CPU usage percentage
|
- Overall CPU usage percentage
|
||||||
- Per-core CPU usage (if enabled in config)
|
- Per-core CPU usage (if enabled in config)
|
||||||
@@ -22,28 +20,28 @@ class CPUMonitorPlugin(MonitorPlugin):
|
|||||||
- Process count
|
- Process count
|
||||||
- CPU frequency (if available)
|
- CPU frequency (if available)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "cpu_monitor"
|
name = "cpu_monitor"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
description = "CPU usage and load monitoring"
|
description = "CPU usage and load monitoring"
|
||||||
interval = 300 # MonitorPlugin: collect every 5 minutes by default
|
interval = 300 # MonitorPlugin: collect every 5 minutes by default
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
super().__init__(config)
|
super().__init__(config)
|
||||||
self.psutil = None
|
self.psutil: Any = None
|
||||||
self.per_core = config.get("per_core", False) if config else False
|
self.per_core = config.get("per_core", False) if config else False
|
||||||
self.interval = config.get("interval", 300) if config else 300
|
self.interval = config.get("interval", 300) if config else 300
|
||||||
|
|
||||||
async def initialize(self) -> bool:
|
async def initialize(self) -> bool:
|
||||||
"""Initialize the CPU monitor plugin.
|
"""Initialize the CPU monitor plugin.
|
||||||
|
|
||||||
Checks if psutil is available.
|
Checks if psutil is available.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if psutil is available, False otherwise
|
True if psutil is available, False otherwise
|
||||||
"""
|
"""
|
||||||
self.logger.info(f"Initializing {self.name} plugin")
|
self.logger.info(f"Initializing {self.name} plugin")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import psutil
|
import psutil
|
||||||
self.psutil = psutil
|
self.psutil = psutil
|
||||||
@@ -54,23 +52,23 @@ class CPUMonitorPlugin(MonitorPlugin):
|
|||||||
"psutil module not available. Install with: pip install psutil"
|
"psutil module not available. Install with: pip install psutil"
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||||
"""Collect CPU metrics.
|
"""Collect CPU metrics.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with CPU metrics
|
Dictionary with CPU metrics
|
||||||
"""
|
"""
|
||||||
if not self.psutil:
|
if not self.psutil:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = {}
|
data = {}
|
||||||
|
|
||||||
# Overall CPU usage percentage (non-blocking, interval=1 for accuracy)
|
# Overall CPU usage percentage (non-blocking, interval=1 for accuracy)
|
||||||
# Note: first call to cpu_percent() returns 0.0, subsequent calls work correctly
|
# Note: first call to cpu_percent() returns 0.0, subsequent calls work correctly
|
||||||
data["cpu_percent"] = self.psutil.cpu_percent(interval=1)
|
data["cpu_percent"] = self.psutil.cpu_percent(interval=1)
|
||||||
|
|
||||||
# Per-core CPU usage (if enabled)
|
# Per-core CPU usage (if enabled)
|
||||||
if self.per_core:
|
if self.per_core:
|
||||||
per_core_percents = self.psutil.cpu_percent(interval=0, percpu=True)
|
per_core_percents = self.psutil.cpu_percent(interval=0, percpu=True)
|
||||||
@@ -79,7 +77,7 @@ class CPUMonitorPlugin(MonitorPlugin):
|
|||||||
else:
|
else:
|
||||||
# Just report core count
|
# Just report core count
|
||||||
data["cpu_core_count"] = self.psutil.cpu_count()
|
data["cpu_core_count"] = self.psutil.cpu_count()
|
||||||
|
|
||||||
# Load average (Unix-like systems only)
|
# Load average (Unix-like systems only)
|
||||||
try:
|
try:
|
||||||
load_avg = self.psutil.getloadavg()
|
load_avg = self.psutil.getloadavg()
|
||||||
@@ -89,13 +87,13 @@ class CPUMonitorPlugin(MonitorPlugin):
|
|||||||
except (AttributeError, OSError):
|
except (AttributeError, OSError):
|
||||||
# Not available on Windows
|
# Not available on Windows
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Process count
|
# Process count
|
||||||
try:
|
try:
|
||||||
data["process_count"] = len(self.psutil.pids())
|
data["process_count"] = len(self.psutil.pids())
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.warning(f"Could not get process count: {e}")
|
self.logger.warning(f"Could not get process count: {e}")
|
||||||
|
|
||||||
# CPU frequency (if available)
|
# CPU frequency (if available)
|
||||||
try:
|
try:
|
||||||
freq = self.psutil.cpu_freq()
|
freq = self.psutil.cpu_freq()
|
||||||
@@ -107,7 +105,7 @@ class CPUMonitorPlugin(MonitorPlugin):
|
|||||||
# Not available on all systems, or may fail on FreeBSD with sysctl issues
|
# Not available on all systems, or may fail on FreeBSD with sysctl issues
|
||||||
self.logger.debug(f"CPU frequency not available: {e}")
|
self.logger.debug(f"CPU frequency not available: {e}")
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# CPU times (user, system, idle, etc.)
|
# CPU times (user, system, idle, etc.)
|
||||||
try:
|
try:
|
||||||
cpu_times = self.psutil.cpu_times_percent(interval=0)
|
cpu_times = self.psutil.cpu_times_percent(interval=0)
|
||||||
@@ -125,12 +123,12 @@ class CPUMonitorPlugin(MonitorPlugin):
|
|||||||
data["uptime_seconds"] = int(time.time() - self.psutil.boot_time())
|
data["uptime_seconds"] = int(time.time() - self.psutil.boot_time())
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.debug(f"Could not get uptime: {e}")
|
self.logger.debug(f"Could not get uptime: {e}")
|
||||||
|
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Collected CPU metrics: {data.get('cpu_percent', 'N/A')}% usage"
|
f"Collected CPU metrics: {data.get('cpu_percent', 'N/A')}% usage"
|
||||||
)
|
)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error collecting CPU metrics: {e}", exc_info=True)
|
self.logger.error(f"Error collecting CPU metrics: {e}", exc_info=True)
|
||||||
return {}
|
return {}
|
||||||
|
|||||||
@@ -5,12 +5,12 @@ Collects disk usage and I/O statistics using psutil.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import Dict, Any, Optional, List
|
from typing import Dict, Any, Optional
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import psutil
|
import psutil
|
||||||
except ImportError:
|
except ImportError:
|
||||||
psutil = None
|
psutil = None # type: ignore[assignment]
|
||||||
|
|
||||||
from hbd.client.plugin import MonitorPlugin
|
from hbd.client.plugin import MonitorPlugin
|
||||||
|
|
||||||
@@ -20,27 +20,27 @@ logger = logging.getLogger(__name__)
|
|||||||
class DiskMonitorPlugin(MonitorPlugin):
|
class DiskMonitorPlugin(MonitorPlugin):
|
||||||
"""
|
"""
|
||||||
Monitor disk usage and I/O statistics.
|
Monitor disk usage and I/O statistics.
|
||||||
|
|
||||||
Collects:
|
Collects:
|
||||||
- Disk partition information
|
- Disk partition information
|
||||||
- Disk usage per partition (total, used, free, percent)
|
- Disk usage per partition (total, used, free, percent)
|
||||||
- Disk I/O counters (read/write bytes, read/write count)
|
- Disk I/O counters (read/write bytes, read/write count)
|
||||||
- Disk I/O time statistics
|
- Disk I/O time statistics
|
||||||
|
|
||||||
Configuration:
|
Configuration:
|
||||||
interval: Collection interval in seconds (default: 300)
|
interval: Collection interval in seconds (default: 300)
|
||||||
partitions: List of mount points to monitor (default: all)
|
partitions: List of mount points to monitor (default: all)
|
||||||
include_io: Include disk I/O statistics (default: True)
|
include_io: Include disk I/O statistics (default: True)
|
||||||
exclude_types: List of filesystem types to exclude (default: tmpfs, devtmpfs, squashfs)
|
exclude_types: List of filesystem types to exclude (default: tmpfs, devtmpfs, squashfs)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "disk_monitor"
|
name = "disk_monitor"
|
||||||
interval = 300 # Collect every 5 minutes by default
|
interval = 300 # Collect every 5 minutes by default
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
"""
|
"""
|
||||||
Initialize the disk monitor plugin.
|
Initialize the disk monitor plugin.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Optional configuration dict with keys:
|
config: Optional configuration dict with keys:
|
||||||
- interval: Collection interval in seconds (default: 300)
|
- interval: Collection interval in seconds (default: 300)
|
||||||
@@ -53,34 +53,34 @@ class DiskMonitorPlugin(MonitorPlugin):
|
|||||||
self.include_io = self.config.get('include_io', True)
|
self.include_io = self.config.get('include_io', True)
|
||||||
self.exclude_types = set(self.config.get('exclude_types', ['tmpfs', 'devtmpfs', 'squashfs']))
|
self.exclude_types = set(self.config.get('exclude_types', ['tmpfs', 'devtmpfs', 'squashfs']))
|
||||||
self.interval = self.config.get('interval', 300)
|
self.interval = self.config.get('interval', 300)
|
||||||
|
|
||||||
if psutil is None:
|
if psutil is None:
|
||||||
raise ImportError("psutil library is required for disk_monitor plugin")
|
raise ImportError("psutil library is required for disk_monitor plugin")
|
||||||
|
|
||||||
# Store previous I/O counters for delta calculation
|
# Store previous I/O counters for delta calculation
|
||||||
self._prev_io = {}
|
self._prev_io: Dict[str, Any] = {}
|
||||||
|
|
||||||
async def initialize(self):
|
async def initialize(self):
|
||||||
"""Initialize the plugin (check psutil availability)."""
|
"""Initialize the plugin (check psutil availability)."""
|
||||||
if psutil is None:
|
if psutil is None:
|
||||||
logger.error("psutil not available - disk_monitor cannot run")
|
logger.error("psutil not available - disk_monitor cannot run")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
logger.info(f"Disk monitor initialized (interval: {self.interval}s, io: {self.include_io})")
|
logger.info(f"Disk monitor initialized (interval: {self.interval}s, io: {self.include_io})")
|
||||||
|
|
||||||
# Initialize I/O counters if available
|
# Initialize I/O counters if available
|
||||||
if self.include_io:
|
if self.include_io:
|
||||||
try:
|
try:
|
||||||
self._prev_io = psutil.disk_io_counters(perdisk=True)
|
self._prev_io = psutil.disk_io_counters(perdisk=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not initialize disk I/O counters: {e}")
|
logger.warning(f"Could not initialize disk I/O counters: {e}")
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def collect(self) -> Dict[str, Any]:
|
async def collect(self) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Collect current disk statistics.
|
Collect current disk statistics.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with disk metrics organized by partition:
|
Dictionary with disk metrics organized by partition:
|
||||||
- partitions: Dict of partition data, keyed by mount point
|
- partitions: Dict of partition data, keyed by mount point
|
||||||
@@ -103,7 +103,7 @@ class DiskMonitorPlugin(MonitorPlugin):
|
|||||||
if psutil is None:
|
if psutil is None:
|
||||||
logger.error("psutil not available")
|
logger.error("psutil not available")
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = await self._collect_metrics()
|
data = await self._collect_metrics()
|
||||||
logger.debug(f"Collected disk metrics: {len(data.get('partitions', {}))} partitions")
|
logger.debug(f"Collected disk metrics: {len(data.get('partitions', {}))} partitions")
|
||||||
@@ -111,24 +111,24 @@ class DiskMonitorPlugin(MonitorPlugin):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error collecting disk metrics: {e}")
|
logger.error(f"Error collecting disk metrics: {e}")
|
||||||
return {"error": str(e)}
|
return {"error": str(e)}
|
||||||
|
|
||||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||||
"""Collect disk metrics from psutil."""
|
"""Collect disk metrics from psutil."""
|
||||||
metrics = {}
|
metrics: Dict[str, Any] = {}
|
||||||
|
|
||||||
# Collect partition usage
|
# Collect partition usage
|
||||||
partitions_data = {}
|
partitions_data = {}
|
||||||
partitions = psutil.disk_partitions(all=False)
|
partitions = psutil.disk_partitions(all=False)
|
||||||
|
|
||||||
for partition in partitions:
|
for partition in partitions:
|
||||||
# Skip unwanted filesystem types
|
# Skip unwanted filesystem types
|
||||||
if partition.fstype in self.exclude_types:
|
if partition.fstype in self.exclude_types:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Skip if we're only monitoring specific partitions
|
# Skip if we're only monitoring specific partitions
|
||||||
if self.partitions and partition.mountpoint not in self.partitions:
|
if self.partitions and partition.mountpoint not in self.partitions:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
usage = psutil.disk_usage(partition.mountpoint)
|
usage = psutil.disk_usage(partition.mountpoint)
|
||||||
partitions_data[partition.mountpoint] = {
|
partitions_data[partition.mountpoint] = {
|
||||||
@@ -145,15 +145,15 @@ class DiskMonitorPlugin(MonitorPlugin):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Error reading {partition.mountpoint}: {e}")
|
logger.warning(f"Error reading {partition.mountpoint}: {e}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
metrics['partitions'] = partitions_data
|
metrics['partitions'] = partitions_data
|
||||||
|
|
||||||
# Collect I/O statistics
|
# Collect I/O statistics
|
||||||
if self.include_io:
|
if self.include_io:
|
||||||
try:
|
try:
|
||||||
io_counters = psutil.disk_io_counters(perdisk=True)
|
io_counters = psutil.disk_io_counters(perdisk=True)
|
||||||
io_data = {}
|
io_data = {}
|
||||||
|
|
||||||
for disk_name, counters in io_counters.items():
|
for disk_name, counters in io_counters.items():
|
||||||
disk_stats = {
|
disk_stats = {
|
||||||
'read_count': counters.read_count,
|
'read_count': counters.read_count,
|
||||||
@@ -161,7 +161,7 @@ class DiskMonitorPlugin(MonitorPlugin):
|
|||||||
'read_bytes': counters.read_bytes,
|
'read_bytes': counters.read_bytes,
|
||||||
'write_bytes': counters.write_bytes,
|
'write_bytes': counters.write_bytes,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Add time statistics if available
|
# Add time statistics if available
|
||||||
if hasattr(counters, 'read_time'):
|
if hasattr(counters, 'read_time'):
|
||||||
disk_stats['read_time'] = counters.read_time
|
disk_stats['read_time'] = counters.read_time
|
||||||
@@ -169,7 +169,7 @@ class DiskMonitorPlugin(MonitorPlugin):
|
|||||||
disk_stats['write_time'] = counters.write_time
|
disk_stats['write_time'] = counters.write_time
|
||||||
if hasattr(counters, 'busy_time'):
|
if hasattr(counters, 'busy_time'):
|
||||||
disk_stats['busy_time'] = counters.busy_time
|
disk_stats['busy_time'] = counters.busy_time
|
||||||
|
|
||||||
# Calculate deltas from previous collection
|
# Calculate deltas from previous collection
|
||||||
if disk_name in self._prev_io:
|
if disk_name in self._prev_io:
|
||||||
prev = self._prev_io[disk_name]
|
prev = self._prev_io[disk_name]
|
||||||
@@ -177,19 +177,19 @@ class DiskMonitorPlugin(MonitorPlugin):
|
|||||||
disk_stats['write_bytes_delta'] = counters.write_bytes - prev.write_bytes
|
disk_stats['write_bytes_delta'] = counters.write_bytes - prev.write_bytes
|
||||||
disk_stats['read_count_delta'] = counters.read_count - prev.read_count
|
disk_stats['read_count_delta'] = counters.read_count - prev.read_count
|
||||||
disk_stats['write_count_delta'] = counters.write_count - prev.write_count
|
disk_stats['write_count_delta'] = counters.write_count - prev.write_count
|
||||||
|
|
||||||
io_data[disk_name] = disk_stats
|
io_data[disk_name] = disk_stats
|
||||||
|
|
||||||
metrics['io_counters'] = io_data
|
metrics['io_counters'] = io_data
|
||||||
|
|
||||||
# Store current counters for next delta calculation
|
# Store current counters for next delta calculation
|
||||||
self._prev_io = io_counters
|
self._prev_io = io_counters
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not collect disk I/O statistics: {e}")
|
logger.warning(f"Could not collect disk I/O statistics: {e}")
|
||||||
|
|
||||||
return metrics
|
return metrics
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
"""Cleanup (nothing to do for this plugin)."""
|
"""Cleanup (nothing to do for this plugin)."""
|
||||||
logger.info("Disk monitor cleanup")
|
logger.info("Disk monitor cleanup")
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from typing import Dict, Any, Optional
|
|||||||
try:
|
try:
|
||||||
import psutil
|
import psutil
|
||||||
except ImportError:
|
except ImportError:
|
||||||
psutil = None
|
psutil = None # type: ignore[assignment]
|
||||||
|
|
||||||
from hbd.client.plugin import InfoPlugin
|
from hbd.client.plugin import InfoPlugin
|
||||||
|
|
||||||
@@ -20,29 +20,29 @@ logger = logging.getLogger(__name__)
|
|||||||
class FilesystemInfoPlugin(InfoPlugin):
|
class FilesystemInfoPlugin(InfoPlugin):
|
||||||
"""
|
"""
|
||||||
Collect filesystem and partition information.
|
Collect filesystem and partition information.
|
||||||
|
|
||||||
This is an InfoPlugin that collects static information once during startup.
|
This is an InfoPlugin that collects static information once during startup.
|
||||||
|
|
||||||
By default, only reports physical mounted filesystems (e.g., ext4, xfs, btrfs).
|
By default, only reports physical mounted filesystems (e.g., ext4, xfs, btrfs).
|
||||||
Set include_pseudo=True to also include pseudo filesystems (proc, sysfs, tmpfs, etc.).
|
Set include_pseudo=True to also include pseudo filesystems (proc, sysfs, tmpfs, etc.).
|
||||||
|
|
||||||
Collects:
|
Collects:
|
||||||
- List of mounted filesystems
|
- List of mounted filesystems
|
||||||
- Partition details (device, mount point, filesystem type, options)
|
- Partition details (device, mount point, filesystem type, options)
|
||||||
- Filesystem capabilities and features
|
- Filesystem capabilities and features
|
||||||
|
|
||||||
Configuration:
|
Configuration:
|
||||||
include_pseudo: Include pseudo/virtual filesystems (default: False)
|
include_pseudo: Include pseudo/virtual filesystems (default: False)
|
||||||
exclude_types: List of additional filesystem types to exclude (default: [])
|
exclude_types: List of additional filesystem types to exclude (default: [])
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "filesystem_info"
|
name = "filesystem_info"
|
||||||
interval = 0 # InfoPlugin - collect once
|
interval = 0 # InfoPlugin - collect once
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
"""
|
"""
|
||||||
Initialize the filesystem info plugin.
|
Initialize the filesystem info plugin.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Optional configuration dict with keys:
|
config: Optional configuration dict with keys:
|
||||||
- include_pseudo: Include pseudo/virtual filesystems (default: False)
|
- include_pseudo: Include pseudo/virtual filesystems (default: False)
|
||||||
@@ -53,25 +53,25 @@ class FilesystemInfoPlugin(InfoPlugin):
|
|||||||
# By default, no exclusions since all=False filters most pseudo filesystems
|
# By default, no exclusions since all=False filters most pseudo filesystems
|
||||||
# Users can add specific types to exclude if needed
|
# Users can add specific types to exclude if needed
|
||||||
self.exclude_types = set(self.config.get('exclude_types', []))
|
self.exclude_types = set(self.config.get('exclude_types', []))
|
||||||
|
|
||||||
if psutil is None:
|
if psutil is None:
|
||||||
raise ImportError("psutil library is required for filesystem_info plugin")
|
raise ImportError("psutil library is required for filesystem_info plugin")
|
||||||
|
|
||||||
async def initialize(self):
|
async def initialize(self):
|
||||||
"""Initialize the plugin (check psutil availability)."""
|
"""Initialize the plugin (check psutil availability)."""
|
||||||
if psutil is None:
|
if psutil is None:
|
||||||
logger.error("psutil not available - filesystem_info cannot run")
|
logger.error("psutil not available - filesystem_info cannot run")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
logger.info(f"Filesystem info initialized (pseudo: {self.include_pseudo})")
|
logger.info(f"Filesystem info initialized (pseudo: {self.include_pseudo})")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def collect(self) -> Dict[str, Any]:
|
async def collect(self) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Collect filesystem information.
|
Collect filesystem information.
|
||||||
|
|
||||||
Returns only physical mounted filesystems by default.
|
Returns only physical mounted filesystems by default.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with filesystem data:
|
Dictionary with filesystem data:
|
||||||
- filesystems: List of filesystem dictionaries:
|
- filesystems: List of filesystem dictionaries:
|
||||||
@@ -87,7 +87,7 @@ class FilesystemInfoPlugin(InfoPlugin):
|
|||||||
if psutil is None:
|
if psutil is None:
|
||||||
logger.error("psutil not available")
|
logger.error("psutil not available")
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = await self._collect_info()
|
data = await self._collect_info()
|
||||||
logger.info(f"Collected filesystem info: {len(data.get('filesystems', []))} filesystems")
|
logger.info(f"Collected filesystem info: {len(data.get('filesystems', []))} filesystems")
|
||||||
@@ -95,30 +95,30 @@ class FilesystemInfoPlugin(InfoPlugin):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error collecting filesystem info: {e}")
|
logger.error(f"Error collecting filesystem info: {e}")
|
||||||
return {"error": str(e)}
|
return {"error": str(e)}
|
||||||
|
|
||||||
async def _collect_info(self) -> Dict[str, Any]:
|
async def _collect_info(self) -> Dict[str, Any]:
|
||||||
"""Collect filesystem information from psutil."""
|
"""Collect filesystem information from psutil."""
|
||||||
info = {}
|
info: Dict[str, Any] = {}
|
||||||
filesystems = []
|
filesystems: list = []
|
||||||
filesystem_types = set()
|
filesystem_types = set()
|
||||||
|
|
||||||
# Get mounted disk partitions
|
# Get mounted disk partitions
|
||||||
# all=False returns only physical devices (real mounted filesystems)
|
# all=False returns only physical devices (real mounted filesystems)
|
||||||
# all=True would include pseudo filesystems (proc, sysfs, etc.)
|
# all=True would include pseudo filesystems (proc, sysfs, etc.)
|
||||||
partitions = psutil.disk_partitions(all=self.include_pseudo)
|
partitions = psutil.disk_partitions(all=self.include_pseudo)
|
||||||
|
|
||||||
for partition in partitions:
|
for partition in partitions:
|
||||||
# Additional filtering if exclude_types is specified
|
# Additional filtering if exclude_types is specified
|
||||||
if partition.fstype in self.exclude_types:
|
if partition.fstype in self.exclude_types:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
fs_info = {
|
fs_info: Dict[str, Any] = {
|
||||||
'device': partition.device,
|
'device': partition.device,
|
||||||
'mountpoint': partition.mountpoint,
|
'mountpoint': partition.mountpoint,
|
||||||
'fstype': partition.fstype,
|
'fstype': partition.fstype,
|
||||||
'opts': partition.opts,
|
'opts': partition.opts,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Try to get filesystem capabilities
|
# Try to get filesystem capabilities
|
||||||
try:
|
try:
|
||||||
# Get path configuration for this mount point
|
# Get path configuration for this mount point
|
||||||
@@ -141,14 +141,14 @@ class FilesystemInfoPlugin(InfoPlugin):
|
|||||||
pass
|
pass
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"Could not get pathconf for {partition.mountpoint}: {e}")
|
logger.debug(f"Could not get pathconf for {partition.mountpoint}: {e}")
|
||||||
|
|
||||||
filesystems.append(fs_info)
|
filesystems.append(fs_info)
|
||||||
filesystem_types.add(partition.fstype)
|
filesystem_types.add(partition.fstype)
|
||||||
|
|
||||||
info['filesystems'] = filesystems
|
info['filesystems'] = filesystems
|
||||||
info['filesystem_types'] = sorted(list(filesystem_types))
|
info['filesystem_types'] = sorted(list(filesystem_types))
|
||||||
info['mount_count'] = len(filesystems)
|
info['mount_count'] = len(filesystems)
|
||||||
|
|
||||||
# Add some additional filesystem statistics
|
# Add some additional filesystem statistics
|
||||||
try:
|
try:
|
||||||
# Get boot time (useful for determining filesystem mount times)
|
# Get boot time (useful for determining filesystem mount times)
|
||||||
@@ -156,9 +156,9 @@ class FilesystemInfoPlugin(InfoPlugin):
|
|||||||
info['boot_time'] = boot_time
|
info['boot_time'] = boot_time
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"Could not get boot time: {e}")
|
logger.debug(f"Could not get boot time: {e}")
|
||||||
|
|
||||||
return info
|
return info
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
"""Cleanup (nothing to do for this plugin)."""
|
"""Cleanup (nothing to do for this plugin)."""
|
||||||
logger.info("Filesystem info cleanup")
|
logger.info("Filesystem info cleanup")
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from typing import Dict, Any, Optional
|
|||||||
try:
|
try:
|
||||||
import psutil
|
import psutil
|
||||||
except ImportError:
|
except ImportError:
|
||||||
psutil = None
|
psutil = None # type: ignore[assignment]
|
||||||
|
|
||||||
from hbd.client.plugin import MonitorPlugin
|
from hbd.client.plugin import MonitorPlugin
|
||||||
|
|
||||||
@@ -32,31 +32,32 @@ def _zfs_arc_bytes() -> int:
|
|||||||
pass
|
pass
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class MemoryMonitorPlugin(MonitorPlugin):
|
class MemoryMonitorPlugin(MonitorPlugin):
|
||||||
"""
|
"""
|
||||||
Monitor memory and swap usage.
|
Monitor memory and swap usage.
|
||||||
|
|
||||||
Collects:
|
Collects:
|
||||||
- Physical memory (RAM) usage and statistics
|
- Physical memory (RAM) usage and statistics
|
||||||
- Virtual memory details
|
- Virtual memory details
|
||||||
- Swap memory usage and statistics
|
- Swap memory usage and statistics
|
||||||
- Memory available for applications
|
- Memory available for applications
|
||||||
|
|
||||||
Configuration:
|
Configuration:
|
||||||
interval: Collection interval in seconds (default: 300)
|
interval: Collection interval in seconds (default: 300)
|
||||||
include_swap: Include swap statistics (default: True)
|
include_swap: Include swap statistics (default: True)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "memory_monitor"
|
name = "memory_monitor"
|
||||||
interval = 300 # Collect every 5 minutes by default
|
interval = 300 # Collect every 5 minutes by default
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
"""
|
"""
|
||||||
Initialize the memory monitor plugin.
|
Initialize the memory monitor plugin.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Optional configuration dict with keys:
|
config: Optional configuration dict with keys:
|
||||||
- interval: Collection interval in seconds (default: 300)
|
- interval: Collection interval in seconds (default: 300)
|
||||||
@@ -65,23 +66,23 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
super().__init__(config)
|
super().__init__(config)
|
||||||
self.include_swap = self.config.get('include_swap', True)
|
self.include_swap = self.config.get('include_swap', True)
|
||||||
self.interval = self.config.get('interval', 300)
|
self.interval = self.config.get('interval', 300)
|
||||||
|
|
||||||
if psutil is None:
|
if psutil is None:
|
||||||
raise ImportError("psutil library is required for memory_monitor plugin")
|
raise ImportError("psutil library is required for memory_monitor plugin")
|
||||||
|
|
||||||
async def initialize(self):
|
async def initialize(self):
|
||||||
"""Initialize the plugin (check psutil availability)."""
|
"""Initialize the plugin (check psutil availability)."""
|
||||||
if psutil is None:
|
if psutil is None:
|
||||||
logger.error("psutil not available - memory_monitor cannot run")
|
logger.error("psutil not available - memory_monitor cannot run")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
logger.info(f"Memory monitor initialized (interval: {self.interval}s, swap: {self.include_swap})")
|
logger.info(f"Memory monitor initialized (interval: {self.interval}s, swap: {self.include_swap})")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def collect(self) -> Dict[str, Any]:
|
async def collect(self) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Collect current memory statistics.
|
Collect current memory statistics.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with memory metrics:
|
Dictionary with memory metrics:
|
||||||
- memory_total: Total physical RAM in bytes
|
- memory_total: Total physical RAM in bytes
|
||||||
@@ -104,7 +105,7 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
if psutil is None:
|
if psutil is None:
|
||||||
logger.error("psutil not available")
|
logger.error("psutil not available")
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = await self._collect_metrics()
|
data = await self._collect_metrics()
|
||||||
logger.debug(f"Collected memory metrics: {len(data)} fields")
|
logger.debug(f"Collected memory metrics: {len(data)} fields")
|
||||||
@@ -112,11 +113,11 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error collecting memory metrics: {e}")
|
logger.error(f"Error collecting memory metrics: {e}")
|
||||||
return {"error": str(e)}
|
return {"error": str(e)}
|
||||||
|
|
||||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||||
"""Collect memory metrics from psutil."""
|
"""Collect memory metrics from psutil."""
|
||||||
metrics = {}
|
metrics: Dict[str, Any] = {}
|
||||||
|
|
||||||
# Virtual (physical) memory statistics
|
# Virtual (physical) memory statistics
|
||||||
vmem = psutil.virtual_memory()
|
vmem = psutil.virtual_memory()
|
||||||
|
|
||||||
@@ -134,7 +135,7 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
metrics['memory_used'] = used
|
metrics['memory_used'] = used
|
||||||
metrics['memory_free'] = vmem.free
|
metrics['memory_free'] = vmem.free
|
||||||
metrics['memory_percent'] = percent
|
metrics['memory_percent'] = percent
|
||||||
|
|
||||||
# Platform-specific memory details
|
# Platform-specific memory details
|
||||||
if hasattr(vmem, 'active'):
|
if hasattr(vmem, 'active'):
|
||||||
metrics['memory_active'] = vmem.active
|
metrics['memory_active'] = vmem.active
|
||||||
@@ -146,7 +147,7 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
metrics['memory_cached'] = vmem.cached
|
metrics['memory_cached'] = vmem.cached
|
||||||
if hasattr(vmem, 'shared'):
|
if hasattr(vmem, 'shared'):
|
||||||
metrics['memory_shared'] = vmem.shared
|
metrics['memory_shared'] = vmem.shared
|
||||||
|
|
||||||
# Swap memory statistics
|
# Swap memory statistics
|
||||||
if self.include_swap:
|
if self.include_swap:
|
||||||
try:
|
try:
|
||||||
@@ -155,7 +156,7 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
metrics['swap_used'] = swap.used
|
metrics['swap_used'] = swap.used
|
||||||
metrics['swap_free'] = swap.free
|
metrics['swap_free'] = swap.free
|
||||||
metrics['swap_percent'] = swap.percent
|
metrics['swap_percent'] = swap.percent
|
||||||
|
|
||||||
# Swap in/out counters (may not be available on all platforms)
|
# Swap in/out counters (may not be available on all platforms)
|
||||||
if hasattr(swap, 'sin'):
|
if hasattr(swap, 'sin'):
|
||||||
metrics['swap_sin'] = swap.sin
|
metrics['swap_sin'] = swap.sin
|
||||||
@@ -163,9 +164,9 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
metrics['swap_sout'] = swap.sout
|
metrics['swap_sout'] = swap.sout
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not collect swap statistics: {e}")
|
logger.warning(f"Could not collect swap statistics: {e}")
|
||||||
|
|
||||||
return metrics
|
return metrics
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
"""Cleanup (nothing to do for this plugin)."""
|
"""Cleanup (nothing to do for this plugin)."""
|
||||||
logger.info("Memory monitor cleanup")
|
logger.info("Memory monitor cleanup")
|
||||||
|
|||||||
@@ -43,10 +43,10 @@ STATUS_NAMES = {
|
|||||||
|
|
||||||
class NagiosRunnerPlugin(MonitorPlugin):
|
class NagiosRunnerPlugin(MonitorPlugin):
|
||||||
"""Run Nagios-compatible monitoring plugins.
|
"""Run Nagios-compatible monitoring plugins.
|
||||||
|
|
||||||
This plugin executes external Nagios plugins and collects their output,
|
This plugin executes external Nagios plugins and collects their output,
|
||||||
including status codes, messages, and performance data.
|
including status codes, messages, and performance data.
|
||||||
|
|
||||||
Configuration:
|
Configuration:
|
||||||
interval: Collection interval in seconds (default: 300)
|
interval: Collection interval in seconds (default: 300)
|
||||||
commands: List of command definitions with 'name' and 'command' keys
|
commands: List of command definitions with 'name' and 'command' keys
|
||||||
@@ -62,20 +62,20 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
- name: check_load
|
- name: check_load
|
||||||
command: /usr/lib/nagios/plugins/check_load -w 5,4,3 -c 10,8,6
|
command: /usr/lib/nagios/plugins/check_load -w 5,4,3 -c 10,8,6
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "nagios_runner"
|
name = "nagios_runner"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
description = "Execute Nagios-compatible monitoring plugins"
|
description = "Execute Nagios-compatible monitoring plugins"
|
||||||
interval = 300 # MonitorPlugin: collect every 5 minutes by default
|
interval = 300 # MonitorPlugin: collect every 5 minutes by default
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
super().__init__(config)
|
super().__init__(config)
|
||||||
|
|
||||||
# Extract configuration
|
# Extract configuration
|
||||||
self.commands: List[Dict[str, str]] = config.get("commands", []) if config else []
|
self.commands: List[Dict[str, str]] = config.get("commands", []) if config else []
|
||||||
self.timeout: int = config.get("timeout", 30) if config else 30
|
self.timeout: int = config.get("timeout", 30) if config else 30
|
||||||
self.interval = config.get("interval", 300) if config else 300
|
self.interval = config.get("interval", 300) if config else 300
|
||||||
|
|
||||||
async def initialize(self) -> bool:
|
async def initialize(self) -> bool:
|
||||||
"""Initialize the Nagios runner plugin.
|
"""Initialize the Nagios runner plugin.
|
||||||
|
|
||||||
@@ -117,14 +117,14 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
)
|
)
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||||
"""Collect metrics from all configured Nagios plugins.
|
"""Collect metrics from all configured Nagios plugins.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with results from all plugins
|
Dictionary with results from all plugins
|
||||||
"""
|
"""
|
||||||
results = {}
|
results: Dict[str, Any] = {}
|
||||||
|
|
||||||
for cmd_config in self.commands:
|
for cmd_config in self.commands:
|
||||||
name = cmd_config.get("name")
|
name = cmd_config.get("name")
|
||||||
@@ -159,7 +159,7 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
results[f"{name}_output"] = str(e)
|
results[f"{name}_output"] = str(e)
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
async def _run_nagios_plugin(
|
async def _run_nagios_plugin(
|
||||||
self,
|
self,
|
||||||
command: str
|
command: str
|
||||||
@@ -181,7 +181,8 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
self.logger.error(f"Command timed out: {command}")
|
self.logger.error(f"Command timed out: {command}")
|
||||||
return NAGIOS_UNKNOWN, f"Command timed out after {self.timeout}s", {}
|
return NAGIOS_UNKNOWN, f"Command timed out after {self.timeout}s", {}
|
||||||
|
|
||||||
status_code = proc.returncode
|
# communicate() has finished, so the process has exited and returncode is set
|
||||||
|
status_code = proc.returncode if proc.returncode is not None else NAGIOS_UNKNOWN
|
||||||
|
|
||||||
if status_code < 0:
|
if status_code < 0:
|
||||||
return NAGIOS_UNKNOWN, f"Process killed by signal {-status_code}", {}
|
return NAGIOS_UNKNOWN, f"Process killed by signal {-status_code}", {}
|
||||||
@@ -210,33 +211,33 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error executing command: {e}")
|
self.logger.error(f"Error executing command: {e}")
|
||||||
return NAGIOS_UNKNOWN, f"Execution error: {str(e)}", {}
|
return NAGIOS_UNKNOWN, f"Execution error: {str(e)}", {}
|
||||||
|
|
||||||
def _parse_perfdata(self, output: str) -> Dict[str, Any]:
|
def _parse_perfdata(self, output: str) -> Dict[str, Any]:
|
||||||
"""Parse Nagios performance data from plugin output.
|
"""Parse Nagios performance data from plugin output.
|
||||||
|
|
||||||
Nagios performance data format:
|
Nagios performance data format:
|
||||||
'label'=value[UOM];[warn];[crit];[min];[max]
|
'label'=value[UOM];[warn];[crit];[min];[max]
|
||||||
|
|
||||||
Multiple metrics separated by spaces.
|
Multiple metrics separated by spaces.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
output: Plugin output string
|
output: Plugin output string
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary of metric_name: value
|
Dictionary of metric_name: value
|
||||||
"""
|
"""
|
||||||
perfdata = {}
|
perfdata: Dict[str, Any] = {}
|
||||||
|
|
||||||
# Performance data comes after the pipe character
|
# Performance data comes after the pipe character
|
||||||
if '|' not in output:
|
if '|' not in output:
|
||||||
return perfdata
|
return perfdata
|
||||||
|
|
||||||
perf_section = output.split('|', 1)[1].strip()
|
perf_section = output.split('|', 1)[1].strip()
|
||||||
|
|
||||||
# Regex to match performance data format
|
# Regex to match performance data format
|
||||||
# Matches: 'label'=value or label=value
|
# Matches: 'label'=value or label=value
|
||||||
perf_regex = r"'?([^'=]+)'?=([\d.]+)([a-zA-Z%]*);?([\d.]*);?([\d.]*);?([\d.]*);?([\d.]*)"
|
perf_regex = r"'?([^'=]+)'?=([\d.]+)([a-zA-Z%]*);?([\d.]*);?([\d.]*);?([\d.]*);?([\d.]*)"
|
||||||
|
|
||||||
for match in re.finditer(perf_regex, perf_section):
|
for match in re.finditer(perf_regex, perf_section):
|
||||||
label = match.group(1).strip()
|
label = match.group(1).strip()
|
||||||
value_str = match.group(2)
|
value_str = match.group(2)
|
||||||
@@ -245,43 +246,43 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
crit = match.group(5)
|
crit = match.group(5)
|
||||||
min_val = match.group(6)
|
min_val = match.group(6)
|
||||||
max_val = match.group(7)
|
max_val = match.group(7)
|
||||||
|
|
||||||
# Convert value to float
|
# Convert value to float
|
||||||
try:
|
try:
|
||||||
value = float(value_str)
|
value = float(value_str)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Store the value
|
# Store the value
|
||||||
perfdata[label] = value
|
perfdata[label] = value
|
||||||
|
|
||||||
# Optionally store UOM as separate field
|
# Optionally store UOM as separate field
|
||||||
if uom:
|
if uom:
|
||||||
perfdata[f"{label}_uom"] = uom
|
perfdata[f"{label}_uom"] = uom
|
||||||
|
|
||||||
# Store thresholds if present
|
# Store thresholds if present
|
||||||
if warn:
|
if warn:
|
||||||
try:
|
try:
|
||||||
perfdata[f"{label}_warn"] = float(warn)
|
perfdata[f"{label}_warn"] = float(warn)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if crit:
|
if crit:
|
||||||
try:
|
try:
|
||||||
perfdata[f"{label}_crit"] = float(crit)
|
perfdata[f"{label}_crit"] = float(crit)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if min_val:
|
if min_val:
|
||||||
try:
|
try:
|
||||||
perfdata[f"{label}_min"] = float(min_val)
|
perfdata[f"{label}_min"] = float(min_val)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if max_val:
|
if max_val:
|
||||||
try:
|
try:
|
||||||
perfdata[f"{label}_max"] = float(max_val)
|
perfdata[f"{label}_max"] = float(max_val)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
return perfdata
|
return perfdata
|
||||||
|
|||||||
@@ -5,12 +5,12 @@ Collects network interface statistics and connection information using psutil.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import Dict, Any, Optional, List
|
from typing import Dict, Any, Optional
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import psutil
|
import psutil
|
||||||
except ImportError:
|
except ImportError:
|
||||||
psutil = None
|
psutil = None # type: ignore[assignment]
|
||||||
|
|
||||||
from hbd.client.plugin import MonitorPlugin
|
from hbd.client.plugin import MonitorPlugin
|
||||||
from hbd.client.connectivity import get_shared
|
from hbd.client.connectivity import get_shared
|
||||||
@@ -21,13 +21,13 @@ logger = logging.getLogger(__name__)
|
|||||||
class NetworkMonitorPlugin(MonitorPlugin):
|
class NetworkMonitorPlugin(MonitorPlugin):
|
||||||
"""
|
"""
|
||||||
Monitor network interface statistics and connections.
|
Monitor network interface statistics and connections.
|
||||||
|
|
||||||
Collects:
|
Collects:
|
||||||
- Network interface I/O counters (bytes sent/received, packets, errors, drops)
|
- Network interface I/O counters (bytes sent/received, packets, errors, drops)
|
||||||
- Per-interface statistics
|
- Per-interface statistics
|
||||||
- Network connection counts by state
|
- Network connection counts by state
|
||||||
- Interface addresses and configuration
|
- Interface addresses and configuration
|
||||||
|
|
||||||
Configuration:
|
Configuration:
|
||||||
interval: Collection interval in seconds (default: 300)
|
interval: Collection interval in seconds (default: 300)
|
||||||
interfaces: List of interfaces to monitor (default: all)
|
interfaces: List of interfaces to monitor (default: all)
|
||||||
@@ -38,14 +38,14 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
|||||||
probe_v6: IPv6 address to ping (default: 2606:4700:4700::1111)
|
probe_v6: IPv6 address to ping (default: 2606:4700:4700::1111)
|
||||||
probe_interval: Seconds between ICMP probes while healthy (default: 60)
|
probe_interval: Seconds between ICMP probes while healthy (default: 60)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "network_monitor"
|
name = "network_monitor"
|
||||||
interval = 300 # Collect every 5 minutes by default
|
interval = 300 # Collect every 5 minutes by default
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
"""
|
"""
|
||||||
Initialize the network monitor plugin.
|
Initialize the network monitor plugin.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Optional configuration dict with keys:
|
config: Optional configuration dict with keys:
|
||||||
- interval: Collection interval in seconds (default: 300)
|
- interval: Collection interval in seconds (default: 300)
|
||||||
@@ -58,34 +58,34 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
|||||||
self.include_connections = self.config.get('include_connections', True)
|
self.include_connections = self.config.get('include_connections', True)
|
||||||
self.include_addresses = self.config.get('include_addresses', False)
|
self.include_addresses = self.config.get('include_addresses', False)
|
||||||
self.interval = self.config.get('interval', 300)
|
self.interval = self.config.get('interval', 300)
|
||||||
|
|
||||||
if psutil is None:
|
if psutil is None:
|
||||||
raise ImportError("psutil library is required for network_monitor plugin")
|
raise ImportError("psutil library is required for network_monitor plugin")
|
||||||
|
|
||||||
# Store previous I/O counters for delta calculation
|
# Store previous I/O counters for delta calculation
|
||||||
self._prev_io = {}
|
self._prev_io: Dict[str, Any] = {}
|
||||||
|
|
||||||
async def initialize(self):
|
async def initialize(self):
|
||||||
"""Initialize the plugin (check psutil availability)."""
|
"""Initialize the plugin (check psutil availability)."""
|
||||||
if psutil is None:
|
if psutil is None:
|
||||||
logger.error("psutil not available - network_monitor cannot run")
|
logger.error("psutil not available - network_monitor cannot run")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
logger.info(f"Network monitor initialized (interval: {self.interval}s, "
|
logger.info(f"Network monitor initialized (interval: {self.interval}s, "
|
||||||
f"connections: {self.include_connections})")
|
f"connections: {self.include_connections})")
|
||||||
|
|
||||||
# Initialize I/O counters
|
# Initialize I/O counters
|
||||||
try:
|
try:
|
||||||
self._prev_io = psutil.net_io_counters(pernic=True)
|
self._prev_io = psutil.net_io_counters(pernic=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not initialize network I/O counters: {e}")
|
logger.warning(f"Could not initialize network I/O counters: {e}")
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def collect(self) -> Dict[str, Any]:
|
async def collect(self) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Collect current network statistics.
|
Collect current network statistics.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with network metrics:
|
Dictionary with network metrics:
|
||||||
- interfaces: Dict of interface statistics, keyed by interface name
|
- interfaces: Dict of interface statistics, keyed by interface name
|
||||||
@@ -113,7 +113,7 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
|||||||
if psutil is None:
|
if psutil is None:
|
||||||
logger.error("psutil not available")
|
logger.error("psutil not available")
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
data = await self._collect_metrics()
|
data = await self._collect_metrics()
|
||||||
logger.debug(f"Collected network metrics: {len(data.get('interfaces', {}))} interfaces")
|
logger.debug(f"Collected network metrics: {len(data.get('interfaces', {}))} interfaces")
|
||||||
@@ -121,21 +121,21 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error collecting network metrics: {e}")
|
logger.error(f"Error collecting network metrics: {e}")
|
||||||
return {"error": str(e)}
|
return {"error": str(e)}
|
||||||
|
|
||||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||||
"""Collect network metrics from psutil."""
|
"""Collect network metrics from psutil."""
|
||||||
metrics = {}
|
metrics: Dict[str, Any] = {}
|
||||||
|
|
||||||
# Collect per-interface I/O counters
|
# Collect per-interface I/O counters
|
||||||
try:
|
try:
|
||||||
io_counters = psutil.net_io_counters(pernic=True)
|
io_counters = psutil.net_io_counters(pernic=True)
|
||||||
interfaces_data = {}
|
interfaces_data = {}
|
||||||
|
|
||||||
for iface_name, counters in io_counters.items():
|
for iface_name, counters in io_counters.items():
|
||||||
# Skip if we're only monitoring specific interfaces
|
# Skip if we're only monitoring specific interfaces
|
||||||
if self.interfaces and iface_name not in self.interfaces:
|
if self.interfaces and iface_name not in self.interfaces:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
iface_stats = {
|
iface_stats = {
|
||||||
'bytes_sent': counters.bytes_sent,
|
'bytes_sent': counters.bytes_sent,
|
||||||
'bytes_recv': counters.bytes_recv,
|
'bytes_recv': counters.bytes_recv,
|
||||||
@@ -146,7 +146,7 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
|||||||
'dropin': counters.dropin,
|
'dropin': counters.dropin,
|
||||||
'dropout': counters.dropout,
|
'dropout': counters.dropout,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Calculate deltas from previous collection
|
# Calculate deltas from previous collection
|
||||||
if iface_name in self._prev_io:
|
if iface_name in self._prev_io:
|
||||||
prev = self._prev_io[iface_name]
|
prev = self._prev_io[iface_name]
|
||||||
@@ -154,46 +154,46 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
|||||||
iface_stats['bytes_recv_delta'] = counters.bytes_recv - prev.bytes_recv
|
iface_stats['bytes_recv_delta'] = counters.bytes_recv - prev.bytes_recv
|
||||||
iface_stats['packets_sent_delta'] = counters.packets_sent - prev.packets_sent
|
iface_stats['packets_sent_delta'] = counters.packets_sent - prev.packets_sent
|
||||||
iface_stats['packets_recv_delta'] = counters.packets_recv - prev.packets_recv
|
iface_stats['packets_recv_delta'] = counters.packets_recv - prev.packets_recv
|
||||||
|
|
||||||
interfaces_data[iface_name] = iface_stats
|
interfaces_data[iface_name] = iface_stats
|
||||||
|
|
||||||
metrics['interfaces'] = interfaces_data
|
metrics['interfaces'] = interfaces_data
|
||||||
|
|
||||||
# Store current counters for next delta calculation
|
# Store current counters for next delta calculation
|
||||||
self._prev_io = io_counters
|
self._prev_io = io_counters
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not collect network I/O counters: {e}")
|
logger.warning(f"Could not collect network I/O counters: {e}")
|
||||||
|
|
||||||
# Collect connection statistics
|
# Collect connection statistics
|
||||||
if self.include_connections:
|
if self.include_connections:
|
||||||
try:
|
try:
|
||||||
connections = psutil.net_connections(kind='inet')
|
connections = psutil.net_connections(kind='inet')
|
||||||
conn_stats = {}
|
conn_stats: Dict[str, int] = {}
|
||||||
|
|
||||||
# Count connections by state
|
# Count connections by state
|
||||||
for conn in connections:
|
for conn in connections:
|
||||||
state = conn.status
|
state = conn.status
|
||||||
conn_stats[state] = conn_stats.get(state, 0) + 1
|
conn_stats[state] = conn_stats.get(state, 0) + 1
|
||||||
|
|
||||||
metrics['connections'] = conn_stats
|
metrics['connections'] = conn_stats
|
||||||
|
|
||||||
except (PermissionError, psutil.AccessDenied):
|
except (PermissionError, psutil.AccessDenied):
|
||||||
logger.debug("Permission denied for net_connections (requires root/admin)")
|
logger.debug("Permission denied for net_connections (requires root/admin)")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not collect connection statistics: {e}")
|
logger.warning(f"Could not collect connection statistics: {e}")
|
||||||
|
|
||||||
# Collect interface addresses
|
# Collect interface addresses
|
||||||
if self.include_addresses:
|
if self.include_addresses:
|
||||||
try:
|
try:
|
||||||
addresses = psutil.net_if_addrs()
|
addresses = psutil.net_if_addrs()
|
||||||
addr_data = {}
|
addr_data = {}
|
||||||
|
|
||||||
for iface_name, addrs in addresses.items():
|
for iface_name, addrs in addresses.items():
|
||||||
# Skip if we're only monitoring specific interfaces
|
# Skip if we're only monitoring specific interfaces
|
||||||
if self.interfaces and iface_name not in self.interfaces:
|
if self.interfaces and iface_name not in self.interfaces:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
iface_addrs = []
|
iface_addrs = []
|
||||||
for addr in addrs:
|
for addr in addrs:
|
||||||
addr_info = {
|
addr_info = {
|
||||||
@@ -205,33 +205,33 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
|||||||
if addr.broadcast:
|
if addr.broadcast:
|
||||||
addr_info['broadcast'] = addr.broadcast
|
addr_info['broadcast'] = addr.broadcast
|
||||||
iface_addrs.append(addr_info)
|
iface_addrs.append(addr_info)
|
||||||
|
|
||||||
addr_data[iface_name] = iface_addrs
|
addr_data[iface_name] = iface_addrs
|
||||||
|
|
||||||
metrics['addresses'] = addr_data
|
metrics['addresses'] = addr_data
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not collect interface addresses: {e}")
|
logger.warning(f"Could not collect interface addresses: {e}")
|
||||||
|
|
||||||
# Add interface stats (up/down status, speed, mtu)
|
# Add interface stats (up/down status, speed, mtu)
|
||||||
try:
|
try:
|
||||||
if_stats = psutil.net_if_stats()
|
if_stats = psutil.net_if_stats()
|
||||||
stats_data = {}
|
stats_data = {}
|
||||||
|
|
||||||
for iface_name, stats in if_stats.items():
|
for iface_name, stats in if_stats.items():
|
||||||
# Skip if we're only monitoring specific interfaces
|
# Skip if we're only monitoring specific interfaces
|
||||||
if self.interfaces and iface_name not in self.interfaces:
|
if self.interfaces and iface_name not in self.interfaces:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
stats_data[iface_name] = {
|
stats_data[iface_name] = {
|
||||||
'isup': stats.isup,
|
'isup': stats.isup,
|
||||||
'duplex': str(stats.duplex) if hasattr(stats, 'duplex') else None,
|
'duplex': str(stats.duplex) if hasattr(stats, 'duplex') else None,
|
||||||
'speed': stats.speed,
|
'speed': stats.speed,
|
||||||
'mtu': stats.mtu,
|
'mtu': stats.mtu,
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics['interface_stats'] = stats_data
|
metrics['interface_stats'] = stats_data
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not collect interface stats: {e}")
|
logger.warning(f"Could not collect interface stats: {e}")
|
||||||
|
|
||||||
@@ -239,9 +239,9 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
|||||||
shared = get_shared()
|
shared = get_shared()
|
||||||
if shared is not None:
|
if shared is not None:
|
||||||
metrics['connectivity'] = shared.as_metric()
|
metrics['connectivity'] = shared.as_metric()
|
||||||
|
|
||||||
return metrics
|
return metrics
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
"""Cleanup (nothing to do for this plugin)."""
|
"""Cleanup (nothing to do for this plugin)."""
|
||||||
logger.info("Network monitor cleanup")
|
logger.info("Network monitor cleanup")
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ kernel, architecture, and distribution details.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import platform
|
import platform
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
@@ -15,7 +14,7 @@ from hbd.client.plugin import InfoPlugin
|
|||||||
|
|
||||||
class OSInfoPlugin(InfoPlugin):
|
class OSInfoPlugin(InfoPlugin):
|
||||||
"""Collect operating system information.
|
"""Collect operating system information.
|
||||||
|
|
||||||
This plugin gathers static OS information that rarely changes:
|
This plugin gathers static OS information that rarely changes:
|
||||||
- OS name and version
|
- OS name and version
|
||||||
- Kernel version
|
- Kernel version
|
||||||
@@ -23,27 +22,27 @@ class OSInfoPlugin(InfoPlugin):
|
|||||||
- Distribution details (for Linux)
|
- Distribution details (for Linux)
|
||||||
- Python version (used by hbc)
|
- Python version (used by hbc)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "os_info"
|
name = "os_info"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
description = "Operating system and platform information"
|
description = "Operating system and platform information"
|
||||||
interval = 0 # InfoPlugin: collect once at startup
|
interval = 0 # InfoPlugin: collect once at startup
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
super().__init__(config)
|
super().__init__(config)
|
||||||
|
|
||||||
async def initialize(self) -> bool:
|
async def initialize(self) -> bool:
|
||||||
"""Initialize the OS info plugin.
|
"""Initialize the OS info plugin.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True (always succeeds - platform module is stdlib)
|
True (always succeeds - platform module is stdlib)
|
||||||
"""
|
"""
|
||||||
self.logger.info(f"Initializing {self.name} plugin")
|
self.logger.info(f"Initializing {self.name} plugin")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
async def _collect_info(self) -> Dict[str, Any]:
|
async def _collect_info(self) -> Dict[str, Any]:
|
||||||
"""Collect OS information.
|
"""Collect OS information.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with OS details
|
Dictionary with OS details
|
||||||
"""
|
"""
|
||||||
@@ -65,15 +64,15 @@ class OSInfoPlugin(InfoPlugin):
|
|||||||
if self.config.get("owner"):
|
if self.config.get("owner"):
|
||||||
self.logger.debug(f"Adding owner from config: {self.config['owner']}")
|
self.logger.debug(f"Adding owner from config: {self.config['owner']}")
|
||||||
data["owner"] = self.config["owner"]
|
data["owner"] = self.config["owner"]
|
||||||
|
|
||||||
# Add Linux-specific distribution info
|
# Add Linux-specific distribution info
|
||||||
if platform.system() == "Linux":
|
if platform.system() == "Linux":
|
||||||
data.update(self._get_linux_distro())
|
data.update(self._get_linux_distro())
|
||||||
|
|
||||||
# Add macOS-specific info
|
# Add macOS-specific info
|
||||||
elif platform.system() == "Darwin":
|
elif platform.system() == "Darwin":
|
||||||
data["macos_version"] = platform.mac_ver()[0]
|
data["macos_version"] = platform.mac_ver()[0]
|
||||||
|
|
||||||
# Add Windows-specific info
|
# Add Windows-specific info
|
||||||
elif platform.system() == "Windows":
|
elif platform.system() == "Windows":
|
||||||
win_ver = platform.win32_ver()
|
win_ver = platform.win32_ver()
|
||||||
@@ -81,22 +80,22 @@ class OSInfoPlugin(InfoPlugin):
|
|||||||
data["windows_version"] = win_ver[1]
|
data["windows_version"] = win_ver[1]
|
||||||
data["windows_sp"] = win_ver[2]
|
data["windows_sp"] = win_ver[2]
|
||||||
data["windows_type"] = win_ver[3]
|
data["windows_type"] = win_ver[3]
|
||||||
|
|
||||||
self.logger.debug(f"Collected OS info: {data['system']} {data['release']}")
|
self.logger.debug(f"Collected OS info: {data['system']} {data['release']}")
|
||||||
return data
|
return data
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.error(f"Error collecting OS info: {e}", exc_info=True)
|
self.logger.error(f"Error collecting OS info: {e}", exc_info=True)
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
def _get_linux_distro(self) -> Dict[str, str]:
|
def _get_linux_distro(self) -> Dict[str, str]:
|
||||||
"""Get Linux distribution information.
|
"""Get Linux distribution information.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with distribution details
|
Dictionary with distribution details
|
||||||
"""
|
"""
|
||||||
distro_info = {}
|
distro_info = {}
|
||||||
|
|
||||||
# Try reading /etc/os-release (standard on modern Linux)
|
# Try reading /etc/os-release (standard on modern Linux)
|
||||||
os_release = Path("/etc/os-release")
|
os_release = Path("/etc/os-release")
|
||||||
if os_release.exists():
|
if os_release.exists():
|
||||||
@@ -121,7 +120,7 @@ class OSInfoPlugin(InfoPlugin):
|
|||||||
distro_info["distro_pretty_name"] = value
|
distro_info["distro_pretty_name"] = value
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.warning(f"Could not read /etc/os-release: {e}")
|
self.logger.warning(f"Could not read /etc/os-release: {e}")
|
||||||
|
|
||||||
# Fallback: try lsb_release (older systems)
|
# Fallback: try lsb_release (older systems)
|
||||||
elif Path("/etc/lsb-release").exists():
|
elif Path("/etc/lsb-release").exists():
|
||||||
try:
|
try:
|
||||||
@@ -138,5 +137,5 @@ class OSInfoPlugin(InfoPlugin):
|
|||||||
distro_info["distro_name"] = value
|
distro_info["distro_name"] = value
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.warning(f"Could not read /etc/lsb-release: {e}")
|
self.logger.warning(f"Could not read /etc/lsb-release: {e}")
|
||||||
|
|
||||||
return distro_info
|
return distro_info
|
||||||
|
|||||||
@@ -67,6 +67,7 @@ class PingMonitorPlugin(MonitorPlugin):
|
|||||||
self.timeout = int(cfg.get("timeout", 5))
|
self.timeout = int(cfg.get("timeout", 5))
|
||||||
# hosts: dict of {hostname: {warning: x, critical: y}} or list of hostnames
|
# hosts: dict of {hostname: {warning: x, critical: y}} or list of hostnames
|
||||||
raw_hosts = cfg.get("hosts", {})
|
raw_hosts = cfg.get("hosts", {})
|
||||||
|
self.hosts: Dict[str, Any]
|
||||||
if isinstance(raw_hosts, list):
|
if isinstance(raw_hosts, list):
|
||||||
self.hosts = {h: {} for h in raw_hosts}
|
self.hosts = {h: {} for h in raw_hosts}
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
"""Common utilities shared between hbc and hbd."""
|
"""Common utilities shared between hbc and hbd."""
|
||||||
|
|
||||||
from hbd import __version__
|
from hbd import __version__ # noqa: F401
|
||||||
|
|||||||
+13
-14
@@ -8,17 +8,17 @@ Message Types:
|
|||||||
PLG: Plugin data message (client -> server)
|
PLG: Plugin data message (client -> server)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Dict, Any, Union
|
from typing import Dict, Any
|
||||||
import json
|
import json
|
||||||
import zlib
|
import zlib
|
||||||
|
|
||||||
|
|
||||||
def encode_value(v: Any) -> str:
|
def encode_value(v: Any) -> str:
|
||||||
"""Encode a value for protocol transmission.
|
"""Encode a value for protocol transmission.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
v: Value to encode (int, float, str, bool, list, dict, etc.)
|
v: Value to encode (int, float, str, bool, list, dict, etc.)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
String representation suitable for protocol
|
String representation suitable for protocol
|
||||||
"""
|
"""
|
||||||
@@ -35,23 +35,23 @@ def encode_value(v: Any) -> str:
|
|||||||
|
|
||||||
def decode_value(val: str) -> Any:
|
def decode_value(val: str) -> Any:
|
||||||
"""Decode a value from protocol format.
|
"""Decode a value from protocol format.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
val: String value from protocol
|
val: String value from protocol
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Decoded Python object
|
Decoded Python object
|
||||||
"""
|
"""
|
||||||
if not val:
|
if not val:
|
||||||
return val
|
return val
|
||||||
|
|
||||||
# Check for JSON-encoded complex types
|
# Check for JSON-encoded complex types
|
||||||
if val.startswith("@"):
|
if val.startswith("@"):
|
||||||
try:
|
try:
|
||||||
return json.loads(val[1:])
|
return json.loads(val[1:])
|
||||||
except Exception:
|
except Exception:
|
||||||
return val[1:] # Return as string without @
|
return val[1:] # Return as string without @
|
||||||
|
|
||||||
# Try numeric conversion (avoid eval to prevent SyntaxWarnings on version strings)
|
# Try numeric conversion (avoid eval to prevent SyntaxWarnings on version strings)
|
||||||
if val[0].isdigit() or (val[0] == '-' and len(val) > 1 and val[1].isdigit()):
|
if val[0].isdigit() or (val[0] == '-' and len(val) > 1 and val[1].isdigit()):
|
||||||
try:
|
try:
|
||||||
@@ -63,7 +63,7 @@ def decode_value(val: str) -> Any:
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
return val
|
return val
|
||||||
|
|
||||||
return val
|
return val
|
||||||
|
|
||||||
|
|
||||||
@@ -92,7 +92,7 @@ def stodict(msg: bytes):
|
|||||||
'!' and decodes accordingly. Returns a dict with key 'ID' set to the
|
'!' and decodes accordingly. Returns a dict with key 'ID' set to the
|
||||||
message ID and the parsed key/value pairs.
|
message ID and the parsed key/value pairs.
|
||||||
"""
|
"""
|
||||||
d = {}
|
d: Dict[str, Any] = {}
|
||||||
if len(msg) > 0 and chr(msg[0]) == "!":
|
if len(msg) > 0 and chr(msg[0]) == "!":
|
||||||
# message is: b'!ID:' + compressed_payload
|
# message is: b'!ID:' + compressed_payload
|
||||||
# original code used msg[1:4].decode() for ID (3 bytes including colon)
|
# original code used msg[1:4].decode() for ID (3 bytes including colon)
|
||||||
@@ -135,12 +135,12 @@ def oldmtodict(msg: bytes):
|
|||||||
|
|
||||||
def encode_plugin_data(plugin_name: str, data: Dict[str, Any]) -> bytes:
|
def encode_plugin_data(plugin_name: str, data: Dict[str, Any]) -> bytes:
|
||||||
"""Encode plugin data into a PLG message.
|
"""Encode plugin data into a PLG message.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
plugin_name: Name of the plugin (e.g., "os_info", "cpu_monitor")
|
plugin_name: Name of the plugin (e.g., "os_info", "cpu_monitor")
|
||||||
data: Plugin data dictionary
|
data: Plugin data dictionary
|
||||||
compress: Whether to compress the payload
|
compress: Whether to compress the payload
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Encoded message bytes
|
Encoded message bytes
|
||||||
"""
|
"""
|
||||||
@@ -151,12 +151,11 @@ def encode_plugin_data(plugin_name: str, data: Dict[str, Any]) -> bytes:
|
|||||||
|
|
||||||
def decode_plugin_data(msg: bytes) -> Dict[str, Any]:
|
def decode_plugin_data(msg: bytes) -> Dict[str, Any]:
|
||||||
"""Decode a PLG message into plugin data.
|
"""Decode a PLG message into plugin data.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
msg: Raw message bytes
|
msg: Raw message bytes
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with 'ID', 'plugin', and plugin data fields
|
Dictionary with 'ID', 'plugin', and plugin data fields
|
||||||
"""
|
"""
|
||||||
return stodict(msg)
|
return stodict(msg)
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
"""HeartBeat Daemon (hbd) - Server/daemon component."""
|
"""HeartBeat Daemon (hbd) - Server/daemon component."""
|
||||||
|
|
||||||
from hbd import __version__
|
from hbd import __version__ # noqa: F401
|
||||||
|
|||||||
+10
-4
@@ -120,7 +120,7 @@ def cmd_passwd(args):
|
|||||||
def cmd_notify(args):
|
def cmd_notify(args):
|
||||||
"""Send a test message via a single notification channel."""
|
"""Send a test message via a single notification channel."""
|
||||||
from .config import load_config
|
from .config import load_config
|
||||||
from .notify import Notification, _dispatch_to_channel, setup
|
from .notify import Notification, setup
|
||||||
|
|
||||||
config = load_config(args.configfile)
|
config = load_config(args.configfile)
|
||||||
setup(config)
|
setup(config)
|
||||||
@@ -193,7 +193,9 @@ def _read_pid(configfile) -> int | None:
|
|||||||
|
|
||||||
|
|
||||||
def cmd_stop(args):
|
def cmd_stop(args):
|
||||||
import os, signal as _signal, time
|
import os
|
||||||
|
import signal as _signal
|
||||||
|
import time
|
||||||
pid = _read_pid(args.configfile)
|
pid = _read_pid(args.configfile)
|
||||||
if pid is None:
|
if pid is None:
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
@@ -212,7 +214,8 @@ def cmd_stop(args):
|
|||||||
|
|
||||||
|
|
||||||
def cmd_reload(args):
|
def cmd_reload(args):
|
||||||
import os, signal as _signal
|
import os
|
||||||
|
import signal as _signal
|
||||||
pid = _read_pid(args.configfile)
|
pid = _read_pid(args.configfile)
|
||||||
if pid is None:
|
if pid is None:
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
@@ -222,7 +225,10 @@ def cmd_reload(args):
|
|||||||
|
|
||||||
|
|
||||||
def cmd_restart(args):
|
def cmd_restart(args):
|
||||||
import os, signal as _signal, time, subprocess
|
import os
|
||||||
|
import signal as _signal
|
||||||
|
import time
|
||||||
|
import subprocess
|
||||||
pid = _read_pid(args.configfile)
|
pid = _read_pid(args.configfile)
|
||||||
if pid is not None:
|
if pid is not None:
|
||||||
print(f"Stopping hbd (pid {pid})...")
|
print(f"Stopping hbd (pid {pid})...")
|
||||||
|
|||||||
+34
-37
@@ -7,7 +7,7 @@ import os
|
|||||||
try:
|
try:
|
||||||
import yaml
|
import yaml
|
||||||
except Exception:
|
except Exception:
|
||||||
yaml = None
|
yaml = None # type: ignore[assignment]
|
||||||
|
|
||||||
SERVER_DEFAULTS = {
|
SERVER_DEFAULTS = {
|
||||||
# Network settings
|
# Network settings
|
||||||
@@ -43,30 +43,30 @@ SERVER_DEFAULTS = {
|
|||||||
|
|
||||||
# Host management
|
# Host management
|
||||||
"hosts": {}, # Unified host definitions
|
"hosts": {}, # Unified host definitions
|
||||||
"dyndomains": ["example.org"], # Domains to update via nsupdate when a host with dyndns: true is updated
|
"dyndomains": ["example.org"], # Domains to update via nsupdate when a host with dyndns: true is updated
|
||||||
|
|
||||||
# DNS updates
|
# DNS updates
|
||||||
"nsupdate_bin": "/usr/bin/nsupdate", # Path to nsupdate binary
|
"nsupdate_bin": "/usr/bin/nsupdate", # Path to nsupdate binary
|
||||||
|
|
||||||
# WebSocket settings
|
# WebSocket settings
|
||||||
"ws_port": 50005,
|
"ws_port": 50005,
|
||||||
"wss_port": None,
|
"wss_port": None,
|
||||||
"cert_path": "/usr/local/etc/ssl/",
|
"cert_path": "/usr/local/etc/ssl/",
|
||||||
"wss_pem": "fullchain.pem",
|
"wss_pem": "fullchain.pem",
|
||||||
"wss_key": "privkey.pem",
|
"wss_key": "privkey.pem",
|
||||||
|
|
||||||
# Message journal configuration
|
# Message journal configuration
|
||||||
"journal_enabled": True,
|
"journal_enabled": True,
|
||||||
"journal_dir": "/var/log/heartbeat",
|
"journal_dir": "/var/log/heartbeat",
|
||||||
"journal_file": "messages.journal",
|
"journal_file": "messages.journal",
|
||||||
"journal_max_size": 100 * 1024 * 1024, # 100MB
|
"journal_max_size": 100 * 1024 * 1024, # 100MB
|
||||||
"journal_max_backups": 10,
|
"journal_max_backups": 10,
|
||||||
|
|
||||||
# Runtime flags
|
# Runtime flags
|
||||||
"foreground": False,
|
"foreground": False,
|
||||||
"verbose": False,
|
"verbose": False,
|
||||||
"debug": 0,
|
"debug": 0,
|
||||||
|
|
||||||
# Plugin/threshold configs (for clients reporting to this server)
|
# Plugin/threshold configs (for clients reporting to this server)
|
||||||
"plugins": {},
|
"plugins": {},
|
||||||
"thresholds": {},
|
"thresholds": {},
|
||||||
@@ -76,11 +76,11 @@ THRESHOLD_DEFAULTS = {
|
|||||||
'thresholds': {
|
'thresholds': {
|
||||||
'cpu_monitor': {
|
'cpu_monitor': {
|
||||||
'cpu_percent': {
|
'cpu_percent': {
|
||||||
'warning': 80.0,
|
'warning': 80.0,
|
||||||
'critical': 90.0
|
'critical': 90.0
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
'memory_monitor': {
|
'memory_monitor': {
|
||||||
'memory_percent': {
|
'memory_percent': {
|
||||||
'warning': 85.0,
|
'warning': 85.0,
|
||||||
'critical': 95.0
|
'critical': 95.0
|
||||||
@@ -90,7 +90,7 @@ THRESHOLD_DEFAULTS = {
|
|||||||
'critical': 75.0
|
'critical': 75.0
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
'disk_monitor': {
|
'disk_monitor': {
|
||||||
'partitions': {
|
'partitions': {
|
||||||
'/': {
|
'/': {
|
||||||
'percent': {
|
'percent': {
|
||||||
@@ -100,18 +100,18 @@ THRESHOLD_DEFAULTS = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
'rtt': {
|
'rtt': {
|
||||||
'warning': 200,
|
'warning': 200,
|
||||||
'critical': 250.0,
|
'critical': 250.0,
|
||||||
'count': 3 # Optional: number of consecutive breaches before alerting
|
'count': 3 # Optional: number of consecutive breaches before alerting
|
||||||
},
|
},
|
||||||
'nagios_runner': {
|
'nagios_runner': {
|
||||||
'status_code': {
|
'status_code': {
|
||||||
'display': '{check_name} {output}',
|
'display': '{check_name} {output}',
|
||||||
'operator': "nagios"
|
'operator': "nagios"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
'zfs_monitor': {
|
'zfs_monitor': {
|
||||||
'pools': {
|
'pools': {
|
||||||
'*': {
|
'*': {
|
||||||
'status': {
|
'status': {
|
||||||
@@ -137,10 +137,10 @@ def load_config(path=None):
|
|||||||
"""Load configuration from a YAML file and merge with server defaults.
|
"""Load configuration from a YAML file and merge with server defaults.
|
||||||
|
|
||||||
If YAML is not available or the file does not exist, defaults are returned.
|
If YAML is not available or the file does not exist, defaults are returned.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
path: Path to YAML config file (default: ~/.hb.yaml)
|
path: Path to YAML config file (default: ~/.hb.yaml)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with configuration
|
Dictionary with configuration
|
||||||
"""
|
"""
|
||||||
@@ -165,16 +165,16 @@ def load_config(path=None):
|
|||||||
|
|
||||||
class ReloadableConfig:
|
class ReloadableConfig:
|
||||||
"""Thread-safe/async-safe configuration wrapper that supports runtime reloading.
|
"""Thread-safe/async-safe configuration wrapper that supports runtime reloading.
|
||||||
|
|
||||||
This class wraps the configuration dictionary and provides:
|
This class wraps the configuration dictionary and provides:
|
||||||
- Thread-safe config reloading via SIGHUP
|
- Thread-safe config reloading via SIGHUP
|
||||||
- Backward-compatible dict-like access
|
- Backward-compatible dict-like access
|
||||||
- Async lock to prevent concurrent reloads
|
- Async lock to prevent concurrent reloads
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, initial_config, config_path=None):
|
def __init__(self, initial_config, config_path=None):
|
||||||
"""Initialize with initial configuration.
|
"""Initialize with initial configuration.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
initial_config: Initial configuration dictionary
|
initial_config: Initial configuration dictionary
|
||||||
config_path: Path to config file for reloading (optional)
|
config_path: Path to config file for reloading (optional)
|
||||||
@@ -183,65 +183,62 @@ class ReloadableConfig:
|
|||||||
self._config_path = config_path
|
self._config_path = config_path
|
||||||
self._lock = asyncio.Lock()
|
self._lock = asyncio.Lock()
|
||||||
self._logger = logging.getLogger(__name__)
|
self._logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
async def reload(self, config_path=None):
|
async def reload(self, config_path=None):
|
||||||
"""Reload configuration from file.
|
"""Reload configuration from file.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config_path: Path to config file (uses stored path if not provided)
|
config_path: Path to config file (uses stored path if not provided)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
New configuration dictionary
|
New configuration dictionary
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception if reload fails (keeps existing config)
|
Exception if reload fails (keeps existing config)
|
||||||
"""
|
"""
|
||||||
path = config_path or self._config_path
|
path = config_path or self._config_path
|
||||||
if not path:
|
if not path:
|
||||||
raise ValueError("No config path specified for reload")
|
raise ValueError("No config path specified for reload")
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
try:
|
try:
|
||||||
# Load new config
|
# Load new config
|
||||||
new_config = load_config(path)
|
new_config = load_config(path)
|
||||||
|
|
||||||
# Store old config for rollback if needed
|
|
||||||
old_config = self._config
|
|
||||||
|
|
||||||
# Update config
|
# Update config
|
||||||
self._config = new_config
|
self._config = new_config
|
||||||
self._logger.info(f"Configuration reloaded from {path}")
|
self._logger.info(f"Configuration reloaded from {path}")
|
||||||
|
|
||||||
return new_config
|
return new_config
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self._logger.error(f"Failed to reload config from {path}: {e}", exc_info=True)
|
self._logger.error(f"Failed to reload config from {path}: {e}", exc_info=True)
|
||||||
# Keep existing config on error
|
# Keep existing config on error
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def get(self, key, default=None):
|
def get(self, key, default=None):
|
||||||
"""Get a config value (dict-compatible)."""
|
"""Get a config value (dict-compatible)."""
|
||||||
return self._config.get(key, default)
|
return self._config.get(key, default)
|
||||||
|
|
||||||
def __getitem__(self, key):
|
def __getitem__(self, key):
|
||||||
"""Get a config value via subscript (dict-compatible)."""
|
"""Get a config value via subscript (dict-compatible)."""
|
||||||
return self._config[key]
|
return self._config[key]
|
||||||
|
|
||||||
def __contains__(self, key):
|
def __contains__(self, key):
|
||||||
"""Check if key exists (dict-compatible)."""
|
"""Check if key exists (dict-compatible)."""
|
||||||
return key in self._config
|
return key in self._config
|
||||||
|
|
||||||
def keys(self):
|
def keys(self):
|
||||||
"""Return config keys (dict-compatible)."""
|
"""Return config keys (dict-compatible)."""
|
||||||
return self._config.keys()
|
return self._config.keys()
|
||||||
|
|
||||||
def items(self):
|
def items(self):
|
||||||
"""Return config items (dict-compatible)."""
|
"""Return config items (dict-compatible)."""
|
||||||
return self._config.items()
|
return self._config.items()
|
||||||
|
|
||||||
def values(self):
|
def values(self):
|
||||||
"""Return config values (dict-compatible)."""
|
"""Return config values (dict-compatible)."""
|
||||||
return self._config.values()
|
return self._config.values()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def config(self):
|
def config(self):
|
||||||
"""Get the underlying config dict (for components that need full dict)."""
|
"""Get the underlying config dict (for components that need full dict)."""
|
||||||
@@ -319,7 +316,7 @@ def get_host_access(config, hostname) -> dict:
|
|||||||
"""
|
"""
|
||||||
host_cfg = get_host_config(config, hostname)
|
host_cfg = get_host_config(config, hostname)
|
||||||
|
|
||||||
owner = host_cfg.get("owner") # or get_default_owner(config)
|
owner = host_cfg.get("owner") # or get_default_owner(config)
|
||||||
|
|
||||||
managers = host_cfg.get("managers", [])
|
managers = host_cfg.get("managers", [])
|
||||||
if isinstance(managers, str):
|
if isinstance(managers, str):
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ def _make_yaml() -> YAML:
|
|||||||
y.preserve_quotes = True
|
y.preserve_quotes = True
|
||||||
return y
|
return y
|
||||||
|
|
||||||
|
|
||||||
# Top-level keys managed by the 'server' logical section
|
# Top-level keys managed by the 'server' logical section
|
||||||
_SERVER_KEYS = [
|
_SERVER_KEYS = [
|
||||||
"hbd_port", "hbd_host", "ws_port", "wss_port", "hb_port",
|
"hbd_port", "hbd_host", "ws_port", "wss_port", "hb_port",
|
||||||
|
|||||||
+6
-4
@@ -1,12 +1,14 @@
|
|||||||
msgs = [] # in-memory list of recent messages for new websocket clients; also logged to file via notify.eventlog
|
msgs: list = [] # in-memory list of recent messages for new websocket clients; also logged to file via notify.eventlog
|
||||||
|
|
||||||
|
|
||||||
class Data:
|
class Data:
|
||||||
|
|
||||||
def __init__(self, config):
|
def __init__(self, config):
|
||||||
self.config = config
|
self.config = config
|
||||||
self.data = {}
|
self.data = {}
|
||||||
|
|
||||||
def update(self, new_data):
|
def update(self, new_data):
|
||||||
self.data.update(new_data)
|
self.data.update(new_data)
|
||||||
|
|
||||||
def get(self, key, default=None):
|
def get(self, key, default=None):
|
||||||
return self.data.get(key, default)
|
return self.data.get(key, default)
|
||||||
|
|||||||
+5
-5
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
from subprocess import Popen, PIPE, STDOUT
|
from subprocess import Popen, PIPE, STDOUT
|
||||||
from typing import Optional
|
from typing import Any, Callable, Optional
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
@@ -79,8 +79,8 @@ async def dns_update_worker(
|
|||||||
hbdclass,
|
hbdclass,
|
||||||
cfg: dict,
|
cfg: dict,
|
||||||
async_queue=None,
|
async_queue=None,
|
||||||
log: Optional[callable] = None,
|
log: Optional[Callable[..., Any]] = None,
|
||||||
pushmsg: Optional[callable] = None,
|
pushmsg: Optional[Callable[..., Any]] = None,
|
||||||
loop: Optional[asyncio.AbstractEventLoop] = None,
|
loop: Optional[asyncio.AbstractEventLoop] = None,
|
||||||
):
|
):
|
||||||
"""Pure async DNS worker that processes updates from asyncio.Queue.
|
"""Pure async DNS worker that processes updates from asyncio.Queue.
|
||||||
@@ -164,7 +164,7 @@ async def dns_update_worker(
|
|||||||
def start_dns_worker(
|
def start_dns_worker(
|
||||||
hbdclass,
|
hbdclass,
|
||||||
cfg: dict,
|
cfg: dict,
|
||||||
log: Optional[callable] = None,
|
log: Optional[Callable[..., Any]] = None,
|
||||||
loop: Optional[asyncio.AbstractEventLoop] = None,
|
loop: Optional[asyncio.AbstractEventLoop] = None,
|
||||||
):
|
):
|
||||||
"""Start the async DNS worker and return the Task.
|
"""Start the async DNS worker and return the Task.
|
||||||
@@ -176,7 +176,7 @@ def start_dns_worker(
|
|||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
# Create asyncio.Queue and wrap in a bridge for thread-safe puts
|
# Create asyncio.Queue and wrap in a bridge for thread-safe puts
|
||||||
async_q = asyncio.Queue()
|
async_q: asyncio.Queue[Any] = asyncio.Queue()
|
||||||
|
|
||||||
class _QueueBridge:
|
class _QueueBridge:
|
||||||
"""Thread-safe wrapper around asyncio.Queue for synchronous callers."""
|
"""Thread-safe wrapper around asyncio.Queue for synchronous callers."""
|
||||||
|
|||||||
+38
-37
@@ -8,6 +8,7 @@ import time
|
|||||||
import json
|
import json
|
||||||
import copy
|
import copy
|
||||||
import queue
|
import queue
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
num = 0
|
num = 0
|
||||||
|
|
||||||
@@ -24,7 +25,7 @@ def log(host, m):
|
|||||||
class Connection:
|
class Connection:
|
||||||
# map of addrs to names
|
# map of addrs to names
|
||||||
|
|
||||||
htab = {}
|
htab: Dict[str, Any] = {}
|
||||||
UNKNOWN = "unknown"
|
UNKNOWN = "unknown"
|
||||||
UP = "up"
|
UP = "up"
|
||||||
DOWN = "down"
|
DOWN = "down"
|
||||||
@@ -43,7 +44,7 @@ class Connection:
|
|||||||
self.statetime = self.lastbeat
|
self.statetime = self.lastbeat
|
||||||
self.deltastatetime = "computed"
|
self.deltastatetime = "computed"
|
||||||
self.state = Connection.UNKNOWN
|
self.state = Connection.UNKNOWN
|
||||||
|
|
||||||
# Timer-based reachability monitoring
|
# Timer-based reachability monitoring
|
||||||
self.overdue_timer = None
|
self.overdue_timer = None
|
||||||
self.overdue_callback = None
|
self.overdue_callback = None
|
||||||
@@ -196,39 +197,39 @@ class Connection:
|
|||||||
if self.host.isDynDns():
|
if self.host.isDynDns():
|
||||||
Host.dnsQ.put((self.host.name, self.addr))
|
Host.dnsQ.put((self.host.name, self.addr))
|
||||||
return r
|
return r
|
||||||
|
|
||||||
def reset_overdue_timer(self, timeout_seconds, callback):
|
def reset_overdue_timer(self, timeout_seconds, callback):
|
||||||
"""Reset the overdue timer for this connection.
|
"""Reset the overdue timer for this connection.
|
||||||
|
|
||||||
Cancels any existing timer and sets a new one that will mark
|
Cancels any existing timer and sets a new one that will mark
|
||||||
the connection as overdue if no heartbeat arrives before timeout.
|
the connection as overdue if no heartbeat arrives before timeout.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
timeout_seconds: Seconds before marking as overdue
|
timeout_seconds: Seconds before marking as overdue
|
||||||
callback: Async function to call when timer expires
|
callback: Async function to call when timer expires
|
||||||
"""
|
"""
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
# Cancel existing timer if any
|
# Cancel existing timer if any
|
||||||
if self.overdue_timer and not self.overdue_timer.cancelled():
|
if self.overdue_timer and not self.overdue_timer.cancelled():
|
||||||
self.overdue_timer.cancel()
|
self.overdue_timer.cancel()
|
||||||
|
|
||||||
# Store parameters for later reference
|
# Store parameters for later reference
|
||||||
self.timeout_duration = timeout_seconds
|
self.timeout_duration = timeout_seconds
|
||||||
self.overdue_callback = callback
|
self.overdue_callback = callback
|
||||||
|
|
||||||
# Create new timer
|
# Create new timer
|
||||||
async def timer_expired():
|
async def timer_expired():
|
||||||
await callback(self)
|
await callback(self)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
self.overdue_timer = loop.call_later(timeout_seconds,
|
self.overdue_timer = loop.call_later(timeout_seconds,
|
||||||
lambda: asyncio.create_task(timer_expired()))
|
lambda: asyncio.create_task(timer_expired()))
|
||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
# No event loop running yet
|
# No event loop running yet
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def cancel_overdue_timer(self):
|
def cancel_overdue_timer(self):
|
||||||
"""Cancel the overdue timer if it exists and clear all timer references."""
|
"""Cancel the overdue timer if it exists and clear all timer references."""
|
||||||
if self.overdue_timer:
|
if self.overdue_timer:
|
||||||
@@ -241,45 +242,45 @@ class Connection:
|
|||||||
self.overdue_timer = None
|
self.overdue_timer = None
|
||||||
self.overdue_callback = None
|
self.overdue_callback = None
|
||||||
self.timeout_duration = None
|
self.timeout_duration = None
|
||||||
|
|
||||||
def get_avg_rtt(self):
|
def get_avg_rtt(self):
|
||||||
"""Get average RTT from recent samples."""
|
"""Get average RTT from recent samples."""
|
||||||
valid_rtts = [r for r in self.rtts if r > 0]
|
valid_rtts = [r for r in self.rtts if r > 0]
|
||||||
if valid_rtts:
|
if valid_rtts:
|
||||||
return sum(valid_rtts) / len(valid_rtts)
|
return sum(valid_rtts) / len(valid_rtts)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
def get_current_rtt(self):
|
def get_current_rtt(self):
|
||||||
"""Get most recent RTT value."""
|
"""Get most recent RTT value."""
|
||||||
return self.rtts[-1] if self.rtts else 0
|
return self.rtts[-1] if self.rtts else 0
|
||||||
|
|
||||||
def check_rtt_threshold(self, warning_threshold=None, critical_threshold=None):
|
def check_rtt_threshold(self, warning_threshold=None, critical_threshold=None):
|
||||||
"""Check if RTT exceeds thresholds.
|
"""Check if RTT exceeds thresholds.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
warning_threshold: RTT in ms for warning level
|
warning_threshold: RTT in ms for warning level
|
||||||
critical_threshold: RTT in ms for critical level
|
critical_threshold: RTT in ms for critical level
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (level, rtt_value) where level is None, 'WARNING', or 'CRITICAL'
|
Tuple of (level, rtt_value) where level is None, 'WARNING', or 'CRITICAL'
|
||||||
"""
|
"""
|
||||||
rtt = self.get_current_rtt()
|
rtt = self.get_current_rtt()
|
||||||
if rtt <= 0:
|
if rtt <= 0:
|
||||||
return (None, rtt)
|
return (None, rtt)
|
||||||
|
|
||||||
if critical_threshold and rtt > critical_threshold:
|
if critical_threshold and rtt > critical_threshold:
|
||||||
return ('CRITICAL', rtt)
|
return ('CRITICAL', rtt)
|
||||||
elif warning_threshold and rtt > warning_threshold:
|
elif warning_threshold and rtt > warning_threshold:
|
||||||
return ('WARNING', rtt)
|
return ('WARNING', rtt)
|
||||||
|
|
||||||
return (None, rtt)
|
return (None, rtt)
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
class Host:
|
class Host:
|
||||||
# Table of Hosts
|
# Table of Hosts
|
||||||
hosts = {}
|
hosts: Dict[str, "Host"] = {}
|
||||||
dnsQ = queue.Queue()
|
dnsQ: "queue.Queue[Any]" = queue.Queue()
|
||||||
|
|
||||||
def __init__(self, name):
|
def __init__(self, name):
|
||||||
global num
|
global num
|
||||||
@@ -328,7 +329,7 @@ class Host:
|
|||||||
d["dyn"] = str(self.dyn)
|
d["dyn"] = str(self.dyn)
|
||||||
d["num"] = self.num
|
d["num"] = self.num
|
||||||
d["status"] = self.status()
|
d["status"] = self.status()
|
||||||
|
|
||||||
# Add alert counts (split by acknowledged status)
|
# Add alert counts (split by acknowledged status)
|
||||||
warning_unacked = 0
|
warning_unacked = 0
|
||||||
warning_acked = 0
|
warning_acked = 0
|
||||||
@@ -347,12 +348,12 @@ class Host:
|
|||||||
critical_acked += 1
|
critical_acked += 1
|
||||||
else:
|
else:
|
||||||
critical_unacked += 1
|
critical_unacked += 1
|
||||||
|
|
||||||
d["alert_warning_unacked"] = warning_unacked
|
d["alert_warning_unacked"] = warning_unacked
|
||||||
d["alert_warning_acked"] = warning_acked
|
d["alert_warning_acked"] = warning_acked
|
||||||
d["alert_critical_unacked"] = critical_unacked
|
d["alert_critical_unacked"] = critical_unacked
|
||||||
d["alert_critical_acked"] = critical_acked
|
d["alert_critical_acked"] = critical_acked
|
||||||
|
|
||||||
for c in ["IPv4", "IPv6"]:
|
for c in ["IPv4", "IPv6"]:
|
||||||
if c in self.connections:
|
if c in self.connections:
|
||||||
cs = self.connections[c].statedict()
|
cs = self.connections[c].statedict()
|
||||||
@@ -409,7 +410,7 @@ class Host:
|
|||||||
ddict[d] = cl
|
ddict[d] = cl
|
||||||
else:
|
else:
|
||||||
ddict[d] = self.__dict__[d]
|
ddict[d] = self.__dict__[d]
|
||||||
|
|
||||||
# Add alert counts (computed from alert_states)
|
# Add alert counts (computed from alert_states)
|
||||||
warning_unacked = 0
|
warning_unacked = 0
|
||||||
warning_acked = 0
|
warning_acked = 0
|
||||||
@@ -428,7 +429,7 @@ class Host:
|
|||||||
critical_acked += 1
|
critical_acked += 1
|
||||||
else:
|
else:
|
||||||
critical_unacked += 1
|
critical_unacked += 1
|
||||||
|
|
||||||
ddict["alert_warning_unacked"] = warning_unacked
|
ddict["alert_warning_unacked"] = warning_unacked
|
||||||
ddict["alert_warning_acked"] = warning_acked
|
ddict["alert_warning_acked"] = warning_acked
|
||||||
ddict["alert_critical_unacked"] = critical_unacked
|
ddict["alert_critical_unacked"] = critical_unacked
|
||||||
@@ -554,7 +555,7 @@ class Host:
|
|||||||
|
|
||||||
def add_plugin_data(self, plugin_name, data, timestamp=None):
|
def add_plugin_data(self, plugin_name, data, timestamp=None):
|
||||||
"""Store plugin data with timestamp.
|
"""Store plugin data with timestamp.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
plugin_name: Name of the plugin (e.g., "cpu_monitor")
|
plugin_name: Name of the plugin (e.g., "cpu_monitor")
|
||||||
data: Dict of plugin data
|
data: Dict of plugin data
|
||||||
@@ -562,24 +563,24 @@ class Host:
|
|||||||
"""
|
"""
|
||||||
if timestamp is None:
|
if timestamp is None:
|
||||||
timestamp = time.time()
|
timestamp = time.time()
|
||||||
|
|
||||||
if plugin_name not in self.plugin_data:
|
if plugin_name not in self.plugin_data:
|
||||||
self.plugin_data[plugin_name] = []
|
self.plugin_data[plugin_name] = []
|
||||||
|
|
||||||
# Add new data
|
# Add new data
|
||||||
self.plugin_data[plugin_name].append((timestamp, data))
|
self.plugin_data[plugin_name].append((timestamp, data))
|
||||||
|
|
||||||
# Enforce retention limit (keep last N samples)
|
# Enforce retention limit (keep last N samples)
|
||||||
if len(self.plugin_data[plugin_name]) > self.plugin_retention:
|
if len(self.plugin_data[plugin_name]) > self.plugin_retention:
|
||||||
self.plugin_data[plugin_name] = self.plugin_data[plugin_name][-self.plugin_retention:]
|
self.plugin_data[plugin_name] = self.plugin_data[plugin_name][-self.plugin_retention:]
|
||||||
|
|
||||||
def get_plugin_data(self, plugin_name, limit=None):
|
def get_plugin_data(self, plugin_name, limit=None):
|
||||||
"""Retrieve plugin data for a specific plugin.
|
"""Retrieve plugin data for a specific plugin.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
plugin_name: Name of the plugin
|
plugin_name: Name of the plugin
|
||||||
limit: Optional limit on number of recent samples to return
|
limit: Optional limit on number of recent samples to return
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of (timestamp, data) tuples, most recent last
|
List of (timestamp, data) tuples, most recent last
|
||||||
"""
|
"""
|
||||||
@@ -587,19 +588,19 @@ class Host:
|
|||||||
if limit and len(data) > limit:
|
if limit and len(data) > limit:
|
||||||
return data[-limit:]
|
return data[-limit:]
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def get_latest_plugin_data(self, plugin_name):
|
def get_latest_plugin_data(self, plugin_name):
|
||||||
"""Get the most recent plugin data for a plugin.
|
"""Get the most recent plugin data for a plugin.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
plugin_name: Name of the plugin
|
plugin_name: Name of the plugin
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
(timestamp, data) tuple or None if no data
|
(timestamp, data) tuple or None if no data
|
||||||
"""
|
"""
|
||||||
data = self.plugin_data.get(plugin_name, [])
|
data = self.plugin_data.get(plugin_name, [])
|
||||||
return data[-1] if data else None
|
return data[-1] if data else None
|
||||||
|
|
||||||
def get_all_plugin_data(self):
|
def get_all_plugin_data(self):
|
||||||
"""Get all plugin data for this host.
|
"""Get all plugin data for this host.
|
||||||
|
|
||||||
|
|||||||
+22
-22
@@ -109,6 +109,7 @@ def _insert_threshold_metric(thresholds: dict, metric_path: str, values: dict) -
|
|||||||
else:
|
else:
|
||||||
thresholds[plugin].setdefault(intermediate, {})[leaf] = cfg
|
thresholds[plugin].setdefault(intermediate, {})[leaf] = cfg
|
||||||
|
|
||||||
|
|
||||||
def _render_template(html_str: str, **context) -> str:
|
def _render_template(html_str: str, **context) -> str:
|
||||||
tmpl = jinja2.Template(html_str)
|
tmpl = jinja2.Template(html_str)
|
||||||
return tmpl.render(**context)
|
return tmpl.render(**context)
|
||||||
@@ -563,7 +564,7 @@ async def start(
|
|||||||
if err:
|
if err:
|
||||||
return err
|
return err
|
||||||
hostname = request.match_info.get("hostname")
|
hostname = request.match_info.get("hostname")
|
||||||
|
|
||||||
if hostname not in hbdclass.Host.hosts:
|
if hostname not in hbdclass.Host.hosts:
|
||||||
return web.json_response({"error": f"Host '{hostname}' not found"}, status=404)
|
return web.json_response({"error": f"Host '{hostname}' not found"}, status=404)
|
||||||
|
|
||||||
@@ -582,7 +583,7 @@ async def start(
|
|||||||
"data": data,
|
"data": data,
|
||||||
"sample_count": len(samples),
|
"sample_count": len(samples),
|
||||||
}
|
}
|
||||||
|
|
||||||
return web.json_response({
|
return web.json_response({
|
||||||
"hostname": hostname,
|
"hostname": hostname,
|
||||||
"plugins": plugins_summary,
|
"plugins": plugins_summary,
|
||||||
@@ -602,23 +603,23 @@ async def start(
|
|||||||
host = hbdclass.Host.hosts[hostname]
|
host = hbdclass.Host.hosts[hostname]
|
||||||
if not _can_view_host(user, host):
|
if not _can_view_host(user, host):
|
||||||
return web.json_response({"error": "Forbidden"}, status=403)
|
return web.json_response({"error": "Forbidden"}, status=403)
|
||||||
|
|
||||||
# Get limit from query parameter
|
# Get limit from query parameter
|
||||||
limit = request.rel_url.query.get("limit", "10")
|
limit = request.rel_url.query.get("limit", "10")
|
||||||
try:
|
try:
|
||||||
limit = int(limit)
|
limit = int(limit)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
limit = 10
|
limit = 10
|
||||||
|
|
||||||
# Get plugin data
|
# Get plugin data
|
||||||
samples = host.get_plugin_data(plugin_name, limit=limit)
|
samples = host.get_plugin_data(plugin_name, limit=limit)
|
||||||
|
|
||||||
if not samples:
|
if not samples:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"error": f"No data for plugin '{plugin_name}' on host '{hostname}'"},
|
{"error": f"No data for plugin '{plugin_name}' on host '{hostname}'"},
|
||||||
status=404
|
status=404
|
||||||
)
|
)
|
||||||
|
|
||||||
# Format samples
|
# Format samples
|
||||||
formatted_samples = [
|
formatted_samples = [
|
||||||
{
|
{
|
||||||
@@ -627,7 +628,7 @@ async def start(
|
|||||||
}
|
}
|
||||||
for ts, data in samples
|
for ts, data in samples
|
||||||
]
|
]
|
||||||
|
|
||||||
return web.json_response({
|
return web.json_response({
|
||||||
"hostname": hostname,
|
"hostname": hostname,
|
||||||
"plugin": plugin_name,
|
"plugin": plugin_name,
|
||||||
@@ -648,17 +649,17 @@ async def start(
|
|||||||
host = hbdclass.Host.hosts[hostname]
|
host = hbdclass.Host.hosts[hostname]
|
||||||
if not _can_view_host(user, host):
|
if not _can_view_host(user, host):
|
||||||
return web.json_response({"error": "Forbidden"}, status=403)
|
return web.json_response({"error": "Forbidden"}, status=403)
|
||||||
|
|
||||||
# Get alert states
|
# Get alert states
|
||||||
alerts = []
|
alerts = []
|
||||||
for metric_path, alert_state in host.alert_states.items():
|
for metric_path, alert_state in host.alert_states.items():
|
||||||
alerts.append(alert_state.to_dict())
|
alerts.append(alert_state.to_dict())
|
||||||
|
|
||||||
# Get summary if threshold_checker available
|
# Get summary if threshold_checker available
|
||||||
summary = {"ok": 0, "warning": 0, "critical": 0, "unknown": 0}
|
summary = {"ok": 0, "warning": 0, "critical": 0, "unknown": 0}
|
||||||
if threshold_checker:
|
if threshold_checker:
|
||||||
summary = threshold_checker.get_alert_summary(host.alert_states)
|
summary = threshold_checker.get_alert_summary(host.alert_states)
|
||||||
|
|
||||||
return web.json_response({
|
return web.json_response({
|
||||||
"hostname": hostname,
|
"hostname": hostname,
|
||||||
"alerts": alerts,
|
"alerts": alerts,
|
||||||
@@ -684,31 +685,31 @@ async def start(
|
|||||||
state for state in host.alert_states.values()
|
state for state in host.alert_states.values()
|
||||||
if state.level != AlertLevel.OK
|
if state.level != AlertLevel.OK
|
||||||
]
|
]
|
||||||
|
|
||||||
for alert in active_alerts:
|
for alert in active_alerts:
|
||||||
alert_dict = alert.to_dict()
|
alert_dict = alert.to_dict()
|
||||||
alert_dict["hostname"] = hostname
|
alert_dict["hostname"] = hostname
|
||||||
all_alerts.append(alert_dict)
|
all_alerts.append(alert_dict)
|
||||||
|
|
||||||
# Sort by level (critical first) then by hostname
|
# Sort by level (critical first) then by hostname
|
||||||
level_order = {"CRITICAL": 0, "WARNING": 1, "UNKNOWN": 2, "OK": 3}
|
level_order = {"CRITICAL": 0, "WARNING": 1, "UNKNOWN": 2, "OK": 3}
|
||||||
all_alerts.sort(
|
all_alerts.sort(
|
||||||
key=lambda a: (level_order.get(a["level"], 99), a["hostname"], a["metric_path"])
|
key=lambda a: (level_order.get(a["level"], 99), a["hostname"], a["metric_path"])
|
||||||
)
|
)
|
||||||
|
|
||||||
# Get summary counts
|
# Get summary counts
|
||||||
summary = {"critical": 0, "warning": 0, "unknown": 0, "total": len(all_alerts)}
|
summary = {"critical": 0, "warning": 0, "unknown": 0, "total": len(all_alerts)}
|
||||||
for alert in all_alerts:
|
for alert in all_alerts:
|
||||||
level = alert["level"].lower()
|
level = alert["level"].lower()
|
||||||
if level in summary:
|
if level in summary:
|
||||||
summary[level] += 1
|
summary[level] += 1
|
||||||
|
|
||||||
return web.json_response({
|
return web.json_response({
|
||||||
"alerts": all_alerts,
|
"alerts": all_alerts,
|
||||||
"summary": summary,
|
"summary": summary,
|
||||||
"host_count": len(hbdclass.Host.hosts),
|
"host_count": len(hbdclass.Host.hosts),
|
||||||
})
|
})
|
||||||
|
|
||||||
async def api_acknowledge_alert(request):
|
async def api_acknowledge_alert(request):
|
||||||
"""Acknowledge an alert to stop reminder notifications."""
|
"""Acknowledge an alert to stop reminder notifications."""
|
||||||
user, err = _require_auth(request)
|
user, err = _require_auth(request)
|
||||||
@@ -721,22 +722,22 @@ async def start(
|
|||||||
{"error": "Invalid JSON in request body"},
|
{"error": "Invalid JSON in request body"},
|
||||||
status=400
|
status=400
|
||||||
)
|
)
|
||||||
|
|
||||||
hostname = data.get("hostname")
|
hostname = data.get("hostname")
|
||||||
metric_path = data.get("metric_path")
|
metric_path = data.get("metric_path")
|
||||||
|
|
||||||
if not hostname or not metric_path:
|
if not hostname or not metric_path:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"error": "Missing required fields: hostname and metric_path"},
|
{"error": "Missing required fields: hostname and metric_path"},
|
||||||
status=400
|
status=400
|
||||||
)
|
)
|
||||||
|
|
||||||
if hostname not in hbdclass.Host.hosts:
|
if hostname not in hbdclass.Host.hosts:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
{"error": f"Host '{hostname}' not found"},
|
{"error": f"Host '{hostname}' not found"},
|
||||||
status=404
|
status=404
|
||||||
)
|
)
|
||||||
|
|
||||||
host = hbdclass.Host.hosts[hostname]
|
host = hbdclass.Host.hosts[hostname]
|
||||||
if not _can_view_host(user, host):
|
if not _can_view_host(user, host):
|
||||||
return web.json_response({"error": "Forbidden"}, status=403)
|
return web.json_response({"error": "Forbidden"}, status=403)
|
||||||
@@ -746,10 +747,10 @@ async def start(
|
|||||||
{"error": f"Alert '{metric_path}' not found for host '{hostname}'"},
|
{"error": f"Alert '{metric_path}' not found for host '{hostname}'"},
|
||||||
status=404
|
status=404
|
||||||
)
|
)
|
||||||
|
|
||||||
alert_state = host.alert_states[metric_path]
|
alert_state = host.alert_states[metric_path]
|
||||||
alert_state.acknowledge()
|
alert_state.acknowledge()
|
||||||
|
|
||||||
return web.json_response({
|
return web.json_response({
|
||||||
"success": True,
|
"success": True,
|
||||||
"hostname": hostname,
|
"hostname": hostname,
|
||||||
@@ -1132,7 +1133,6 @@ async def start(
|
|||||||
elif is_mon:
|
elif is_mon:
|
||||||
monitored.append(hostname)
|
monitored.append(hostname)
|
||||||
|
|
||||||
|
|
||||||
# Resolve notification channel configs for display
|
# Resolve notification channel configs for display
|
||||||
notif_channels = []
|
notif_channels = []
|
||||||
if current_user:
|
if current_user:
|
||||||
|
|||||||
+54
-54
@@ -11,7 +11,7 @@ import os
|
|||||||
import asyncio
|
import asyncio
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, Any, Optional, Callable, Iterable, List, Tuple, Union
|
from typing import Dict, Any, Optional, Callable, Iterable, List, TextIO, Tuple, Union
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -19,14 +19,14 @@ logger = logging.getLogger(__name__)
|
|||||||
class MessageJournal:
|
class MessageJournal:
|
||||||
"""
|
"""
|
||||||
Journal logger for heartbeat messages with size-based rotation.
|
Journal logger for heartbeat messages with size-based rotation.
|
||||||
|
|
||||||
Features:
|
Features:
|
||||||
- Logs all received messages in JSON format
|
- Logs all received messages in JSON format
|
||||||
- Automatic rotation when file size exceeds threshold
|
- Automatic rotation when file size exceeds threshold
|
||||||
- Keeps configurable number of rotated logs
|
- Keeps configurable number of rotated logs
|
||||||
- Thread-safe and async-safe operation
|
- Thread-safe and async-safe operation
|
||||||
- Configurable log directory and file naming
|
- Configurable log directory and file naming
|
||||||
|
|
||||||
Configuration:
|
Configuration:
|
||||||
journal_dir: Directory for journal files (default: /var/log/heartbeat/)
|
journal_dir: Directory for journal files (default: /var/log/heartbeat/)
|
||||||
journal_file: Base filename (default: messages.journal)
|
journal_file: Base filename (default: messages.journal)
|
||||||
@@ -34,69 +34,69 @@ class MessageJournal:
|
|||||||
max_backups: Number of backup files to keep (default: 10)
|
max_backups: Number of backup files to keep (default: 10)
|
||||||
enabled: Enable/disable journaling (default: True)
|
enabled: Enable/disable journaling (default: True)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
"""
|
"""
|
||||||
Initialize the message journal.
|
Initialize the message journal.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Configuration dictionary with journal settings
|
config: Configuration dictionary with journal settings
|
||||||
"""
|
"""
|
||||||
self.config = config or {}
|
self.config = config or {}
|
||||||
|
|
||||||
# Configuration options
|
# Configuration options
|
||||||
self.journal_dir = Path(self.config.get('journal_dir', '/var/log/heartbeat'))
|
self.journal_dir = Path(self.config.get('journal_dir', '/var/log/heartbeat'))
|
||||||
self.journal_file = self.config.get('journal_file', 'messages.journal')
|
self.journal_file = self.config.get('journal_file', 'messages.journal')
|
||||||
self.max_size = self.config.get('journal_max_size', 100 * 1024 * 1024) # 100MB default
|
self.max_size = self.config.get('journal_max_size', 100 * 1024 * 1024) # 100MB default
|
||||||
self.max_backups = self.config.get('journal_max_backups', 10)
|
self.max_backups = self.config.get('journal_max_backups', 10)
|
||||||
self.enabled = self.config.get('journal_enabled', True)
|
self.enabled = self.config.get('journal_enabled', True)
|
||||||
|
|
||||||
# Runtime state
|
# Runtime state
|
||||||
self._file_handle = None
|
self._file_handle: Optional[TextIO] = None
|
||||||
self._current_size = 0
|
self._current_size = 0
|
||||||
self._lock = asyncio.Lock()
|
self._lock = asyncio.Lock()
|
||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
# Full path to current journal file
|
# Full path to current journal file
|
||||||
self.journal_path = self.journal_dir / self.journal_file
|
self.journal_path = self.journal_dir / self.journal_file
|
||||||
|
|
||||||
async def initialize(self) -> bool:
|
async def initialize(self) -> bool:
|
||||||
"""
|
"""
|
||||||
Initialize the journal.
|
Initialize the journal.
|
||||||
|
|
||||||
Creates journal directory if needed and opens the journal file.
|
Creates journal directory if needed and opens the journal file.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if initialization successful, False otherwise
|
True if initialization successful, False otherwise
|
||||||
"""
|
"""
|
||||||
if not self.enabled:
|
if not self.enabled:
|
||||||
logger.info("Message journal disabled in configuration")
|
logger.info("Message journal disabled in configuration")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Create journal directory if it doesn't exist
|
# Create journal directory if it doesn't exist
|
||||||
self.journal_dir.mkdir(parents=True, exist_ok=True)
|
self.journal_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# Open journal file in append mode
|
# Open journal file in append mode
|
||||||
self._file_handle = open(self.journal_path, 'a', encoding='utf-8')
|
self._file_handle = open(self.journal_path, 'a', encoding='utf-8')
|
||||||
|
|
||||||
# Get current file size
|
# Get current file size
|
||||||
try:
|
try:
|
||||||
self._current_size = os.path.getsize(self.journal_path)
|
self._current_size = os.path.getsize(self.journal_path)
|
||||||
except OSError:
|
except OSError:
|
||||||
self._current_size = 0
|
self._current_size = 0
|
||||||
|
|
||||||
self._initialized = True
|
self._initialized = True
|
||||||
logger.info(f"Message journal initialized: {self.journal_path} "
|
logger.info(f"Message journal initialized: {self.journal_path} "
|
||||||
f"(current size: {self._current_size:,} bytes, "
|
f"(current size: {self._current_size:,} bytes, "
|
||||||
f"max: {self.max_size:,} bytes)")
|
f"max: {self.max_size:,} bytes)")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Failed to initialize message journal: {e}")
|
logger.error(f"Failed to initialize message journal: {e}")
|
||||||
self.enabled = False
|
self.enabled = False
|
||||||
return False
|
return False
|
||||||
|
|
||||||
async def log_message(
|
async def log_message(
|
||||||
self,
|
self,
|
||||||
msg: Dict[str, Any],
|
msg: Dict[str, Any],
|
||||||
@@ -105,7 +105,7 @@ class MessageJournal:
|
|||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Log a received message to the journal.
|
Log a received message to the journal.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
msg: Parsed message dictionary
|
msg: Parsed message dictionary
|
||||||
addr: Source address (ip, port) tuple
|
addr: Source address (ip, port) tuple
|
||||||
@@ -113,19 +113,19 @@ class MessageJournal:
|
|||||||
"""
|
"""
|
||||||
if not self.enabled or not self._initialized:
|
if not self.enabled or not self._initialized:
|
||||||
return
|
return
|
||||||
|
|
||||||
# Skip HTB (heartbeat) messages - too verbose
|
# Skip HTB (heartbeat) messages - too verbose
|
||||||
msg_id = msg.get('ID', '')
|
msg_id = msg.get('ID', '')
|
||||||
if msg_id == 'HTB':
|
if msg_id == 'HTB':
|
||||||
return
|
return
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
try:
|
try:
|
||||||
# Prepare journal entry
|
# Prepare journal entry
|
||||||
if timestamp is None:
|
if timestamp is None:
|
||||||
import time
|
import time
|
||||||
timestamp = time.time()
|
timestamp = time.time()
|
||||||
|
|
||||||
entry = {
|
entry = {
|
||||||
'timestamp': timestamp,
|
'timestamp': timestamp,
|
||||||
'datetime': datetime.fromtimestamp(timestamp).isoformat(),
|
'datetime': datetime.fromtimestamp(timestamp).isoformat(),
|
||||||
@@ -133,21 +133,21 @@ class MessageJournal:
|
|||||||
'source_port': addr[1] if isinstance(addr, (tuple, list)) and len(addr) > 1 else None,
|
'source_port': addr[1] if isinstance(addr, (tuple, list)) and len(addr) > 1 else None,
|
||||||
'message': msg
|
'message': msg
|
||||||
}
|
}
|
||||||
|
|
||||||
# Serialize to JSON (one line per entry)
|
# Serialize to JSON (one line per entry)
|
||||||
json_line = json.dumps(entry, separators=(',', ':')) + '\n'
|
json_line = json.dumps(entry, separators=(',', ':')) + '\n'
|
||||||
json_bytes = json_line.encode('utf-8')
|
json_bytes = json_line.encode('utf-8')
|
||||||
|
|
||||||
# Check if rotation is needed
|
# Check if rotation is needed
|
||||||
if self._current_size + len(json_bytes) > self.max_size:
|
if self._current_size + len(json_bytes) > self.max_size:
|
||||||
await self._rotate()
|
await self._rotate()
|
||||||
|
|
||||||
# Write to journal
|
# Write to journal
|
||||||
if self._file_handle:
|
if self._file_handle:
|
||||||
self._file_handle.write(json_line)
|
self._file_handle.write(json_line)
|
||||||
self._file_handle.flush() # Ensure data is written
|
self._file_handle.flush() # Ensure data is written
|
||||||
self._current_size += len(json_bytes)
|
self._current_size += len(json_bytes)
|
||||||
|
|
||||||
logger.debug(f"Logged message from {addr[0]}: {msg.get('ID', 'UNKNOWN')}")
|
logger.debug(f"Logged message from {addr[0]}: {msg.get('ID', 'UNKNOWN')}")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -181,7 +181,7 @@ class MessageJournal:
|
|||||||
async def _rotate(self):
|
async def _rotate(self):
|
||||||
"""
|
"""
|
||||||
Rotate the journal file.
|
Rotate the journal file.
|
||||||
|
|
||||||
Renames current file with timestamp, opens new file, and removes
|
Renames current file with timestamp, opens new file, and removes
|
||||||
old backups exceeding max_backups limit.
|
old backups exceeding max_backups limit.
|
||||||
"""
|
"""
|
||||||
@@ -190,25 +190,25 @@ class MessageJournal:
|
|||||||
if self._file_handle:
|
if self._file_handle:
|
||||||
self._file_handle.close()
|
self._file_handle.close()
|
||||||
self._file_handle = None
|
self._file_handle = None
|
||||||
|
|
||||||
# Generate backup filename with timestamp
|
# Generate backup filename with timestamp
|
||||||
timestamp_str = datetime.now().strftime('%Y%m%d-%H%M%S')
|
timestamp_str = datetime.now().strftime('%Y%m%d-%H%M%S')
|
||||||
backup_name = f"{self.journal_file}.{timestamp_str}"
|
backup_name = f"{self.journal_file}.{timestamp_str}"
|
||||||
backup_path = self.journal_dir / backup_name
|
backup_path = self.journal_dir / backup_name
|
||||||
|
|
||||||
# Rename current file to backup
|
# Rename current file to backup
|
||||||
if self.journal_path.exists():
|
if self.journal_path.exists():
|
||||||
self.journal_path.rename(backup_path)
|
self.journal_path.rename(backup_path)
|
||||||
logger.info(f"Rotated journal: {backup_path} "
|
logger.info(f"Rotated journal: {backup_path} "
|
||||||
f"(size: {self._current_size:,} bytes)")
|
f"(size: {self._current_size:,} bytes)")
|
||||||
|
|
||||||
# Open new journal file
|
# Open new journal file
|
||||||
self._file_handle = open(self.journal_path, 'a', encoding='utf-8')
|
self._file_handle = open(self.journal_path, 'a', encoding='utf-8')
|
||||||
self._current_size = 0
|
self._current_size = 0
|
||||||
|
|
||||||
# Clean up old backups
|
# Clean up old backups
|
||||||
await self._cleanup_old_backups()
|
await self._cleanup_old_backups()
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error rotating journal: {e}")
|
logger.error(f"Error rotating journal: {e}")
|
||||||
# Try to reopen the file even if rotation failed
|
# Try to reopen the file even if rotation failed
|
||||||
@@ -217,18 +217,18 @@ class MessageJournal:
|
|||||||
except Exception as e2:
|
except Exception as e2:
|
||||||
logger.error(f"Failed to reopen journal after rotation error: {e2}")
|
logger.error(f"Failed to reopen journal after rotation error: {e2}")
|
||||||
self.enabled = False
|
self.enabled = False
|
||||||
|
|
||||||
async def _cleanup_old_backups(self):
|
async def _cleanup_old_backups(self):
|
||||||
"""
|
"""
|
||||||
Remove old backup files exceeding max_backups limit.
|
Remove old backup files exceeding max_backups limit.
|
||||||
|
|
||||||
Keeps only the most recent backups based on filename (which includes timestamp).
|
Keeps only the most recent backups based on filename (which includes timestamp).
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
# Find all backup files
|
# Find all backup files
|
||||||
backup_pattern = f"{self.journal_file}.*"
|
backup_pattern = f"{self.journal_file}.*"
|
||||||
backup_files = sorted(self.journal_dir.glob(backup_pattern))
|
backup_files = sorted(self.journal_dir.glob(backup_pattern))
|
||||||
|
|
||||||
# Remove oldest backups if we have too many
|
# Remove oldest backups if we have too many
|
||||||
if len(backup_files) > self.max_backups:
|
if len(backup_files) > self.max_backups:
|
||||||
files_to_remove = backup_files[:len(backup_files) - self.max_backups]
|
files_to_remove = backup_files[:len(backup_files) - self.max_backups]
|
||||||
@@ -238,10 +238,10 @@ class MessageJournal:
|
|||||||
logger.info(f"Removed old backup: {backup_file.name}")
|
logger.info(f"Removed old backup: {backup_file.name}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed to remove old backup {backup_file}: {e}")
|
logger.warning(f"Failed to remove old backup {backup_file}: {e}")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error cleaning up old backups: {e}")
|
logger.error(f"Error cleaning up old backups: {e}")
|
||||||
|
|
||||||
async def log_threshold_event(
|
async def log_threshold_event(
|
||||||
self,
|
self,
|
||||||
host_name: str,
|
host_name: str,
|
||||||
@@ -253,7 +253,7 @@ class MessageJournal:
|
|||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Log a threshold state change event.
|
Log a threshold state change event.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
host_name: Name of the host
|
host_name: Name of the host
|
||||||
metric_path: Full metric path (e.g., "cpu_monitor.cpu_percent")
|
metric_path: Full metric path (e.g., "cpu_monitor.cpu_percent")
|
||||||
@@ -264,11 +264,11 @@ class MessageJournal:
|
|||||||
"""
|
"""
|
||||||
if not self.enabled or not self._initialized:
|
if not self.enabled or not self._initialized:
|
||||||
return
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if timestamp is None:
|
if timestamp is None:
|
||||||
timestamp = __import__('time').time()
|
timestamp = __import__('time').time()
|
||||||
|
|
||||||
event = {
|
event = {
|
||||||
'timestamp': timestamp,
|
'timestamp': timestamp,
|
||||||
'iso_time': datetime.fromtimestamp(timestamp).isoformat(),
|
'iso_time': datetime.fromtimestamp(timestamp).isoformat(),
|
||||||
@@ -279,30 +279,30 @@ class MessageJournal:
|
|||||||
'new_level': new_level,
|
'new_level': new_level,
|
||||||
'value': value,
|
'value': value,
|
||||||
}
|
}
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
if not self._file_handle:
|
if not self._file_handle:
|
||||||
return
|
return
|
||||||
|
|
||||||
# Check if rotation is needed
|
# Check if rotation is needed
|
||||||
if self._current_size >= self.max_size:
|
if self._current_size >= self.max_size:
|
||||||
await self._rotate()
|
await self._rotate()
|
||||||
|
|
||||||
# Write event
|
# Write event
|
||||||
line = json.dumps(event) + '\n'
|
line = json.dumps(event) + '\n'
|
||||||
self._file_handle.write(line)
|
self._file_handle.write(line)
|
||||||
self._file_handle.flush()
|
self._file_handle.flush()
|
||||||
|
|
||||||
# Update size
|
# Update size
|
||||||
self._current_size += len(line.encode('utf-8'))
|
self._current_size += len(line.encode('utf-8'))
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error logging threshold event: {e}")
|
logger.error(f"Error logging threshold event: {e}")
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
"""
|
"""
|
||||||
Close the journal and release resources.
|
Close the journal and release resources.
|
||||||
|
|
||||||
Should be called during shutdown.
|
Should be called during shutdown.
|
||||||
"""
|
"""
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
@@ -315,11 +315,11 @@ class MessageJournal:
|
|||||||
finally:
|
finally:
|
||||||
self._file_handle = None
|
self._file_handle = None
|
||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
def get_stats(self) -> Dict[str, Any]:
|
def get_stats(self) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Get journal statistics.
|
Get journal statistics.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with journal stats
|
Dictionary with journal stats
|
||||||
"""
|
"""
|
||||||
@@ -423,10 +423,10 @@ _journal_instance: Optional[MessageJournal] = None
|
|||||||
def get_journal(config: Optional[Dict[str, Any]] = None) -> MessageJournal:
|
def get_journal(config: Optional[Dict[str, Any]] = None) -> MessageJournal:
|
||||||
"""
|
"""
|
||||||
Get or create the global journal instance.
|
Get or create the global journal instance.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Configuration dictionary (only used on first call)
|
config: Configuration dictionary (only used on first call)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
MessageJournal instance
|
MessageJournal instance
|
||||||
"""
|
"""
|
||||||
|
|||||||
+19
-20
@@ -5,8 +5,6 @@ import logging
|
|||||||
import socket
|
import socket
|
||||||
import time
|
import time
|
||||||
import signal
|
import signal
|
||||||
import sys
|
|
||||||
import ssl
|
|
||||||
from . import __version__
|
from . import __version__
|
||||||
|
|
||||||
from . import udp
|
from . import udp
|
||||||
@@ -23,6 +21,7 @@ eventlog = notify_mod.eventlog
|
|||||||
|
|
||||||
# shared runtime collections and helpers
|
# shared runtime collections and helpers
|
||||||
|
|
||||||
|
|
||||||
def save_state(config, hbdclass):
|
def save_state(config, hbdclass):
|
||||||
"""Save current state to pickle file. Safe to call at any time."""
|
"""Save current state to pickle file. Safe to call at any time."""
|
||||||
import pickle
|
import pickle
|
||||||
@@ -68,21 +67,21 @@ def cleanup_function(config, hbdclass):
|
|||||||
|
|
||||||
async def reload_configuration(config_obj, config_path, components):
|
async def reload_configuration(config_obj, config_path, components):
|
||||||
"""Reload configuration and update all components.
|
"""Reload configuration and update all components.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config_obj: ReloadableConfig instance
|
config_obj: ReloadableConfig instance
|
||||||
config_path: Path to config file
|
config_path: Path to config file
|
||||||
components: Dict with threshold_checker and other components
|
components: Dict with threshold_checker and other components
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if reload succeeded, False otherwise
|
True if reload succeeded, False otherwise
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
logger.info("Starting configuration reload...")
|
logger.info("Starting configuration reload...")
|
||||||
|
|
||||||
# Reload config file
|
# Reload config file
|
||||||
new_config = await config_obj.reload(config_path)
|
new_config = await config_obj.reload(config_path)
|
||||||
|
|
||||||
# Update notify module
|
# Update notify module
|
||||||
notify_mod.reload_config(new_config)
|
notify_mod.reload_config(new_config)
|
||||||
|
|
||||||
@@ -103,23 +102,23 @@ async def reload_configuration(config_obj, config_path, components):
|
|||||||
if 'threshold_checker' in components:
|
if 'threshold_checker' in components:
|
||||||
components['threshold_checker'].reload(new_config)
|
components['threshold_checker'].reload(new_config)
|
||||||
components['threshold_checker'].purge_stale_alerts(hbdclass)
|
components['threshold_checker'].purge_stale_alerts(hbdclass)
|
||||||
|
|
||||||
# Note: Changes to the following require restart:
|
# Note: Changes to the following require restart:
|
||||||
# - hb_port, hbd_port, ws_port (already bound)
|
# - hb_port, hbd_port, ws_port (already bound)
|
||||||
# - SSL certificates (already loaded)
|
# - SSL certificates (already loaded)
|
||||||
# - pickfile (already opened)
|
# - pickfile (already opened)
|
||||||
# - journal settings (journal already initialized)
|
# - journal settings (journal already initialized)
|
||||||
|
|
||||||
# These are reloadable and effective immediately:
|
# These are reloadable and effective immediately:
|
||||||
# - notification_channels
|
# - notification_channels
|
||||||
# - threshold_configs
|
# - threshold_configs
|
||||||
# - hosts (watchhosts, dyndns, notification_channels)
|
# - hosts (watchhosts, dyndns, notification_channels)
|
||||||
# - grace period (used on next heartbeat)
|
# - grace period (used on next heartbeat)
|
||||||
# - debug/verbose flags (used on next message)
|
# - debug/verbose flags (used on next message)
|
||||||
|
|
||||||
logger.info("Configuration reload completed successfully")
|
logger.info("Configuration reload completed successfully")
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("=" * 60)
|
logger.error("=" * 60)
|
||||||
logger.error(f"Failed to reload configuration: {e}", exc_info=True)
|
logger.error(f"Failed to reload configuration: {e}", exc_info=True)
|
||||||
@@ -142,7 +141,7 @@ async def _run_async(config, config_path=None):
|
|||||||
sig_name = signal.Signals(signum).name if hasattr(signal, "Signals") else signum
|
sig_name = signal.Signals(signum).name if hasattr(signal, "Signals") else signum
|
||||||
logger.info(f"Received {sig_name}, initiating shutdown...")
|
logger.info(f"Received {sig_name}, initiating shutdown...")
|
||||||
loop.call_soon_threadsafe(shutdown_event.set)
|
loop.call_soon_threadsafe(shutdown_event.set)
|
||||||
|
|
||||||
def reload_handler(signum, frame):
|
def reload_handler(signum, frame):
|
||||||
sig_name = signal.Signals(signum).name if hasattr(signal, "Signals") else signum
|
sig_name = signal.Signals(signum).name if hasattr(signal, "Signals") else signum
|
||||||
logger.info(f"Received {sig_name}, initiating config reload...")
|
logger.info(f"Received {sig_name}, initiating config reload...")
|
||||||
@@ -181,7 +180,7 @@ async def _run_async(config, config_path=None):
|
|||||||
journal=msg_journal,
|
journal=msg_journal,
|
||||||
)
|
)
|
||||||
logger.info("Threshold checker initialized")
|
logger.info("Threshold checker initialized")
|
||||||
|
|
||||||
# Components dict for reload orchestration
|
# Components dict for reload orchestration
|
||||||
components = {
|
components = {
|
||||||
'threshold_checker': threshold_checker,
|
'threshold_checker': threshold_checker,
|
||||||
@@ -324,7 +323,7 @@ async def _run_async(config, config_path=None):
|
|||||||
],
|
],
|
||||||
return_when=asyncio.FIRST_COMPLETED
|
return_when=asyncio.FIRST_COMPLETED
|
||||||
)
|
)
|
||||||
|
|
||||||
# Check which event was triggered
|
# Check which event was triggered
|
||||||
if shutdown_event.is_set():
|
if shutdown_event.is_set():
|
||||||
logger.info("Shutdown signal received, stopping services...")
|
logger.info("Shutdown signal received, stopping services...")
|
||||||
@@ -332,24 +331,24 @@ async def _run_async(config, config_path=None):
|
|||||||
for task in pending:
|
for task in pending:
|
||||||
task.cancel()
|
task.cancel()
|
||||||
break
|
break
|
||||||
|
|
||||||
if reload_event.is_set():
|
if reload_event.is_set():
|
||||||
# Clear the event for next reload
|
# Clear the event for next reload
|
||||||
reload_event.clear()
|
reload_event.clear()
|
||||||
|
|
||||||
# Cancel pending wait tasks
|
# Cancel pending wait tasks
|
||||||
for task in pending:
|
for task in pending:
|
||||||
task.cancel()
|
task.cancel()
|
||||||
|
|
||||||
# Perform reload if config_path is available
|
# Perform reload if config_path is available
|
||||||
if config_path:
|
if config_path:
|
||||||
await reload_configuration(config, config_path, components)
|
await reload_configuration(config, config_path, components)
|
||||||
else:
|
else:
|
||||||
logger.warning("Cannot reload: no config path available")
|
logger.warning("Cannot reload: no config path available")
|
||||||
|
|
||||||
# Continue main loop
|
# Continue main loop
|
||||||
continue
|
continue
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("Error in main loop: %s", e)
|
logger.exception("Error in main loop: %s", e)
|
||||||
finally:
|
finally:
|
||||||
@@ -381,7 +380,7 @@ async def _run_async(config, config_path=None):
|
|||||||
logger.warning("Timeout waiting for tasks to cancel")
|
logger.warning("Timeout waiting for tasks to cancel")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("Exception during task cancellation: %s", e)
|
logger.debug("Exception during task cancellation: %s", e)
|
||||||
|
|
||||||
# Close message journal
|
# Close message journal
|
||||||
try:
|
try:
|
||||||
await msg_journal.close()
|
await msg_journal.close()
|
||||||
@@ -487,7 +486,7 @@ def run(config, config_path=None):
|
|||||||
"""Start the hbd service (blocking).
|
"""Start the hbd service (blocking).
|
||||||
|
|
||||||
Manually manages the event loop to ensure clean shutdown.
|
Manually manages the event loop to ensure clean shutdown.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Configuration dictionary
|
config: Configuration dictionary
|
||||||
config_path: Path to config file (for reload support)
|
config_path: Path to config file (for reload support)
|
||||||
|
|||||||
@@ -1,15 +1,13 @@
|
|||||||
"""Monitor helper for heartbeat daemon.
|
"""Monitor helper for heartbeat daemon.
|
||||||
|
|
||||||
This module provides monitoring tasks for the heartbeat daemon.
|
This module provides monitoring tasks for the heartbeat daemon.
|
||||||
The primary reachability monitoring is now event-driven (timers set/reset
|
The primary reachability monitoring is now event-driven (timers set/reset
|
||||||
on HTB arrival in udp.py) rather than periodic polling.
|
on HTB arrival in udp.py) rather than periodic polling.
|
||||||
|
|
||||||
This module can be extended for additional monitoring tasks.
|
This module can be extended for additional monitoring tasks.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
import asyncio
|
|
||||||
import time
|
|
||||||
from . import notify as notify_mod
|
from . import notify as notify_mod
|
||||||
|
|
||||||
DROPOVERDUE = 7 * 24 * 3600
|
DROPOVERDUE = 7 * 24 * 3600
|
||||||
@@ -18,11 +16,10 @@ eventlog = notify_mod.eventlog
|
|||||||
|
|
||||||
async def cleanup_connections(hbdclass):
|
async def cleanup_connections(hbdclass):
|
||||||
"""Clean up connection timers on shutdown.
|
"""Clean up connection timers on shutdown.
|
||||||
|
|
||||||
Cancels all active overdue timers to prevent callbacks after shutdown.
|
Cancels all active overdue timers to prevent callbacks after shutdown.
|
||||||
"""
|
"""
|
||||||
for hostname, host in list(hbdclass.Host.hosts.items()):
|
for hostname, host in list(hbdclass.Host.hosts.items()):
|
||||||
for conn_type, conn in host.connections.items():
|
for conn_type, conn in host.connections.items():
|
||||||
if hasattr(conn, 'cancel_overdue_timer'):
|
if hasattr(conn, 'cancel_overdue_timer'):
|
||||||
conn.cancel_overdue_timer()
|
conn.cancel_overdue_timer()
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import smtplib
|
|||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import time
|
||||||
import sys
|
import sys
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from . import data
|
from . import data
|
||||||
@@ -49,6 +49,7 @@ logf = None
|
|||||||
|
|
||||||
_LEVEL_ORDER = {"RECOVER": 0, "INFO": 0, "WARNING": 1, "CRITICAL": 2}
|
_LEVEL_ORDER = {"RECOVER": 0, "INFO": 0, "WARNING": 1, "CRITICAL": 2}
|
||||||
|
|
||||||
|
|
||||||
def _level_value(level: str) -> int:
|
def _level_value(level: str) -> int:
|
||||||
return _LEVEL_ORDER.get(level.upper(), 0)
|
return _LEVEL_ORDER.get(level.upper(), 0)
|
||||||
|
|
||||||
@@ -103,7 +104,6 @@ def initlog(logfile):
|
|||||||
|
|
||||||
|
|
||||||
def closelog():
|
def closelog():
|
||||||
global logf
|
|
||||||
if logf and logf != sys.stderr:
|
if logf and logf != sys.stderr:
|
||||||
try:
|
try:
|
||||||
logf.close()
|
logf.close()
|
||||||
@@ -331,8 +331,6 @@ async def _send_sms_voipms_async(channel_cfg: dict, notif: Notification) -> bool
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
|
async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
|
||||||
"""Send a Matrix message using matrix-nio."""
|
"""Send a Matrix message using matrix-nio."""
|
||||||
try:
|
try:
|
||||||
@@ -341,7 +339,6 @@ async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
|
|||||||
logger.error("matrix-nio not installed; pip install matrix-nio")
|
logger.error("matrix-nio not installed; pip install matrix-nio")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
from nio import AsyncClient
|
|
||||||
homeserver = channel_cfg.get("homeserver", "")
|
homeserver = channel_cfg.get("homeserver", "")
|
||||||
access_token = channel_cfg.get("access_token", "")
|
access_token = channel_cfg.get("access_token", "")
|
||||||
room_id = channel_cfg.get("room_id", "")
|
room_id = channel_cfg.get("room_id", "")
|
||||||
@@ -359,7 +356,6 @@ async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
|
|||||||
client = AsyncClient(homeserver)
|
client = AsyncClient(homeserver)
|
||||||
client.access_token = access_token
|
client.access_token = access_token
|
||||||
try:
|
try:
|
||||||
from nio import RoomSendResponse
|
|
||||||
content = {
|
content = {
|
||||||
"msgtype": "m.text",
|
"msgtype": "m.text",
|
||||||
"body": text,
|
"body": text,
|
||||||
|
|||||||
+1
-1
@@ -128,7 +128,7 @@ def get_providers(config: dict) -> list[ResolvedProvider]:
|
|||||||
Entries with missing required fields or unknown types are skipped with
|
Entries with missing required fields or unknown types are skipped with
|
||||||
a warning log. Order follows config declaration order.
|
a warning log. Order follows config declaration order.
|
||||||
"""
|
"""
|
||||||
result = []
|
result: list = []
|
||||||
oauth_cfg = config.get("oauth", {})
|
oauth_cfg = config.get("oauth", {})
|
||||||
if not isinstance(oauth_cfg, dict):
|
if not isinstance(oauth_cfg, dict):
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -420,7 +420,7 @@ def get_settings_sections(config: dict, threshold_checker=None, user=None) -> li
|
|||||||
"fields": [
|
"fields": [
|
||||||
field("journal_enabled", "Enabled", "boolean",
|
field("journal_enabled", "Enabled", "boolean",
|
||||||
"Turn journalling on or off.", editable=True),
|
"Turn journalling on or off.", editable=True),
|
||||||
field("journal_dir", "Journal directory","path",
|
field("journal_dir", "Journal directory", "path",
|
||||||
"Directory where journal files are written.", editable=True),
|
"Directory where journal files are written.", editable=True),
|
||||||
field("journal_file", "Journal filename", "text",
|
field("journal_file", "Journal filename", "text",
|
||||||
"Base filename for the journal (rotated copies get a numeric suffix)."),
|
"Base filename for the journal (rotated copies get a numeric suffix)."),
|
||||||
|
|||||||
+120
-117
@@ -13,13 +13,14 @@ import asyncio
|
|||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Dict, List, Any, Optional, Tuple, Callable
|
from typing import Dict, List, Any, Optional, Tuple
|
||||||
from . import notify as notify_mod
|
from . import notify as notify_mod
|
||||||
from .config import THRESHOLD_DEFAULTS
|
from .config import THRESHOLD_DEFAULTS
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
eventlog = notify_mod.eventlog
|
eventlog = notify_mod.eventlog
|
||||||
|
|
||||||
|
|
||||||
class AlertLevel(Enum):
|
class AlertLevel(Enum):
|
||||||
"""Alert severity levels."""
|
"""Alert severity levels."""
|
||||||
OK = 0
|
OK = 0
|
||||||
@@ -52,42 +53,42 @@ class AlertState:
|
|||||||
self.metric_path = metric_path
|
self.metric_path = metric_path
|
||||||
self.level = AlertLevel.OK
|
self.level = AlertLevel.OK
|
||||||
self.since = time.time()
|
self.since = time.time()
|
||||||
self.last_value = None
|
self.last_value: Any = None
|
||||||
self.last_check = time.time()
|
self.last_check = time.time()
|
||||||
self.notification_count = 0
|
self.notification_count = 0
|
||||||
self.last_notification = None
|
self.last_notification: Optional[float] = None
|
||||||
self.threshold_value = None # The threshold value that triggered alert
|
self.threshold_value: Optional[float] = None # The threshold value that triggered alert
|
||||||
self.operator = None # The comparison operator (>, <, >=, etc.)
|
self.operator: Optional[str] = None # The comparison operator (>, <, >=, etc.)
|
||||||
self.hysteresis: Optional[float] = None # Hysteresis fraction used for recovery
|
self.hysteresis: Optional[float] = None # Hysteresis fraction used for recovery
|
||||||
self.formatted_message = None # Formatted display message for UI
|
self.formatted_message: Optional[str] = None # Formatted display message for UI
|
||||||
self.acknowledged = False # Whether alert has been acknowledged
|
self.acknowledged = False # Whether alert has been acknowledged
|
||||||
self.acknowledged_at = None # Timestamp when acknowledged
|
self.acknowledged_at: Optional[float] = None # Timestamp when acknowledged
|
||||||
self.consecutive_count = 0 # Consecutive exceedances while still OK (for count gating)
|
self.consecutive_count = 0 # Consecutive exceedances while still OK (for count gating)
|
||||||
self.pending_since: Optional[float] = None # non-None while waiting out grace period before notifying
|
self.pending_since: Optional[float] = None # non-None while waiting out grace period before notifying
|
||||||
|
|
||||||
def update(
|
def update(
|
||||||
self,
|
self,
|
||||||
level: AlertLevel,
|
level: AlertLevel,
|
||||||
value: Any,
|
value: Any,
|
||||||
threshold_value: Optional[float] = None,
|
threshold_value: Optional[float] = None,
|
||||||
operator: Optional[str] = None
|
operator: Optional[str] = None
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""
|
"""
|
||||||
Update alert state.
|
Update alert state.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
level: New alert level
|
level: New alert level
|
||||||
value: Current metric value
|
value: Current metric value
|
||||||
threshold_value: The threshold value that was exceeded (if applicable)
|
threshold_value: The threshold value that was exceeded (if applicable)
|
||||||
operator: The comparison operator (>, <, >=, etc.)
|
operator: The comparison operator (>, <, >=, etc.)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if state changed (notification needed), False otherwise
|
True if state changed (notification needed), False otherwise
|
||||||
"""
|
"""
|
||||||
now = time.time()
|
now = time.time()
|
||||||
self.last_check = now
|
self.last_check = now
|
||||||
self.last_value = value
|
self.last_value = value
|
||||||
|
|
||||||
# Update threshold info when alert is active
|
# Update threshold info when alert is active
|
||||||
if level != AlertLevel.OK:
|
if level != AlertLevel.OK:
|
||||||
self.threshold_value = threshold_value
|
self.threshold_value = threshold_value
|
||||||
@@ -96,7 +97,7 @@ class AlertState:
|
|||||||
# Clear threshold info when returning to OK
|
# Clear threshold info when returning to OK
|
||||||
self.threshold_value = None
|
self.threshold_value = None
|
||||||
self.operator = None
|
self.operator = None
|
||||||
|
|
||||||
# Check if state changed
|
# Check if state changed
|
||||||
if level != self.level:
|
if level != self.level:
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -116,13 +117,13 @@ class AlertState:
|
|||||||
self.acknowledged = False
|
self.acknowledged = False
|
||||||
self.acknowledged_at = None
|
self.acknowledged_at = None
|
||||||
return True
|
return True
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def to_dict(self) -> dict:
|
def to_dict(self) -> dict:
|
||||||
"""Convert alert state to dictionary for serialization."""
|
"""Convert alert state to dictionary for serialization."""
|
||||||
import math
|
import math
|
||||||
|
|
||||||
# Helper to sanitize numeric values for JSON (handle inf/nan)
|
# Helper to sanitize numeric values for JSON (handle inf/nan)
|
||||||
def sanitize_value(val):
|
def sanitize_value(val):
|
||||||
if isinstance(val, float):
|
if isinstance(val, float):
|
||||||
@@ -131,7 +132,7 @@ class AlertState:
|
|||||||
if math.isnan(val):
|
if math.isnan(val):
|
||||||
return None
|
return None
|
||||||
return val
|
return val
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
"metric_path": self.metric_path,
|
"metric_path": self.metric_path,
|
||||||
"level": self.level.name,
|
"level": self.level.name,
|
||||||
@@ -141,11 +142,11 @@ class AlertState:
|
|||||||
"notification_count": self.notification_count,
|
"notification_count": self.notification_count,
|
||||||
"acknowledged": self.acknowledged,
|
"acknowledged": self.acknowledged,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Include acknowledgment timestamp if acknowledged
|
# Include acknowledgment timestamp if acknowledged
|
||||||
if self.acknowledged_at is not None:
|
if self.acknowledged_at is not None:
|
||||||
result["acknowledged_at"] = self.acknowledged_at
|
result["acknowledged_at"] = self.acknowledged_at
|
||||||
|
|
||||||
# Include threshold info if available
|
# Include threshold info if available
|
||||||
if self.threshold_value is not None:
|
if self.threshold_value is not None:
|
||||||
result["threshold_value"] = sanitize_value(self.threshold_value)
|
result["threshold_value"] = sanitize_value(self.threshold_value)
|
||||||
@@ -164,7 +165,7 @@ class AlertState:
|
|||||||
result["recovery_threshold"] = round(self.threshold_value + ha, 4)
|
result["recovery_threshold"] = round(self.threshold_value + ha, 4)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def __setstate__(self, state):
|
def __setstate__(self, state):
|
||||||
"""Restore from pickle, backfilling fields added after the pickle was written."""
|
"""Restore from pickle, backfilling fields added after the pickle was written."""
|
||||||
self.__dict__.update(state)
|
self.__dict__.update(state)
|
||||||
@@ -182,6 +183,7 @@ class AlertState:
|
|||||||
def __str__(self):
|
def __str__(self):
|
||||||
return self.to_dict().__str__()
|
return self.to_dict().__str__()
|
||||||
|
|
||||||
|
|
||||||
class ThresholdConfig:
|
class ThresholdConfig:
|
||||||
"""Configuration for a single threshold check."""
|
"""Configuration for a single threshold check."""
|
||||||
|
|
||||||
@@ -218,7 +220,7 @@ class ThresholdConfig:
|
|||||||
self.display = display
|
self.display = display
|
||||||
self.count = max(1, int(count))
|
self.count = max(1, int(count))
|
||||||
self.grace = float(grace) if grace is not None else None
|
self.grace = float(grace) if grace is not None else None
|
||||||
|
|
||||||
# Parse operator
|
# Parse operator
|
||||||
try:
|
try:
|
||||||
self.operator = ComparisonOperator(operator)
|
self.operator = ComparisonOperator(operator)
|
||||||
@@ -229,7 +231,7 @@ class ThresholdConfig:
|
|||||||
metric_path
|
metric_path
|
||||||
)
|
)
|
||||||
self.operator = ComparisonOperator.GT
|
self.operator = ComparisonOperator.GT
|
||||||
|
|
||||||
def evaluate(self, value: float) -> AlertLevel:
|
def evaluate(self, value: float) -> AlertLevel:
|
||||||
"""
|
"""
|
||||||
Evaluate a value against this threshold.
|
Evaluate a value against this threshold.
|
||||||
@@ -271,7 +273,7 @@ class ThresholdConfig:
|
|||||||
return AlertLevel.WARNING
|
return AlertLevel.WARNING
|
||||||
|
|
||||||
return AlertLevel.OK
|
return AlertLevel.OK
|
||||||
|
|
||||||
def evaluate_with_hysteresis(
|
def evaluate_with_hysteresis(
|
||||||
self,
|
self,
|
||||||
value: float,
|
value: float,
|
||||||
@@ -279,11 +281,11 @@ class ThresholdConfig:
|
|||||||
) -> AlertLevel:
|
) -> AlertLevel:
|
||||||
"""
|
"""
|
||||||
Evaluate with hysteresis to prevent flapping.
|
Evaluate with hysteresis to prevent flapping.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
value: Current metric value
|
value: Current metric value
|
||||||
current_level: Current alert level
|
current_level: Current alert level
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
New alert level considering hysteresis
|
New alert level considering hysteresis
|
||||||
"""
|
"""
|
||||||
@@ -296,7 +298,7 @@ class ThresholdConfig:
|
|||||||
# If no hysteresis, return new level
|
# If no hysteresis, return new level
|
||||||
if self.hysteresis == 0.0:
|
if self.hysteresis == 0.0:
|
||||||
return new_level
|
return new_level
|
||||||
|
|
||||||
# If improving (going to a lower severity), apply hysteresis
|
# If improving (going to a lower severity), apply hysteresis
|
||||||
if new_level.value < current_level.value:
|
if new_level.value < current_level.value:
|
||||||
# For recovery, value must be better by hysteresis amount
|
# For recovery, value must be better by hysteresis amount
|
||||||
@@ -306,10 +308,10 @@ class ThresholdConfig:
|
|||||||
threshold = self.warning
|
threshold = self.warning
|
||||||
else:
|
else:
|
||||||
return new_level
|
return new_level
|
||||||
|
|
||||||
# Calculate hysteresis threshold
|
# Calculate hysteresis threshold
|
||||||
hysteresis_amount = abs(threshold * self.hysteresis)
|
hysteresis_amount = abs(threshold * self.hysteresis)
|
||||||
|
|
||||||
if self.operator in [ComparisonOperator.GT, ComparisonOperator.GTE]:
|
if self.operator in [ComparisonOperator.GT, ComparisonOperator.GTE]:
|
||||||
# For "greater than" thresholds, value must go below by hysteresis
|
# For "greater than" thresholds, value must go below by hysteresis
|
||||||
recovery_threshold = threshold - hysteresis_amount
|
recovery_threshold = threshold - hysteresis_amount
|
||||||
@@ -322,9 +324,9 @@ class ThresholdConfig:
|
|||||||
if value <= recovery_threshold:
|
if value <= recovery_threshold:
|
||||||
# Not enough improvement, keep current level
|
# Not enough improvement, keep current level
|
||||||
return current_level
|
return current_level
|
||||||
|
|
||||||
return new_level
|
return new_level
|
||||||
|
|
||||||
def _compare(self, value: float, threshold: float) -> bool:
|
def _compare(self, value: float, threshold: float) -> bool:
|
||||||
"""Perform comparison based on operator."""
|
"""Perform comparison based on operator."""
|
||||||
if self.operator == ComparisonOperator.GT:
|
if self.operator == ComparisonOperator.GT:
|
||||||
@@ -344,7 +346,7 @@ class ThresholdConfig:
|
|||||||
|
|
||||||
class ThresholdChecker:
|
class ThresholdChecker:
|
||||||
"""Main threshold checking and alerting system."""
|
"""Main threshold checking and alerting system."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
config: Dict[str, Any],
|
config: Dict[str, Any],
|
||||||
@@ -353,34 +355,34 @@ class ThresholdChecker:
|
|||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Initialize threshold checker.
|
Initialize threshold checker.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: Threshold configuration dictionary from YAML
|
config: Threshold configuration dictionary from YAML
|
||||||
renotify_interval: Seconds between repeat notifications (default: 1 hour)
|
renotify_interval: Seconds between repeat notifications (default: 1 hour)
|
||||||
journal: Optional MessageJournal instance for logging threshold events
|
journal: Optional MessageJournal instance for logging threshold events
|
||||||
"""
|
"""
|
||||||
# Named threshold configurations (pre-merged: defaults + overrides): {config_name: {metric_path: ThresholdConfig}}
|
# Named threshold configurations (pre-merged: defaults + overrides): {config_name: {metric_path: ThresholdConfig}}
|
||||||
self.threshold_configs = {}
|
self.threshold_configs: Dict[str, Dict[str, ThresholdConfig]] = {}
|
||||||
|
|
||||||
# Raw overrides only for each named config (no defaults baked in): {config_name: {metric_path: ThresholdConfig}}
|
# Raw overrides only for each named config (no defaults baked in): {config_name: {metric_path: ThresholdConfig}}
|
||||||
self.threshold_raw_configs: Dict[str, Dict[str, ThresholdConfig]] = {}
|
self.threshold_raw_configs: Dict[str, Dict[str, ThresholdConfig]] = {}
|
||||||
|
|
||||||
# Single threshold set for backward compatibility: {metric_path: ThresholdConfig}
|
# Single threshold set for backward compatibility: {metric_path: ThresholdConfig}
|
||||||
self.thresholds = {}
|
self.thresholds: Dict[str, ThresholdConfig] = {}
|
||||||
|
|
||||||
# Host to ordered list of config names: {host_name: [config_name, ...]}
|
# Host to ordered list of config names: {host_name: [config_name, ...]}
|
||||||
self.host_config_mapping: Dict[str, List[str]] = {}
|
self.host_config_mapping: Dict[str, List[str]] = {}
|
||||||
|
|
||||||
# Default config name to use when no mapping exists
|
# Default config name to use when no mapping exists
|
||||||
self.default_config = "default"
|
self.default_config = "default"
|
||||||
|
|
||||||
self.renotify_interval = renotify_interval
|
self.renotify_interval = renotify_interval
|
||||||
self.grace_seconds: float = float(config.get("grace", 2))
|
self.grace_seconds: float = float(config.get("grace", 2))
|
||||||
self.journal = journal
|
self.journal = journal
|
||||||
|
|
||||||
# Parse configuration
|
# Parse configuration
|
||||||
self._parse_config(config)
|
self._parse_config(config)
|
||||||
|
|
||||||
total_thresholds = sum(len(cfg) for cfg in self.threshold_configs.values())
|
total_thresholds = sum(len(cfg) for cfg in self.threshold_configs.values())
|
||||||
if total_thresholds == 0 and len(self.thresholds) > 0:
|
if total_thresholds == 0 and len(self.thresholds) > 0:
|
||||||
# Backward compatibility: using single threshold set
|
# Backward compatibility: using single threshold set
|
||||||
@@ -392,18 +394,18 @@ class ThresholdChecker:
|
|||||||
len(self.threshold_configs),
|
len(self.threshold_configs),
|
||||||
total_thresholds
|
total_thresholds
|
||||||
)
|
)
|
||||||
|
|
||||||
def reload(self, config: Dict[str, Any]):
|
def reload(self, config: Dict[str, Any]):
|
||||||
"""Reload threshold configuration from new config dict.
|
"""Reload threshold configuration from new config dict.
|
||||||
|
|
||||||
This clears all existing thresholds and re-parses from the new configuration.
|
This clears all existing thresholds and re-parses from the new configuration.
|
||||||
Alert states are preserved to maintain hysteresis across reloads.
|
Alert states are preserved to maintain hysteresis across reloads.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config: New configuration dictionary
|
config: New configuration dictionary
|
||||||
"""
|
"""
|
||||||
logger.info("Reloading threshold configuration...")
|
logger.info("Reloading threshold configuration...")
|
||||||
|
|
||||||
# Clear old configuration
|
# Clear old configuration
|
||||||
self.threshold_configs.clear()
|
self.threshold_configs.clear()
|
||||||
self.threshold_raw_configs.clear()
|
self.threshold_raw_configs.clear()
|
||||||
@@ -413,13 +415,13 @@ class ThresholdChecker:
|
|||||||
|
|
||||||
# Parse new configuration
|
# Parse new configuration
|
||||||
self._parse_config(config)
|
self._parse_config(config)
|
||||||
|
|
||||||
total_thresholds = sum(len(cfg) for cfg in self.threshold_configs.values())
|
total_thresholds = sum(len(cfg) for cfg in self.threshold_configs.values())
|
||||||
if total_thresholds == 0 and len(self.thresholds) > 0:
|
if total_thresholds == 0 and len(self.thresholds) > 0:
|
||||||
total_thresholds = len(self.thresholds)
|
total_thresholds = len(self.thresholds)
|
||||||
|
|
||||||
logger.info("Threshold configuration reloaded: %d total thresholds", total_thresholds)
|
logger.info("Threshold configuration reloaded: %d total thresholds", total_thresholds)
|
||||||
|
|
||||||
def _parse_config(self, config: Dict[str, Any]):
|
def _parse_config(self, config: Dict[str, Any]):
|
||||||
"""Parse threshold configuration from YAML structure.
|
"""Parse threshold configuration from YAML structure.
|
||||||
|
|
||||||
@@ -449,7 +451,7 @@ class ThresholdChecker:
|
|||||||
self._parse_legacy_config(config)
|
self._parse_legacy_config(config)
|
||||||
else:
|
else:
|
||||||
logger.info("No thresholds configured")
|
logger.info("No thresholds configured")
|
||||||
|
|
||||||
def _parse_multi_config(self, config: Dict[str, Any]):
|
def _parse_multi_config(self, config: Dict[str, Any]):
|
||||||
"""Parse multiple named threshold configurations."""
|
"""Parse multiple named threshold configurations."""
|
||||||
threshold_configs = config.get("threshold_configs", {})
|
threshold_configs = config.get("threshold_configs", {})
|
||||||
@@ -545,34 +547,34 @@ class ThresholdChecker:
|
|||||||
if isinstance(legacy_mapping, dict):
|
if isinstance(legacy_mapping, dict):
|
||||||
for host_name, value in legacy_mapping.items():
|
for host_name, value in legacy_mapping.items():
|
||||||
self.host_config_mapping[host_name] = _normalise(value)
|
self.host_config_mapping[host_name] = _normalise(value)
|
||||||
|
|
||||||
# Set default config (first one alphabetically or explicitly set)
|
# Set default config (first one alphabetically or explicitly set)
|
||||||
self.default_config = config.get("default_threshold_config", "default")
|
self.default_config = config.get("default_threshold_config", "default")
|
||||||
if self.default_config not in self.threshold_configs and self.threshold_configs:
|
if self.default_config not in self.threshold_configs and self.threshold_configs:
|
||||||
# Use first available config as default
|
# Use first available config as default
|
||||||
self.default_config = sorted(self.threshold_configs.keys())[0]
|
self.default_config = sorted(self.threshold_configs.keys())[0]
|
||||||
logger.info("Using '%s' as default threshold config", self.default_config)
|
logger.info("Using '%s' as default threshold config", self.default_config)
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Loaded %d threshold configurations with %d host mappings",
|
"Loaded %d threshold configurations with %d host mappings",
|
||||||
len(self.threshold_configs),
|
len(self.threshold_configs),
|
||||||
len(self.host_config_mapping)
|
len(self.host_config_mapping)
|
||||||
)
|
)
|
||||||
|
|
||||||
def _parse_legacy_config(self, config: Dict[str, Any]):
|
def _parse_legacy_config(self, config: Dict[str, Any]):
|
||||||
"""Parse legacy single threshold configuration for backward compatibility."""
|
"""Parse legacy single threshold configuration for backward compatibility."""
|
||||||
if not config or "thresholds" not in config:
|
if not config or "thresholds" not in config:
|
||||||
logger.info("No thresholds configured")
|
logger.info("No thresholds configured")
|
||||||
return
|
return
|
||||||
|
|
||||||
thresholds_config = config["thresholds"]
|
thresholds_config = config["thresholds"]
|
||||||
|
|
||||||
for plugin_name, plugin_thresholds in thresholds_config.items():
|
for plugin_name, plugin_thresholds in thresholds_config.items():
|
||||||
if not isinstance(plugin_thresholds, dict):
|
if not isinstance(plugin_thresholds, dict):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=self.thresholds)
|
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=self.thresholds)
|
||||||
|
|
||||||
def _parse_plugin_thresholds(
|
def _parse_plugin_thresholds(
|
||||||
self,
|
self,
|
||||||
plugin_name: str,
|
plugin_name: str,
|
||||||
@@ -580,7 +582,7 @@ class ThresholdChecker:
|
|||||||
target_dict: Optional[Dict[str, ThresholdConfig]] = None
|
target_dict: Optional[Dict[str, ThresholdConfig]] = None
|
||||||
):
|
):
|
||||||
"""Parse thresholds for a specific plugin.
|
"""Parse thresholds for a specific plugin.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
plugin_name: Name of the plugin
|
plugin_name: Name of the plugin
|
||||||
thresholds: Threshold configuration dictionary
|
thresholds: Threshold configuration dictionary
|
||||||
@@ -588,7 +590,7 @@ class ThresholdChecker:
|
|||||||
"""
|
"""
|
||||||
if target_dict is None:
|
if target_dict is None:
|
||||||
target_dict = self.thresholds
|
target_dict = self.thresholds
|
||||||
|
|
||||||
# Special handling for RTT thresholds (per-host)
|
# Special handling for RTT thresholds (per-host)
|
||||||
if plugin_name == "rtt":
|
if plugin_name == "rtt":
|
||||||
self._parse_rtt_thresholds(thresholds, target_dict)
|
self._parse_rtt_thresholds(thresholds, target_dict)
|
||||||
@@ -606,7 +608,7 @@ class ThresholdChecker:
|
|||||||
for metric_name, threshold_config in thresholds.items():
|
for metric_name, threshold_config in thresholds.items():
|
||||||
if not isinstance(threshold_config, dict):
|
if not isinstance(threshold_config, dict):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Handle nested metrics (e.g., partitions./.percent or pools.*.status)
|
# Handle nested metrics (e.g., partitions./.percent or pools.*.status)
|
||||||
if metric_name == "partitions":
|
if metric_name == "partitions":
|
||||||
self._parse_partition_thresholds(plugin_name, threshold_config, target_dict)
|
self._parse_partition_thresholds(plugin_name, threshold_config, target_dict)
|
||||||
@@ -614,9 +616,9 @@ class ThresholdChecker:
|
|||||||
if metric_name == "pools":
|
if metric_name == "pools":
|
||||||
self._parse_pool_thresholds(plugin_name, threshold_config, target_dict)
|
self._parse_pool_thresholds(plugin_name, threshold_config, target_dict)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
metric_path = f"{plugin_name}.{metric_name}"
|
metric_path = f"{plugin_name}.{metric_name}"
|
||||||
|
|
||||||
# Extract threshold values
|
# Extract threshold values
|
||||||
warning = threshold_config.get("warning")
|
warning = threshold_config.get("warning")
|
||||||
critical = threshold_config.get("critical")
|
critical = threshold_config.get("critical")
|
||||||
@@ -643,7 +645,7 @@ class ThresholdChecker:
|
|||||||
display=display,
|
display=display,
|
||||||
grace=grace,
|
grace=grace,
|
||||||
)
|
)
|
||||||
|
|
||||||
target_dict[metric_path] = threshold
|
target_dict[metric_path] = threshold
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Registered threshold for %s: warn=%s, crit=%s, op=%s",
|
"Registered threshold for %s: warn=%s, crit=%s, op=%s",
|
||||||
@@ -652,7 +654,7 @@ class ThresholdChecker:
|
|||||||
critical,
|
critical,
|
||||||
operator
|
operator
|
||||||
)
|
)
|
||||||
|
|
||||||
def _parse_partition_thresholds(
|
def _parse_partition_thresholds(
|
||||||
self,
|
self,
|
||||||
plugin_name: str,
|
plugin_name: str,
|
||||||
@@ -660,7 +662,7 @@ class ThresholdChecker:
|
|||||||
target_dict: Optional[Dict[str, ThresholdConfig]] = None
|
target_dict: Optional[Dict[str, ThresholdConfig]] = None
|
||||||
):
|
):
|
||||||
"""Parse partition-specific thresholds for disk monitoring.
|
"""Parse partition-specific thresholds for disk monitoring.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
plugin_name: Name of the plugin
|
plugin_name: Name of the plugin
|
||||||
partitions: Partition threshold configuration
|
partitions: Partition threshold configuration
|
||||||
@@ -668,18 +670,18 @@ class ThresholdChecker:
|
|||||||
"""
|
"""
|
||||||
if target_dict is None:
|
if target_dict is None:
|
||||||
target_dict = self.thresholds
|
target_dict = self.thresholds
|
||||||
|
|
||||||
for partition, metrics in partitions.items():
|
for partition, metrics in partitions.items():
|
||||||
if not isinstance(metrics, dict):
|
if not isinstance(metrics, dict):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for metric_name, threshold_config in metrics.items():
|
for metric_name, threshold_config in metrics.items():
|
||||||
if not isinstance(threshold_config, dict):
|
if not isinstance(threshold_config, dict):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Create metric path like "disk_monitor./dev/sda1.percent"
|
# Create metric path like "disk_monitor./dev/sda1.percent"
|
||||||
metric_path = f"{plugin_name}.{partition}.{metric_name}"
|
metric_path = f"{plugin_name}.{partition}.{metric_name}"
|
||||||
|
|
||||||
warning = threshold_config.get("warning")
|
warning = threshold_config.get("warning")
|
||||||
critical = threshold_config.get("critical")
|
critical = threshold_config.get("critical")
|
||||||
operator = threshold_config.get("operator", ">")
|
operator = threshold_config.get("operator", ">")
|
||||||
@@ -700,7 +702,7 @@ class ThresholdChecker:
|
|||||||
display=display,
|
display=display,
|
||||||
grace=grace,
|
grace=grace,
|
||||||
)
|
)
|
||||||
|
|
||||||
target_dict[metric_path] = threshold
|
target_dict[metric_path] = threshold
|
||||||
|
|
||||||
def _parse_pool_thresholds(
|
def _parse_pool_thresholds(
|
||||||
@@ -761,26 +763,26 @@ class ThresholdChecker:
|
|||||||
target_dict: Optional[Dict[str, ThresholdConfig]] = None
|
target_dict: Optional[Dict[str, ThresholdConfig]] = None
|
||||||
):
|
):
|
||||||
"""Parse RTT thresholds (network latency thresholds).
|
"""Parse RTT thresholds (network latency thresholds).
|
||||||
|
|
||||||
RTT thresholds are configured as:
|
RTT thresholds are configured as:
|
||||||
thresholds:
|
thresholds:
|
||||||
rtt:
|
rtt:
|
||||||
warning: 100.0 # ms
|
warning: 100.0 # ms
|
||||||
critical: 500.0 # ms
|
critical: 500.0 # ms
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
rtt_thresholds: RTT threshold configuration
|
rtt_thresholds: RTT threshold configuration
|
||||||
target_dict: Dictionary to store parsed thresholds
|
target_dict: Dictionary to store parsed thresholds
|
||||||
"""
|
"""
|
||||||
if target_dict is None:
|
if target_dict is None:
|
||||||
target_dict = self.thresholds
|
target_dict = self.thresholds
|
||||||
|
|
||||||
if not isinstance(rtt_thresholds, dict):
|
if not isinstance(rtt_thresholds, dict):
|
||||||
return
|
return
|
||||||
|
|
||||||
# Metric path is simply "rtt" (not per-host)
|
# Metric path is simply "rtt" (not per-host)
|
||||||
metric_path = "rtt"
|
metric_path = "rtt"
|
||||||
|
|
||||||
warning = rtt_thresholds.get("warning")
|
warning = rtt_thresholds.get("warning")
|
||||||
critical = rtt_thresholds.get("critical")
|
critical = rtt_thresholds.get("critical")
|
||||||
operator = rtt_thresholds.get("operator", ">")
|
operator = rtt_thresholds.get("operator", ">")
|
||||||
@@ -813,7 +815,7 @@ class ThresholdChecker:
|
|||||||
critical,
|
critical,
|
||||||
count,
|
count,
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_thresholds_for_host(self, host_name: str) -> Dict[str, ThresholdConfig]:
|
def get_thresholds_for_host(self, host_name: str) -> Dict[str, ThresholdConfig]:
|
||||||
"""Get the effective threshold configuration for a host.
|
"""Get the effective threshold configuration for a host.
|
||||||
|
|
||||||
@@ -827,7 +829,7 @@ class ThresholdChecker:
|
|||||||
Dictionary of thresholds for this host
|
Dictionary of thresholds for this host
|
||||||
"""
|
"""
|
||||||
# Legacy mode: single threshold set for all hosts
|
# Legacy mode: single threshold set for all hosts
|
||||||
if self.thresholds and not self.threshold_configs:
|
if self.thresholds:
|
||||||
return self.thresholds
|
return self.thresholds
|
||||||
|
|
||||||
if not self.threshold_configs:
|
if not self.threshold_configs:
|
||||||
@@ -864,7 +866,7 @@ class ThresholdChecker:
|
|||||||
else:
|
else:
|
||||||
result.update(raw)
|
result.update(raw)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def check_value(
|
def check_value(
|
||||||
self,
|
self,
|
||||||
host_name: str,
|
host_name: str,
|
||||||
@@ -874,30 +876,30 @@ class ThresholdChecker:
|
|||||||
) -> Optional[Tuple[AlertLevel, AlertLevel]]:
|
) -> Optional[Tuple[AlertLevel, AlertLevel]]:
|
||||||
"""
|
"""
|
||||||
Check a single value against configured threshold.
|
Check a single value against configured threshold.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
host_name: Name of the host
|
host_name: Name of the host
|
||||||
metric_path: Full metric path (e.g., "rtt.hostname")
|
metric_path: Full metric path (e.g., "rtt.hostname")
|
||||||
value: The metric value to check
|
value: The metric value to check
|
||||||
alert_states: Host's alert_states dictionary
|
alert_states: Host's alert_states dictionary
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (old_level, new_level) if state changed, None otherwise
|
Tuple of (old_level, new_level) if state changed, None otherwise
|
||||||
"""
|
"""
|
||||||
# Get host-specific thresholds
|
# Get host-specific thresholds
|
||||||
thresholds = self.get_thresholds_for_host(host_name)
|
thresholds = self.get_thresholds_for_host(host_name)
|
||||||
|
|
||||||
if metric_path not in thresholds:
|
if metric_path not in thresholds:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
threshold = thresholds[metric_path]
|
threshold = thresholds[metric_path]
|
||||||
|
|
||||||
# Get or create alert state
|
# Get or create alert state
|
||||||
if metric_path not in alert_states:
|
if metric_path not in alert_states:
|
||||||
alert_states[metric_path] = AlertState(metric_path)
|
alert_states[metric_path] = AlertState(metric_path)
|
||||||
|
|
||||||
alert_state = alert_states[metric_path]
|
alert_state = alert_states[metric_path]
|
||||||
|
|
||||||
# Evaluate threshold with hysteresis
|
# Evaluate threshold with hysteresis
|
||||||
new_level = threshold.evaluate_with_hysteresis(
|
new_level = threshold.evaluate_with_hysteresis(
|
||||||
value,
|
value,
|
||||||
@@ -946,6 +948,7 @@ class ThresholdChecker:
|
|||||||
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, None)
|
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, None)
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _find_threshold(
|
def _find_threshold(
|
||||||
self, thresholds: Dict[str, "ThresholdConfig"], metric_path: str
|
self, thresholds: Dict[str, "ThresholdConfig"], metric_path: str
|
||||||
) -> Tuple[Optional["ThresholdConfig"], Optional[str]]:
|
) -> Tuple[Optional["ThresholdConfig"], Optional[str]]:
|
||||||
@@ -985,21 +988,21 @@ class ThresholdChecker:
|
|||||||
) -> list:
|
) -> list:
|
||||||
"""
|
"""
|
||||||
Check plugin data against configured thresholds.
|
Check plugin data against configured thresholds.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
host_name: Name of the host
|
host_name: Name of the host
|
||||||
plugin_name: Name of the plugin
|
plugin_name: Name of the plugin
|
||||||
data: Plugin data dictionary
|
data: Plugin data dictionary
|
||||||
alert_states: Host's alert_states dictionary
|
alert_states: Host's alert_states dictionary
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of (metric_path, old_level, new_level, value) tuples for state changes
|
List of (metric_path, old_level, new_level, value) tuples for state changes
|
||||||
"""
|
"""
|
||||||
state_changes = []
|
state_changes = []
|
||||||
|
|
||||||
# Get host-specific thresholds
|
# Get host-specific thresholds
|
||||||
thresholds = self.get_thresholds_for_host(host_name)
|
thresholds = self.get_thresholds_for_host(host_name)
|
||||||
|
|
||||||
# Check flat metrics
|
# Check flat metrics
|
||||||
for metric_name, value in data.items():
|
for metric_name, value in data.items():
|
||||||
metric_path = f"{plugin_name}.{metric_name}"
|
metric_path = f"{plugin_name}.{metric_name}"
|
||||||
@@ -1045,9 +1048,9 @@ class ThresholdChecker:
|
|||||||
alert_states,
|
alert_states,
|
||||||
state_changes
|
state_changes
|
||||||
)
|
)
|
||||||
|
|
||||||
return state_changes
|
return state_changes
|
||||||
|
|
||||||
def _check_nested_metrics(
|
def _check_nested_metrics(
|
||||||
self,
|
self,
|
||||||
host_name: str,
|
host_name: str,
|
||||||
@@ -1059,7 +1062,7 @@ class ThresholdChecker:
|
|||||||
"""Check nested metrics like partition-specific thresholds."""
|
"""Check nested metrics like partition-specific thresholds."""
|
||||||
# Get host-specific thresholds
|
# Get host-specific thresholds
|
||||||
thresholds = self.get_thresholds_for_host(host_name)
|
thresholds = self.get_thresholds_for_host(host_name)
|
||||||
|
|
||||||
# ZFS pool health checks
|
# ZFS pool health checks
|
||||||
if plugin_name == "zfs_monitor" and "pools" in data:
|
if plugin_name == "zfs_monitor" and "pools" in data:
|
||||||
pools = data["pools"]
|
pools = data["pools"]
|
||||||
@@ -1103,29 +1106,29 @@ class ThresholdChecker:
|
|||||||
partitions = data["partitions"]
|
partitions = data["partitions"]
|
||||||
if not isinstance(partitions, dict):
|
if not isinstance(partitions, dict):
|
||||||
return
|
return
|
||||||
|
|
||||||
for partition, metrics in partitions.items():
|
for partition, metrics in partitions.items():
|
||||||
if not isinstance(metrics, dict):
|
if not isinstance(metrics, dict):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for metric_name, value in metrics.items():
|
for metric_name, value in metrics.items():
|
||||||
metric_path = f"{plugin_name}.{partition}.{metric_name}"
|
metric_path = f"{plugin_name}.{partition}.{metric_name}"
|
||||||
|
|
||||||
if metric_path not in thresholds:
|
if metric_path not in thresholds:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
threshold = thresholds[metric_path]
|
threshold = thresholds[metric_path]
|
||||||
|
|
||||||
if metric_path not in alert_states:
|
if metric_path not in alert_states:
|
||||||
alert_states[metric_path] = AlertState(metric_path)
|
alert_states[metric_path] = AlertState(metric_path)
|
||||||
|
|
||||||
alert_state = alert_states[metric_path]
|
alert_state = alert_states[metric_path]
|
||||||
|
|
||||||
new_level = threshold.evaluate_with_hysteresis(
|
new_level = threshold.evaluate_with_hysteresis(
|
||||||
value,
|
value,
|
||||||
alert_state.level
|
alert_state.level
|
||||||
)
|
)
|
||||||
|
|
||||||
# Determine which threshold was exceeded
|
# Determine which threshold was exceeded
|
||||||
threshold_value = None
|
threshold_value = None
|
||||||
if new_level == AlertLevel.CRITICAL and threshold.critical is not None:
|
if new_level == AlertLevel.CRITICAL and threshold.critical is not None:
|
||||||
@@ -1141,7 +1144,7 @@ class ThresholdChecker:
|
|||||||
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, data)
|
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, data)
|
||||||
elif new_level != AlertLevel.OK:
|
elif new_level != AlertLevel.OK:
|
||||||
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, data)
|
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, data)
|
||||||
|
|
||||||
def _trigger_notification(
|
def _trigger_notification(
|
||||||
self,
|
self,
|
||||||
host_name: str,
|
host_name: str,
|
||||||
@@ -1155,7 +1158,7 @@ class ThresholdChecker:
|
|||||||
metric_name: Optional[str] = None,
|
metric_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
"""Trigger a notification for an alert state change.
|
"""Trigger a notification for an alert state change.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
host_name: Name of the host
|
host_name: Name of the host
|
||||||
metric_path: Full metric path
|
metric_path: Full metric path
|
||||||
@@ -1171,7 +1174,7 @@ class ThresholdChecker:
|
|||||||
threshold_value = threshold.critical
|
threshold_value = threshold.critical
|
||||||
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||||
threshold_value = threshold.warning
|
threshold_value = threshold.warning
|
||||||
|
|
||||||
# Format operator symbol
|
# Format operator symbol
|
||||||
op_symbol = threshold.operator.value
|
op_symbol = threshold.operator.value
|
||||||
|
|
||||||
@@ -1223,7 +1226,7 @@ class ThresholdChecker:
|
|||||||
formatted_threshold_msg = _fmt() if has_display and new_level != AlertLevel.OK else None
|
formatted_threshold_msg = _fmt() if has_display and new_level != AlertLevel.OK else None
|
||||||
|
|
||||||
return lvl, message, formatted_threshold_msg
|
return lvl, message, formatted_threshold_msg
|
||||||
|
|
||||||
def _send_notification(
|
def _send_notification(
|
||||||
self,
|
self,
|
||||||
host_name: str,
|
host_name: str,
|
||||||
@@ -1254,7 +1257,7 @@ class ThresholdChecker:
|
|||||||
service=short_path,
|
service=short_path,
|
||||||
),
|
),
|
||||||
))
|
))
|
||||||
|
|
||||||
# Log to journal
|
# Log to journal
|
||||||
if self.journal is not None:
|
if self.journal is not None:
|
||||||
try:
|
try:
|
||||||
@@ -1270,10 +1273,10 @@ class ThresholdChecker:
|
|||||||
logger.debug(f"Failed to log threshold event to journal: {e}")
|
logger.debug(f"Failed to log threshold event to journal: {e}")
|
||||||
# Log to eventlog as well
|
# Log to eventlog as well
|
||||||
eventlog(host_name, lvl, message, service="threshold")
|
eventlog(host_name, lvl, message, service="threshold")
|
||||||
|
|
||||||
def _format_display(
|
def _format_display(
|
||||||
self,
|
self,
|
||||||
display_format: str,
|
display_format: Optional[str],
|
||||||
value: Any,
|
value: Any,
|
||||||
threshold_value: Optional[float],
|
threshold_value: Optional[float],
|
||||||
op_symbol: str,
|
op_symbol: str,
|
||||||
@@ -1330,7 +1333,7 @@ class ThresholdChecker:
|
|||||||
status = plugin_data.get(f"{check_name}_status")
|
status = plugin_data.get(f"{check_name}_status")
|
||||||
if status is not None:
|
if status is not None:
|
||||||
format_context['status'] = status
|
format_context['status'] = status
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Format the display string
|
# Format the display string
|
||||||
return display_format.format(**format_context)
|
return display_format.format(**format_context)
|
||||||
@@ -1349,7 +1352,7 @@ class ThresholdChecker:
|
|||||||
e
|
e
|
||||||
)
|
)
|
||||||
return f"(threshold: {op_symbol} {threshold_value})"
|
return f"(threshold: {op_symbol} {threshold_value})"
|
||||||
|
|
||||||
def _apply_grace(
|
def _apply_grace(
|
||||||
self,
|
self,
|
||||||
host_name: str,
|
host_name: str,
|
||||||
@@ -1472,7 +1475,7 @@ class ThresholdChecker:
|
|||||||
metric_name: Optional[str] = None,
|
metric_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
"""Check if we should send a repeat notification.
|
"""Check if we should send a repeat notification.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
host_name: Name of the host
|
host_name: Name of the host
|
||||||
alert_state: Current alert state
|
alert_state: Current alert state
|
||||||
@@ -1487,16 +1490,16 @@ class ThresholdChecker:
|
|||||||
# Skip reminders if alert has been acknowledged
|
# Skip reminders if alert has been acknowledged
|
||||||
if alert_state.acknowledged:
|
if alert_state.acknowledged:
|
||||||
return
|
return
|
||||||
|
|
||||||
now = time.time()
|
now = time.time()
|
||||||
|
|
||||||
# Check if we should re-notify
|
# Check if we should re-notify
|
||||||
if alert_state.last_notification is None:
|
if alert_state.last_notification is None:
|
||||||
# First notification already sent during state change
|
# First notification already sent during state change
|
||||||
alert_state.last_notification = now
|
alert_state.last_notification = now
|
||||||
alert_state.notification_count = 1
|
alert_state.notification_count = 1
|
||||||
return
|
return
|
||||||
|
|
||||||
if (now - alert_state.last_notification) >= self.renotify_interval:
|
if (now - alert_state.last_notification) >= self.renotify_interval:
|
||||||
# Determine which threshold is active
|
# Determine which threshold is active
|
||||||
threshold_value = None
|
threshold_value = None
|
||||||
@@ -1504,7 +1507,7 @@ class ThresholdChecker:
|
|||||||
threshold_value = threshold.critical
|
threshold_value = threshold.critical
|
||||||
elif alert_state.level == AlertLevel.WARNING and threshold.warning is not None:
|
elif alert_state.level == AlertLevel.WARNING and threshold.warning is not None:
|
||||||
threshold_value = threshold.warning
|
threshold_value = threshold.warning
|
||||||
|
|
||||||
# Format operator symbol
|
# Format operator symbol
|
||||||
op_symbol = threshold.operator.value
|
op_symbol = threshold.operator.value
|
||||||
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
|
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
|
||||||
@@ -1541,7 +1544,7 @@ class ThresholdChecker:
|
|||||||
logger.info("Re-notification sent: %s", message)
|
logger.info("Re-notification sent: %s", message)
|
||||||
alert_state.last_notification = now
|
alert_state.last_notification = now
|
||||||
alert_state.notification_count += 1
|
alert_state.notification_count += 1
|
||||||
|
|
||||||
def purge_stale_alerts(self, hbdclass) -> None:
|
def purge_stale_alerts(self, hbdclass) -> None:
|
||||||
"""Remove alert states that have no matching threshold configuration.
|
"""Remove alert states that have no matching threshold configuration.
|
||||||
|
|
||||||
@@ -1578,10 +1581,10 @@ class ThresholdChecker:
|
|||||||
def get_active_alerts(self, alert_states: Dict[str, AlertState]) -> list:
|
def get_active_alerts(self, alert_states: Dict[str, AlertState]) -> list:
|
||||||
"""
|
"""
|
||||||
Get all currently active (non-OK) alerts.
|
Get all currently active (non-OK) alerts.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
alert_states: Host's alert_states dictionary
|
alert_states: Host's alert_states dictionary
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of AlertState objects that are not OK
|
List of AlertState objects that are not OK
|
||||||
"""
|
"""
|
||||||
@@ -1589,19 +1592,19 @@ class ThresholdChecker:
|
|||||||
state for state in alert_states.values()
|
state for state in alert_states.values()
|
||||||
if state.level != AlertLevel.OK
|
if state.level != AlertLevel.OK
|
||||||
]
|
]
|
||||||
|
|
||||||
def get_alert_summary(self, alert_states: Dict[str, AlertState]) -> Dict[str, int]:
|
def get_alert_summary(self, alert_states: Dict[str, AlertState]) -> Dict[str, int]:
|
||||||
"""
|
"""
|
||||||
Get summary counts of alert levels.
|
Get summary counts of alert levels.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
alert_states: Host's alert_states dictionary
|
alert_states: Host's alert_states dictionary
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Dictionary with counts: {"ok": N, "warning": N, "critical": N}
|
Dictionary with counts: {"ok": N, "warning": N, "critical": N}
|
||||||
"""
|
"""
|
||||||
summary = {"ok": 0, "warning": 0, "critical": 0, "unknown": 0}
|
summary = {"ok": 0, "warning": 0, "critical": 0, "unknown": 0}
|
||||||
|
|
||||||
for state in alert_states.values():
|
for state in alert_states.values():
|
||||||
if state.level == AlertLevel.OK:
|
if state.level == AlertLevel.OK:
|
||||||
summary["ok"] += 1
|
summary["ok"] += 1
|
||||||
@@ -1611,5 +1614,5 @@ class ThresholdChecker:
|
|||||||
summary["critical"] += 1
|
summary["critical"] += 1
|
||||||
elif state.level == AlertLevel.UNKNOWN:
|
elif state.level == AlertLevel.UNKNOWN:
|
||||||
summary["unknown"] += 1
|
summary["unknown"] += 1
|
||||||
|
|
||||||
return summary
|
return summary
|
||||||
|
|||||||
+25
-17
@@ -5,6 +5,7 @@ import socket
|
|||||||
import struct
|
import struct
|
||||||
import time
|
import time
|
||||||
import zlib
|
import zlib
|
||||||
|
from typing import Optional
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from platform import system as platform_system
|
from platform import system as platform_system
|
||||||
@@ -23,14 +24,15 @@ _STALE_INTERVAL_MULTIPLIER = 3
|
|||||||
|
|
||||||
# SO_TIMESTAMP: kernel attaches a struct timeval to each received datagram.
|
# SO_TIMESTAMP: kernel attaches a struct timeval to each received datagram.
|
||||||
# Supported on Linux, FreeBSD, and macOS. The constant is not exposed by
|
# Supported on Linux, FreeBSD, and macOS. The constant is not exposed by
|
||||||
# Python's socket module on all platforms
|
# Python's socket module on all platforms
|
||||||
platform = platform_system()
|
platform = platform_system()
|
||||||
|
_SO_TIMESTAMP: Optional[int]
|
||||||
if platform == "Darwin":
|
if platform == "Darwin":
|
||||||
_SO_TIMESTAMP = 1024 # SO_TIMESTAMP on macOS (not in Python's socket module)
|
_SO_TIMESTAMP = 1024 # SO_TIMESTAMP on macOS (not in Python's socket module)
|
||||||
elif platform == "Linux":
|
elif platform == "Linux":
|
||||||
_SO_TIMESTAMP = 29 # Linux value (not in older Python versions)
|
_SO_TIMESTAMP = 29 # Linux value (not in older Python versions)
|
||||||
elif platform == "FreeBSD":
|
elif platform == "FreeBSD":
|
||||||
_SO_TIMESTAMP = 32 # FreeBSD value (not in older Python versions)
|
_SO_TIMESTAMP = 32 # FreeBSD value (not in older Python versions)
|
||||||
else:
|
else:
|
||||||
logger.warning("SO_TIMESTAMP may not be supported on this platform (%s)", platform)
|
logger.warning("SO_TIMESTAMP may not be supported on this platform (%s)", platform)
|
||||||
_SO_TIMESTAMP = None
|
_SO_TIMESTAMP = None
|
||||||
@@ -45,6 +47,8 @@ def enable_kernel_timestamps(sock) -> bool:
|
|||||||
Returns True if the kernel will supply receive timestamps, False otherwise
|
Returns True if the kernel will supply receive timestamps, False otherwise
|
||||||
(unsupported platform, older kernel, or insufficient permissions).
|
(unsupported platform, older kernel, or insufficient permissions).
|
||||||
"""
|
"""
|
||||||
|
if _SO_TIMESTAMP is None:
|
||||||
|
return False
|
||||||
try:
|
try:
|
||||||
sock.setsockopt(socket.SOL_SOCKET, _SO_TIMESTAMP, 1)
|
sock.setsockopt(socket.SOL_SOCKET, _SO_TIMESTAMP, 1)
|
||||||
return True
|
return True
|
||||||
@@ -71,6 +75,7 @@ class RecvmsgTransport:
|
|||||||
Exposes the same sendto() / close() interface as asyncio's DatagramTransport
|
Exposes the same sendto() / close() interface as asyncio's DatagramTransport
|
||||||
so the rest of the code does not need to know which path is in use.
|
so the rest of the code does not need to know which path is in use.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, loop, sock):
|
def __init__(self, loop, sock):
|
||||||
self._loop = loop
|
self._loop = loop
|
||||||
self._sock = sock
|
self._sock = sock
|
||||||
@@ -410,7 +415,7 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
if not msg:
|
if not msg:
|
||||||
return
|
return
|
||||||
now = ctx.get("recv_ts") or time.time()
|
now = ctx.get("recv_ts") or time.time()
|
||||||
|
|
||||||
# Log message to journal
|
# Log message to journal
|
||||||
msg_journal = ctx.get("msg_journal")
|
msg_journal = ctx.get("msg_journal")
|
||||||
if msg_journal:
|
if msg_journal:
|
||||||
@@ -421,16 +426,21 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
loop.create_task(msg_journal.log_message(msg, addr, now))
|
loop.create_task(msg_journal.log_message(msg, addr, now))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug(f"Failed to log message to journal: {e}")
|
logger.debug(f"Failed to log message to journal: {e}")
|
||||||
|
|
||||||
|
# Only named HTB/PLG datagrams come from clients; anything else (e.g. an internet scanner's
|
||||||
|
# SIP probe) is journaled above but must not sign on a host.
|
||||||
|
if msg.get("ID") not in ("HTB", "PLG") or not msg.get("name"):
|
||||||
|
logger.debug("Ignoring non-client datagram from %s", addr)
|
||||||
|
return
|
||||||
|
|
||||||
cfg = ctx.get("config", {})
|
cfg = ctx.get("config", {})
|
||||||
hbdcls = ctx.get("hbdclass")
|
hbdcls = ctx["hbdclass"]
|
||||||
msg_to_websockets = ctx.get("msg_to_websockets")
|
msg_to_websockets = ctx.get("msg_to_websockets")
|
||||||
DEBUG = ctx.get("DEBUG", 0)
|
DEBUG = ctx.get("DEBUG", 0)
|
||||||
verbose = ctx.get("verbose", False)
|
|
||||||
|
|
||||||
# normalize addr (ip, port)
|
# normalize addr (ip, port)
|
||||||
ip = addr[0] if isinstance(addr, (list, tuple)) else addr
|
ip = addr[0] if isinstance(addr, (list, tuple)) else addr
|
||||||
name = msg.get("name", "unknown")
|
name = msg["name"]
|
||||||
from ..common.utils import shortname
|
from ..common.utils import shortname
|
||||||
from . import config as config_mod
|
from . import config as config_mod
|
||||||
|
|
||||||
@@ -451,12 +461,10 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
else:
|
else:
|
||||||
host = hbdcls.Host.hosts[uname]
|
host = hbdcls.Host.hosts[uname]
|
||||||
newh = False
|
newh = False
|
||||||
|
|
||||||
cid = msg.get("id", 0)
|
cid = msg.get("id", 0)
|
||||||
try:
|
rtt_raw = msg.get("rtt")
|
||||||
rtt = float(msg.get("rtt"))
|
rtt = float(rtt_raw) if rtt_raw is not None else None
|
||||||
except TypeError:
|
|
||||||
rtt = None
|
|
||||||
|
|
||||||
if msg.get("ID") == "HTB":
|
if msg.get("ID") == "HTB":
|
||||||
host.doesack = msg.get("acks", -1)
|
host.doesack = msg.get("acks", -1)
|
||||||
@@ -517,14 +525,14 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
|
|
||||||
# If os_info reports an owner and none is configured server-side, apply it
|
# If os_info reports an owner and none is configured server-side, apply it
|
||||||
if plugin_name == "os_info":
|
if plugin_name == "os_info":
|
||||||
config_owner = config_mod.get_host_access(cfg, uname).get("owner")
|
config_owner = config_mod.get_host_access(cfg, uname).get("owner")
|
||||||
default_owner = config_mod.get_default_owner(cfg)
|
default_owner = config_mod.get_default_owner(cfg)
|
||||||
inferred_owner = config_owner or plugin_data.get("owner") or default_owner
|
inferred_owner = config_owner or plugin_data.get("owner") or default_owner
|
||||||
host.owner = inferred_owner
|
host.owner = inferred_owner
|
||||||
logger.info(f"owner for {uname} is {host.owner}")
|
logger.info(f"owner for {uname} is {host.owner}")
|
||||||
if DEBUG > 1:
|
if DEBUG > 1:
|
||||||
print(f"Stored plugin data for {uname}: {plugin_name}")
|
print(f"Stored plugin data for {uname}: {plugin_name}")
|
||||||
|
|
||||||
# Check thresholds if checker is available
|
# Check thresholds if checker is available
|
||||||
threshold_checker = ctx.get("threshold_checker")
|
threshold_checker = ctx.get("threshold_checker")
|
||||||
if threshold_checker:
|
if threshold_checker:
|
||||||
@@ -539,7 +547,7 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
print(f"Threshold state changes for {uname}: {state_changes}")
|
print(f"Threshold state changes for {uname}: {state_changes}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error checking thresholds for {uname}.{plugin_name}: {e}")
|
logger.error(f"Error checking thresholds for {uname}.{plugin_name}: {e}")
|
||||||
|
|
||||||
# Notify websockets of plugin update
|
# Notify websockets of plugin update
|
||||||
if msg_to_websockets:
|
if msg_to_websockets:
|
||||||
try:
|
try:
|
||||||
@@ -665,13 +673,13 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
timeout_seconds = interval + grace
|
timeout_seconds = interval + grace
|
||||||
on_overdue, _ = _make_timer_callbacks(uname, host, ctx)
|
on_overdue, _ = _make_timer_callbacks(uname, host, ctx)
|
||||||
conn.reset_overdue_timer(timeout_seconds, on_overdue)
|
conn.reset_overdue_timer(timeout_seconds, on_overdue)
|
||||||
|
|
||||||
# Check RTT thresholds using the threshold checker
|
# Check RTT thresholds using the threshold checker
|
||||||
threshold_checker = ctx.get("threshold_checker")
|
threshold_checker = ctx.get("threshold_checker")
|
||||||
if threshold_checker and rtt and rtt > 0:
|
if threshold_checker and rtt and rtt > 0:
|
||||||
# Metric path for RTT is simply "rtt"
|
# Metric path for RTT is simply "rtt"
|
||||||
metric_path = "rtt"
|
metric_path = "rtt"
|
||||||
|
|
||||||
# Check against configured thresholds (handles alerts, notifications, etc.)
|
# Check against configured thresholds (handles alerts, notifications, etc.)
|
||||||
threshold_checker.check_value(
|
threshold_checker.check_value(
|
||||||
host_name=uname,
|
host_name=uname,
|
||||||
|
|||||||
+14
-3
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "hbd"
|
name = "hbd"
|
||||||
version = "5.4.5"
|
version = "5.4.7"
|
||||||
description = "Heartbeat monitoring system — client (hbc) and server (hbd)"
|
description = "Heartbeat monitoring system — client (hbc) and server (hbd)"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
@@ -58,9 +58,12 @@ all = [
|
|||||||
# Development dependencies
|
# Development dependencies
|
||||||
dev = [
|
dev = [
|
||||||
"pytest>=7.0",
|
"pytest>=7.0",
|
||||||
|
"pytest-asyncio>=0.23",
|
||||||
"pytest-cov>=4.0",
|
"pytest-cov>=4.0",
|
||||||
"flake8>=5.0",
|
"flake8>=5.0",
|
||||||
"mypy>=1.10",
|
"mypy>=1.10",
|
||||||
|
"types-PyYAML",
|
||||||
|
"types-psutil",
|
||||||
"black>=23.0",
|
"black>=23.0",
|
||||||
"isort>=5.0",
|
"isort>=5.0",
|
||||||
"tox>=4.0",
|
"tox>=4.0",
|
||||||
@@ -83,10 +86,18 @@ include = ["hbd*"]
|
|||||||
|
|
||||||
|
|
||||||
[tool.black]
|
[tool.black]
|
||||||
line-length = 111
|
line-length = 165
|
||||||
|
|
||||||
[tool.flake8]
|
[tool.flake8]
|
||||||
max-line-length = 111
|
max-line-length = 165
|
||||||
|
|
||||||
[tool.pylint.format]
|
[tool.pylint.format]
|
||||||
max-line-length = 111
|
max-line-length = 111
|
||||||
|
|
||||||
|
[[tool.mypy.overrides]]
|
||||||
|
# optional notification libraries that ship neither stubs nor py.typed
|
||||||
|
module = ["nio", "nio.*", "mattermostdriver", "mattermostdriver.*"]
|
||||||
|
ignore_missing_imports = true
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
CC ?= cc
|
CC ?= cc
|
||||||
CFLAGS = -O2 -Wall -Wextra -std=c11
|
CFLAGS = -O2 -Wall -Wextra -std=c11
|
||||||
LDFLAGS = -lz -lpthread -lm
|
LDFLAGS = -lz -lpthread -lm
|
||||||
|
# res_search() lives in libresolv on older glibc; in libc on newer glibc, musl and the BSDs
|
||||||
|
LDFLAGS += $(shell echo 'int main(void){return 0;}' | $(CC) -x c - -lresolv -o /dev/null 2>/dev/null && echo -lresolv)
|
||||||
TARGET = hbc_mini
|
TARGET = hbc_mini
|
||||||
SRC = hbc_mini.c
|
SRC = hbc_mini.c
|
||||||
|
|
||||||
|
|||||||
+204
-1
@@ -20,6 +20,7 @@
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
#include <arpa/inet.h>
|
#include <arpa/inet.h>
|
||||||
|
#include <arpa/nameser.h>
|
||||||
#include <netinet/in.h>
|
#include <netinet/in.h>
|
||||||
#include <ctype.h>
|
#include <ctype.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
@@ -27,6 +28,7 @@
|
|||||||
#include <math.h>
|
#include <math.h>
|
||||||
#include <netdb.h>
|
#include <netdb.h>
|
||||||
#include <pthread.h>
|
#include <pthread.h>
|
||||||
|
#include <resolv.h>
|
||||||
#include <signal.h>
|
#include <signal.h>
|
||||||
#include <stdarg.h>
|
#include <stdarg.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
@@ -576,6 +578,7 @@ static int conn_send(conn_t *c, const char *msg_id, kvdict_t *d) {
|
|||||||
|
|
||||||
struct sockaddr_storage sa; socklen_t salen = 0;
|
struct sockaddr_storage sa; socklen_t salen = 0;
|
||||||
memset(&sa, 0, sizeof(sa));
|
memset(&sa, 0, sizeof(sa));
|
||||||
|
pthread_mutex_lock(&c->mu); /* conn_retarget() may change addr/af/sockfd */
|
||||||
if (c->af == AF_INET) {
|
if (c->af == AF_INET) {
|
||||||
struct sockaddr_in *s4 = (struct sockaddr_in *)&sa;
|
struct sockaddr_in *s4 = (struct sockaddr_in *)&sa;
|
||||||
s4->sin_family = AF_INET;
|
s4->sin_family = AF_INET;
|
||||||
@@ -590,7 +593,6 @@ static int conn_send(conn_t *c, const char *msg_id, kvdict_t *d) {
|
|||||||
salen = sizeof(*s6);
|
salen = sizeof(*s6);
|
||||||
}
|
}
|
||||||
|
|
||||||
pthread_mutex_lock(&c->mu);
|
|
||||||
c->last_send = now_ts();
|
c->last_send = now_ts();
|
||||||
ssize_t sent = sendto(c->sockfd, wire, wlen, 0, (struct sockaddr *)&sa, salen);
|
ssize_t sent = sendto(c->sockfd, wire, wlen, 0, (struct sockaddr *)&sa, salen);
|
||||||
pthread_mutex_unlock(&c->mu);
|
pthread_mutex_unlock(&c->mu);
|
||||||
@@ -1655,6 +1657,199 @@ static void usage(const char *prog) {
|
|||||||
" -h Help\n", prog);
|
" -h Help\n", prog);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ============================================================
|
||||||
|
* Re-resolution of the server hosts at the TTL of their A/AAAA records
|
||||||
|
* getaddrinfo() hides the TTL, so a res_search() query is made only to learn it; the
|
||||||
|
* addresses still come from getaddrinfo(). TTL unknown (IP literal, /etc/hosts name, no
|
||||||
|
* resolv.conf, timeout) -> DNS_DEFAULT_TTL. Runs in a detached thread so a slow resolver
|
||||||
|
* cannot delay heartbeats; the main loop applies the result.
|
||||||
|
* ============================================================ */
|
||||||
|
|
||||||
|
#define DNS_DEFAULT_TTL 300 /* used when the real TTL is unknown */
|
||||||
|
#define DNS_MIN_TTL 30 /* a TTL of 0 must not turn into a resolution loop */
|
||||||
|
#define DNS_FAILED_RETRY 60
|
||||||
|
|
||||||
|
typedef struct { int af; char addr[INET6_ADDRSTRLEN]; } dns_addr_t;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
const char **hosts;
|
||||||
|
int nhost, port, af_filter;
|
||||||
|
} dns_ctx_t;
|
||||||
|
|
||||||
|
static struct {
|
||||||
|
pthread_mutex_t mu;
|
||||||
|
bool ready; /* a result is waiting for the main loop */
|
||||||
|
int n;
|
||||||
|
dns_addr_t addrs[MAX_HOSTS];
|
||||||
|
} g_dns = { PTHREAD_MUTEX_INITIALIZER, false, 0, {{0, ""}} };
|
||||||
|
|
||||||
|
/* Smallest A/AAAA TTL (CNAMEs included) of host, or -1 if it cannot be determined. */
|
||||||
|
static long dns_host_ttl(const char *host, int af_filter) {
|
||||||
|
unsigned char probe[sizeof(struct in6_addr)];
|
||||||
|
if (inet_pton(AF_INET, host, probe) == 1 || inet_pton(AF_INET6, host, probe) == 1) return -1;
|
||||||
|
|
||||||
|
long best = -1;
|
||||||
|
int types[2] = { ns_t_a, ns_t_aaaa };
|
||||||
|
for (int t = 0; t < 2; t++) {
|
||||||
|
if (af_filter == AF_INET && t == 1) continue;
|
||||||
|
if (af_filter == AF_INET6 && t == 0) continue;
|
||||||
|
unsigned char ans[NS_PACKETSZ * 2];
|
||||||
|
int len = res_search(host, ns_c_in, types[t], ans, sizeof(ans));
|
||||||
|
ns_msg msg;
|
||||||
|
if (len < 0 || ns_initparse(ans, len, &msg) < 0) continue;
|
||||||
|
long min = -1; bool found = false;
|
||||||
|
for (int i = 0; i < ns_msg_count(msg, ns_s_an); i++) {
|
||||||
|
ns_rr rr;
|
||||||
|
if (ns_parserr(&msg, ns_s_an, i, &rr) < 0) continue;
|
||||||
|
int ty = ns_rr_type(rr);
|
||||||
|
if (ty != types[t] && ty != ns_t_cname) continue;
|
||||||
|
if (ty == types[t]) found = true;
|
||||||
|
if (min < 0 || (long)ns_rr_ttl(rr) < min) min = (long)ns_rr_ttl(rr);
|
||||||
|
}
|
||||||
|
if (found && (best < 0 || min < best)) best = min;
|
||||||
|
}
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Seconds until the next resolution: the smallest known TTL, floored at DNS_MIN_TTL. */
|
||||||
|
static int dns_refresh_delay(const dns_ctx_t *x) {
|
||||||
|
long best = -1;
|
||||||
|
for (int i = 0; i < x->nhost; i++) {
|
||||||
|
long t = dns_host_ttl(x->hosts[i], x->af_filter);
|
||||||
|
if (t >= 0 && (best < 0 || t < best)) best = t;
|
||||||
|
}
|
||||||
|
if (best < 0) return DNS_DEFAULT_TTL;
|
||||||
|
return best < DNS_MIN_TTL ? DNS_MIN_TTL : (best > 86400 ? 86400 : (int)best);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Every (family, address) the hosts resolve to, deduplicated; -1 if any lookup fails. */
|
||||||
|
static int dns_resolve_all(const dns_ctx_t *x, dns_addr_t *out, int max) {
|
||||||
|
int n = 0;
|
||||||
|
for (int i = 0; i < x->nhost; i++) {
|
||||||
|
struct addrinfo hints = {0}, *res = NULL;
|
||||||
|
hints.ai_socktype = SOCK_DGRAM;
|
||||||
|
hints.ai_protocol = IPPROTO_UDP;
|
||||||
|
hints.ai_family = x->af_filter;
|
||||||
|
char ps[16]; snprintf(ps, sizeof(ps), "%d", x->port);
|
||||||
|
if (getaddrinfo(x->hosts[i], ps, &hints, &res) != 0) {
|
||||||
|
LOGW("cannot resolve %s", x->hosts[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
for (struct addrinfo *ai = res; ai && n < max; ai = ai->ai_next) {
|
||||||
|
dns_addr_t a; a.af = ai->ai_family;
|
||||||
|
void *ip = (ai->ai_family == AF_INET)
|
||||||
|
? (void *)&((struct sockaddr_in *)ai->ai_addr)->sin_addr
|
||||||
|
: (void *)&((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr;
|
||||||
|
inet_ntop(ai->ai_family, ip, a.addr, sizeof(a.addr));
|
||||||
|
bool dup = false;
|
||||||
|
for (int k = 0; k < n; k++) if (strcmp(out[k].addr, a.addr) == 0) dup = true;
|
||||||
|
if (!dup) out[n++] = a;
|
||||||
|
}
|
||||||
|
freeaddrinfo(res);
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void *dns_thread(void *arg) {
|
||||||
|
const dns_ctx_t *x = arg;
|
||||||
|
int delay = dns_refresh_delay(x);
|
||||||
|
while (g_running) {
|
||||||
|
for (int s = 0; s < delay && g_running; s++) sleep(1);
|
||||||
|
if (!g_running) break;
|
||||||
|
dns_addr_t tmp[MAX_HOSTS];
|
||||||
|
int n = dns_resolve_all(x, tmp, MAX_HOSTS);
|
||||||
|
if (n <= 0) { delay = DNS_FAILED_RETRY; continue; }
|
||||||
|
pthread_mutex_lock(&g_dns.mu);
|
||||||
|
memcpy(g_dns.addrs, tmp, sizeof(tmp));
|
||||||
|
g_dns.n = n; g_dns.ready = true;
|
||||||
|
pthread_mutex_unlock(&g_dns.mu);
|
||||||
|
delay = dns_refresh_delay(x);
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Point c at a new server address; the socket is replaced if the family changed. */
|
||||||
|
static void conn_retarget(conn_t *c, int af, const char *addr) {
|
||||||
|
LOGI("server address %s -> %s", c->addr, addr);
|
||||||
|
pthread_mutex_lock(&c->mu);
|
||||||
|
if (af != c->af) {
|
||||||
|
int fd = socket(af, SOCK_DGRAM, 0);
|
||||||
|
if (fd < 0) { LOGE("socket: %s", strerror(errno)); pthread_mutex_unlock(&c->mu); return; }
|
||||||
|
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
|
||||||
|
if (c->sockfd >= 0) close(c->sockfd);
|
||||||
|
c->sockfd = fd; c->af = af;
|
||||||
|
}
|
||||||
|
snprintf(c->addr, sizeof(c->addr), "%s", addr);
|
||||||
|
pthread_mutex_unlock(&c->mu);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Main-loop side: follow the latest resolution. Mirrors hbd.client.dnsttl.retarget(). Gone
|
||||||
|
* addresses are re-pointed at new ones in list order, so g_conns[0] (the one plugin data
|
||||||
|
* uses) survives as long as anything resolves; the rest are retired, surplus new ones added. */
|
||||||
|
static void dns_apply(int *conn_id, const char *iam, int port) {
|
||||||
|
dns_addr_t r[MAX_HOSTS]; int nr;
|
||||||
|
pthread_mutex_lock(&g_dns.mu);
|
||||||
|
if (!g_dns.ready) { pthread_mutex_unlock(&g_dns.mu); return; }
|
||||||
|
memcpy(r, g_dns.addrs, sizeof(r)); nr = g_dns.n; g_dns.ready = false;
|
||||||
|
pthread_mutex_unlock(&g_dns.mu);
|
||||||
|
|
||||||
|
int gone[MAX_HOSTS], ng = 0, fresh[MAX_HOSTS], nf = 0;
|
||||||
|
for (int i = 0; i < g_nconns; i++) {
|
||||||
|
bool wanted = false;
|
||||||
|
for (int j = 0; j < nr; j++) if (strcmp(g_conns[i].addr, r[j].addr) == 0) wanted = true;
|
||||||
|
if (!wanted) gone[ng++] = i;
|
||||||
|
}
|
||||||
|
for (int j = 0; j < nr; j++) {
|
||||||
|
bool have = false;
|
||||||
|
for (int i = 0; i < g_nconns; i++) if (strcmp(g_conns[i].addr, r[j].addr) == 0) have = true;
|
||||||
|
if (!have) fresh[nf++] = j;
|
||||||
|
}
|
||||||
|
|
||||||
|
int paired = ng < nf ? ng : nf;
|
||||||
|
for (int k = 0; k < paired; k++) conn_retarget(&g_conns[gone[k]], r[fresh[k]].af, r[fresh[k]].addr);
|
||||||
|
|
||||||
|
/* drop[] holds connection indexes to retire, in ascending order */
|
||||||
|
int drop[MAX_HOSTS], nd = 0;
|
||||||
|
for (int k = paired; k < ng; k++) drop[nd++] = gone[k];
|
||||||
|
if (nd > 0 && drop[0] == 0) {
|
||||||
|
int keep = -1;
|
||||||
|
for (int i = 1; i < g_nconns && keep < 0; i++) {
|
||||||
|
bool isgone = false;
|
||||||
|
for (int k = 0; k < ng; k++) if (gone[k] == i) isgone = true;
|
||||||
|
if (!isgone) keep = i;
|
||||||
|
}
|
||||||
|
if (keep > 0) {
|
||||||
|
conn_retarget(&g_conns[0], g_conns[keep].af, g_conns[keep].addr);
|
||||||
|
drop[0] = keep;
|
||||||
|
for (int a = 0; a < nd - 1 && drop[a] > drop[a + 1]; a++) {
|
||||||
|
int tmp = drop[a]; drop[a] = drop[a + 1]; drop[a + 1] = tmp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (int k = nd - 1; k >= 0; k--) {
|
||||||
|
int i = drop[k];
|
||||||
|
if (i == 0) continue;
|
||||||
|
LOGI("dropping server address %s", g_conns[i].addr);
|
||||||
|
conn_close(&g_conns[i]);
|
||||||
|
pthread_mutex_destroy(&g_conns[i].mu);
|
||||||
|
int last = --g_nconns;
|
||||||
|
if (i != last) {
|
||||||
|
memcpy(&g_conns[i], &g_conns[last], sizeof(conn_t));
|
||||||
|
pthread_mutex_init(&g_conns[i].mu, NULL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int k = paired; k < nf && g_nconns < MAX_HOSTS; k++) {
|
||||||
|
conn_t *c = &g_conns[g_nconns];
|
||||||
|
memset(c, 0, sizeof(*c));
|
||||||
|
c->conn_id = (*conn_id)++; c->port = port;
|
||||||
|
c->af = r[fresh[k]].af; c->sockfd = -1;
|
||||||
|
snprintf(c->name, sizeof(c->name), "%s", iam);
|
||||||
|
snprintf(c->addr, sizeof(c->addr), "%s", r[fresh[k]].addr);
|
||||||
|
if (conn_open(c)) { g_nconns++; LOGI("adding server address %s", c->addr); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
int main(int argc, char **argv) {
|
||||||
bool do_boot = false;
|
bool do_boot = false;
|
||||||
bool do_daemon = false;
|
bool do_daemon = false;
|
||||||
@@ -1776,9 +1971,17 @@ int main(int argc, char **argv) {
|
|||||||
else LOGW("pthread_create(probe): %s", strerror(prc));
|
else LOGW("pthread_create(probe): %s", strerror(prc));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Re-resolution at the DNS TTL; detached, the process exit reaps it */
|
||||||
|
dns_ctx_t dns_ctx = { hosts, nhost, cfg.hb_port, af_filter };
|
||||||
|
pthread_t dtid;
|
||||||
|
int drc = pthread_create(&dtid, NULL, dns_thread, &dns_ctx);
|
||||||
|
if (drc == 0) pthread_detach(dtid);
|
||||||
|
else LOGW("pthread_create(dns): %s", strerror(drc));
|
||||||
|
|
||||||
/* Main heartbeat + receive loop */
|
/* Main heartbeat + receive loop */
|
||||||
time_t last_hb = 0;
|
time_t last_hb = 0;
|
||||||
while (g_running) {
|
while (g_running) {
|
||||||
|
dns_apply(&conn_id, iam, cfg.hb_port);
|
||||||
time_t now = time(NULL);
|
time_t now = time(NULL);
|
||||||
if (now - last_hb >= cfg.interval) {
|
if (now - last_hb >= cfg.interval) {
|
||||||
for (int i = 0; i < g_nconns; i++) {
|
for (int i = 0; i < g_nconns; i++) {
|
||||||
|
|||||||
@@ -226,6 +226,55 @@ static int live_mode(void) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---- server address re-resolution (dns_apply) ---- */
|
||||||
|
|
||||||
|
static void dns_conn(int i, int af, const char *addr) {
|
||||||
|
memset(&g_conns[i], 0, sizeof(conn_t));
|
||||||
|
g_conns[i].conn_id = i + 1; g_conns[i].af = af; g_conns[i].port = 5;
|
||||||
|
snprintf(g_conns[i].addr, sizeof(g_conns[i].addr), "%s", addr);
|
||||||
|
conn_open(&g_conns[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void dns_push(int n, const dns_addr_t *a) {
|
||||||
|
memcpy(g_dns.addrs, a, n * sizeof(*a)); g_dns.n = n; g_dns.ready = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_dns_apply_follows_changes(void) {
|
||||||
|
int id = 3;
|
||||||
|
dns_conn(0, AF_INET, "1.1.1.1"); dns_conn(1, AF_INET6, "2001:db8::1"); g_nconns = 2;
|
||||||
|
|
||||||
|
dns_addr_t moved[2] = {{AF_INET, "9.9.9.9"}, {AF_INET6, "2001:db8::9"}};
|
||||||
|
dns_push(2, moved); dns_apply(&id, "me", 5);
|
||||||
|
CHECK(g_nconns == 2);
|
||||||
|
CHECK_STR(g_conns[0].addr, "9.9.9.9");
|
||||||
|
CHECK_STR(g_conns[1].addr, "2001:db8::9");
|
||||||
|
|
||||||
|
/* the first connection's address vanishes and nothing replaces it: it survives, the other goes */
|
||||||
|
dns_addr_t v6only[1] = {{AF_INET6, "2001:db8::9"}};
|
||||||
|
dns_push(1, v6only); dns_apply(&id, "me", 5);
|
||||||
|
CHECK(g_nconns == 1);
|
||||||
|
CHECK_STR(g_conns[0].addr, "2001:db8::9");
|
||||||
|
CHECK(g_conns[0].af == AF_INET6);
|
||||||
|
|
||||||
|
dns_addr_t grown[2] = {{AF_INET6, "2001:db8::9"}, {AF_INET, "5.5.5.5"}};
|
||||||
|
dns_push(2, grown); dns_apply(&id, "me", 5);
|
||||||
|
CHECK(g_nconns == 2);
|
||||||
|
CHECK_STR(g_conns[1].addr, "5.5.5.5");
|
||||||
|
CHECK(g_conns[1].conn_id == 3);
|
||||||
|
|
||||||
|
dns_push(2, grown); dns_apply(&id, "me", 5); /* unchanged: nothing happens */
|
||||||
|
CHECK(g_nconns == 2 && id == 4);
|
||||||
|
for (int i = 0; i < g_nconns; i++) conn_close(&g_conns[i]);
|
||||||
|
g_nconns = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_dns_ttl_unknown_for_ip_literals(void) {
|
||||||
|
CHECK(dns_host_ttl("192.0.2.1", 0) == -1);
|
||||||
|
CHECK(dns_host_ttl("2001:db8::1", 0) == -1);
|
||||||
|
const char *h[] = {"192.0.2.1"}; dns_ctx_t x = {h, 1, 5, 0};
|
||||||
|
CHECK(dns_refresh_delay(&x) == DNS_DEFAULT_TTL);
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char **argv) {
|
int main(int argc, char **argv) {
|
||||||
if (argc > 1 && !strcmp(argv[1], "--live")) return live_mode();
|
if (argc > 1 && !strcmp(argv[1], "--live")) return live_mode();
|
||||||
test_gateway_parsers();
|
test_gateway_parsers();
|
||||||
@@ -238,6 +287,8 @@ int main(int argc, char **argv) {
|
|||||||
test_route_loss_then_return_pings_immediately();
|
test_route_loss_then_return_pings_immediately();
|
||||||
test_missing_ping_stays_unknown_even_after_route_loss();
|
test_missing_ping_stays_unknown_even_after_route_loss();
|
||||||
test_unsupported_family_is_unknown();
|
test_unsupported_family_is_unknown();
|
||||||
|
test_dns_apply_follows_changes();
|
||||||
|
test_dns_ttl_unknown_for_ip_literals();
|
||||||
printf("%d checks, %d failed\n", g_run, g_fail);
|
printf("%d checks, %d failed\n", g_run, g_fail);
|
||||||
return g_fail ? 1 : 0;
|
return g_fail ? 1 : 0;
|
||||||
}
|
}
|
||||||
|
|||||||
+215
-1
@@ -28,6 +28,7 @@ IPv4) path as an outage. Config, in the network_monitor section of ~/.hbc.json:
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import ipaddress
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
@@ -36,6 +37,7 @@ import re
|
|||||||
import shutil
|
import shutil
|
||||||
import signal
|
import signal
|
||||||
import socket
|
import socket
|
||||||
|
import struct
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
@@ -48,7 +50,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple
|
from typing import Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple
|
||||||
|
|
||||||
# updated by scripts/bumpminor.sh
|
# updated by scripts/bumpminor.sh
|
||||||
__version__ = "5.4.5"
|
__version__ = "5.4.7"
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Protocol (mirrors hbd/common/proto.py)
|
# Protocol (mirrors hbd/common/proto.py)
|
||||||
@@ -1164,6 +1166,14 @@ class AsyncConnection:
|
|||||||
self._transport.close()
|
self._transport.close()
|
||||||
self._transport = None
|
self._transport = None
|
||||||
|
|
||||||
|
def set_target(self, addr: str, af: int):
|
||||||
|
"""Re-point at a new server address; the socket is reopened on the next send if the family changed."""
|
||||||
|
if af != self.af:
|
||||||
|
self.close()
|
||||||
|
self.af = af
|
||||||
|
self.addr = addr
|
||||||
|
self._log = logging.getLogger(f"hbc.conn.{addr}")
|
||||||
|
|
||||||
def _handle_ack(self, msg: Dict[str, Any], now: float):
|
def _handle_ack(self, msg: Dict[str, Any], now: float):
|
||||||
rtt = (now - self.lastsend) * 1000.0
|
rtt = (now - self.lastsend) * 1000.0
|
||||||
self.rtts.append(rtt)
|
self.rtts.append(rtt)
|
||||||
@@ -1188,6 +1198,207 @@ class AsyncConnection:
|
|||||||
self.lastsend = time.time()
|
self.lastsend = time.time()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Re-resolution of the server hosts at the TTL of their A/AAAA records
|
||||||
|
# getaddrinfo() hides the TTL, so a minimal stub-resolver query is made only to learn it; the
|
||||||
|
# addresses still come from getaddrinfo(). TTL unknown (IP literal, /etc/hosts name, no
|
||||||
|
# resolv.conf, timeout) -> _DNS_DEFAULT_TTL.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
log = logging.getLogger("hbc.dns")
|
||||||
|
|
||||||
|
_DNS_DEFAULT_TTL = 300 # used when the real TTL is unknown
|
||||||
|
_DNS_MIN_TTL = 30 # a TTL of 0 must not turn into a resolution loop
|
||||||
|
_DNS_FAILED_RETRY = 60 # re-try delay after a failed resolution
|
||||||
|
_DNS_QUERY_TIMEOUT = 2.0
|
||||||
|
|
||||||
|
_DNS_TYPE_A, _DNS_TYPE_CNAME, _DNS_TYPE_AAAA = 1, 5, 28
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_parse_resolv_conf(path: str = "/etc/resolv.conf") -> Tuple[List[str], List[str]]:
|
||||||
|
"""Return (nameservers, search domains)."""
|
||||||
|
servers: List[str] = []
|
||||||
|
search: List[str] = []
|
||||||
|
try:
|
||||||
|
with open(path) as f:
|
||||||
|
for line in f:
|
||||||
|
parts = line.split()
|
||||||
|
if len(parts) < 2:
|
||||||
|
continue
|
||||||
|
if parts[0] == "nameserver":
|
||||||
|
servers.append(parts[1].split("%")[0])
|
||||||
|
elif parts[0] in ("search", "domain"):
|
||||||
|
search = parts[1:]
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return servers, search
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_build_query(name: str, qtype: int, qid: int) -> bytes:
|
||||||
|
out = struct.pack("!HHHHHH", qid, 0x0100, 1, 0, 0, 0)
|
||||||
|
for label in name.rstrip(".").split("."):
|
||||||
|
out += bytes([len(label)]) + label.encode("idna")
|
||||||
|
return out + b"\x00" + struct.pack("!HH", qtype, 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_skip_name(buf: bytes, off: int) -> int:
|
||||||
|
while True:
|
||||||
|
n = buf[off]
|
||||||
|
if n == 0:
|
||||||
|
return off + 1
|
||||||
|
if n & 0xC0:
|
||||||
|
return off + 2
|
||||||
|
off += n + 1
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_min_answer_ttl(resp: bytes, qid: int, qtype: int) -> Optional[int]:
|
||||||
|
"""Smallest TTL among the answer RRs (CNAMEs included), or None if there is no qtype answer."""
|
||||||
|
try:
|
||||||
|
rid, flags, qd, an = struct.unpack("!HHHH", resp[:8])
|
||||||
|
if rid != qid or flags & 0x000F:
|
||||||
|
return None
|
||||||
|
off = 12
|
||||||
|
for _ in range(qd):
|
||||||
|
off = _dns_skip_name(resp, off) + 4
|
||||||
|
ttls: List[int] = []
|
||||||
|
found = False
|
||||||
|
for _ in range(an):
|
||||||
|
off = _dns_skip_name(resp, off)
|
||||||
|
rtype, _cls, ttl, rdlen = struct.unpack("!HHIH", resp[off:off + 10])
|
||||||
|
off += 10 + rdlen
|
||||||
|
if rtype in (qtype, _DNS_TYPE_CNAME):
|
||||||
|
ttls.append(ttl)
|
||||||
|
found = found or rtype == qtype
|
||||||
|
return min(ttls) if found else None
|
||||||
|
except (struct.error, IndexError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_query_ttl(server: str, name: str, qtype: int) -> Optional[int]:
|
||||||
|
qid = struct.unpack("!H", os.urandom(2))[0]
|
||||||
|
try:
|
||||||
|
af = socket.AF_INET6 if ":" in server else socket.AF_INET
|
||||||
|
with socket.socket(af, socket.SOCK_DGRAM) as s:
|
||||||
|
s.settimeout(_DNS_QUERY_TIMEOUT)
|
||||||
|
s.sendto(_dns_build_query(name, qtype, qid), (server, 53))
|
||||||
|
return _dns_min_answer_ttl(s.recv(4096), qid, qtype)
|
||||||
|
except (OSError, UnicodeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_lookup_ttl(host: str, af_filter: int = 0) -> Optional[int]:
|
||||||
|
"""Blocking. Smallest A/AAAA TTL for host, or None when it cannot be determined."""
|
||||||
|
try:
|
||||||
|
ipaddress.ip_address(host)
|
||||||
|
return None
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
servers, search = _dns_parse_resolv_conf()
|
||||||
|
if host.endswith("."):
|
||||||
|
names = [host]
|
||||||
|
elif "." in host:
|
||||||
|
names = [host] + [f"{host}.{d}" for d in search]
|
||||||
|
else:
|
||||||
|
names = [f"{host}.{d}" for d in search] + [host]
|
||||||
|
qtypes = [t for t, af in ((_DNS_TYPE_A, socket.AF_INET), (_DNS_TYPE_AAAA, socket.AF_INET6))
|
||||||
|
if af_filter in (0, af)]
|
||||||
|
for name in names:
|
||||||
|
ttls = []
|
||||||
|
for qtype in qtypes:
|
||||||
|
for server in servers:
|
||||||
|
ttl = _dns_query_ttl(server, name, qtype)
|
||||||
|
if ttl is not None:
|
||||||
|
ttls.append(ttl)
|
||||||
|
break
|
||||||
|
if ttls:
|
||||||
|
return min(ttls)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_refresh_delay(ttls: List[Optional[int]]) -> int:
|
||||||
|
"""Seconds until the next resolution: the smallest known TTL, floored at _DNS_MIN_TTL."""
|
||||||
|
known = [t for t in ttls if t is not None]
|
||||||
|
return max(min(known), _DNS_MIN_TTL) if known else _DNS_DEFAULT_TTL
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_resolve(hosts: List[str], port: int, af_filter: int) -> Optional[List[Tuple[int, str]]]:
|
||||||
|
"""Blocking. Every (family, address) the hosts resolve to, deduplicated, or None if any lookup fails."""
|
||||||
|
found: List[Tuple[int, str]] = []
|
||||||
|
for host in hosts:
|
||||||
|
try:
|
||||||
|
infos = socket.getaddrinfo(host, port, af_filter, 0, socket.SOL_UDP)
|
||||||
|
except socket.gaierror as e:
|
||||||
|
log.warning("cannot resolve %s: %s", host, e)
|
||||||
|
return None
|
||||||
|
for info in infos:
|
||||||
|
entry = (info[0], str(info[4][0]))
|
||||||
|
if entry not in found:
|
||||||
|
found.append(entry)
|
||||||
|
return found or None
|
||||||
|
|
||||||
|
|
||||||
|
def _dns_retarget(connections: list, resolved: List[Tuple[int, str]]) -> Tuple[List[Tuple[int, str]], list]:
|
||||||
|
"""Point the connections at the resolved addresses, in place.
|
||||||
|
|
||||||
|
A connection whose address is gone is re-pointed at an address that is new, in list order, so
|
||||||
|
the first connection (the one plugin data uses) survives as long as anything resolves.
|
||||||
|
Returns (new addresses still needing a connection, connections to drop).
|
||||||
|
"""
|
||||||
|
current = {c.addr for c in connections}
|
||||||
|
wanted = {addr for _, addr in resolved}
|
||||||
|
new = [(af, addr) for af, addr in resolved if addr not in current]
|
||||||
|
gone = [c for c in connections if c.addr not in wanted]
|
||||||
|
paired = min(len(gone), len(new))
|
||||||
|
for conn, (af, addr) in zip(gone, new):
|
||||||
|
log.info("server address %s -> %s", conn.addr, addr)
|
||||||
|
conn.set_target(addr, af)
|
||||||
|
drop = gone[paired:]
|
||||||
|
if connections and connections[0] in drop:
|
||||||
|
# plugin data goes out on the first connection: hand it a surviving address and drop that one instead
|
||||||
|
keep = next(c for c in connections if c not in gone)
|
||||||
|
connections[0].set_target(keep.addr, keep.af)
|
||||||
|
drop.remove(connections[0])
|
||||||
|
drop.append(keep)
|
||||||
|
return new[paired:], drop
|
||||||
|
|
||||||
|
|
||||||
|
async def _dns_refresher(connections: List["AsyncConnection"], hosts: List[str], port: int, af_filter: int,
|
||||||
|
name: str, interval: int, probe: Optional[_Connectivity], next_id: int):
|
||||||
|
"""Re-resolve the server hosts every TTL seconds and follow address changes.
|
||||||
|
|
||||||
|
A failed lookup leaves the connections untouched. Addresses that disappear are re-pointed at
|
||||||
|
new ones where possible; the rest are retired, and surplus new ones get a connection.
|
||||||
|
"""
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
|
||||||
|
def ttl_delay() -> int:
|
||||||
|
return _dns_refresh_delay([_dns_lookup_ttl(h, af_filter) for h in hosts])
|
||||||
|
|
||||||
|
delay = await loop.run_in_executor(None, ttl_delay)
|
||||||
|
while _running:
|
||||||
|
await _sleep(delay)
|
||||||
|
if not _running:
|
||||||
|
break
|
||||||
|
resolved = await loop.run_in_executor(None, _dns_resolve, hosts, port, af_filter)
|
||||||
|
if resolved is None:
|
||||||
|
delay = _DNS_FAILED_RETRY
|
||||||
|
continue
|
||||||
|
new, gone = _dns_retarget(connections, resolved)
|
||||||
|
for conn in gone:
|
||||||
|
if len(connections) > 1:
|
||||||
|
log.info("dropping server address %s", conn.addr)
|
||||||
|
conn._dead = True
|
||||||
|
conn.close()
|
||||||
|
connections.remove(conn)
|
||||||
|
for af, addr in new:
|
||||||
|
conn = AsyncConnection(next_id, addr, port, af, name)
|
||||||
|
next_id += 1
|
||||||
|
log.info("adding server address %s", addr)
|
||||||
|
connections.append(conn)
|
||||||
|
_active_tasks.append(asyncio.create_task(_heartbeat_sender(conn, interval, probe)))
|
||||||
|
delay = await loop.run_in_executor(None, ttl_delay)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Server command handlers
|
# Server command handlers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -1463,6 +1674,9 @@ async def _async_main(args, cfg: Dict[str, Any]) -> int:
|
|||||||
for conn in connections:
|
for conn in connections:
|
||||||
_active_tasks.append(asyncio.create_task(_heartbeat_sender(conn, interval, _probe)))
|
_active_tasks.append(asyncio.create_task(_heartbeat_sender(conn, interval, _probe)))
|
||||||
|
|
||||||
|
_active_tasks.append(asyncio.create_task(
|
||||||
|
_dns_refresher(connections, args.hosts, port, af_filter, iam, interval, _probe, conn_id)))
|
||||||
|
|
||||||
if plugins and connections:
|
if plugins and connections:
|
||||||
_active_tasks.append(asyncio.create_task(_plugin_collector(connections[0], plugins)))
|
_active_tasks.append(asyncio.create_task(_plugin_collector(connections[0], plugins)))
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
"""TTL-driven re-resolution of the server hostnames in the full client."""
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
|
||||||
|
from hbd.client import dnsttl
|
||||||
|
|
||||||
|
|
||||||
|
def _rr(rtype, ttl, rdata):
|
||||||
|
return b"\xc0\x0c" + struct.pack("!HHIH", rtype, 1, ttl, len(rdata)) + rdata
|
||||||
|
|
||||||
|
|
||||||
|
def _response(qid, qtype, answers, rcode=0):
|
||||||
|
q = dnsttl.build_query("hbd.example.com", qtype, qid)[12:]
|
||||||
|
return struct.pack("!HHHHHH", qid, 0x8180 | rcode, 1, len(answers), 0, 0) + q + b"".join(answers)
|
||||||
|
|
||||||
|
|
||||||
|
def test_min_ttl_covers_cname_and_address():
|
||||||
|
resp = _response(7, dnsttl.TYPE_A, [_rr(dnsttl.TYPE_CNAME, 60, b"\x01x\x00"),
|
||||||
|
_rr(dnsttl.TYPE_A, 300, b"\x01\x02\x03\x04")])
|
||||||
|
assert dnsttl.min_answer_ttl(resp, 7, dnsttl.TYPE_A) == 60
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_answer_of_requested_type_is_unknown():
|
||||||
|
assert dnsttl.min_answer_ttl(_response(7, dnsttl.TYPE_AAAA, []), 7, dnsttl.TYPE_AAAA) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_rcode_and_foreign_id_are_unknown():
|
||||||
|
ans = [_rr(dnsttl.TYPE_A, 300, b"\x01\x02\x03\x04")]
|
||||||
|
assert dnsttl.min_answer_ttl(_response(7, 1, ans, rcode=3), 7, 1) is None
|
||||||
|
assert dnsttl.min_answer_ttl(_response(7, 1, ans), 8, 1) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_truncated_response_is_unknown():
|
||||||
|
assert dnsttl.min_answer_ttl(b"\x00\x07\x81", 7, 1) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_refresh_delay():
|
||||||
|
assert dnsttl.refresh_delay([300, 120, None]) == 120
|
||||||
|
assert dnsttl.refresh_delay([0]) == dnsttl.MIN_TTL
|
||||||
|
assert dnsttl.refresh_delay([None]) == dnsttl.DEFAULT_TTL
|
||||||
|
|
||||||
|
|
||||||
|
def test_ip_literal_has_no_ttl():
|
||||||
|
assert dnsttl.lookup_ttl("192.0.2.1") is None
|
||||||
|
assert dnsttl.lookup_ttl("2001:db8::1") is None
|
||||||
|
|
||||||
|
|
||||||
|
class _C:
|
||||||
|
def __init__(self, addr, af=socket.AF_INET):
|
||||||
|
self.addr, self.af = addr, af
|
||||||
|
|
||||||
|
def set_target(self, addr, af):
|
||||||
|
self.addr, self.af = addr, af
|
||||||
|
|
||||||
|
|
||||||
|
V4, V6 = socket.AF_INET, socket.AF_INET6
|
||||||
|
|
||||||
|
|
||||||
|
def test_retarget_follows_address_change_in_place():
|
||||||
|
first, second = _C("1.1.1.1"), _C("2001:db8::1", V6)
|
||||||
|
new, gone = dnsttl.retarget([first, second], [(V4, "9.9.9.9"), (V6, "2001:db8::9")])
|
||||||
|
assert (new, gone) == ([], [])
|
||||||
|
assert (first.addr, second.addr) == ("9.9.9.9", "2001:db8::9")
|
||||||
|
|
||||||
|
|
||||||
|
def test_retarget_unchanged_is_a_noop():
|
||||||
|
conns = [_C("1.1.1.1")]
|
||||||
|
assert dnsttl.retarget(conns, [(V4, "1.1.1.1")]) == ([], [])
|
||||||
|
|
||||||
|
|
||||||
|
def test_retarget_reports_surplus_new_and_gone():
|
||||||
|
a, b = _C("1.1.1.1"), _C("2.2.2.2")
|
||||||
|
new, gone = dnsttl.retarget([a, b], [(V4, "3.3.3.3")])
|
||||||
|
assert new == [] and gone == [b] and a.addr == "3.3.3.3"
|
||||||
|
new, gone = dnsttl.retarget([a], [(V4, "3.3.3.3"), (V4, "4.4.4.4")])
|
||||||
|
assert new == [(V4, "4.4.4.4")] and gone == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_retarget_never_drops_the_first_connection():
|
||||||
|
a, b = _C("1.1.1.1"), _C("2.2.2.2")
|
||||||
|
new, gone = dnsttl.retarget([a, b], [(V4, "2.2.2.2")])
|
||||||
|
assert new == [] and gone == [b] and a.addr == "2.2.2.2"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_failure_returns_none(monkeypatch):
|
||||||
|
def boom(*a):
|
||||||
|
raise socket.gaierror("nope")
|
||||||
|
monkeypatch.setattr(socket, "getaddrinfo", boom)
|
||||||
|
assert dnsttl.resolve(["x.invalid"], 1, 0) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_refresher_follows_a_change_and_adds_a_family(monkeypatch):
|
||||||
|
import asyncio
|
||||||
|
from hbd.client import main as hbc
|
||||||
|
|
||||||
|
answers = [[(V4, "9.9.9.9"), (V6, "2001:db8::9")], None, [(V4, "9.9.9.9")]]
|
||||||
|
monkeypatch.setattr(dnsttl, "resolve", lambda *a: answers.pop(0) if len(answers) > 1 else answers[0])
|
||||||
|
monkeypatch.setattr(dnsttl, "lookup_ttl", lambda *a: None)
|
||||||
|
monkeypatch.setattr(dnsttl, "DEFAULT_TTL", 0.01)
|
||||||
|
monkeypatch.setattr(dnsttl, "FAILED_RETRY", 0.01)
|
||||||
|
|
||||||
|
async def idle(*a):
|
||||||
|
await asyncio.sleep(10)
|
||||||
|
monkeypatch.setattr(hbc, "heartbeat_sender", idle)
|
||||||
|
|
||||||
|
async def go():
|
||||||
|
hbc.shutdown_event = asyncio.Event()
|
||||||
|
hbc.active_tasks = []
|
||||||
|
conns = [hbc.AsyncConnection(1, "1.1.1.1", 5, V4, "me")]
|
||||||
|
task = asyncio.create_task(hbc.dns_refresher(conns, ["h"], 5, 0, "me", 10, None, 2))
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
|
hbc.shutdown_event.set()
|
||||||
|
await task
|
||||||
|
for t in hbc.active_tasks:
|
||||||
|
t.cancel()
|
||||||
|
return conns
|
||||||
|
|
||||||
|
conns = asyncio.run(go())
|
||||||
|
assert [(c.conn_id, c.addr) for c in conns] == [(1, "9.9.9.9")]
|
||||||
@@ -1,5 +1,3 @@
|
|||||||
import glob
|
|
||||||
import os
|
|
||||||
import pytest
|
import pytest
|
||||||
from hbd.server import configio
|
from hbd.server import configio
|
||||||
|
|
||||||
|
|||||||
+37
-82
@@ -1,9 +1,8 @@
|
|||||||
import time
|
import asyncio
|
||||||
import queue
|
|
||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import patch, MagicMock
|
from unittest.mock import patch, MagicMock
|
||||||
|
|
||||||
import hbd.dns as dns
|
import hbd.server.dns as dns
|
||||||
|
|
||||||
|
|
||||||
class TestDNS(unittest.TestCase):
|
class TestDNS(unittest.TestCase):
|
||||||
@@ -17,7 +16,7 @@ class TestDNS(unittest.TestCase):
|
|||||||
p = dns.create_nsupdate_payload("host", "2001:db8::1", "example")
|
p = dns.create_nsupdate_payload("host", "2001:db8::1", "example")
|
||||||
self.assertIn("update add host.dy.example 5 AAAA 2001:db8::1", p)
|
self.assertIn("update add host.dy.example 5 AAAA 2001:db8::1", p)
|
||||||
|
|
||||||
@patch("hbd.dns.Popen")
|
@patch("hbd.server.dns.Popen")
|
||||||
def test_nsupdate_success(self, mock_popen):
|
def test_nsupdate_success(self, mock_popen):
|
||||||
proc = MagicMock()
|
proc = MagicMock()
|
||||||
proc.communicate.return_value = (b"status: NOERROR", None)
|
proc.communicate.return_value = (b"status: NOERROR", None)
|
||||||
@@ -41,7 +40,7 @@ class TestDNS(unittest.TestCase):
|
|||||||
stderr=dns.STDOUT,
|
stderr=dns.STDOUT,
|
||||||
)
|
)
|
||||||
|
|
||||||
@patch("hbd.dns.Popen")
|
@patch("hbd.server.dns.Popen")
|
||||||
def test_nsupdate_failure(self, mock_popen):
|
def test_nsupdate_failure(self, mock_popen):
|
||||||
proc = MagicMock()
|
proc = MagicMock()
|
||||||
proc.communicate.return_value = (b"some error", None)
|
proc.communicate.return_value = (b"some error", None)
|
||||||
@@ -57,83 +56,7 @@ class TestDNS(unittest.TestCase):
|
|||||||
self.assertIsNotNone(err)
|
self.assertIsNotNone(err)
|
||||||
self.assertIn("some error", err)
|
self.assertIn("some error", err)
|
||||||
|
|
||||||
def test_dnsupdatethread_processes_queue(self):
|
@patch("hbd.server.dns.Popen")
|
||||||
# patch nsupdate to succeed
|
|
||||||
with patch("hbd.dns.nsupdate", return_value=None):
|
|
||||||
logs = []
|
|
||||||
|
|
||||||
def log(h, m):
|
|
||||||
logs.append((h, m))
|
|
||||||
|
|
||||||
emails = []
|
|
||||||
|
|
||||||
def email(s, m):
|
|
||||||
emails.append((s, m))
|
|
||||||
|
|
||||||
class FakeHost:
|
|
||||||
dnsQ = queue.Queue()
|
|
||||||
|
|
||||||
class FakeHbd:
|
|
||||||
Host = FakeHost
|
|
||||||
|
|
||||||
# start the thread (daemon) that processes the queue
|
|
||||||
t = dns.start_dns_thread(
|
|
||||||
FakeHbd, {"dyndomains": ["example"]}, log=log, email=email
|
|
||||||
)
|
|
||||||
self.assertTrue(t.is_alive())
|
|
||||||
|
|
||||||
# enqueue one item and wait for it to be processed (polling with timeout)
|
|
||||||
FakeHbd.Host.dnsQ.put(("testhost", "1.2.3.4"))
|
|
||||||
|
|
||||||
for _ in range(30):
|
|
||||||
if logs:
|
|
||||||
break
|
|
||||||
time.sleep(0.1)
|
|
||||||
|
|
||||||
self.assertTrue(logs, "dnsupdatethread did not call log")
|
|
||||||
self.assertTrue(
|
|
||||||
any("changed address" in m or "DNS updated" in m for (_h, m) in logs)
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_dnsupdatethread_calls_email_on_failure(self):
|
|
||||||
# patch nsupdate to fail with an error message
|
|
||||||
with patch("hbd.dns.nsupdate", return_value="error: failed"):
|
|
||||||
logs = []
|
|
||||||
|
|
||||||
def log(h, m):
|
|
||||||
logs.append((h, m))
|
|
||||||
|
|
||||||
emails = []
|
|
||||||
|
|
||||||
def email(s, m):
|
|
||||||
emails.append((s, m))
|
|
||||||
|
|
||||||
class FakeHost:
|
|
||||||
dnsQ = queue.Queue()
|
|
||||||
|
|
||||||
class FakeHbd:
|
|
||||||
Host = FakeHost
|
|
||||||
|
|
||||||
dns.start_dns_thread(
|
|
||||||
FakeHbd, {"dyndomains": ["example"]}, log=log, email=email
|
|
||||||
)
|
|
||||||
# enqueue and wait for the email to be sent
|
|
||||||
FakeHbd.Host.dnsQ.put(("testhost", "1.2.3.4"))
|
|
||||||
|
|
||||||
for _ in range(30):
|
|
||||||
if emails:
|
|
||||||
break
|
|
||||||
time.sleep(0.1)
|
|
||||||
|
|
||||||
self.assertTrue(emails, "dnsupdatethread did not call email on failure")
|
|
||||||
self.assertTrue(
|
|
||||||
any(
|
|
||||||
"nsupdate failed" in s or "nsupdate failed" in m or "error" in m
|
|
||||||
for (s, m) in emails
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
@patch("hbd.dns.Popen")
|
|
||||||
def test_nsupdate_raises_oserror(self, mock_popen):
|
def test_nsupdate_raises_oserror(self, mock_popen):
|
||||||
mock_popen.side_effect = OSError("noexec")
|
mock_popen.side_effect = OSError("noexec")
|
||||||
err = dns.nsupdate(
|
err = dns.nsupdate(
|
||||||
@@ -147,5 +70,37 @@ class TestDNS(unittest.TestCase):
|
|||||||
self.assertIn("execution failed", err)
|
self.assertIn("execution failed", err)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDNSWorker(unittest.IsolatedAsyncioTestCase):
|
||||||
|
|
||||||
|
async def _run_worker(self, nsupdate_result):
|
||||||
|
"""Feed one update through dns_update_worker; returns the (name, level, message) log calls."""
|
||||||
|
logs = []
|
||||||
|
queue = asyncio.Queue()
|
||||||
|
await queue.put(("testhost", "1.2.3.4"))
|
||||||
|
await queue.put(None)
|
||||||
|
with patch("hbd.server.dns.nsupdate", return_value=nsupdate_result):
|
||||||
|
await dns.dns_update_worker(
|
||||||
|
None, {"dyndomains": ["example"]}, async_queue=queue,
|
||||||
|
log=lambda name, level, msg: logs.append((name, level, msg)),
|
||||||
|
)
|
||||||
|
return logs
|
||||||
|
|
||||||
|
async def test_worker_logs_successful_update(self):
|
||||||
|
logs = await self._run_worker(None)
|
||||||
|
self.assertEqual(len(logs), 1)
|
||||||
|
name, level, msg = logs[0]
|
||||||
|
self.assertEqual((name, level), ("testhost", "INFO"))
|
||||||
|
self.assertIn("DNS updated testhost.dy.example", msg)
|
||||||
|
self.assertIn("1.2.3.4", msg)
|
||||||
|
|
||||||
|
async def test_worker_logs_error_on_failed_update(self):
|
||||||
|
logs = await self._run_worker("error: failed")
|
||||||
|
self.assertEqual(len(logs), 1)
|
||||||
|
name, level, msg = logs[0]
|
||||||
|
self.assertEqual((name, level), ("testhost", "ERROR"))
|
||||||
|
self.assertIn("DNS update failed", msg)
|
||||||
|
self.assertIn("error: failed", msg)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from hbd.udp import handle_datagram, parse_message
|
from hbd.server.udp import handle_datagram, parse_message
|
||||||
from hbd.proto import dicttos
|
from hbd.common.proto import dicttos, stodict
|
||||||
|
|
||||||
|
|
||||||
class FakeTransport:
|
class FakeTransport:
|
||||||
@@ -17,7 +17,7 @@ def dummy_noop(*a, **k):
|
|||||||
def test_handle_cmd_sends_command():
|
def test_handle_cmd_sends_command():
|
||||||
ftr = FakeTransport()
|
ftr = FakeTransport()
|
||||||
# prepare ctx linking to hbdclass
|
# prepare ctx linking to hbdclass
|
||||||
import hbdclass
|
from hbd.server import hbdclass
|
||||||
|
|
||||||
ctx = {
|
ctx = {
|
||||||
"config": {"watchhosts": []},
|
"config": {"watchhosts": []},
|
||||||
@@ -34,7 +34,7 @@ def test_handle_cmd_sends_command():
|
|||||||
# create host by sending initial heartbeat
|
# create host by sending initial heartbeat
|
||||||
msg = parse_message(dicttos("HTB", {"name": "cmdhost", "interval": 10}))
|
msg = parse_message(dicttos("HTB", {"name": "cmdhost", "interval": 10}))
|
||||||
handle_datagram(msg, ("127.0.0.1", 50000), ftr, ctx)
|
handle_datagram(msg, ("127.0.0.1", 50000), ftr, ctx)
|
||||||
assert ftr.sent[0][0] == b"ACK"
|
assert ftr.sent[0][0].startswith(b"!ACK:")
|
||||||
|
|
||||||
# queue a CMD for the host and send another heartbeat; expect command sent
|
# queue a CMD for the host and send another heartbeat; expect command sent
|
||||||
h = hbdclass.Host.hosts["cmdhost"]
|
h = hbdclass.Host.hosts["cmdhost"]
|
||||||
@@ -42,8 +42,6 @@ def test_handle_cmd_sends_command():
|
|||||||
ftr.sent.clear()
|
ftr.sent.clear()
|
||||||
msg2 = parse_message(dicttos("HTB", {"name": "cmdhost", "interval": 10}))
|
msg2 = parse_message(dicttos("HTB", {"name": "cmdhost", "interval": 10}))
|
||||||
handle_datagram(msg2, ("127.0.0.1", 50000), ftr, ctx)
|
handle_datagram(msg2, ("127.0.0.1", 50000), ftr, ctx)
|
||||||
# should have sent ACK and the command; last send should be non-empty
|
# the ACK and the queued command are both sent; the command is a compressed CMD packet
|
||||||
assert len(ftr.sent) >= 1
|
decoded = [stodict(data) for data, _addr in ftr.sent]
|
||||||
# the command for ??0 will be sent as raw cmd string
|
assert any(d["ID"] == "CMD" and d["cmd"] == "doit" for d in decoded)
|
||||||
# so at least one send contains b'doit' or similar
|
|
||||||
assert any(b"doit" in s[0] for s in ftr.sent)
|
|
||||||
|
|||||||
@@ -0,0 +1,72 @@
|
|||||||
|
"""TTL-driven re-resolution in the standalone scripts/hbc_mini.py."""
|
||||||
|
import asyncio
|
||||||
|
import importlib.util
|
||||||
|
import pathlib
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
|
||||||
|
_PATH = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "hbc_mini.py"
|
||||||
|
_SPEC = importlib.util.spec_from_file_location("hbc_mini", _PATH)
|
||||||
|
mini = importlib.util.module_from_spec(_SPEC)
|
||||||
|
sys.modules["hbc_mini"] = mini
|
||||||
|
_SPEC.loader.exec_module(mini)
|
||||||
|
|
||||||
|
V4, V6 = socket.AF_INET, socket.AF_INET6
|
||||||
|
|
||||||
|
|
||||||
|
def _rr(rtype, ttl, rdata):
|
||||||
|
return b"\xc0\x0c" + struct.pack("!HHIH", rtype, 1, ttl, len(rdata)) + rdata
|
||||||
|
|
||||||
|
|
||||||
|
def _response(qid, qtype, answers, rcode=0):
|
||||||
|
q = mini._dns_build_query("hbd.example.com", qtype, qid)[12:]
|
||||||
|
return struct.pack("!HHHHHH", qid, 0x8180 | rcode, 1, len(answers), 0, 0) + q + b"".join(answers)
|
||||||
|
|
||||||
|
|
||||||
|
def test_min_ttl_covers_cname_and_address():
|
||||||
|
resp = _response(7, 1, [_rr(5, 60, b"\x01x\x00"), _rr(1, 300, b"\x01\x02\x03\x04")])
|
||||||
|
assert mini._dns_min_answer_ttl(resp, 7, 1) == 60
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_ttl_cases():
|
||||||
|
assert mini._dns_min_answer_ttl(_response(7, 28, []), 7, 28) is None
|
||||||
|
assert mini._dns_min_answer_ttl(_response(7, 1, [_rr(1, 9, b"1234")], rcode=3), 7, 1) is None
|
||||||
|
assert mini._dns_min_answer_ttl(b"\x00\x07\x81", 7, 1) is None
|
||||||
|
assert mini._dns_lookup_ttl("192.0.2.1") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_refresh_delay():
|
||||||
|
assert mini._dns_refresh_delay([300, 120, None]) == 120
|
||||||
|
assert mini._dns_refresh_delay([0]) == mini._DNS_MIN_TTL
|
||||||
|
assert mini._dns_refresh_delay([None]) == mini._DNS_DEFAULT_TTL
|
||||||
|
|
||||||
|
|
||||||
|
def test_refresher_follows_a_change_and_adds_a_family(monkeypatch):
|
||||||
|
answers = [[(V4, "9.9.9.9"), (V6, "2001:db8::9")], None, [(V4, "9.9.9.9")]]
|
||||||
|
monkeypatch.setattr(mini, "_dns_resolve", lambda *a: answers.pop(0) if len(answers) > 1 else answers[0])
|
||||||
|
monkeypatch.setattr(mini, "_dns_lookup_ttl", lambda *a: None)
|
||||||
|
monkeypatch.setattr(mini, "_DNS_DEFAULT_TTL", 0.01)
|
||||||
|
monkeypatch.setattr(mini, "_DNS_FAILED_RETRY", 0.01)
|
||||||
|
|
||||||
|
async def idle(*a):
|
||||||
|
await asyncio.sleep(10)
|
||||||
|
monkeypatch.setattr(mini, "_heartbeat_sender", idle)
|
||||||
|
|
||||||
|
async def go():
|
||||||
|
mini._running = True
|
||||||
|
mini._shutdown_event = asyncio.Event()
|
||||||
|
mini._active_tasks = []
|
||||||
|
conns = [mini.AsyncConnection(1, "1.1.1.1", 5, V4, "me")]
|
||||||
|
task = asyncio.create_task(mini._dns_refresher(conns, ["h"], 5, 0, "me", 10, None, 2))
|
||||||
|
await asyncio.sleep(0.2)
|
||||||
|
mini._running = False
|
||||||
|
mini._shutdown_event.set()
|
||||||
|
await task
|
||||||
|
for t in mini._active_tasks:
|
||||||
|
t.cancel()
|
||||||
|
return conns
|
||||||
|
|
||||||
|
conns = asyncio.run(go())
|
||||||
|
assert [(c.conn_id, c.addr) for c in conns] == [(1, "9.9.9.9")]
|
||||||
|
assert conns[0].af == V4
|
||||||
@@ -1,5 +1,4 @@
|
|||||||
"""Tests for the config read/write API helpers in http.py."""
|
"""Tests for the config read/write API helpers in http.py."""
|
||||||
import pytest
|
|
||||||
from hbd.server import http
|
from hbd.server import http
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
"""Tests for PUT /api/0/users/me logic."""
|
"""Tests for PUT /api/0/users/me logic."""
|
||||||
import pytest
|
|
||||||
from hbd.server import users as users_mod
|
from hbd.server import users as users_mod
|
||||||
|
|
||||||
|
|
||||||
@@ -87,7 +86,6 @@ def test_put_users_me_notification_channels(tmp_path):
|
|||||||
|
|
||||||
def test_visible_channels_excludes_private_from_others():
|
def test_visible_channels_excludes_private_from_others():
|
||||||
"""Private channels owned by another user must not appear in the visible set."""
|
"""Private channels owned by another user must not appear in the visible set."""
|
||||||
from hbd.server import settings as settings_mod
|
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
"notification_channels": {
|
"notification_channels": {
|
||||||
@@ -105,7 +103,7 @@ def test_visible_channels_excludes_private_from_others():
|
|||||||
self.admin = admin
|
self.admin = admin
|
||||||
|
|
||||||
alice = FakeUser("alice")
|
alice = FakeUser("alice")
|
||||||
bob = FakeUser("bob")
|
bob = FakeUser("bob")
|
||||||
admin = FakeUser("admin", admin=True)
|
admin = FakeUser("admin", admin=True)
|
||||||
|
|
||||||
# Simulate _visible_channels_for_user logic (mirrors http.py implementation)
|
# Simulate _visible_channels_for_user logic (mirrors http.py implementation)
|
||||||
@@ -119,5 +117,5 @@ def test_visible_channels_excludes_private_from_others():
|
|||||||
}
|
}
|
||||||
|
|
||||||
assert visible(alice) == {"public_ch", "alice_priv"}
|
assert visible(alice) == {"public_ch", "alice_priv"}
|
||||||
assert visible(bob) == {"public_ch", "bob_priv"}
|
assert visible(bob) == {"public_ch", "bob_priv"}
|
||||||
assert visible(admin) == {"public_ch", "alice_priv", "bob_priv"}
|
assert visible(admin) == {"public_ch", "alice_priv", "bob_priv"}
|
||||||
|
|||||||
@@ -1,15 +1,11 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
import stat
|
import stat
|
||||||
|
|
||||||
from hbd.client.plugins.nagios_runner import (
|
from hbd.client.plugins.nagios_runner import NagiosRunnerPlugin, NAGIOS_UNKNOWN
|
||||||
NagiosRunnerPlugin,
|
|
||||||
NAGIOS_OK,
|
# Nagios plugin exit codes (the module itself only needs UNKNOWN)
|
||||||
NAGIOS_WARNING,
|
NAGIOS_OK, NAGIOS_WARNING, NAGIOS_CRITICAL = 0, 1, 2
|
||||||
NAGIOS_CRITICAL,
|
|
||||||
NAGIOS_UNKNOWN,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_no_commands_sets_skip_reason():
|
def test_no_commands_sets_skip_reason():
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
"""Tests for notification channel CRUD via configio helpers and visibility logic."""
|
"""Tests for notification channel CRUD via configio helpers and visibility logic."""
|
||||||
import pytest
|
|
||||||
from hbd.server import configio, settings as settings_mod
|
from hbd.server import configio, settings as settings_mod
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ def reset_users_dict():
|
|||||||
users_mod.users = original
|
users_mod.users = original
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def test_make_state_returns_unique_tokens():
|
def test_make_state_returns_unique_tokens():
|
||||||
s1 = oauth.make_state()
|
s1 = oauth.make_state()
|
||||||
s2 = oauth.make_state()
|
s2 = oauth.make_state()
|
||||||
@@ -124,7 +123,6 @@ def test_provision_oauth_user_survives_config_reload():
|
|||||||
assert "oauthonly" in users_mod.users
|
assert "oauthonly" in users_mod.users
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Integration-style tests: callback logic chain
|
# Integration-style tests: callback logic chain
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
+8
-8
@@ -1,19 +1,19 @@
|
|||||||
from hbd.proto import dicttos, stodict, oldmtodict
|
from hbd.common.proto import dicttos, stodict, oldmtodict
|
||||||
|
|
||||||
|
|
||||||
def test_dicttos_and_stodict_uncompressed():
|
def test_dicttos_and_stodict_roundtrip():
|
||||||
msg = dicttos("HTB", {"name": "host.example", "interval": 10})
|
msg = dicttos("HTB", {"name": "host.example", "interval": 10})
|
||||||
|
assert msg.startswith(b"!HTB:")
|
||||||
d = stodict(msg)
|
d = stodict(msg)
|
||||||
assert d["ID"].startswith("HTB")
|
assert d["ID"] == "HTB"
|
||||||
assert d["name"] == "host.example"
|
assert d["name"] == "host.example"
|
||||||
assert d["interval"] == 10
|
assert d["interval"] == 10
|
||||||
|
|
||||||
|
|
||||||
def test_dicttos_and_stodict_compressed():
|
def test_dicttos_and_stodict_ack():
|
||||||
msg = dicttos("ACK", {"time": 12345}, compress=True)
|
d = stodict(dicttos("ACK", {"time": 12345}))
|
||||||
d = stodict(msg)
|
assert d["ID"] == "ACK"
|
||||||
# for compressed the original code included the colon in ID slice; ensure no crash
|
assert d["time"] == 12345
|
||||||
assert "ID" in d
|
|
||||||
|
|
||||||
|
|
||||||
def test_oldmtodict():
|
def test_oldmtodict():
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import pytest
|
|
||||||
from hbd.server import settings as settings_mod
|
from hbd.server import settings as settings_mod
|
||||||
|
|
||||||
CFG = {
|
CFG = {
|
||||||
|
|||||||
+84
-80
@@ -13,13 +13,12 @@ Tests cover:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import time
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# Add parent directory to path for imports
|
# Add parent directory to path for imports
|
||||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||||
|
|
||||||
from hbd.threshold import (
|
from hbd.server.threshold import ( # noqa: E402
|
||||||
ThresholdChecker,
|
ThresholdChecker,
|
||||||
ThresholdConfig,
|
ThresholdConfig,
|
||||||
AlertLevel,
|
AlertLevel,
|
||||||
@@ -31,37 +30,37 @@ from hbd.threshold import (
|
|||||||
def test_threshold_config_basic():
|
def test_threshold_config_basic():
|
||||||
"""Test basic threshold configuration."""
|
"""Test basic threshold configuration."""
|
||||||
print("Test 1: Basic threshold configuration...")
|
print("Test 1: Basic threshold configuration...")
|
||||||
|
|
||||||
config = ThresholdConfig(
|
config = ThresholdConfig(
|
||||||
metric_path="cpu_monitor.cpu_percent",
|
metric_path="cpu_monitor.cpu_percent",
|
||||||
warning=80.0,
|
warning=80.0,
|
||||||
critical=90.0,
|
critical=90.0,
|
||||||
operator=">",
|
operator=">",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Test below warning
|
# Test below warning
|
||||||
result = config.evaluate(50.0)
|
result = config.evaluate(50.0)
|
||||||
assert result == AlertLevel.OK, f"Expected OK, got {result}"
|
assert result == AlertLevel.OK, f"Expected OK, got {result}"
|
||||||
|
|
||||||
# Test at warning
|
# Test at warning
|
||||||
result = config.evaluate(80.0)
|
result = config.evaluate(80.0)
|
||||||
assert result == AlertLevel.OK, f"Expected OK at boundary, got {result}"
|
assert result == AlertLevel.OK, f"Expected OK at boundary, got {result}"
|
||||||
|
|
||||||
# Test above warning but below critical
|
# Test above warning but below critical
|
||||||
result = config.evaluate(85.0)
|
result = config.evaluate(85.0)
|
||||||
assert result == AlertLevel.WARNING, f"Expected WARNING, got {result}"
|
assert result == AlertLevel.WARNING, f"Expected WARNING, got {result}"
|
||||||
|
|
||||||
# Test above critical
|
# Test above critical
|
||||||
result = config.evaluate(95.0)
|
result = config.evaluate(95.0)
|
||||||
assert result == AlertLevel.CRITICAL, f"Expected CRITICAL, got {result}"
|
assert result == AlertLevel.CRITICAL, f"Expected CRITICAL, got {result}"
|
||||||
|
|
||||||
print(" ✓ Basic threshold configuration works")
|
print(" ✓ Basic threshold configuration works")
|
||||||
|
|
||||||
|
|
||||||
def test_threshold_operators():
|
def test_threshold_operators():
|
||||||
"""Test all comparison operators."""
|
"""Test all comparison operators."""
|
||||||
print("\nTest 2: Comparison operators...")
|
print("\nTest 2: Comparison operators...")
|
||||||
|
|
||||||
# Greater than operator
|
# Greater than operator
|
||||||
config_gt = ThresholdConfig(
|
config_gt = ThresholdConfig(
|
||||||
metric_path="test.metric",
|
metric_path="test.metric",
|
||||||
@@ -71,7 +70,7 @@ def test_threshold_operators():
|
|||||||
)
|
)
|
||||||
assert config_gt.evaluate(85.0) == AlertLevel.WARNING
|
assert config_gt.evaluate(85.0) == AlertLevel.WARNING
|
||||||
assert config_gt.evaluate(75.0) == AlertLevel.OK
|
assert config_gt.evaluate(75.0) == AlertLevel.OK
|
||||||
|
|
||||||
# Less than operator (for inverse thresholds like available memory)
|
# Less than operator (for inverse thresholds like available memory)
|
||||||
config_lt = ThresholdConfig(
|
config_lt = ThresholdConfig(
|
||||||
metric_path="memory.available_mb",
|
metric_path="memory.available_mb",
|
||||||
@@ -82,7 +81,7 @@ def test_threshold_operators():
|
|||||||
assert config_lt.evaluate(800) == AlertLevel.WARNING, "Should warn when below 1000"
|
assert config_lt.evaluate(800) == AlertLevel.WARNING, "Should warn when below 1000"
|
||||||
assert config_lt.evaluate(400) == AlertLevel.CRITICAL, "Should be critical when below 500"
|
assert config_lt.evaluate(400) == AlertLevel.CRITICAL, "Should be critical when below 500"
|
||||||
assert config_lt.evaluate(1500) == AlertLevel.OK, "Should be OK when above 1000"
|
assert config_lt.evaluate(1500) == AlertLevel.OK, "Should be OK when above 1000"
|
||||||
|
|
||||||
# Greater than or equal
|
# Greater than or equal
|
||||||
config_gte = ThresholdConfig(
|
config_gte = ThresholdConfig(
|
||||||
metric_path="test.metric",
|
metric_path="test.metric",
|
||||||
@@ -91,7 +90,7 @@ def test_threshold_operators():
|
|||||||
)
|
)
|
||||||
assert config_gte.evaluate(80.0) == AlertLevel.WARNING
|
assert config_gte.evaluate(80.0) == AlertLevel.WARNING
|
||||||
assert config_gte.evaluate(79.9) == AlertLevel.OK
|
assert config_gte.evaluate(79.9) == AlertLevel.OK
|
||||||
|
|
||||||
# Less than or equal
|
# Less than or equal
|
||||||
config_lte = ThresholdConfig(
|
config_lte = ThresholdConfig(
|
||||||
metric_path="test.metric",
|
metric_path="test.metric",
|
||||||
@@ -100,14 +99,14 @@ def test_threshold_operators():
|
|||||||
)
|
)
|
||||||
assert config_lte.evaluate(20.0) == AlertLevel.WARNING
|
assert config_lte.evaluate(20.0) == AlertLevel.WARNING
|
||||||
assert config_lte.evaluate(20.1) == AlertLevel.OK
|
assert config_lte.evaluate(20.1) == AlertLevel.OK
|
||||||
|
|
||||||
print(" ✓ All comparison operators work correctly")
|
print(" ✓ All comparison operators work correctly")
|
||||||
|
|
||||||
|
|
||||||
def test_hysteresis():
|
def test_hysteresis():
|
||||||
"""Test hysteresis to prevent flapping."""
|
"""Test hysteresis to prevent flapping."""
|
||||||
print("\nTest 3: Hysteresis...")
|
print("\nTest 3: Hysteresis...")
|
||||||
|
|
||||||
config = ThresholdConfig(
|
config = ThresholdConfig(
|
||||||
metric_path="cpu_monitor.cpu_percent",
|
metric_path="cpu_monitor.cpu_percent",
|
||||||
warning=80.0,
|
warning=80.0,
|
||||||
@@ -115,87 +114,87 @@ def test_hysteresis():
|
|||||||
operator=">",
|
operator=">",
|
||||||
hysteresis=0.1, # 10% hysteresis
|
hysteresis=0.1, # 10% hysteresis
|
||||||
)
|
)
|
||||||
|
|
||||||
# Start at OK, go to WARNING
|
# Start at OK, go to WARNING
|
||||||
result = config.evaluate_with_hysteresis(85.0, AlertLevel.OK)
|
result = config.evaluate_with_hysteresis(85.0, AlertLevel.OK)
|
||||||
assert result == AlertLevel.WARNING, "Should enter WARNING state"
|
assert result == AlertLevel.WARNING, "Should enter WARNING state"
|
||||||
|
|
||||||
# Try to recover with insufficient improvement (within hysteresis)
|
# Try to recover with insufficient improvement (within hysteresis)
|
||||||
# Warning threshold is 80, hysteresis is 10%, so need to go below 80 - 8 = 72
|
# Warning threshold is 80, hysteresis is 10%, so need to go below 80 - 8 = 72
|
||||||
result = config.evaluate_with_hysteresis(77.0, AlertLevel.WARNING)
|
result = config.evaluate_with_hysteresis(77.0, AlertLevel.WARNING)
|
||||||
assert result == AlertLevel.WARNING, "Should stay in WARNING (hysteresis)"
|
assert result == AlertLevel.WARNING, "Should stay in WARNING (hysteresis)"
|
||||||
|
|
||||||
# Recover with sufficient improvement
|
# Recover with sufficient improvement
|
||||||
result = config.evaluate_with_hysteresis(70.0, AlertLevel.WARNING)
|
result = config.evaluate_with_hysteresis(70.0, AlertLevel.WARNING)
|
||||||
assert result == AlertLevel.OK, "Should recover to OK"
|
assert result == AlertLevel.OK, "Should recover to OK"
|
||||||
|
|
||||||
# Test critical hysteresis
|
# Test critical hysteresis
|
||||||
result = config.evaluate_with_hysteresis(95.0, AlertLevel.WARNING)
|
result = config.evaluate_with_hysteresis(95.0, AlertLevel.WARNING)
|
||||||
assert result == AlertLevel.CRITICAL, "Should escalate to CRITICAL"
|
assert result == AlertLevel.CRITICAL, "Should escalate to CRITICAL"
|
||||||
|
|
||||||
# Try to recover from critical with insufficient improvement
|
# Try to recover from critical with insufficient improvement
|
||||||
# Critical threshold is 90, hysteresis is 10%, so need to go below 90 - 9 = 81
|
# Critical threshold is 90, hysteresis is 10%, so need to go below 90 - 9 = 81
|
||||||
result = config.evaluate_with_hysteresis(85.0, AlertLevel.CRITICAL)
|
result = config.evaluate_with_hysteresis(85.0, AlertLevel.CRITICAL)
|
||||||
assert result == AlertLevel.CRITICAL, "Should stay CRITICAL (hysteresis, still above 81)"
|
assert result == AlertLevel.CRITICAL, "Should stay CRITICAL (hysteresis, still above 81)"
|
||||||
|
|
||||||
# Sufficient improvement to drop from CRITICAL (below 81)
|
# Sufficient improvement to drop from CRITICAL (below 81)
|
||||||
result = config.evaluate_with_hysteresis(75.0, AlertLevel.CRITICAL)
|
result = config.evaluate_with_hysteresis(75.0, AlertLevel.CRITICAL)
|
||||||
assert result == AlertLevel.OK, "Should drop to OK (below warning threshold)"
|
assert result == AlertLevel.OK, "Should drop to OK (below warning threshold)"
|
||||||
|
|
||||||
# Now test dropping from CRITICAL to WARNING
|
# Now test dropping from CRITICAL to WARNING
|
||||||
result = config.evaluate_with_hysteresis(95.0, AlertLevel.OK)
|
result = config.evaluate_with_hysteresis(95.0, AlertLevel.OK)
|
||||||
assert result == AlertLevel.CRITICAL, "Should go to CRITICAL"
|
assert result == AlertLevel.CRITICAL, "Should go to CRITICAL"
|
||||||
|
|
||||||
# Drop to between warning and critical, but still in hysteresis zone
|
# Drop to between warning and critical, but still in hysteresis zone
|
||||||
result = config.evaluate_with_hysteresis(82.0, AlertLevel.CRITICAL)
|
result = config.evaluate_with_hysteresis(82.0, AlertLevel.CRITICAL)
|
||||||
assert result == AlertLevel.CRITICAL, "Should stay CRITICAL (in hysteresis)"
|
assert result == AlertLevel.CRITICAL, "Should stay CRITICAL (in hysteresis)"
|
||||||
|
|
||||||
# Drop below critical hysteresis but still above warning threshold
|
# Drop below critical hysteresis but still above warning threshold
|
||||||
# At 80.1, we're above WARNING (80) so should evaluate to WARNING
|
# At 80.1, we're above WARNING (80) so should evaluate to WARNING
|
||||||
result = config.evaluate_with_hysteresis(80.5, AlertLevel.CRITICAL)
|
result = config.evaluate_with_hysteresis(80.5, AlertLevel.CRITICAL)
|
||||||
assert result == AlertLevel.WARNING, "Should drop to WARNING when below critical hysteresis"
|
assert result == AlertLevel.WARNING, "Should drop to WARNING when below critical hysteresis"
|
||||||
|
|
||||||
print(" ✓ Hysteresis prevents flapping")
|
print(" ✓ Hysteresis prevents flapping")
|
||||||
|
|
||||||
|
|
||||||
def test_alert_state():
|
def test_alert_state():
|
||||||
"""Test alert state tracking."""
|
"""Test alert state tracking."""
|
||||||
print("\nTest 4: Alert state tracking...")
|
print("\nTest 4: Alert state tracking...")
|
||||||
|
|
||||||
alert = AlertState("cpu_monitor.cpu_percent")
|
alert = AlertState("cpu_monitor.cpu_percent")
|
||||||
|
|
||||||
# Initial state
|
# Initial state
|
||||||
assert alert.level == AlertLevel.OK
|
assert alert.level == AlertLevel.OK
|
||||||
assert alert.notification_count == 0
|
assert alert.notification_count == 0
|
||||||
|
|
||||||
# Update to WARNING - should trigger notification
|
# Update to WARNING - should trigger notification
|
||||||
changed = alert.update(AlertLevel.WARNING, 85.0)
|
changed = alert.update(AlertLevel.WARNING, 85.0)
|
||||||
assert changed == True, "State change should return True"
|
assert changed is True, "State change should return True"
|
||||||
assert alert.level == AlertLevel.WARNING
|
assert alert.level == AlertLevel.WARNING
|
||||||
assert alert.last_value == 85.0
|
assert alert.last_value == 85.0
|
||||||
|
|
||||||
# Update with same level - no notification
|
# Update with same level - no notification
|
||||||
changed = alert.update(AlertLevel.WARNING, 86.0)
|
changed = alert.update(AlertLevel.WARNING, 86.0)
|
||||||
assert changed == False, "No state change should return False"
|
assert changed is False, "No state change should return False"
|
||||||
assert alert.last_value == 86.0
|
assert alert.last_value == 86.0
|
||||||
|
|
||||||
# Escalate to CRITICAL
|
# Escalate to CRITICAL
|
||||||
changed = alert.update(AlertLevel.CRITICAL, 95.0)
|
changed = alert.update(AlertLevel.CRITICAL, 95.0)
|
||||||
assert changed == True, "Escalation should trigger notification"
|
assert changed is True, "Escalation should trigger notification"
|
||||||
assert alert.level == AlertLevel.CRITICAL
|
assert alert.level == AlertLevel.CRITICAL
|
||||||
|
|
||||||
# Recover to OK
|
# Recover to OK
|
||||||
changed = alert.update(AlertLevel.OK, 50.0)
|
changed = alert.update(AlertLevel.OK, 50.0)
|
||||||
assert changed == True, "Recovery should trigger notification"
|
assert changed is True, "Recovery should trigger notification"
|
||||||
assert alert.level == AlertLevel.OK
|
assert alert.level == AlertLevel.OK
|
||||||
|
|
||||||
print(" ✓ Alert state tracking works correctly")
|
print(" ✓ Alert state tracking works correctly")
|
||||||
|
|
||||||
|
|
||||||
def test_threshold_checker_parsing():
|
def test_threshold_checker_parsing():
|
||||||
"""Test parsing threshold configuration from YAML structure."""
|
"""Test parsing threshold configuration from YAML structure."""
|
||||||
print("\nTest 5: Configuration parsing...")
|
print("\nTest 5: Configuration parsing...")
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
"thresholds": {
|
"thresholds": {
|
||||||
"cpu_monitor": {
|
"cpu_monitor": {
|
||||||
@@ -239,9 +238,9 @@ def test_threshold_checker_parsing():
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
checker = ThresholdChecker(config)
|
checker = ThresholdChecker(config)
|
||||||
|
|
||||||
# Verify thresholds were parsed
|
# Verify thresholds were parsed
|
||||||
assert "cpu_monitor.cpu_percent" in checker.thresholds
|
assert "cpu_monitor.cpu_percent" in checker.thresholds
|
||||||
assert "cpu_monitor.load_1min" in checker.thresholds
|
assert "cpu_monitor.load_1min" in checker.thresholds
|
||||||
@@ -249,18 +248,27 @@ def test_threshold_checker_parsing():
|
|||||||
assert "memory_monitor.available_mb" in checker.thresholds
|
assert "memory_monitor.available_mb" in checker.thresholds
|
||||||
assert "disk_monitor./.percent" in checker.thresholds
|
assert "disk_monitor./.percent" in checker.thresholds
|
||||||
assert "disk_monitor./home.percent" in checker.thresholds
|
assert "disk_monitor./home.percent" in checker.thresholds
|
||||||
|
|
||||||
# Verify operators were parsed correctly
|
# Verify operators were parsed correctly
|
||||||
assert checker.thresholds["cpu_monitor.cpu_percent"].operator == ComparisonOperator.GT
|
assert checker.thresholds["cpu_monitor.cpu_percent"].operator == ComparisonOperator.GT
|
||||||
assert checker.thresholds["memory_monitor.available_mb"].operator == ComparisonOperator.LT
|
assert checker.thresholds["memory_monitor.available_mb"].operator == ComparisonOperator.LT
|
||||||
|
|
||||||
print(f" ✓ Parsed {len(checker.thresholds)} thresholds correctly")
|
print(f" ✓ Parsed {len(checker.thresholds)} thresholds correctly")
|
||||||
|
|
||||||
|
|
||||||
def test_check_plugin_data():
|
def _checker_with_notifications(config, notifications, monkeypatch):
|
||||||
|
"""ThresholdChecker with no grace period that records notifications instead of sending them."""
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ThresholdChecker, "_send_notification",
|
||||||
|
lambda self, host, lvl, message, *a: notifications.append(f"[{lvl}] {host} {message}"),
|
||||||
|
)
|
||||||
|
return ThresholdChecker({"grace": 0, **config})
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_plugin_data(monkeypatch):
|
||||||
"""Test checking plugin data against thresholds."""
|
"""Test checking plugin data against thresholds."""
|
||||||
print("\nTest 6: Plugin data checking...")
|
print("\nTest 6: Plugin data checking...")
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
"thresholds": {
|
"thresholds": {
|
||||||
"cpu_monitor": {
|
"cpu_monitor": {
|
||||||
@@ -275,57 +283,53 @@ def test_check_plugin_data():
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
notifications = []
|
notifications = []
|
||||||
|
checker = _checker_with_notifications(config, notifications, monkeypatch)
|
||||||
def notification_callback(msg):
|
|
||||||
notifications.append(msg)
|
|
||||||
|
|
||||||
checker = ThresholdChecker(config, notification_callback=notification_callback)
|
|
||||||
alert_states = {}
|
alert_states = {}
|
||||||
|
|
||||||
# First check - OK
|
# First check - OK
|
||||||
plugin_data = {
|
plugin_data = {
|
||||||
"cpu_percent": 50.0,
|
"cpu_percent": 50.0,
|
||||||
"load_1min": 2.0,
|
"load_1min": 2.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
state_changes = checker.check_plugin_data(
|
state_changes = checker.check_plugin_data(
|
||||||
host_name="testhost",
|
host_name="testhost",
|
||||||
plugin_name="cpu_monitor",
|
plugin_name="cpu_monitor",
|
||||||
data=plugin_data,
|
data=plugin_data,
|
||||||
alert_states=alert_states,
|
alert_states=alert_states,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert len(state_changes) == 0, "No thresholds violated, no state changes"
|
assert len(state_changes) == 0, "No thresholds violated, no state changes"
|
||||||
assert len(notifications) == 0, "No notifications should be sent"
|
assert len(notifications) == 0, "No notifications should be sent"
|
||||||
|
|
||||||
# Second check - WARNING
|
# Second check - WARNING
|
||||||
plugin_data = {
|
plugin_data = {
|
||||||
"cpu_percent": 85.0,
|
"cpu_percent": 85.0,
|
||||||
"load_1min": 2.0,
|
"load_1min": 2.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
state_changes = checker.check_plugin_data(
|
state_changes = checker.check_plugin_data(
|
||||||
host_name="testhost",
|
host_name="testhost",
|
||||||
plugin_name="cpu_monitor",
|
plugin_name="cpu_monitor",
|
||||||
data=plugin_data,
|
data=plugin_data,
|
||||||
alert_states=alert_states,
|
alert_states=alert_states,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert len(state_changes) == 1, "One metric should change state"
|
assert len(state_changes) == 1, "One metric should change state"
|
||||||
assert state_changes[0][0] == "cpu_monitor.cpu_percent"
|
assert state_changes[0][0] == "cpu_monitor.cpu_percent"
|
||||||
assert state_changes[0][2] == AlertLevel.WARNING
|
assert state_changes[0][2] == AlertLevel.WARNING
|
||||||
assert len(notifications) == 1, "One notification should be sent"
|
assert len(notifications) == 1, "One notification should be sent"
|
||||||
assert "WARNING" in notifications[0]
|
assert "WARNING" in notifications[0]
|
||||||
assert "testhost" in notifications[0]
|
assert "testhost" in notifications[0]
|
||||||
|
|
||||||
# Third check - CRITICAL
|
# Third check - CRITICAL
|
||||||
plugin_data = {
|
plugin_data = {
|
||||||
"cpu_percent": 95.0,
|
"cpu_percent": 95.0,
|
||||||
"load_1min": 9.0,
|
"load_1min": 9.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
notifications.clear()
|
notifications.clear()
|
||||||
state_changes = checker.check_plugin_data(
|
state_changes = checker.check_plugin_data(
|
||||||
host_name="testhost",
|
host_name="testhost",
|
||||||
@@ -333,16 +337,16 @@ def test_check_plugin_data():
|
|||||||
data=plugin_data,
|
data=plugin_data,
|
||||||
alert_states=alert_states,
|
alert_states=alert_states,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert len(state_changes) == 2, "Two metrics should change state"
|
assert len(state_changes) == 2, "Two metrics should change state"
|
||||||
assert len(notifications) == 2, "Two notifications should be sent"
|
assert len(notifications) == 2, "Two notifications should be sent"
|
||||||
|
|
||||||
# Fourth check - Recovery
|
# Fourth check - Recovery
|
||||||
plugin_data = {
|
plugin_data = {
|
||||||
"cpu_percent": 50.0,
|
"cpu_percent": 50.0,
|
||||||
"load_1min": 1.0,
|
"load_1min": 1.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
notifications.clear()
|
notifications.clear()
|
||||||
state_changes = checker.check_plugin_data(
|
state_changes = checker.check_plugin_data(
|
||||||
host_name="testhost",
|
host_name="testhost",
|
||||||
@@ -350,18 +354,18 @@ def test_check_plugin_data():
|
|||||||
data=plugin_data,
|
data=plugin_data,
|
||||||
alert_states=alert_states,
|
alert_states=alert_states,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert len(state_changes) == 2, "Two metrics should recover"
|
assert len(state_changes) == 2, "Two metrics should recover"
|
||||||
assert len(notifications) == 2, "Two recovery notifications"
|
assert len(notifications) == 2, "Two recovery notifications"
|
||||||
assert any("RECOVERED" in n for n in notifications), "Should have recovery notification"
|
assert any("RECOVER" in n for n in notifications), "Should have recovery notification"
|
||||||
|
|
||||||
print(" ✓ Plugin data checking and notifications work")
|
print(" ✓ Plugin data checking and notifications work")
|
||||||
|
|
||||||
|
|
||||||
def test_nested_metrics():
|
def test_nested_metrics(monkeypatch):
|
||||||
"""Test checking nested metrics like disk partitions."""
|
"""Test checking nested metrics like disk partitions."""
|
||||||
print("\nTest 7: Nested metrics (partitions)...")
|
print("\nTest 7: Nested metrics (partitions)...")
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
"thresholds": {
|
"thresholds": {
|
||||||
"disk_monitor": {
|
"disk_monitor": {
|
||||||
@@ -382,11 +386,11 @@ def test_nested_metrics():
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
notifications = []
|
notifications = []
|
||||||
checker = ThresholdChecker(config, notification_callback=lambda m: notifications.append(m))
|
checker = _checker_with_notifications(config, notifications, monkeypatch)
|
||||||
alert_states = {}
|
alert_states = {}
|
||||||
|
|
||||||
plugin_data = {
|
plugin_data = {
|
||||||
"partitions": {
|
"partitions": {
|
||||||
"/": {
|
"/": {
|
||||||
@@ -399,26 +403,26 @@ def test_nested_metrics():
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
state_changes = checker.check_plugin_data(
|
state_changes = checker.check_plugin_data(
|
||||||
host_name="testhost",
|
host_name="testhost",
|
||||||
plugin_name="disk_monitor",
|
plugin_name="disk_monitor",
|
||||||
data=plugin_data,
|
data=plugin_data,
|
||||||
alert_states=alert_states,
|
alert_states=alert_states,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert len(state_changes) == 1, "One partition should trigger alert"
|
assert len(state_changes) == 1, "One partition should trigger alert"
|
||||||
assert "/home" in state_changes[0][0], "Should be /home partition"
|
assert "/home" in state_changes[0][0], "Should be /home partition"
|
||||||
assert state_changes[0][2] == AlertLevel.WARNING
|
assert state_changes[0][2] == AlertLevel.WARNING
|
||||||
assert len(notifications) == 1
|
assert len(notifications) == 1
|
||||||
|
|
||||||
print(" ✓ Nested metric checking works")
|
print(" ✓ Nested metric checking works")
|
||||||
|
|
||||||
|
|
||||||
def test_alert_summary():
|
def test_alert_summary():
|
||||||
"""Test getting alert summaries."""
|
"""Test getting alert summaries."""
|
||||||
print("\nTest 8: Alert summaries...")
|
print("\nTest 8: Alert summaries...")
|
||||||
|
|
||||||
config = {
|
config = {
|
||||||
"thresholds": {
|
"thresholds": {
|
||||||
"cpu_monitor": {
|
"cpu_monitor": {
|
||||||
@@ -430,31 +434,31 @@ def test_alert_summary():
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
checker = ThresholdChecker(config)
|
checker = ThresholdChecker(config)
|
||||||
alert_states = {}
|
alert_states = {}
|
||||||
|
|
||||||
# Create some alert states
|
# Create some alert states
|
||||||
plugin_data = {
|
plugin_data = {
|
||||||
"cpu_percent": 85.0, # WARNING
|
"cpu_percent": 85.0, # WARNING
|
||||||
"load_1min": 9.0, # CRITICAL
|
"load_1min": 9.0, # CRITICAL
|
||||||
}
|
}
|
||||||
checker.check_plugin_data("testhost", "cpu_monitor", plugin_data, alert_states)
|
checker.check_plugin_data("testhost", "cpu_monitor", plugin_data, alert_states)
|
||||||
|
|
||||||
plugin_data = {
|
plugin_data = {
|
||||||
"percent": 96.0, # CRITICAL
|
"percent": 96.0, # CRITICAL
|
||||||
}
|
}
|
||||||
checker.check_plugin_data("testhost", "memory_monitor", plugin_data, alert_states)
|
checker.check_plugin_data("testhost", "memory_monitor", plugin_data, alert_states)
|
||||||
|
|
||||||
# Get summary
|
# Get summary
|
||||||
summary = checker.get_alert_summary(alert_states)
|
summary = checker.get_alert_summary(alert_states)
|
||||||
assert summary["warning"] == 1, "Should have 1 warning"
|
assert summary["warning"] == 1, "Should have 1 warning"
|
||||||
assert summary["critical"] == 2, "Should have 2 critical"
|
assert summary["critical"] == 2, "Should have 2 critical"
|
||||||
|
|
||||||
# Get active alerts
|
# Get active alerts
|
||||||
active = checker.get_active_alerts(alert_states)
|
active = checker.get_active_alerts(alert_states)
|
||||||
assert len(active) == 3, "Should have 3 active alerts"
|
assert len(active) == 3, "Should have 3 active alerts"
|
||||||
|
|
||||||
print(" ✓ Alert summaries work correctly")
|
print(" ✓ Alert summaries work correctly")
|
||||||
|
|
||||||
|
|
||||||
@@ -463,7 +467,7 @@ def run_all_tests():
|
|||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
print("THRESHOLD SYSTEM TEST SUITE")
|
print("THRESHOLD SYSTEM TEST SUITE")
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
test_threshold_config_basic()
|
test_threshold_config_basic()
|
||||||
test_threshold_operators()
|
test_threshold_operators()
|
||||||
@@ -473,12 +477,12 @@ def run_all_tests():
|
|||||||
test_check_plugin_data()
|
test_check_plugin_data()
|
||||||
test_nested_metrics()
|
test_nested_metrics()
|
||||||
test_alert_summary()
|
test_alert_summary()
|
||||||
|
|
||||||
print("\n" + "=" * 70)
|
print("\n" + "=" * 70)
|
||||||
print("✓ ALL TESTS PASSED")
|
print("✓ ALL TESTS PASSED")
|
||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
except AssertionError as e:
|
except AssertionError as e:
|
||||||
print(f"\n✗ TEST FAILED: {e}")
|
print(f"\n✗ TEST FAILED: {e}")
|
||||||
import traceback
|
import traceback
|
||||||
|
|||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
from hbd.udp import parse_message
|
from hbd.server.udp import parse_message
|
||||||
from hbd.proto import dicttos
|
from hbd.common.proto import dicttos
|
||||||
|
|
||||||
|
|
||||||
def test_parse_message_uncompressed():
|
def test_parse_message_uncompressed():
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""Datagrams that are not heartbeat traffic (internet scanners) must not create hosts."""
|
||||||
|
from hbd.common.proto import dicttos
|
||||||
|
from hbd.server import hbdclass
|
||||||
|
from hbd.server.udp import handle_datagram, parse_message
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTransport:
|
||||||
|
sent = 0
|
||||||
|
|
||||||
|
def sendto(self, data, addr):
|
||||||
|
_FakeTransport.sent += 1
|
||||||
|
|
||||||
|
|
||||||
|
def _ctx():
|
||||||
|
return {"config": {}, "hbdclass": hbdclass, "msg_to_websockets": None,
|
||||||
|
"DEBUG": 0, "verbose": False}
|
||||||
|
|
||||||
|
|
||||||
|
def _handle(raw):
|
||||||
|
handle_datagram(parse_message(raw), ("192.0.2.9", 50000), _FakeTransport(), _ctx())
|
||||||
|
|
||||||
|
|
||||||
|
def test_sip_invite_does_not_create_a_host():
|
||||||
|
hbdclass.Host.hosts.pop("unknown", None)
|
||||||
|
_handle(b"INVITE sip:10000@192.0.2.1:50003 SIP/2.0\r\nTo: <sip:10000@192.0.2.1>\r\n\r\n")
|
||||||
|
assert "unknown" not in hbdclass.Host.hosts
|
||||||
|
|
||||||
|
|
||||||
|
def test_nameless_heartbeat_does_not_create_a_host_or_get_acked():
|
||||||
|
hbdclass.Host.hosts.pop("unknown", None)
|
||||||
|
_FakeTransport.sent = 0
|
||||||
|
_handle(dicttos("HTB", {"id": 0}))
|
||||||
|
assert "unknown" not in hbdclass.Host.hosts
|
||||||
|
assert _FakeTransport.sent == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_heartbeat_still_creates_its_host():
|
||||||
|
hbdclass.Host.hosts.pop("stray-ok", None)
|
||||||
|
_handle(dicttos("HTB", {"name": "stray-ok", "id": 0}))
|
||||||
|
assert "stray-ok" in hbdclass.Host.hosts
|
||||||
Reference in New Issue
Block a user