feat(client): per-protocol connectivity probe
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Uc9V1v7iXruubpoLiSE24M
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,202 @@
|
||||
"""Per-protocol internet presence probe.
|
||||
|
||||
Tier 1 (every tier1_interval seconds, sends no packets): a UDP connect() to the probe
|
||||
address reveals whether a route exists and which source address would be used.
|
||||
Tier 2 (every probe_interval seconds, sooner after a failure or a change of address or
|
||||
gateway): one ICMP echo to the probe address. Absence needs fail_limit consecutive
|
||||
failures (or no route at all); presence needs one success.
|
||||
|
||||
The result is read by the heartbeat sender (p4/p6 flags on every HTB) and by the
|
||||
network_monitor plugin (connectivity metric) via the shared instance.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional, Set, Tuple
|
||||
|
||||
from hbd.client import gateway as gateway_mod
|
||||
from hbd.client.pingutil import PingUnavailable, run_ping
|
||||
|
||||
logger = logging.getLogger("hbc.connectivity")
|
||||
|
||||
FAMILIES = ("ipv4", "ipv6")
|
||||
FLAG_KEYS = {"ipv4": "p4", "ipv6": "p6"}
|
||||
DEFAULT_PROBES = {"ipv4": "1.1.1.1", "ipv6": "2606:4700:4700::1111"}
|
||||
|
||||
|
||||
class ProbeUnavailable(Exception):
|
||||
"""The address family cannot be probed on this host (e.g. no IPv6 support)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FamilyState:
|
||||
present: Optional[bool] = None # None = unknown (probe cannot run yet/at all)
|
||||
addr: Optional[str] = None
|
||||
gateway: Optional[str] = None
|
||||
rtt: Optional[float] = None
|
||||
|
||||
|
||||
def local_address(family: str, target: str) -> Optional[str]:
|
||||
"""Source address the kernel would use to reach target, or None if there is no route.
|
||||
|
||||
A UDP connect() sends nothing. Raises ProbeUnavailable if the family is unsupported.
|
||||
"""
|
||||
af = socket.AF_INET if family == "ipv4" else socket.AF_INET6
|
||||
try:
|
||||
sock = socket.socket(af, socket.SOCK_DGRAM)
|
||||
except OSError as e:
|
||||
raise ProbeUnavailable(str(e)) from e
|
||||
try:
|
||||
sock.connect((target, 9))
|
||||
return str(sock.getsockname()[0])
|
||||
except OSError:
|
||||
return None
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
async def ping_rtt(target: str, family: str) -> Optional[float]:
|
||||
"""One ICMP echo; returns RTT in ms or None if no reply."""
|
||||
result = await run_ping(target, 1, 2, 4 if family == "ipv4" else 6)
|
||||
rtt = result["rtt_avg"]
|
||||
return None if rtt == float("inf") else rtt
|
||||
|
||||
|
||||
class Connectivity:
|
||||
def __init__(
|
||||
self,
|
||||
probes: Optional[Dict[str, str]] = None,
|
||||
probe_interval: float = 60.0,
|
||||
retry_interval: float = 10.0,
|
||||
tier1_interval: float = 5.0,
|
||||
fail_limit: int = 2,
|
||||
local_address_fn: Callable[[str, str], Optional[str]] = local_address,
|
||||
ping_fn: Callable[[str, str], Awaitable[Optional[float]]] = ping_rtt,
|
||||
gateway_fn: Callable[[str], Awaitable[Optional[str]]] = gateway_mod.default_gateway,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
):
|
||||
self.probes = {**DEFAULT_PROBES, **(probes or {})}
|
||||
self.probe_interval = probe_interval
|
||||
self.retry_interval = retry_interval
|
||||
self.tier1_interval = tier1_interval
|
||||
self.fail_limit = fail_limit
|
||||
self._local_address = local_address_fn
|
||||
self._ping = ping_fn
|
||||
self._gateway_fn = gateway_fn
|
||||
self._clock = clock
|
||||
# Reading the gateway is a cheap /proc read on Linux but a subprocess elsewhere,
|
||||
# so elsewhere it is refreshed only when a tier-2 probe is due.
|
||||
self._cheap_gateway = sys.platform.startswith("linux")
|
||||
self.states: Dict[str, FamilyState] = {f: FamilyState() for f in FAMILIES}
|
||||
self._fails: Dict[str, int] = {f: 0 for f in FAMILIES}
|
||||
self._next_ping: Dict[str, float] = {f: 0.0 for f in FAMILIES}
|
||||
self._last_key: Dict[str, Optional[Tuple[Optional[str], Optional[str]]]] = {
|
||||
f: None for f in FAMILIES}
|
||||
self._ping_disabled: Set[str] = set()
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, cfg: Optional[Dict[str, Any]]) -> "Connectivity":
|
||||
cfg = cfg or {}
|
||||
probes = {}
|
||||
if cfg.get("probe_v4"):
|
||||
probes["ipv4"] = str(cfg["probe_v4"])
|
||||
if cfg.get("probe_v6"):
|
||||
probes["ipv6"] = str(cfg["probe_v6"])
|
||||
return cls(probes=probes, probe_interval=float(cfg.get("probe_interval", 60)))
|
||||
|
||||
# ---- readers (cheap; safe to call on every heartbeat) --------------------------
|
||||
def snapshot(self) -> Dict[str, FamilyState]:
|
||||
return {f: replace(s) for f, s in self.states.items()}
|
||||
|
||||
def flags(self) -> Dict[str, bool]:
|
||||
"""HTB flags for families whose presence is known."""
|
||||
return {FLAG_KEYS[f]: s.present for f, s in self.states.items() if s.present is not None}
|
||||
|
||||
def as_metric(self) -> Dict[str, Dict[str, Any]]:
|
||||
return {f: asdict(s) for f, s in self.states.items()}
|
||||
|
||||
# ---- probing -------------------------------------------------------------------
|
||||
async def refresh(self) -> None:
|
||||
for fam in FAMILIES:
|
||||
try:
|
||||
await self._refresh_family(fam)
|
||||
except Exception:
|
||||
logger.exception("connectivity probe failed for %s", fam)
|
||||
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
while not stop.is_set():
|
||||
await self.refresh()
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=self.tier1_interval)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
async def _gateway(self, fam: str) -> Optional[str]:
|
||||
try:
|
||||
return await self._gateway_fn(fam)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
async def _refresh_family(self, fam: str) -> None:
|
||||
st = self.states[fam]
|
||||
try:
|
||||
addr = self._local_address(fam, self.probes[fam])
|
||||
except ProbeUnavailable:
|
||||
st.present = None
|
||||
return
|
||||
|
||||
if addr is None: # tier 1: no route => absent immediately
|
||||
self._fails[fam] = 0
|
||||
self._next_ping[fam] = 0.0
|
||||
self._last_key[fam] = None
|
||||
st.present, st.addr, st.gateway, st.rtt = False, None, None, None
|
||||
return
|
||||
|
||||
now = self._clock()
|
||||
gw = await self._gateway(fam) if self._cheap_gateway else st.gateway
|
||||
key = (addr, gw)
|
||||
due = key != self._last_key[fam] or now >= self._next_ping[fam]
|
||||
if due and not self._cheap_gateway:
|
||||
gw = await self._gateway(fam)
|
||||
key = (addr, gw)
|
||||
st.addr, st.gateway = addr, gw
|
||||
self._last_key[fam] = key
|
||||
if due and fam not in self._ping_disabled:
|
||||
await self._tier2(fam, st, now)
|
||||
|
||||
async def _tier2(self, fam: str, st: FamilyState, now: float) -> None:
|
||||
try:
|
||||
rtt = await self._ping(self.probes[fam], fam)
|
||||
except PingUnavailable:
|
||||
logger.warning("ping unavailable; %s presence stays unknown", fam)
|
||||
self._ping_disabled.add(fam)
|
||||
st.present = None
|
||||
return
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
rtt = None
|
||||
if rtt is None:
|
||||
self._fails[fam] += 1
|
||||
self._next_ping[fam] = now + self.retry_interval
|
||||
if self._fails[fam] >= self.fail_limit:
|
||||
st.present, st.rtt = False, None
|
||||
else:
|
||||
self._fails[fam] = 0
|
||||
self._next_ping[fam] = now + self.probe_interval
|
||||
st.present, st.rtt = True, rtt
|
||||
|
||||
|
||||
# ---- shared instance (heartbeat sender and network_monitor plugin) -----------------
|
||||
_shared: Optional[Connectivity] = None
|
||||
|
||||
|
||||
def set_shared(conn: Optional[Connectivity]) -> None:
|
||||
global _shared
|
||||
_shared = conn
|
||||
|
||||
|
||||
def get_shared() -> Optional[Connectivity]:
|
||||
return _shared
|
||||
Reference in New Issue
Block a user