feat(client): re-resolve the server hostnames at the DNS TTL
hbc, hbc_mini.py and hbc_mini.c resolved the server once at startup, so an address change needed a client restart. They now re-resolve every TTL seconds (smallest A/AAAA TTL, floor 30s, 300s when unknown) and re-point, add or drop connections to follow the new addresses; a failed lookup keeps the current ones. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5.5
@@ -28,6 +28,7 @@ IPv4) path as an outage. Config, in the network_monitor section of ~/.hbc.json:
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -36,6 +37,7 @@ import re
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
@@ -1164,6 +1166,14 @@ class AsyncConnection:
|
||||
self._transport.close()
|
||||
self._transport = None
|
||||
|
||||
def set_target(self, addr: str, af: int):
|
||||
"""Re-point at a new server address; the socket is reopened on the next send if the family changed."""
|
||||
if af != self.af:
|
||||
self.close()
|
||||
self.af = af
|
||||
self.addr = addr
|
||||
self._log = logging.getLogger(f"hbc.conn.{addr}")
|
||||
|
||||
def _handle_ack(self, msg: Dict[str, Any], now: float):
|
||||
rtt = (now - self.lastsend) * 1000.0
|
||||
self.rtts.append(rtt)
|
||||
@@ -1188,6 +1198,207 @@ class AsyncConnection:
|
||||
self.lastsend = time.time()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Re-resolution of the server hosts at the TTL of their A/AAAA records
|
||||
# getaddrinfo() hides the TTL, so a minimal stub-resolver query is made only to learn it; the
|
||||
# addresses still come from getaddrinfo(). TTL unknown (IP literal, /etc/hosts name, no
|
||||
# resolv.conf, timeout) -> _DNS_DEFAULT_TTL.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log = logging.getLogger("hbc.dns")
|
||||
|
||||
_DNS_DEFAULT_TTL = 300 # used when the real TTL is unknown
|
||||
_DNS_MIN_TTL = 30 # a TTL of 0 must not turn into a resolution loop
|
||||
_DNS_FAILED_RETRY = 60 # re-try delay after a failed resolution
|
||||
_DNS_QUERY_TIMEOUT = 2.0
|
||||
|
||||
_DNS_TYPE_A, _DNS_TYPE_CNAME, _DNS_TYPE_AAAA = 1, 5, 28
|
||||
|
||||
|
||||
def _dns_parse_resolv_conf(path: str = "/etc/resolv.conf") -> Tuple[List[str], List[str]]:
|
||||
"""Return (nameservers, search domains)."""
|
||||
servers: List[str] = []
|
||||
search: List[str] = []
|
||||
try:
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
if parts[0] == "nameserver":
|
||||
servers.append(parts[1].split("%")[0])
|
||||
elif parts[0] in ("search", "domain"):
|
||||
search = parts[1:]
|
||||
except OSError:
|
||||
pass
|
||||
return servers, search
|
||||
|
||||
|
||||
def _dns_build_query(name: str, qtype: int, qid: int) -> bytes:
|
||||
out = struct.pack("!HHHHHH", qid, 0x0100, 1, 0, 0, 0)
|
||||
for label in name.rstrip(".").split("."):
|
||||
out += bytes([len(label)]) + label.encode("idna")
|
||||
return out + b"\x00" + struct.pack("!HH", qtype, 1)
|
||||
|
||||
|
||||
def _dns_skip_name(buf: bytes, off: int) -> int:
|
||||
while True:
|
||||
n = buf[off]
|
||||
if n == 0:
|
||||
return off + 1
|
||||
if n & 0xC0:
|
||||
return off + 2
|
||||
off += n + 1
|
||||
|
||||
|
||||
def _dns_min_answer_ttl(resp: bytes, qid: int, qtype: int) -> Optional[int]:
|
||||
"""Smallest TTL among the answer RRs (CNAMEs included), or None if there is no qtype answer."""
|
||||
try:
|
||||
rid, flags, qd, an = struct.unpack("!HHHH", resp[:8])
|
||||
if rid != qid or flags & 0x000F:
|
||||
return None
|
||||
off = 12
|
||||
for _ in range(qd):
|
||||
off = _dns_skip_name(resp, off) + 4
|
||||
ttls: List[int] = []
|
||||
found = False
|
||||
for _ in range(an):
|
||||
off = _dns_skip_name(resp, off)
|
||||
rtype, _cls, ttl, rdlen = struct.unpack("!HHIH", resp[off:off + 10])
|
||||
off += 10 + rdlen
|
||||
if rtype in (qtype, _DNS_TYPE_CNAME):
|
||||
ttls.append(ttl)
|
||||
found = found or rtype == qtype
|
||||
return min(ttls) if found else None
|
||||
except (struct.error, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _dns_query_ttl(server: str, name: str, qtype: int) -> Optional[int]:
|
||||
qid = struct.unpack("!H", os.urandom(2))[0]
|
||||
try:
|
||||
af = socket.AF_INET6 if ":" in server else socket.AF_INET
|
||||
with socket.socket(af, socket.SOCK_DGRAM) as s:
|
||||
s.settimeout(_DNS_QUERY_TIMEOUT)
|
||||
s.sendto(_dns_build_query(name, qtype, qid), (server, 53))
|
||||
return _dns_min_answer_ttl(s.recv(4096), qid, qtype)
|
||||
except (OSError, UnicodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _dns_lookup_ttl(host: str, af_filter: int = 0) -> Optional[int]:
|
||||
"""Blocking. Smallest A/AAAA TTL for host, or None when it cannot be determined."""
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
return None
|
||||
except ValueError:
|
||||
pass
|
||||
servers, search = _dns_parse_resolv_conf()
|
||||
if host.endswith("."):
|
||||
names = [host]
|
||||
elif "." in host:
|
||||
names = [host] + [f"{host}.{d}" for d in search]
|
||||
else:
|
||||
names = [f"{host}.{d}" for d in search] + [host]
|
||||
qtypes = [t for t, af in ((_DNS_TYPE_A, socket.AF_INET), (_DNS_TYPE_AAAA, socket.AF_INET6))
|
||||
if af_filter in (0, af)]
|
||||
for name in names:
|
||||
ttls = []
|
||||
for qtype in qtypes:
|
||||
for server in servers:
|
||||
ttl = _dns_query_ttl(server, name, qtype)
|
||||
if ttl is not None:
|
||||
ttls.append(ttl)
|
||||
break
|
||||
if ttls:
|
||||
return min(ttls)
|
||||
return None
|
||||
|
||||
|
||||
def _dns_refresh_delay(ttls: List[Optional[int]]) -> int:
|
||||
"""Seconds until the next resolution: the smallest known TTL, floored at _DNS_MIN_TTL."""
|
||||
known = [t for t in ttls if t is not None]
|
||||
return max(min(known), _DNS_MIN_TTL) if known else _DNS_DEFAULT_TTL
|
||||
|
||||
|
||||
def _dns_resolve(hosts: List[str], port: int, af_filter: int) -> Optional[List[Tuple[int, str]]]:
|
||||
"""Blocking. Every (family, address) the hosts resolve to, deduplicated, or None if any lookup fails."""
|
||||
found: List[Tuple[int, str]] = []
|
||||
for host in hosts:
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, port, af_filter, 0, socket.SOL_UDP)
|
||||
except socket.gaierror as e:
|
||||
log.warning("cannot resolve %s: %s", host, e)
|
||||
return None
|
||||
for info in infos:
|
||||
entry = (info[0], str(info[4][0]))
|
||||
if entry not in found:
|
||||
found.append(entry)
|
||||
return found or None
|
||||
|
||||
|
||||
def _dns_retarget(connections: list, resolved: List[Tuple[int, str]]) -> Tuple[List[Tuple[int, str]], list]:
|
||||
"""Point the connections at the resolved addresses, in place.
|
||||
|
||||
A connection whose address is gone is re-pointed at an address that is new, in list order, so
|
||||
the first connection (the one plugin data uses) survives as long as anything resolves.
|
||||
Returns (new addresses still needing a connection, connections to drop).
|
||||
"""
|
||||
current = {c.addr for c in connections}
|
||||
wanted = {addr for _, addr in resolved}
|
||||
new = [(af, addr) for af, addr in resolved if addr not in current]
|
||||
gone = [c for c in connections if c.addr not in wanted]
|
||||
paired = min(len(gone), len(new))
|
||||
for conn, (af, addr) in zip(gone, new):
|
||||
log.info("server address %s -> %s", conn.addr, addr)
|
||||
conn.set_target(addr, af)
|
||||
drop = gone[paired:]
|
||||
if connections and connections[0] in drop:
|
||||
# plugin data goes out on the first connection: hand it a surviving address and drop that one instead
|
||||
keep = next(c for c in connections if c not in gone)
|
||||
connections[0].set_target(keep.addr, keep.af)
|
||||
drop.remove(connections[0])
|
||||
drop.append(keep)
|
||||
return new[paired:], drop
|
||||
|
||||
|
||||
async def _dns_refresher(connections: List["AsyncConnection"], hosts: List[str], port: int, af_filter: int,
|
||||
name: str, interval: int, probe: Optional[_Connectivity], next_id: int):
|
||||
"""Re-resolve the server hosts every TTL seconds and follow address changes.
|
||||
|
||||
A failed lookup leaves the connections untouched. Addresses that disappear are re-pointed at
|
||||
new ones where possible; the rest are retired, and surplus new ones get a connection.
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def ttl_delay() -> int:
|
||||
return _dns_refresh_delay([_dns_lookup_ttl(h, af_filter) for h in hosts])
|
||||
|
||||
delay = await loop.run_in_executor(None, ttl_delay)
|
||||
while _running:
|
||||
await _sleep(delay)
|
||||
if not _running:
|
||||
break
|
||||
resolved = await loop.run_in_executor(None, _dns_resolve, hosts, port, af_filter)
|
||||
if resolved is None:
|
||||
delay = _DNS_FAILED_RETRY
|
||||
continue
|
||||
new, gone = _dns_retarget(connections, resolved)
|
||||
for conn in gone:
|
||||
if len(connections) > 1:
|
||||
log.info("dropping server address %s", conn.addr)
|
||||
conn._dead = True
|
||||
conn.close()
|
||||
connections.remove(conn)
|
||||
for af, addr in new:
|
||||
conn = AsyncConnection(next_id, addr, port, af, name)
|
||||
next_id += 1
|
||||
log.info("adding server address %s", addr)
|
||||
connections.append(conn)
|
||||
_active_tasks.append(asyncio.create_task(_heartbeat_sender(conn, interval, probe)))
|
||||
delay = await loop.run_in_executor(None, ttl_delay)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Server command handlers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1463,6 +1674,9 @@ async def _async_main(args, cfg: Dict[str, Any]) -> int:
|
||||
for conn in connections:
|
||||
_active_tasks.append(asyncio.create_task(_heartbeat_sender(conn, interval, _probe)))
|
||||
|
||||
_active_tasks.append(asyncio.create_task(
|
||||
_dns_refresher(connections, args.hosts, port, af_filter, iam, interval, _probe, conn_id)))
|
||||
|
||||
if plugins and connections:
|
||||
_active_tasks.append(asyncio.create_task(_plugin_collector(connections[0], plugins)))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user