"""monitor helper and thread for heartbeat daemon.""" from __future__ import annotations import asyncio import threading import subprocess import time from subprocess import Popen, PIPE, STDOUT from typing import Optional from . import hbdclass DROPOVERDUE = 7 * 24 * 3600 def checkoverdue(config: dict, hbdclass, log: callable, pushmsg: callable, msg_to_websockets: callable): now = time.time() for h in list(hbdclass.Host.hosts.keys()): pmsg = [] for c in hbdclass.Host.hosts[h].connections: conn = hbdclass.Host.hosts[h].connections[c] if conn.state == hbdclass.Connection.DOWN: continue timeout = hbdclass.Host.hosts[h].interval + config.get("grace", 10) if conn.state == hbdclass.Connection.UP and (now - conn.lastbeat) > timeout: conn.newstate(hbdclass.Connection.OVERDUE, now, config.get("grace", 10)) pmsg.append(conn.afam) if ( conn.state == hbdclass.Connection.OVERDUE and (now - conn.lastbeat) > DROPOVERDUE ): conn.newstate(hbdclass.Connection.UNKNOWN, conn.lastbeat) if pmsg != []: if h in config.get("watchhosts", []): pushmsg("%s %s overdue" % (h, " and ".join(pmsg))) log(h, "%s overdue" % " and ".join(pmsg)) msg_to_websockets("host", hbdclass.Host.hosts[h].stateinfo()) async def start( config: dict, hbdclass: callable, log=None, pushmsg=None, msg_to_websockets=None, ): """ start a monitor loop that checks for overdue hosts every minute """ while True: await asyncio.sleep(15) # 15 seconds between checks checkoverdue(config, hbdclass, log, pushmsg, msg_to_websockets)