51 lines
1.6 KiB
Python
51 lines
1.6 KiB
Python
"""monitor helper and thread for heartbeat daemon."""
|
|
|
|
from __future__ import annotations
|
|
import asyncio
|
|
import time
|
|
|
|
DROPOVERDUE = 7 * 24 * 3600
|
|
|
|
|
|
def checkoverdue(
|
|
config: dict,
|
|
hbdclass,
|
|
log: callable,
|
|
pushmsg: callable,
|
|
msg_to_websockets: callable,
|
|
):
|
|
now = time.time()
|
|
for h in list(hbdclass.Host.hosts.keys()):
|
|
pmsg = []
|
|
for c in hbdclass.Host.hosts[h].connections:
|
|
conn = hbdclass.Host.hosts[h].connections[c]
|
|
if conn.state == hbdclass.Connection.DOWN:
|
|
continue
|
|
timeout = hbdclass.Host.hosts[h].interval + config.get("grace", 10)
|
|
if conn.state == hbdclass.Connection.UP and (now - conn.lastbeat) > timeout:
|
|
conn.newstate(hbdclass.Connection.OVERDUE, now, config.get("grace", 10))
|
|
pmsg.append(conn.afam)
|
|
if (
|
|
conn.state == hbdclass.Connection.OVERDUE
|
|
and (now - conn.lastbeat) > DROPOVERDUE
|
|
):
|
|
conn.newstate(hbdclass.Connection.UNKNOWN, conn.lastbeat)
|
|
if pmsg != []:
|
|
if h in config.get("watchhosts", []):
|
|
pushmsg("%s %s overdue" % (h, " and ".join(pmsg)))
|
|
log(h, "%s overdue" % " and ".join(pmsg))
|
|
msg_to_websockets("host", hbdclass.Host.hosts[h].stateinfo())
|
|
|
|
|
|
async def start(
|
|
config: dict,
|
|
hbdclass: callable,
|
|
log=None,
|
|
pushmsg=None,
|
|
msg_to_websockets=None,
|
|
):
|
|
"""start a monitor loop that checks for overdue hosts every minute"""
|
|
while True:
|
|
await asyncio.sleep(15) # 15 seconds between checks
|
|
checkoverdue(config, hbdclass, log, pushmsg, msg_to_websockets)
|