47 lines
1.8 KiB
Python
47 lines
1.8 KiB
Python
"""monitor helper and thread for heartbeat daemon."""
|
|
from __future__ import annotations
|
|
import asyncio
|
|
import threading
|
|
import subprocess
|
|
import time
|
|
from subprocess import Popen, PIPE, STDOUT
|
|
from typing import Optional
|
|
from . import hbdclass
|
|
DROPOVERDUE = 7 * 24 * 3600
|
|
|
|
def checkoverdue(config: dict, hbdclass, log: callable, email: callable, pushmsg: callable, msg_to_websockets: callable):
|
|
now = time.time()
|
|
for h in list(hbdclass.Host.hosts.keys()):
|
|
pmsg = []
|
|
for c in hbdclass.Host.hosts[h].connections:
|
|
conn = hbdclass.Host.hosts[h].connections[c]
|
|
if conn.state == hbdclass.Connection.DOWN:
|
|
continue
|
|
timeout = hbdclass.Host.hosts[h].interval + config.get("grace", 10)
|
|
if conn.state == hbdclass.Connection.UP and (now - conn.lastbeat) > timeout:
|
|
conn.newstate(hbdclass.Connection.OVERDUE, now, config.get("grace", 10))
|
|
pmsg.append(conn.afam)
|
|
if (
|
|
conn.state == hbdclass.Connection.OVERDUE and (now - conn.lastbeat) > DROPOVERDUE
|
|
):
|
|
conn.newstate(hbdclass.Connection.UNKNOWN, conn.lastbeat)
|
|
if pmsg != []:
|
|
if h in config.get("watchhosts", []):
|
|
email("overdue", "%s overdue" % " and ".join(pmsg))
|
|
pushmsg("%s %s overdue" % (h, " and ".join(pmsg)))
|
|
log(h, "%s overdue" % " and ".join(pmsg))
|
|
msg_to_websockets("host", hbdclass.Host.hosts[h].stateinfo())
|
|
|
|
async def start(
|
|
config: dict,
|
|
hbdclass: callable,
|
|
log=None,
|
|
email=None,
|
|
pushmsg=None,
|
|
msg_to_websockets=None,
|
|
):
|
|
""" start a monitor loop that checks for overdue hosts every minute """
|
|
while True:
|
|
await asyncio.sleep(15) # 15 seconds between checks
|
|
checkoverdue(config, hbdclass, log, email, pushmsg, msg_to_websockets)
|