refactor monitor, add threshold rtesting

hbd/server/main.py

@@ -14,28 +14,35 @@ from . import hbdclass
 
 from . import ws as ws_mod
 from . import notify as notify_mod
+from . import data 
 
 logger = logging.getLogger(__name__)
 msg_to_websockets = ws_mod.broadcast
-eventlog = notify_mod.log
-
-lastfm = ["", "", ""]
+eventlog = notify_mod.eventlog
 
 # shared runtime collections and helpers
-msgs = notify_mod.msgs
 
-def cleanup_function(config):
+def cleanup_function(config, hbdclass):
     """This function will be executed upon program exit."""
     logger.info("Running cleanup function...")
     import pickle
 
+    # Ensure all timer references are cleared before pickling
+    for hostname, host in list(hbdclass.Host.hosts.items()):
+        for conn_type, conn in host.connections.items():
+            if hasattr(conn, 'overdue_timer'):
+                conn.overdue_timer = None
+            if hasattr(conn, 'overdue_callback'):
+                conn.overdue_callback = None
+            if hasattr(conn, 'timeout_duration'):
+                conn.timeout_duration = None
+
     pickfile = config.get("pickfile", "hbd.pickle")
 
     pickf = open(pickfile, "wb")
     pick = pickle.Pickler(pickf)
     pick.dump(hbdclass.Host.hosts)
-    pick.dump(msgs)
-    pick.dump(lastfm)
+    pick.dump(data.msgs)
     pickf.close()
 
     logger.info("Cleanup complete.")
@@ -124,7 +131,6 @@ async def _run_async(config):
                 port=config.get("hbd_port", 50004),
                 config=config,
                 hbdclass=hbdclass,
-                msgs_getter=lambda: msgs,
                 log=eventlog,
                 pushmsg=pushmsg,
                 msg_to_websockets=msg_to_websockets,
@@ -186,7 +192,7 @@ async def _run_async(config):
                     hbdclass.Host.hosts[h].stateinfo()
                     for h in sorted(hbdclass.Host.hosts)
                 ],
-                get_msgs=lambda: msgs,
+#                get_msgs=lambda: msgs,
                 config=config,
             )
         )
@@ -200,7 +206,6 @@ async def _run_async(config):
             monitor_mod.start(
                 config=config,
                 hbdclass=hbdclass,
-                log=eventlog,
                 pushmsg=pushmsg,
                 msg_to_websockets=msg_to_websockets,
             )
@@ -216,6 +221,13 @@ async def _run_async(config):
     except Exception as e:
         logger.exception("Error in main loop: %s", e)
     finally:
+        # Clean up connection timers
+        try:
+            logger.info("Cleaning up connection timers...")
+            await monitor_mod.cleanup_connections(hbdclass)
+        except Exception as e:
+            logger.warning("Error cleaning up connection timers: %s", e)
+        
         # Cancel all running tasks
         logger.info("Cancelling tasks...")
         try:
@@ -279,7 +291,6 @@ async def _run_async(config):
 
 def load_pickled_hosts(config, hbdclass):
     """Load pickled hosts from file, if available."""
-    global lastfm, msgs
     import os
     import pickle
 
@@ -294,11 +305,7 @@ def load_pickled_hosts(config, hbdclass):
         pick = pickle.Unpickler(pickf)
         try:
             hbdclass.Host.hosts = pick.load()
-            msgs = pick.load()
-            try:
-                lastfm = pick.load()
-            except Exception:
-                lastfm = ["", "", ""]
+            data.msgs = pick.load()
             pickf.close()
         except Exception as e:
             logger.exception("load pickled failed: %s", e)
@@ -331,7 +338,7 @@ def run(config):
     load_pickled_hosts(config, hbdclass)
 
     notify_mod.initlog(logfile=config.get("logfile", "messages.log"))
-    eventlog(None, f"hbd version {__version__} starting up")
+    eventlog(None, "INFO", f"hbd version {__version__} starting up")
 
     # Create and set the event loop manually
     loop = asyncio.new_event_loop()
@@ -344,8 +351,9 @@ def run(config):
     except Exception as e:
         logger.exception("Unhandled exception in main: %s", e)
     finally:
-        cleanup_function(config)
+        cleanup_function(config, hbdclass)
         logger.info("hbd shutdown complete")
+        eventlog(None, "INFO", f"hbd version {__version__} shutdown")
         notify_mod.closelog()
         # Explicitly close the loop
         try: