thermal: alert on recovery (crit→warn, warn→normal)

van-thermal already logged falling crossings to the journal but only paged
Pushover on rising ones. announce() now returns the crossing direction, and a
falling crossing pages an "all-clear" when leaving a level we'd have alerted on,
so the phone that got the rising alert also gets the recovery. thermal-config
gains the pushover_level + shared credentials_path keys. Ignore __pycache__.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.gitignore

@@ -1 +1,3 @@
 .claude/*
+__pycache__/
+*.pyc

power/thermal-config.json

@@ -3,6 +3,8 @@
   "log_interval": 60,
   "log_path": "/var/log/van-thermal.csv",
   "log_max_bytes": 5242880,
+  "pushover_level": "warn",
+  "credentials_path": "/etc/van-battery/pushover.json",
   "sensors": [
     { "name": "cpu",  "hwmon": "coretemp", "label": "Package id 0", "warn": 80, "crit": 95, "clear_margin": 5 },
     { "name": "nvme", "hwmon": "nvme",     "label": "Composite",    "warn": 65, "crit": 70, "clear_margin": 5 }

power/van-thermal

@@ -5,17 +5,23 @@ One small daemon that does three jobs off a single sysfs sample loop:
   1. publishes /run/van-thermal/state.json  (the Cockpit "Temps" card reads this,
      exactly like van-failover's state.json — no shelling out to `sensors` per refresh)
   2. warns to the journal on threshold crossings, with hysteresis so a sensor hovering
-     on the line doesn't spam (journalctl -u van-thermal)
+     on the line doesn't spam (journalctl -u van-thermal), and sends a Pushover alert
+     on each *rising* crossing into warn/crit (same hysteresis debounces the phone)
   3. appends throttled CSV history to /var/log/van-thermal.csv with self-rotation
 
 Sensors are resolved by hwmon *name* + *label* at runtime, never by hwmonN index
-(that number is assigned at boot and is not stable). Stdlib only.
+(that number is assigned at boot and is not stable). Pushover credentials are shared
+with van-battery (/etc/van-battery/pushover.json, 0600); missing/placeholder creds
+disable sending but nothing else. Stdlib only.
 """
 
 import json
 import os
+import socket
 import sys
 import time
+import urllib.parse
+import urllib.request
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -23,12 +29,18 @@ CONFIG_PATH = os.environ.get("VAN_THERMAL_CONFIG", "/etc/van-thermal/config.json
 STATE_DIR = Path("/run/van-thermal")
 STATE_PATH = STATE_DIR / "state.json"
 HWMON = Path("/sys/class/hwmon")
+HOST = socket.gethostname()
 
 DEFAULTS = {
     "sample_interval": 10,      # seconds between sysfs reads (alerting cadence)
     "log_interval": 60,         # seconds between CSV rows (history cadence)
     "log_path": "/var/log/van-thermal.csv",
     "log_max_bytes": 5 * 1024 * 1024,   # rotate to .1 past this, keep one old file
+    # Lowest level whose rising crossing sends Pushover ("warn" or "crit"). Journal
+    # logging happens at every crossing regardless of this.
+    "pushover_level": "warn",
+    # Shared with van-battery so there's a single secret to maintain.
+    "credentials_path": "/etc/van-battery/pushover.json",
     "sensors": [
         {"name": "cpu",  "hwmon": "coretemp", "label": "Package id 0",
          "warn": 85, "crit": 95, "clear_margin": 5},
@@ -38,6 +50,7 @@ DEFAULTS = {
 }
 
 LEVELS = ("ok", "warn", "crit")
+PLACEHOLDERS = {"", "REPLACE_ME", "your-token-here", "your-user-key-here"}
 
 
 def log(msg, level="info"):
@@ -46,6 +59,42 @@ def log(msg, level="info"):
     print(pri + msg, flush=True)
 
 
+def load_creds(cfg):
+    try:
+        c = json.loads(Path(cfg["credentials_path"]).read_text())
+        token, user = str(c.get("token", "")).strip(), str(c.get("user", "")).strip()
+        if token in PLACEHOLDERS or user in PLACEHOLDERS:
+            return None
+        return token, user
+    except FileNotFoundError:
+        return None
+    except Exception as e:
+        log(f"credentials {cfg['credentials_path']} unreadable ({e})", "warn")
+        return None
+
+
+def pushover(cfg, title, message, priority=0):
+    creds = load_creds(cfg)
+    if not creds:
+        log(f"pushover skipped (no credentials): {title} — {message}", "warn")
+        return False
+    token, user = creds
+    data = urllib.parse.urlencode({
+        "token": token, "user": user, "title": title,
+        "message": message, "priority": priority,
+    }).encode()
+    req = urllib.request.Request("https://api.pushover.net/1/messages.json", data=data)
+    try:
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            ok = resp.status == 200
+        if not ok:
+            log(f"pushover HTTP {resp.status}", "warn")
+        return ok
+    except Exception as e:
+        log(f"pushover send failed: {e}", "warn")
+        return False
+
+
 def load_config():
     cfg = dict(DEFAULTS)
     try:
@@ -120,12 +169,16 @@ def classify(temp, spec, prev_level):
 
 
 def announce(name, temp, old, new):
+    """Log a level change to the journal. Returns the crossing direction
+    ("rising" / "falling") so the caller can decide whether to page via
+    Pushover, or None when the level is unchanged."""
     if old == new:
-        return
+        return None
     rising = LEVELS.index(new) > LEVELS.index(old)
     sev = {"crit": "crit", "warn": "warn", "ok": "info"}[new]
     arrow = "rose to" if rising else "fell back to"
     log(f"{name} {arrow} {new.upper()} ({temp:.1f}°C)", sev if rising else "info")
+    return "rising" if rising else "falling"
 
 
 def rotate_log(path, max_bytes):
@@ -171,8 +224,11 @@ def main():
     cache = {}
     levels = {s["name"]: "ok" for s in cfg["sensors"]}
     last_log = 0.0
+    # Minimum level (by severity index) whose rising crossing pages via Pushover.
+    alert_idx = LEVELS.index(cfg["pushover_level"]) if cfg["pushover_level"] in LEVELS else 1
     log(f"van-thermal up: sampling every {cfg['sample_interval']}s, "
-        f"logging every {cfg['log_interval']}s to {cfg['log_path']}")
+        f"logging every {cfg['log_interval']}s to {cfg['log_path']}; "
+        f"Pushover on rising >= {LEVELS[alert_idx].upper()}")
 
     while True:
         readings = {}
@@ -180,7 +236,23 @@ def main():
             temp = read_temp(s, cache)
             new = classify(temp, s, levels[s["name"]])
             if temp is not None:
-                announce(s["name"], temp, levels[s["name"]], new)
+                old = levels[s["name"]]
+                direction = announce(s["name"], temp, old, new)
+                if direction == "rising" and LEVELS.index(new) >= alert_idx:
+                    icon = "🔥" if new == "crit" else "🌡"
+                    thr = s["crit"] if new == "crit" else s["warn"]
+                    pushover(cfg, f"{icon} {HOST}: {s['name']} {new.upper()} {temp:.1f}°C",
+                             f"{s['name']} temperature {temp:.1f}°C crossed {new.upper()} "
+                             f"threshold ({thr}°C). warn {s['warn']}, crit {s['crit']}.",
+                             priority=1 if new == "crit" else 0)
+                elif direction == "falling" and LEVELS.index(old) >= alert_idx:
+                    # Recovery: page only when leaving a level we'd have paged about,
+                    # so the phone that got the rising alert also gets the all-clear.
+                    label = "NORMAL" if new == "ok" else new.upper()
+                    pushover(cfg, f"✅ {HOST}: {s['name']} back to {label} {temp:.1f}°C",
+                             f"{s['name']} temperature {temp:.1f}°C dropped back to {label} "
+                             f"(warn {s['warn']}, crit {s['crit']}).",
+                             priority=0)
             levels[s["name"]] = new
             readings[s["name"]] = {
                 "temp": None if temp is None else round(temp, 1),