thermal: watch NVMe/RP1 temps, fan stall, and undervoltage on the Pi 5

van-thermal grows sensor kinds beyond plain temperatures: "fan" alerts when
pwmfan is commanded on (pwm > 0) but reads 0 RPM (warn on first sample, crit
if it persists — 0 RPM with pwm 0 is just the firmware idling a cool SoC),
and "undervolt" goes crit on the live rpi_volt alarm plus a sticky warn off
the firmware's latched since-boot bit, so dips shorter than the 10s sample
interval still surface once. All kinds share the existing hysteresis /
journal / Pushover machinery (fan 🌀, undervolt ⚡).

Config adds nvme Composite (65/70), rp1_adc (80/85 — the RP1 die drives all
USB/eth I/O), fan, and undervolt to the existing cpu sensor. The CSV logger
now derives per-kind columns and self-rotates when the header changes; the
Cockpit Temps card renders RPM + pwm duty and undervoltage state alongside
the temperatures.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

cockpit/vanrouter/vanrouter.js

@@ -143,14 +143,26 @@ function renderThermal(th) {
         return;
     }
     const pillClass = { ok: "ok", warn: "warn", crit: "bad" };
-    let html = `<table><thead><tr><th>Sensor</th><th>Temp</th><th>Status</th><th>Warn / Crit</th></tr></thead><tbody>`;
+    let html = `<table><thead><tr><th>Sensor</th><th>Value</th><th>Status</th><th>Limits</th></tr></thead><tbody>`;
     Object.keys(th.sensors).forEach(name => {
         const s = th.sensors[name];
-        const temp = s.temp == null ? "—" : `${esc(s.temp)} °C`;
+        let value, limits;
+        if (s.kind === "fan") {
+            value = s.rpm == null ? "—" : `${esc(s.rpm)} RPM`;
+            limits = s.pwm == null ? "—" : `pwm ${esc(s.pwm)}/255`;
+        } else if (s.kind === "undervolt") {
+            value = s.now == null ? "—"
+                  : s.now ? "undervoltage"
+                  : s.since_boot ? "dip since boot" : "ok";
+            limits = "—";
+        } else {
+            value = s.temp == null ? "—" : `${esc(s.temp)} °C`;
+            limits = `${esc(s.warn)} / ${esc(s.crit)} °C`;
+        }
         const lvl = s.level || "ok";
-        html += `<tr><td>${esc(name.toUpperCase())}</td><td>${temp}</td>` +
+        html += `<tr><td>${esc(name.toUpperCase())}</td><td>${value}</td>` +
                 `<td><span class="pill ${pillClass[lvl] || ""}">${esc(lvl)}</span></td>` +
-                `<td class="muted">${esc(s.warn)} / ${esc(s.crit)} °C</td></tr>`;
+                `<td class="muted">${limits}</td></tr>`;
     });
     html += `</tbody></table>`;
     html += `<p class="muted">History: <code>/var/log/van-thermal.csv</code> · alerts: <code>journalctl -u van-thermal</code></p>`;

power/thermal-config.json

@@ -12,6 +12,31 @@
       "warn": 80,
       "crit": 85,
       "clear_margin": 5
+    },
+    {
+      "name": "nvme",
+      "hwmon": "nvme",
+      "label": "Composite",
+      "warn": 65,
+      "crit": 70,
+      "clear_margin": 5
+    },
+    {
+      "name": "rp1",
+      "hwmon": "rp1_adc",
+      "warn": 80,
+      "crit": 85,
+      "clear_margin": 5
+    },
+    {
+      "name": "fan",
+      "kind": "fan",
+      "hwmon": "pwmfan"
+    },
+    {
+      "name": "undervolt",
+      "kind": "undervolt",
+      "hwmon": "rpi_volt"
     }
   ]
 }

power/van-thermal

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""van-thermal — temperature monitor for the campervan router (wayback).
+"""van-thermal — temperature + health monitor for the campervan router.
 
 One small daemon that does three jobs off a single sysfs sample loop:
   1. publishes /run/van-thermal/state.json  (the Cockpit "Temps" card reads this,
@@ -10,7 +10,10 @@ One small daemon that does three jobs off a single sysfs sample loop:
   3. appends throttled CSV history to /var/log/van-thermal.csv with self-rotation
 
 Sensors are resolved by hwmon *name* + *label* at runtime, never by hwmonN index
-(that number is assigned at boot and is not stable). Pushover credentials are shared
+(that number is assigned at boot and is not stable). Besides temperatures a sensor
+spec may set "kind": "fan" (alerts when the fan is commanded on but reads 0 RPM) or
+"kind": "undervolt" (live rpi_volt alarm, plus the firmware's latched since-boot bit
+so dips between samples still surface). Pushover credentials are shared
 with van-battery (/etc/van-battery/pushover.json, 0600); missing/placeholder creds
 disable sending but nothing else. Stdlib only.
 """
@@ -18,6 +21,7 @@ disable sending but nothing else. Stdlib only.
 import json
 import os
 import socket
+import subprocess
 import sys
 import time
 import urllib.parse
@@ -50,6 +54,7 @@ DEFAULTS = {
 }
 
 LEVELS = ("ok", "warn", "crit")
+ICONS = {"fan": "🌀", "undervolt": "⚡"}   # pushover title icons for non-temp kinds
 PLACEHOLDERS = {"", "REPLACE_ME", "your-token-here", "your-user-key-here"}
 
 
@@ -126,15 +131,28 @@ def find_hwmon(name):
     return None
 
 
+def resolve_hwmon(spec, cache):
+    """Cached hwmon-dir lookup by name; re-resolved when the dir vanished."""
+    d = cache.get(spec["hwmon"])
+    if d is None or not d.exists():
+        d = find_hwmon(spec["hwmon"])
+        cache[spec["hwmon"]] = d
+    return d
+
+
+def read_hwmon_int(d, fname):
+    try:
+        return int((d / fname).read_text().strip())
+    except (OSError, ValueError):
+        return None
+
+
 def read_temp(spec, cache):
     """Read one sensor's temperature in °C, or None if unavailable.
 
     spec: {hwmon, label}. Resolves hwmon dir + the tempN whose *_label matches,
     falling back to temp1 when the chip exposes no labels."""
-    d = cache.get(spec["hwmon"])
-    if d is None or not d.exists():
-        d = find_hwmon(spec["hwmon"])
-        cache[spec["hwmon"]] = d
+    d = resolve_hwmon(spec, cache)
     if d is None:
         return None
 
@@ -159,6 +177,65 @@ def read_temp(spec, cache):
         return None
 
 
+def read_fan(spec, cache):
+    """(rpm, pwm) for a pwmfan hwmon, either None when unavailable."""
+    d = resolve_hwmon(spec, cache)
+    if d is None:
+        return None, None
+    rpm = read_hwmon_int(d, "fan1_input")
+    if rpm is None:
+        cache[spec["hwmon"]] = None   # hwmon may have re-enumerated
+        return None, None
+    return rpm, read_hwmon_int(d, "pwm1")
+
+
+def read_undervolt(spec, cache):
+    """(now, since_boot) undervoltage flags.
+
+    Live alarm from the rpi_volt hwmon; the firmware's latched since-boot bit
+    (get_throttled bit 16) catches dips shorter than the sample interval."""
+    now = None
+    d = resolve_hwmon(spec, cache)
+    if d is not None:
+        v = read_hwmon_int(d, "in0_lcrit_alarm")
+        if v is None:
+            cache[spec["hwmon"]] = None
+        else:
+            now = bool(v)
+    since_boot = None
+    try:
+        out = subprocess.run(["vcgencmd", "get_throttled"], capture_output=True,
+                             text=True, timeout=5).stdout
+        bits = int(out.split("=")[1], 16)
+        since_boot = bool(bits & 0x10000)
+        if now is None:
+            now = bool(bits & 0x1)
+    except Exception:
+        pass
+    return now, since_boot
+
+
+def classify_fan(rpm, pwm, prev_level):
+    """A fan commanded on (pwm > 0) reading 0 RPM is stalled/unplugged: first
+    such sample is warn, a consecutive one escalates to crit. pwm == 0 with
+    0 RPM is the firmware idling the fan on a cool SoC — that's ok."""
+    if rpm is None:
+        return prev_level
+    if rpm > 0 or pwm is None or pwm == 0:
+        return "ok"
+    return "crit" if prev_level in ("warn", "crit") else "warn"
+
+
+def classify_undervolt(now, since_boot, prev_level):
+    """crit while actively under-volted; warn (sticky until reboot) once a dip
+    has been latched, so a transient still gets one page + a yellow pill."""
+    if now is None:
+        return prev_level
+    if now:
+        return "crit"
+    return "warn" if since_boot else "ok"
+
+
 def classify(temp, spec, prev_level):
     """Level with hysteresis: step up at the threshold, step down only after
     dropping clear_margin below it, so a sensor on the line doesn't oscillate."""
@@ -175,7 +252,7 @@ def classify(temp, spec, prev_level):
     return "ok"
 
 
-def announce(name, temp, old, new):
+def announce(name, disp, old, new):
     """Log a level change to the journal. Returns the crossing direction
     ("rising" / "falling") so the caller can decide whether to page via
     Pushover, or None when the level is unchanged."""
@@ -184,7 +261,7 @@ def announce(name, temp, old, new):
     rising = LEVELS.index(new) > LEVELS.index(old)
     sev = {"crit": "crit", "warn": "warn", "ok": "info"}[new]
     arrow = "rose to" if rising else "fell back to"
-    log(f"{name} {arrow} {new.upper()} ({temp:.1f}°C)", sev if rising else "info")
+    log(f"{name} {arrow} {new.upper()} ({disp})", sev if rising else "info")
     return "rising" if rising else "falling"
 
 
@@ -196,20 +273,39 @@ def rotate_log(path, max_bytes):
         log(f"log rotate failed: {e}", "warn")
 
 
+def csv_columns(s):
+    unit = {"temp": "c", "fan": "rpm", "undervolt": "uv"}.get(s.get("kind", "temp"), "v")
+    return [f"{s['name']}_{unit}", f"{s['name']}_lvl"]
+
+
+def csv_value(r):
+    kind = r.get("kind", "temp")
+    if kind == "fan":
+        return "" if r["rpm"] is None else str(r["rpm"])
+    if kind == "undervolt":
+        return "" if r["now"] is None else str(int(r["now"]))
+    return "" if r["temp"] is None else f"{r['temp']:.1f}"
+
+
 def write_csv(cfg, readings):
     path = Path(cfg["log_path"])
     rotate_log(path, cfg["log_max_bytes"])
-    new = not path.exists()
+    header = ",".join(["time"] + sum([csv_columns(s) for s in cfg["sensors"]], []))
     try:
         path.parent.mkdir(parents=True, exist_ok=True)
+        if path.exists():
+            with open(path) as f:
+                if f.readline().rstrip("\n") != header:
+                    # Sensor set changed — rotate so columns stay aligned with the header.
+                    path.replace(path.with_suffix(path.suffix + ".1"))
+        new = not path.exists()
         with open(path, "a") as f:
             if new:
-                f.write(",".join(["time"] + sum(
-                    [[f"{s['name']}_c", f"{s['name']}_lvl"] for s in cfg["sensors"]], [])) + "\n")
+                f.write(header + "\n")
             row = [datetime.now(timezone.utc).isoformat(timespec="seconds")]
             for s in cfg["sensors"]:
                 r = readings[s["name"]]
-                row += ["" if r["temp"] is None else f"{r['temp']:.1f}", r["level"]]
+                row += [csv_value(r), r["level"]]
             f.write(",".join(row) + "\n")
     except OSError as e:
         log(f"csv write failed: {e}", "warn")
@@ -240,31 +336,56 @@ def main():
     while True:
         readings = {}
         for s in cfg["sensors"]:
+            name, kind = s["name"], s.get("kind", "temp")
+            old = levels[name]
+
+            if kind == "fan":
+                rpm, pwm = read_fan(s, cache)
+                new = classify_fan(rpm, pwm, old)
+                disp = None if rpm is None else f"{rpm} RPM"
+                detail = (f"fan turning at {rpm} RPM (pwm {pwm}/255)." if new == "ok" else
+                          f"fan reads 0 RPM while commanded on (pwm {pwm}/255) — "
+                          "stalled, blocked, or unplugged.")
+                reading = {"kind": kind, "rpm": rpm, "pwm": pwm}
+            elif kind == "undervolt":
+                uv_now, uv_boot = read_undervolt(s, cache)
+                new = classify_undervolt(uv_now, uv_boot, old)
+                disp = None if uv_now is None else (
+                    "UNDERVOLTAGE" if uv_now else
+                    "dip since boot" if uv_boot else "supply ok")
+                detail = {"crit": "supply voltage below threshold right now — check PSU and cabling.",
+                          "warn": "an undervoltage dip was latched since boot (supply ok now; "
+                                  "latch clears on reboot).",
+                          "ok": "supply voltage ok."}[new]
+                reading = {"kind": kind, "now": uv_now, "since_boot": uv_boot}
+            else:
                 temp = read_temp(s, cache)
-            new = classify(temp, s, levels[s["name"]])
-            if temp is not None:
-                old = levels[s["name"]]
-                direction = announce(s["name"], temp, old, new)
-                if direction == "rising" and LEVELS.index(new) >= alert_idx:
-                    icon = "🔥" if new == "crit" else "🌡"
+                new = classify(temp, s, old)
+                disp = None if temp is None else f"{temp:.1f}°C"
                 thr = s["crit"] if new == "crit" else s["warn"]
-                    pushover(cfg, f"{icon} {HOST}: {s['name']} {new.upper()} {temp:.1f}°C",
-                             f"{s['name']} temperature {temp:.1f}°C crossed {new.upper()} "
-                             f"threshold ({thr}°C). warn {s['warn']}, crit {s['crit']}.",
+                detail = (f"below warn ({s['warn']}°C)." if new == "ok" else
+                          f"crossed the {new.upper()} threshold ({thr}°C); "
+                          f"warn {s['warn']}, crit {s['crit']}.")
+                reading = {"kind": kind, "temp": None if temp is None else round(temp, 1),
+                           "warn": s["warn"], "crit": s["crit"]}
+
+            if disp is not None:
+                direction = announce(name, disp, old, new)
+                if direction == "rising" and LEVELS.index(new) >= alert_idx:
+                    icon = ICONS.get(kind, "🔥" if new == "crit" else "🌡")
+                    pushover(cfg, f"{icon} {HOST}: {name} {new.upper()} — {disp}",
+                             f"{name}: {detail}",
                              priority=1 if new == "crit" else 0)
                 elif direction == "falling" and LEVELS.index(old) >= alert_idx:
                     # Recovery: page only when leaving a level we'd have paged about,
                     # so the phone that got the rising alert also gets the all-clear.
                     label = "NORMAL" if new == "ok" else new.upper()
-                    pushover(cfg, f"✅ {HOST}: {s['name']} back to {label} {temp:.1f}°C",
-                             f"{s['name']} temperature {temp:.1f}°C dropped back to {label} "
-                             f"(warn {s['warn']}, crit {s['crit']}).",
-                             priority=0)
-            levels[s["name"]] = new
-            readings[s["name"]] = {
-                "temp": None if temp is None else round(temp, 1),
-                "level": new, "warn": s["warn"], "crit": s["crit"],
-            }
+                    pushover(cfg, f"✅ {HOST}: {name} back to {label} — {disp}",
+                             f"{name}: {detail}", priority=0)
+
+            levels[name] = new
+            reading["level"] = new
+            readings[name] = reading
 
         try:
             write_state(readings)

power/van-thermal.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Temperature monitor (CPU + NVMe) for the campervan router
+Description=Thermal + health monitor (temps, fan, undervoltage) for the campervan router
 # Order after network-online so a boot-time threshold crossing (common: the Pi
 # boots hot) can actually reach Pushover — the very first sample fires within
 # seconds of start. If no WAN comes up, wait-online times out and we start anyway.