diff --git a/cockpit/vanrouter/vanrouter.js b/cockpit/vanrouter/vanrouter.js index 205ded1..dfd589d 100644 --- a/cockpit/vanrouter/vanrouter.js +++ b/cockpit/vanrouter/vanrouter.js @@ -143,14 +143,26 @@ function renderThermal(th) { return; } const pillClass = { ok: "ok", warn: "warn", crit: "bad" }; - let html = ``; + let html = `
SensorTempStatusWarn / Crit
`; Object.keys(th.sensors).forEach(name => { const s = th.sensors[name]; - const temp = s.temp == null ? "—" : `${esc(s.temp)} °C`; + let value, limits; + if (s.kind === "fan") { + value = s.rpm == null ? "—" : `${esc(s.rpm)} RPM`; + limits = s.pwm == null ? "—" : `pwm ${esc(s.pwm)}/255`; + } else if (s.kind === "undervolt") { + value = s.now == null ? "—" + : s.now ? "undervoltage" + : s.since_boot ? "dip since boot" : "ok"; + limits = "—"; + } else { + value = s.temp == null ? "—" : `${esc(s.temp)} °C`; + limits = `${esc(s.warn)} / ${esc(s.crit)} °C`; + } const lvl = s.level || "ok"; - html += `` + + html += `` + `` + - ``; + ``; }); html += `
SensorValueStatusLimits
${esc(name.toUpperCase())}${temp}
${esc(name.toUpperCase())}${value}${esc(lvl)}${esc(s.warn)} / ${esc(s.crit)} °C
${limits}
`; html += `

History: /var/log/van-thermal.csv · alerts: journalctl -u van-thermal

`; diff --git a/power/thermal-config.json b/power/thermal-config.json index c780a9a..2fad9a2 100644 --- a/power/thermal-config.json +++ b/power/thermal-config.json @@ -12,6 +12,31 @@ "warn": 80, "crit": 85, "clear_margin": 5 + }, + { + "name": "nvme", + "hwmon": "nvme", + "label": "Composite", + "warn": 65, + "crit": 70, + "clear_margin": 5 + }, + { + "name": "rp1", + "hwmon": "rp1_adc", + "warn": 80, + "crit": 85, + "clear_margin": 5 + }, + { + "name": "fan", + "kind": "fan", + "hwmon": "pwmfan" + }, + { + "name": "undervolt", + "kind": "undervolt", + "hwmon": "rpi_volt" } ] } diff --git a/power/van-thermal b/power/van-thermal index 7a3b71d..c323d20 100644 --- a/power/van-thermal +++ b/power/van-thermal @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""van-thermal — temperature monitor for the campervan router (wayback). +"""van-thermal — temperature + health monitor for the campervan router. One small daemon that does three jobs off a single sysfs sample loop: 1. publishes /run/van-thermal/state.json (the Cockpit "Temps" card reads this, @@ -10,7 +10,10 @@ One small daemon that does three jobs off a single sysfs sample loop: 3. appends throttled CSV history to /var/log/van-thermal.csv with self-rotation Sensors are resolved by hwmon *name* + *label* at runtime, never by hwmonN index -(that number is assigned at boot and is not stable). Pushover credentials are shared +(that number is assigned at boot and is not stable). Besides temperatures a sensor +spec may set "kind": "fan" (alerts when the fan is commanded on but reads 0 RPM) or +"kind": "undervolt" (live rpi_volt alarm, plus the firmware's latched since-boot bit +so dips between samples still surface). Pushover credentials are shared with van-battery (/etc/van-battery/pushover.json, 0600); missing/placeholder creds disable sending but nothing else. Stdlib only. """ @@ -18,6 +21,7 @@ disable sending but nothing else. Stdlib only. import json import os import socket +import subprocess import sys import time import urllib.parse @@ -50,6 +54,7 @@ DEFAULTS = { } LEVELS = ("ok", "warn", "crit") +ICONS = {"fan": "🌀", "undervolt": "⚡"} # pushover title icons for non-temp kinds PLACEHOLDERS = {"", "REPLACE_ME", "your-token-here", "your-user-key-here"} @@ -126,15 +131,28 @@ def find_hwmon(name): return None +def resolve_hwmon(spec, cache): + """Cached hwmon-dir lookup by name; re-resolved when the dir vanished.""" + d = cache.get(spec["hwmon"]) + if d is None or not d.exists(): + d = find_hwmon(spec["hwmon"]) + cache[spec["hwmon"]] = d + return d + + +def read_hwmon_int(d, fname): + try: + return int((d / fname).read_text().strip()) + except (OSError, ValueError): + return None + + def read_temp(spec, cache): """Read one sensor's temperature in °C, or None if unavailable. spec: {hwmon, label}. Resolves hwmon dir + the tempN whose *_label matches, falling back to temp1 when the chip exposes no labels.""" - d = cache.get(spec["hwmon"]) - if d is None or not d.exists(): - d = find_hwmon(spec["hwmon"]) - cache[spec["hwmon"]] = d + d = resolve_hwmon(spec, cache) if d is None: return None @@ -159,6 +177,65 @@ def read_temp(spec, cache): return None +def read_fan(spec, cache): + """(rpm, pwm) for a pwmfan hwmon, either None when unavailable.""" + d = resolve_hwmon(spec, cache) + if d is None: + return None, None + rpm = read_hwmon_int(d, "fan1_input") + if rpm is None: + cache[spec["hwmon"]] = None # hwmon may have re-enumerated + return None, None + return rpm, read_hwmon_int(d, "pwm1") + + +def read_undervolt(spec, cache): + """(now, since_boot) undervoltage flags. + + Live alarm from the rpi_volt hwmon; the firmware's latched since-boot bit + (get_throttled bit 16) catches dips shorter than the sample interval.""" + now = None + d = resolve_hwmon(spec, cache) + if d is not None: + v = read_hwmon_int(d, "in0_lcrit_alarm") + if v is None: + cache[spec["hwmon"]] = None + else: + now = bool(v) + since_boot = None + try: + out = subprocess.run(["vcgencmd", "get_throttled"], capture_output=True, + text=True, timeout=5).stdout + bits = int(out.split("=")[1], 16) + since_boot = bool(bits & 0x10000) + if now is None: + now = bool(bits & 0x1) + except Exception: + pass + return now, since_boot + + +def classify_fan(rpm, pwm, prev_level): + """A fan commanded on (pwm > 0) reading 0 RPM is stalled/unplugged: first + such sample is warn, a consecutive one escalates to crit. pwm == 0 with + 0 RPM is the firmware idling the fan on a cool SoC — that's ok.""" + if rpm is None: + return prev_level + if rpm > 0 or pwm is None or pwm == 0: + return "ok" + return "crit" if prev_level in ("warn", "crit") else "warn" + + +def classify_undervolt(now, since_boot, prev_level): + """crit while actively under-volted; warn (sticky until reboot) once a dip + has been latched, so a transient still gets one page + a yellow pill.""" + if now is None: + return prev_level + if now: + return "crit" + return "warn" if since_boot else "ok" + + def classify(temp, spec, prev_level): """Level with hysteresis: step up at the threshold, step down only after dropping clear_margin below it, so a sensor on the line doesn't oscillate.""" @@ -175,7 +252,7 @@ def classify(temp, spec, prev_level): return "ok" -def announce(name, temp, old, new): +def announce(name, disp, old, new): """Log a level change to the journal. Returns the crossing direction ("rising" / "falling") so the caller can decide whether to page via Pushover, or None when the level is unchanged.""" @@ -184,7 +261,7 @@ def announce(name, temp, old, new): rising = LEVELS.index(new) > LEVELS.index(old) sev = {"crit": "crit", "warn": "warn", "ok": "info"}[new] arrow = "rose to" if rising else "fell back to" - log(f"{name} {arrow} {new.upper()} ({temp:.1f}°C)", sev if rising else "info") + log(f"{name} {arrow} {new.upper()} ({disp})", sev if rising else "info") return "rising" if rising else "falling" @@ -196,20 +273,39 @@ def rotate_log(path, max_bytes): log(f"log rotate failed: {e}", "warn") +def csv_columns(s): + unit = {"temp": "c", "fan": "rpm", "undervolt": "uv"}.get(s.get("kind", "temp"), "v") + return [f"{s['name']}_{unit}", f"{s['name']}_lvl"] + + +def csv_value(r): + kind = r.get("kind", "temp") + if kind == "fan": + return "" if r["rpm"] is None else str(r["rpm"]) + if kind == "undervolt": + return "" if r["now"] is None else str(int(r["now"])) + return "" if r["temp"] is None else f"{r['temp']:.1f}" + + def write_csv(cfg, readings): path = Path(cfg["log_path"]) rotate_log(path, cfg["log_max_bytes"]) - new = not path.exists() + header = ",".join(["time"] + sum([csv_columns(s) for s in cfg["sensors"]], [])) try: path.parent.mkdir(parents=True, exist_ok=True) + if path.exists(): + with open(path) as f: + if f.readline().rstrip("\n") != header: + # Sensor set changed — rotate so columns stay aligned with the header. + path.replace(path.with_suffix(path.suffix + ".1")) + new = not path.exists() with open(path, "a") as f: if new: - f.write(",".join(["time"] + sum( - [[f"{s['name']}_c", f"{s['name']}_lvl"] for s in cfg["sensors"]], [])) + "\n") + f.write(header + "\n") row = [datetime.now(timezone.utc).isoformat(timespec="seconds")] for s in cfg["sensors"]: r = readings[s["name"]] - row += ["" if r["temp"] is None else f"{r['temp']:.1f}", r["level"]] + row += [csv_value(r), r["level"]] f.write(",".join(row) + "\n") except OSError as e: log(f"csv write failed: {e}", "warn") @@ -240,31 +336,56 @@ def main(): while True: readings = {} for s in cfg["sensors"]: - temp = read_temp(s, cache) - new = classify(temp, s, levels[s["name"]]) - if temp is not None: - old = levels[s["name"]] - direction = announce(s["name"], temp, old, new) + name, kind = s["name"], s.get("kind", "temp") + old = levels[name] + + if kind == "fan": + rpm, pwm = read_fan(s, cache) + new = classify_fan(rpm, pwm, old) + disp = None if rpm is None else f"{rpm} RPM" + detail = (f"fan turning at {rpm} RPM (pwm {pwm}/255)." if new == "ok" else + f"fan reads 0 RPM while commanded on (pwm {pwm}/255) — " + "stalled, blocked, or unplugged.") + reading = {"kind": kind, "rpm": rpm, "pwm": pwm} + elif kind == "undervolt": + uv_now, uv_boot = read_undervolt(s, cache) + new = classify_undervolt(uv_now, uv_boot, old) + disp = None if uv_now is None else ( + "UNDERVOLTAGE" if uv_now else + "dip since boot" if uv_boot else "supply ok") + detail = {"crit": "supply voltage below threshold right now — check PSU and cabling.", + "warn": "an undervoltage dip was latched since boot (supply ok now; " + "latch clears on reboot).", + "ok": "supply voltage ok."}[new] + reading = {"kind": kind, "now": uv_now, "since_boot": uv_boot} + else: + temp = read_temp(s, cache) + new = classify(temp, s, old) + disp = None if temp is None else f"{temp:.1f}°C" + thr = s["crit"] if new == "crit" else s["warn"] + detail = (f"below warn ({s['warn']}°C)." if new == "ok" else + f"crossed the {new.upper()} threshold ({thr}°C); " + f"warn {s['warn']}, crit {s['crit']}.") + reading = {"kind": kind, "temp": None if temp is None else round(temp, 1), + "warn": s["warn"], "crit": s["crit"]} + + if disp is not None: + direction = announce(name, disp, old, new) if direction == "rising" and LEVELS.index(new) >= alert_idx: - icon = "🔥" if new == "crit" else "🌡" - thr = s["crit"] if new == "crit" else s["warn"] - pushover(cfg, f"{icon} {HOST}: {s['name']} {new.upper()} {temp:.1f}°C", - f"{s['name']} temperature {temp:.1f}°C crossed {new.upper()} " - f"threshold ({thr}°C). warn {s['warn']}, crit {s['crit']}.", + icon = ICONS.get(kind, "🔥" if new == "crit" else "🌡") + pushover(cfg, f"{icon} {HOST}: {name} {new.upper()} — {disp}", + f"{name}: {detail}", priority=1 if new == "crit" else 0) elif direction == "falling" and LEVELS.index(old) >= alert_idx: # Recovery: page only when leaving a level we'd have paged about, # so the phone that got the rising alert also gets the all-clear. label = "NORMAL" if new == "ok" else new.upper() - pushover(cfg, f"✅ {HOST}: {s['name']} back to {label} {temp:.1f}°C", - f"{s['name']} temperature {temp:.1f}°C dropped back to {label} " - f"(warn {s['warn']}, crit {s['crit']}).", - priority=0) - levels[s["name"]] = new - readings[s["name"]] = { - "temp": None if temp is None else round(temp, 1), - "level": new, "warn": s["warn"], "crit": s["crit"], - } + pushover(cfg, f"✅ {HOST}: {name} back to {label} — {disp}", + f"{name}: {detail}", priority=0) + + levels[name] = new + reading["level"] = new + readings[name] = reading try: write_state(readings) diff --git a/power/van-thermal.service b/power/van-thermal.service index ff8e79f..f9b028a 100644 --- a/power/van-thermal.service +++ b/power/van-thermal.service @@ -1,5 +1,5 @@ [Unit] -Description=Temperature monitor (CPU + NVMe) for the campervan router +Description=Thermal + health monitor (temps, fan, undervoltage) for the campervan router # Order after network-online so a boot-time threshold crossing (common: the Pi # boots hot) can actually reach Pushover — the very first sample fires within # seconds of start. If no WAN comes up, wait-online times out and we start anyway.