From e7a82126f2f642f8a319d146b8488da18a7a6f5b Mon Sep 17 00:00:00 2001 From: Andreas Wrede Date: Tue, 4 Aug 2026 08:44:53 -0400 Subject: [PATCH] =?UTF-8?q?power:=20add=20van-nvme-watch=20=E2=80=94=20pag?= =?UTF-8?q?es=20on=20NVMe=20I/O-timeout/reset=20recurrence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tails journalctl -kf for the "nvme nvmeN: I/O tag ... timeout, reset controller" signature that crashed and corrupted the root fs on 2026-08-02 (and recurred 2026-08-04, that time self-healing). Watches a short grace window to tell a clean self-heal from an escalation (repeated timeout or a following ext4 error) before paging via the existing Pushover credentials, with a live SMART/superblock snapshot in the alert body. --- README.md | 13 ++ deploy.sh | 9 +- power/nvme-watch-config.json | 7 + power/van-nvme-watch | 261 +++++++++++++++++++++++++++++++++++ power/van-nvme-watch.service | 13 ++ 5 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 power/nvme-watch-config.json create mode 100644 power/van-nvme-watch create mode 100644 power/van-nvme-watch.service diff --git a/README.md b/README.md index dd5e541..0ee5d66 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ This directory is the source of truth. The live system files live under `/etc`, | NAT + forwarding | **nftables** + sysctl | | WAN health + failover | **van-failover** daemon | | Temperature monitor / alert / log | **van-thermal** daemon | +| NVMe I/O-timeout/reset watchdog + alert | **van-nvme-watch** daemon | | Battery monitor / low-charge alert + shutdown | **van-battery** daemon | | Auto-reboot on hang | **systemd hardware watchdog** (`intel_oc_wdt`) | | Liveness / dead-man's switch + metrics | **hbc** heartbeat client → hbd.wrede.pvt | @@ -121,6 +122,9 @@ This directory is the source of truth. The live system files live under `/etc`, | `van-thermal` | `/usr/local/sbin/van-thermal` | temperature daemon (Python): publishes state, alerts, logs history | | `thermal-config.json` | `/etc/van-thermal/config.json` | sensors + warn/crit thresholds + sample/log intervals | | `van-thermal.service` | `/etc/systemd/system/van-thermal.service` | `Restart=always` | +| `van-nvme-watch` | `/usr/local/sbin/van-nvme-watch` | NVMe watchdog (Python): tails `journalctl -k` for I/O-timeout/reset events, Pushover alerts | +| `nvme-watch-config.json` | `/etc/van-nvme-watch/config.json` | grace period, cooldown, device paths | +| `van-nvme-watch.service` | `/etc/systemd/system/van-nvme-watch.service` | `Restart=always` | | `van-battery` | `/usr/local/sbin/van-battery` | battery daemon (Python): Pushover low-charge alerts + safe shutdown | | `battery-config.json` | `/etc/van-battery/config.json` | warn levels, shutdown level, poll interval, paths | | `van-battery.service` | `/etc/systemd/system/van-battery.service` | `Restart=always` | @@ -180,6 +184,15 @@ The `homeassistant` LAN name comes from `ap/van-ap-dnsmasq.conf` (`host-record` - **Tuning**: edit `/etc/van-thermal/config.json` (thresholds, intervals, sensor list), then `systemctl restart van-thermal`. Defaults: CPU warn 80 / crit 95 °C (silicon crit is 100), NVMe warn 65 / crit 70 °C (drive crit ~71). - Status: `systemctl status van-thermal` or `cat /run/van-thermal/state.json`. +### NVMe watchdog (`van-nvme-watch`) +- Event-driven, not polled: tails `journalctl -kf` for the `nvme nvmeN: I/O tag ... timeout, reset controller` signature that crashed and corrupted the root fs on 2026-08-02 (recurred 2026-08-04, self-healed — see the memory notes for that investigation). There's no sensor to sample, only a log line to catch. +- On a match it watches a `grace_period` (default 20 s) for either a clean re-init (self-healed) or a second timeout / an ext4 error following it (escalated) before alerting, so the Pushover message already says which outcome happened — no need to SSH in during a scare. +- The alert body includes a live `smartctl`/`tune2fs` snapshot (SMART health, critical-warning flag, media error count, filesystem state). +- **Live state**: `/run/van-nvme-watch/state.json` (last event, same convention as van-failover/van-thermal). +- Shares Pushover credentials with van-thermal/van-battery (`/etc/van-battery/pushover.json`). +- **Tuning**: edit `/etc/van-nvme-watch/config.json` (`grace_period`, `cooldown`, device paths), then `systemctl restart van-nvme-watch`. +- Status: `systemctl status van-nvme-watch`, `journalctl -u van-nvme-watch -f`, or `cat /run/van-nvme-watch/state.json`. + ### Battery monitor (`van-battery`) - Watches mains vs battery via `/sys/class/power_supply/AC0/online` (0 = on battery) and charge via `BAT0/capacity`. Both resolve by `type` (Mains/Battery) if those names ever differ. - **Only while on battery**, it sends escalating **Pushover** alerts at **25 / 20 / 15 %**, and at **10 %** sends a final alert and runs `systemctl poweroff` (after `shutdown_grace`, default 8 s, so the alert flushes first). diff --git a/deploy.sh b/deploy.sh index 0f549a0..a548691 100755 --- a/deploy.sh +++ b/deploy.sh @@ -192,6 +192,11 @@ if [ ! -f /etc/van-battery/pushover.json ]; then warn "seeded /etc/van-battery/pushover.json (EDIT IT: add Pushover token + user)" fi +echo "== nvme watchdog ==" +install -D -m0755 power/van-nvme-watch /usr/local/sbin/van-nvme-watch +install -D -m0644 power/nvme-watch-config.json /etc/van-nvme-watch/config.json +install -D -m0644 power/van-nvme-watch.service /etc/systemd/system/van-nvme-watch.service + echo "== home assistant ==" # Native HA (Podman Quadlet, replaced the ha_van VM). daemon-reload below # regenerates homeassistant.service; started (not restarted) at the end so a @@ -221,12 +226,13 @@ systemctl restart systemd-resolved # its wait-online would just stall network-online.target. NM-wait-online covers WANs. systemctl mask systemd-networkd-wait-online.service >/dev/null 2>&1 || true systemctl unmask hostapd >/dev/null 2>&1 || true -systemctl enable regdomain.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-gps-owntracks >/dev/null 2>&1 || true +systemctl enable regdomain.service hostapd hostapd-2g van-ap-dnsmasq nftables systemd-networkd van-failover van-thermal van-nvme-watch van-ap-watchdog van-ap-watchdog-2g van-wlan-watchdog van-gps-owntracks >/dev/null 2>&1 || true # bluetooth: host BlueZ serves the onboard hci0 to the HA container over D-Bus systemctl enable --now bluetooth >/dev/null 2>&1 || true systemctl start homeassistant || warn "homeassistant failed to start (podman/quadlet — check journalctl -u homeassistant)" systemctl start esphome || warn "esphome failed to start (podman/quadlet — check journalctl -u esphome)" systemctl restart van-thermal +systemctl restart van-nvme-watch systemctl restart van-gps-owntracks # Pick up unmanaged-devices changes so NM releases/keeps the right interfaces. nmcli general reload 2>/dev/null || systemctl reload NetworkManager 2>/dev/null || true @@ -252,3 +258,4 @@ echo " iw dev $WIFI_5G_IFACE info | grep -E 'ssid|channel|width'" echo " iw dev $WIFI_2G_IFACE info | grep -E 'ssid|channel|width'" echo " cat /run/van-failover/state.json" echo " cat /run/van-thermal/state.json" +echo " cat /run/van-nvme-watch/state.json" diff --git a/power/nvme-watch-config.json b/power/nvme-watch-config.json new file mode 100644 index 0000000..4146b8c --- /dev/null +++ b/power/nvme-watch-config.json @@ -0,0 +1,7 @@ +{ + "nvme_device": "nvme0", + "fs_device": "/dev/nvme0n1p2", + "grace_period": 20, + "cooldown": 300, + "credentials_path": "/etc/van-battery/pushover.json" +} diff --git a/power/van-nvme-watch b/power/van-nvme-watch new file mode 100644 index 0000000..2d759fb --- /dev/null +++ b/power/van-nvme-watch @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""van-nvme-watch — pages when the NVMe root hits an I/O-timeout/reset event. + +This exact kernel signature ("nvme nvmeN: I/O tag ... timeout, reset +controller") crashed and corrupted the NVMe root on 2026-08-02, then recurred +2026-08-04 and self-healed. Root cause is still unconfirmed (suspected +USB/PCIe host-bandwidth contention during heavy interface churn), so this +just watches for it recurring rather than trying to prevent it. + +Tails `journalctl -kf` (event-driven, not polled) rather than sysfs like +van-thermal, since there's no sensor to sample — only a log line to catch. +On a match it watches a short grace window for either a clean controller +re-init (self-healed) or a follow-up ext4 error / a second timeout (escalating) +before paging, and includes a live SMART + superblock snapshot in the message +so the phone alert already answers "is the filesystem actually at risk". + +Publishes /run/van-nvme-watch/state.json (last event, same convention as +van-failover/van-thermal). Pushover credentials shared with van-battery/ +van-thermal (/etc/van-battery/pushover.json, 0600). Stdlib only. +""" + +import json +import os +import queue +import re +import socket +import subprocess +import sys +import threading +import time +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +CONFIG_PATH = os.environ.get("VAN_NVME_WATCH_CONFIG", "/etc/van-nvme-watch/config.json") +STATE_DIR = Path("/run/van-nvme-watch") +STATE_PATH = STATE_DIR / "state.json" +HOST = socket.gethostname() + +DEFAULTS = { + "nvme_device": "nvme0", # controller name as it appears in dmesg / smartctl target + "fs_device": "/dev/nvme0n1p2", # for the superblock snapshot + "grace_period": 20, # seconds to watch for escalation after a timeout line + "cooldown": 300, # minimum seconds between Pushover sends + "credentials_path": "/etc/van-battery/pushover.json", +} + +# The exact crash/recurrence signature, kept nvme-scoped and loose enough to +# catch variant opcodes/tags/queue ids without matching unrelated nvme lines. +TIMEOUT_RE = re.compile(r"nvme (nvme\d+): .*\b(?:timeout|reset controller)\b", re.IGNORECASE) +# Signs the event is more than a clean self-heal: an actual filesystem error, +# or forced read-only remount, following the reset. +ESCALATION_RE = re.compile( + r"EXT4-fs error|EXT4-fs.*remount.*read-only|Remounting filesystem read-only|" + r"Buffer I/O error|I/O error, dev nvme", + re.IGNORECASE, +) + +PLACEHOLDERS = {"", "REPLACE_ME", "your-token-here", "your-user-key-here"} + + +def log(msg, level="info"): + pri = {"info": "<6>", "warn": "<4>", "crit": "<2>"}.get(level, "<6>") + print(pri + msg, flush=True) + + +def load_creds(cfg): + try: + c = json.loads(Path(cfg["credentials_path"]).read_text()) + token, user = str(c.get("token", "")).strip(), str(c.get("user", "")).strip() + if token in PLACEHOLDERS or user in PLACEHOLDERS: + return None + return token, user + except FileNotFoundError: + return None + except Exception as e: + log(f"credentials {cfg['credentials_path']} unreadable ({e})", "warn") + return None + + +def pushover(cfg, title, message, priority=0, attempts=3, retry_delay=15): + creds = load_creds(cfg) + if not creds: + log(f"pushover skipped (no credentials): {title} — {message}", "warn") + return False + token, user = creds + data = urllib.parse.urlencode({ + "token": token, "user": user, "title": title, + "message": message, "priority": priority, + }).encode() + req = urllib.request.Request("https://api.pushover.net/1/messages.json", data=data) + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(req, timeout=10) as resp: + ok = resp.status == 200 + if not ok: + log(f"pushover HTTP {resp.status}", "warn") + return ok + except Exception as e: + last = attempt == attempts + log(f"pushover send failed ({attempt}/{attempts}): {e}", "warn") + if not last: + time.sleep(retry_delay) + return False + + +def load_config(): + cfg = dict(DEFAULTS) + try: + with open(CONFIG_PATH) as f: + cfg.update(json.load(f)) + except FileNotFoundError: + log(f"config {CONFIG_PATH} not found, using built-in defaults") + except Exception as e: + log(f"config {CONFIG_PATH} unreadable ({e}), using built-in defaults", "warn") + return cfg + + +def snapshot_health(cfg): + """Best-effort SMART + superblock snapshot for the alert body — answers + "did this one actually hurt the filesystem" without needing to SSH in.""" + info = {} + try: + out = subprocess.run(["smartctl", "-a", f"/dev/{cfg['nvme_device']}"], + capture_output=True, text=True, timeout=15).stdout + for key, pattern in ( + ("smart_health", r"SMART overall-health self-assessment test result:\s*(\S+)"), + ("smart_critical_warning", r"Critical Warning:\s*(\S+)"), + ("smart_media_errors", r"Media and Data Integrity Errors:\s*(\d+)"), + ): + m = re.search(pattern, out) + info[key] = m.group(1) if m else "?" + except Exception as e: + info["smart_error"] = str(e) + try: + out = subprocess.run(["tune2fs", "-l", cfg["fs_device"]], + capture_output=True, text=True, timeout=15).stdout + m = re.search(r"Filesystem state:\s*(\S+)", out) + info["fs_state"] = m.group(1) if m else "?" + except Exception as e: + info["fs_state_error"] = str(e) + return info + + +def write_state(payload): + STATE_DIR.mkdir(parents=True, exist_ok=True) + tmp = STATE_PATH.with_suffix(".tmp") + tmp.write_text(json.dumps(payload)) + tmp.replace(STATE_PATH) + + +def journal_reader(q): + """Runs forever in a background thread, pushing new kernel log lines onto + q. journalctl itself can exit (journald restart, etc.) — respawn it.""" + while True: + try: + proc = subprocess.Popen( + ["journalctl", "-kf", "-n", "0", "-o", "cat"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, bufsize=1) + for line in proc.stdout: + q.put(line.rstrip("\n")) + proc.wait() + log(f"journalctl exited (code {proc.returncode}), restarting in 5s", "warn") + except Exception as e: + log(f"journalctl reader error: {e}, restarting in 5s", "warn") + time.sleep(5) + + +def finalize(cfg, pending, escalated, reason, alert_state): + """Wrap up a pending event: snapshot health, log, publish state, and + Pushover (subject to cooldown). alert_state is a {"last_alert": monotonic} + box shared across calls so the cooldown persists across events.""" + health = snapshot_health(cfg) + outcome = "ESCALATED" if escalated else "self-healed" + icon = "🚨" if escalated else "⚠️" + priority = 1 if escalated else 0 + + lines = [f"first: {pending['line']}"] + if pending.get("count", 1) > 1: + lines.append(f"repeated {pending['count']}x within the grace window") + if pending.get("escalation_line"): + lines.append(f"then: {pending['escalation_line']}") + lines.append(f"fs state: {health.get('fs_state', '?')} | " + f"SMART: {health.get('smart_health', '?')}, " + f"critical_warning={health.get('smart_critical_warning', '?')}, " + f"media_errors={health.get('smart_media_errors', '?')}") + message = "\n".join(lines) + + log(f"nvme event {outcome} ({reason}): {message}", "crit" if escalated else "warn") + + write_state({ + "updated": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "device": pending["device"], + "count": pending.get("count", 1), + "escalated": escalated, + "reason": reason, + "first_line": pending["line"], + "escalation_line": pending.get("escalation_line"), + "health": health, + }) + + if time.monotonic() - alert_state["last_alert"] >= cfg["cooldown"]: + pushover(cfg, f"{icon} {HOST}: NVMe timeout — {outcome}", message, priority=priority) + alert_state["last_alert"] = time.monotonic() + else: + log("pushover suppressed (cooldown)", "warn") + + +def main(): + cfg = load_config() + log(f"van-nvme-watch up: tailing journalctl -k for nvme timeout/reset events " + f"(grace period {cfg['grace_period']}s, cooldown {cfg['cooldown']}s)") + + q = queue.Queue() + threading.Thread(target=journal_reader, args=(q,), daemon=True).start() + + pending = None + alert_state = {"last_alert": 0.0} + + while True: + if pending is not None: + remaining = pending["deadline"] - time.monotonic() + if remaining <= 0: + finalize(cfg, pending, False, "grace period elapsed", alert_state) + pending = None + continue + try: + line = q.get(timeout=remaining) + except queue.Empty: + continue + else: + line = q.get() + + if pending is None: + m = TIMEOUT_RE.search(line) + if m: + pending = {"device": m.group(1), "line": line, "count": 1, + "deadline": time.monotonic() + cfg["grace_period"]} + log(f"nvme timeout detected: {line}", "warn") + continue + + if TIMEOUT_RE.search(line): + pending["count"] += 1 + if pending["count"] >= 2: + finalize(cfg, pending, True, "repeated timeouts", alert_state) + pending = None + continue + + if ESCALATION_RE.search(line): + pending["escalation_line"] = line + finalize(cfg, pending, True, "filesystem error followed", alert_state) + pending = None + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(0) diff --git a/power/van-nvme-watch.service b/power/van-nvme-watch.service new file mode 100644 index 0000000..75d36c1 --- /dev/null +++ b/power/van-nvme-watch.service @@ -0,0 +1,13 @@ +[Unit] +Description=NVMe I/O-timeout/reset watchdog + Pushover alert for the campervan router +After=local-fs.target network-online.target systemd-journald.service +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/sbin/van-nvme-watch +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target