fix: preserve chart history across reconnects, show gaps for missing data
Only wipe real (non-RTT) plugin data on an actual client reboot (boot flag), not on every ordinary OVERDUE/DOWN -> UP recovery. A transient network blip no longer erases CPU/memory/etc. history. Also split the shared time-series chart into separate line/area segments wherever the gap between samples is much larger than the typical spacing, so missing data (host overdue, or history that simply hasn't accumulated across a drop) renders as a visual gap instead of an interpolated line. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DEimzMv4Q5EjFg3hoiZ69T
This commit is contained in:
@@ -847,16 +847,38 @@
|
|||||||
const yRange = yHigh - yLow || 1;
|
const yRange = yHigh - yLow || 1;
|
||||||
const y = v => PAD.top + cH - ((v - yLow) / yRange) * cH;
|
const y = v => PAD.top + cH - ((v - yLow) / yRange) * cH;
|
||||||
|
|
||||||
// Build polyline points and filled area path
|
|
||||||
const linePoints = pts.map(p => `${x(p.t).toFixed(1)},${y(p.v).toFixed(1)}`).join(' ');
|
|
||||||
const areaPath = `M${x(pts[0].t).toFixed(1)},${(PAD.top + cH).toFixed(1)} ` +
|
|
||||||
pts.map(p => `L${x(p.t).toFixed(1)},${y(p.v).toFixed(1)}`).join(' ') +
|
|
||||||
` L${x(pts[pts.length-1].t).toFixed(1)},${(PAD.top + cH).toFixed(1)} Z`;
|
|
||||||
|
|
||||||
// Color based on latest value
|
// Color based on latest value
|
||||||
const latest = pts[pts.length - 1].v;
|
const latest = pts[pts.length - 1].v;
|
||||||
const { stroke: strokeColor, fill: fillColor } = opts.colorFor(latest);
|
const { stroke: strokeColor, fill: fillColor } = opts.colorFor(latest);
|
||||||
|
|
||||||
|
// Split into segments wherever the gap between consecutive samples is
|
||||||
|
// much larger than the typical spacing (e.g. the host was overdue/down
|
||||||
|
// for a while) — draw each segment separately so missing data reads as
|
||||||
|
// a visual gap instead of an interpolated line across dead time.
|
||||||
|
const deltas = [];
|
||||||
|
for (let i = 1; i < pts.length; i++) deltas.push(pts[i].t - pts[i - 1].t);
|
||||||
|
deltas.sort((a, b) => a - b);
|
||||||
|
const medianDelta = deltas[Math.floor(deltas.length / 2)];
|
||||||
|
const gapThreshold = medianDelta * 2.5;
|
||||||
|
|
||||||
|
const segments = [[pts[0]]];
|
||||||
|
for (let i = 1; i < pts.length; i++) {
|
||||||
|
if (pts[i].t - pts[i - 1].t > gapThreshold) segments.push([]);
|
||||||
|
segments[segments.length - 1].push(pts[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
let linePolylines = '';
|
||||||
|
let areaPaths = '';
|
||||||
|
for (const seg of segments) {
|
||||||
|
if (seg.length < 2) continue;
|
||||||
|
const segPoints = seg.map(p => `${x(p.t).toFixed(1)},${y(p.v).toFixed(1)}`).join(' ');
|
||||||
|
linePolylines += `<polyline points="${segPoints}" fill="none" stroke="${strokeColor}" stroke-width="1.5" stroke-linejoin="round"/>`;
|
||||||
|
const segArea = `M${x(seg[0].t).toFixed(1)},${(PAD.top + cH).toFixed(1)} ` +
|
||||||
|
seg.map(p => `L${x(p.t).toFixed(1)},${y(p.v).toFixed(1)}`).join(' ') +
|
||||||
|
` L${x(seg[seg.length-1].t).toFixed(1)},${(PAD.top + cH).toFixed(1)} Z`;
|
||||||
|
areaPaths += `<path d="${segArea}" fill="${fillColor}" opacity="0.6"/>`;
|
||||||
|
}
|
||||||
|
|
||||||
// Compute nice tick step for ~3-5 grid lines
|
// Compute nice tick step for ~3-5 grid lines
|
||||||
const rawStep = yRange / 4;
|
const rawStep = yRange / 4;
|
||||||
const mag = Math.pow(10, Math.floor(Math.log10(rawStep || 1)));
|
const mag = Math.pow(10, Math.floor(Math.log10(rawStep || 1)));
|
||||||
@@ -890,8 +912,8 @@
|
|||||||
<line x1="${PAD.left}" y1="${PAD.top}" x2="${PAD.left}" y2="${PAD.top + cH}" stroke="#ccc" stroke-width="1"/>
|
<line x1="${PAD.left}" y1="${PAD.top}" x2="${PAD.left}" y2="${PAD.top + cH}" stroke="#ccc" stroke-width="1"/>
|
||||||
<line x1="${PAD.left}" y1="${PAD.top + cH}" x2="${PAD.left + cW}" y2="${PAD.top + cH}" stroke="#ccc" stroke-width="1"/>
|
<line x1="${PAD.left}" y1="${PAD.top + cH}" x2="${PAD.left + cW}" y2="${PAD.top + cH}" stroke="#ccc" stroke-width="1"/>
|
||||||
<g clip-path="url(#${opts.clipId})">
|
<g clip-path="url(#${opts.clipId})">
|
||||||
<path d="${areaPath}" fill="${fillColor}" opacity="0.6"/>
|
${areaPaths}
|
||||||
<polyline points="${linePoints}" fill="none" stroke="${strokeColor}" stroke-width="1.5" stroke-linejoin="round"/>
|
${linePolylines}
|
||||||
</g>
|
</g>
|
||||||
${xLabels}
|
${xLabels}
|
||||||
</svg>`;
|
</svg>`;
|
||||||
|
|||||||
+5
-1
@@ -521,9 +521,13 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
# Transition to UP and log/notify if appropriate
|
# Transition to UP and log/notify if appropriate
|
||||||
lasts = conn.state
|
lasts = conn.state
|
||||||
d = conn.newstate(hbdcls.Connection.UP, now)
|
d = conn.newstate(hbdcls.Connection.UP, now)
|
||||||
|
if boot:
|
||||||
# On reboot, pre-boot plugin data and derived alerts are stale.
|
# On reboot, pre-boot plugin data and derived alerts are stale.
|
||||||
# Cancel all plugin timers and wipe plugin state so timers restart
|
# Cancel all plugin timers and wipe plugin state so timers restart
|
||||||
# cleanly from the first two post-boot samples.
|
# cleanly from the first two post-boot samples. An ordinary
|
||||||
|
# reconnect (no boot flag) doesn't invalidate the client's
|
||||||
|
# already-collected data, so it's left alone — this keeps chart
|
||||||
|
# history intact across a transient network blip.
|
||||||
for pname in list(host.plugin_timers):
|
for pname in list(host.plugin_timers):
|
||||||
host.cancel_plugin_timer(pname)
|
host.cancel_plugin_timer(pname)
|
||||||
for pname in [k for k in host.plugin_data if not _is_rtt_key(k)]:
|
for pname in [k for k in host.plugin_data if not _is_rtt_key(k)]:
|
||||||
|
|||||||
@@ -14,10 +14,12 @@ class _FakeTransport:
|
|||||||
self.sent.append((data, addr))
|
self.sent.append((data, addr))
|
||||||
|
|
||||||
|
|
||||||
def _htb(name, rtt=None, interval=0):
|
def _htb(name, rtt=None, interval=0, boot=0):
|
||||||
d = {"name": name, "interval": interval, "id": 0}
|
d = {"name": name, "interval": interval, "id": 0}
|
||||||
if rtt is not None:
|
if rtt is not None:
|
||||||
d["rtt"] = rtt
|
d["rtt"] = rtt
|
||||||
|
if boot:
|
||||||
|
d["boot"] = boot
|
||||||
return parse_message(dicttos("HTB", d))
|
return parse_message(dicttos("HTB", d))
|
||||||
|
|
||||||
|
|
||||||
@@ -94,10 +96,11 @@ def test_request_update_fires_on_recovery_even_with_rtt_history():
|
|||||||
assert ack.get("request_update")
|
assert ack.get("request_update")
|
||||||
|
|
||||||
|
|
||||||
def test_recovery_clears_real_plugin_data_but_preserves_rtt_history():
|
def test_ordinary_recovery_preserves_real_plugin_data_and_rtt_history():
|
||||||
"""Regression for Finding 2: host.plugin_data.clear() on recovery must
|
"""An ordinary reconnect (no boot flag) — e.g. OVERDUE->UP after a
|
||||||
wipe real client-collected plugin data while leaving rtt_* history
|
transient network blip — must NOT wipe already-collected real plugin
|
||||||
samples intact.
|
data (e.g. cpu_monitor, os_info) or rtt_* history. Only an actual
|
||||||
|
client reboot invalidates that data (see the boot-flag test below).
|
||||||
"""
|
"""
|
||||||
hbdclass.Host.hosts.pop("rtt-hist-host5", None)
|
hbdclass.Host.hosts.pop("rtt-hist-host5", None)
|
||||||
transport = _FakeTransport()
|
transport = _FakeTransport()
|
||||||
@@ -116,8 +119,35 @@ def test_recovery_clears_real_plugin_data_but_preserves_rtt_history():
|
|||||||
host.add_plugin_data("os_info", {"os": "linux"}, timestamp=time.time())
|
host.add_plugin_data("os_info", {"os": "linux"}, timestamp=time.time())
|
||||||
assert "os_info" in host.plugin_data
|
assert "os_info" in host.plugin_data
|
||||||
|
|
||||||
# Recovery heartbeat.
|
# Ordinary recovery heartbeat — no boot flag.
|
||||||
handle_datagram(_htb("rtt-hist-host5", rtt=13.0), ("127.0.0.1", 50000), transport, ctx)
|
handle_datagram(_htb("rtt-hist-host5", rtt=13.0), ("127.0.0.1", 50000), transport, ctx)
|
||||||
|
|
||||||
|
assert "os_info" in host.plugin_data
|
||||||
|
assert len(host.plugin_data["rtt_ipv4"]) == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_boot_recovery_clears_real_plugin_data_but_preserves_rtt_history():
|
||||||
|
"""A recovery heartbeat carrying the boot flag (client process actually
|
||||||
|
restarted) must still wipe stale real plugin data, while rtt_* history
|
||||||
|
(still a valid measurement, unaffected by a client reboot) survives.
|
||||||
|
"""
|
||||||
|
hbdclass.Host.hosts.pop("rtt-hist-host6", None)
|
||||||
|
transport = _FakeTransport()
|
||||||
|
ctx = _base_ctx()
|
||||||
|
|
||||||
|
for rtt in (10.0, 11.0, 12.0):
|
||||||
|
handle_datagram(_htb("rtt-hist-host6", rtt=rtt), ("127.0.0.1", 50000), transport, ctx)
|
||||||
|
|
||||||
|
host = hbdclass.Host.hosts["rtt-hist-host6"]
|
||||||
|
assert len(host.plugin_data["rtt_ipv4"]) == 3
|
||||||
|
|
||||||
|
conn = host.connections["IPv4"]
|
||||||
|
conn.state = hbdclass.Connection.DOWN
|
||||||
|
host.add_plugin_data("os_info", {"os": "linux"}, timestamp=time.time())
|
||||||
|
assert "os_info" in host.plugin_data
|
||||||
|
|
||||||
|
# Recovery heartbeat with boot=1 — client process actually restarted.
|
||||||
|
handle_datagram(_htb("rtt-hist-host6", rtt=13.0, boot=1), ("127.0.0.1", 50000), transport, ctx)
|
||||||
|
|
||||||
assert "os_info" not in host.plugin_data
|
assert "os_info" not in host.plugin_data
|
||||||
assert len(host.plugin_data["rtt_ipv4"]) == 4
|
assert len(host.plugin_data["rtt_ipv4"]) == 4
|
||||||
|
|||||||
Reference in New Issue
Block a user