Andreas WredeandClaude Opus 4.8
4414967bdc
A (host, service) pair is flapping once it exceeds flap_count warning or critical notifications within flap_interval minutes. The notification that trips the state carries "Now flapping!! No more messages!" and every later one is dropped, including RECOVER. The state ends silently flap_interval minutes after a RECOVER, provided no further alert arrived meanwhile. Hooked into notify.send_notification, the single choke point for channel delivery, so only outbound notifications are suppressed — eventlog keeps recording, leaving the journal and /log with the full history of the flap. Threshold alerts key on their metric path, so a flapping disk check cannot silence a CPU alert; connectivity, boot and shutdown events key on the host itself. State lives at module level in flap.py and is never pickled. Flapping pairs surface in Host.stateinfo() and render as an amber badge on the live dashboard. Config: flap_count (5), flap_interval (10 minutes), 0 in either disables; both editable on the settings page. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
409 lines
18 KiB
HTML
409 lines
18 KiB
HTML
<!DOCTYPE html>
|
||
<html>
|
||
{% include 'head.html' %}
|
||
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||
<script src="/static/hbd-ui.js"></script>
|
||
|
||
<style>
|
||
html, body { height: auto; overflow: visible; }
|
||
body { background: var(--st-paper); padding: 60px 0 0; }
|
||
|
||
.live-main { max-width: 1180px; margin: 0 auto; padding: 0 20px 70px; }
|
||
|
||
.st-toolbar input {
|
||
font: 12px var(--st-mono); color: var(--st-ink);
|
||
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||
padding: 5px 12px; min-width: 160px;
|
||
}
|
||
.live-dot { display: inline-flex; align-items: center; gap: 6px; font-size: 12px; font-weight: 600; color: var(--st-ok); }
|
||
.live-dot::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: var(--st-ok); }
|
||
.live-dot.reconnecting { color: var(--st-crit); }
|
||
.live-dot.reconnecting::before { background: var(--st-crit); }
|
||
|
||
/* ── host rows: 6 columns ── */
|
||
.hrow, .hhead {
|
||
display: grid;
|
||
grid-template-columns: 175px 105px 70px 125px 85px minmax(0,1fr);
|
||
gap: 4px 14px; align-items: center;
|
||
padding: 6px 14px; border-bottom: 1px solid var(--st-line-soft);
|
||
}
|
||
.hrow:last-child { border-bottom: none; }
|
||
.hrow:hover { background: var(--st-surface-2); }
|
||
.hhead {
|
||
font-family: var(--st-mono); font-size: 10.5px; text-transform: uppercase; letter-spacing: .05em;
|
||
color: var(--st-faint); background: var(--st-surface-2); border-radius: 8px 8px 0 0;
|
||
}
|
||
.hhead .num, .hrow .num { text-align: right; }
|
||
|
||
.hrow.sev-warn { border-left: 3px solid var(--st-warn); padding-left: 11px; }
|
||
.hrow.sev-crit { border-left: 3px solid var(--st-crit); padding-left: 11px; }
|
||
|
||
.host { font-family: var(--st-mono); font-size: 13px; font-weight: 400; display: flex; align-items: center; gap: 7px; min-width: 0; }
|
||
.host.watched { font-weight: 700; }
|
||
.host a { color: inherit; text-decoration: none; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||
.host a:hover { color: var(--st-accent); }
|
||
.dot.down { background: var(--st-crit); box-shadow: 0 0 0 3px var(--st-crit-soft); }
|
||
.dot.overdue { background: transparent; border: 2px solid var(--st-warn); }
|
||
.stale { color: var(--st-faint); font-size: 10px; border: 1px solid var(--st-line); border-radius: 4px; padding: 1px 4px; font-family: var(--st-mono); white-space: nowrap; }
|
||
.flap { color: var(--st-warn); font-size: 10px; border: 1px solid var(--st-warn); border-radius: 4px; padding: 1px 4px; font-family: var(--st-mono); white-space: nowrap; }
|
||
.chip.outline { background: transparent; border: 1px solid var(--st-line); }
|
||
|
||
.lat { font-family: var(--st-mono); font-size: 12px; color: var(--st-muted); font-variant-numeric: tabular-nums; }
|
||
.since { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-faint); font-variant-numeric: tabular-nums; }
|
||
|
||
.lastalert { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||
.lastalert .lvl { font-weight: 700; }
|
||
.lastalert .lvl.crit { color: var(--st-crit); }
|
||
.lastalert .lvl.warn { color: var(--st-warn); }
|
||
.lastalert .ago { color: var(--st-faint); }
|
||
.lastalert.quiet { color: var(--st-faint); }
|
||
|
||
/* ── hover card on the status chip ── */
|
||
.tip { position: relative; display: inline-flex; }
|
||
.tipbox {
|
||
display: none; position: absolute; z-index: 40; top: calc(100% + 6px); left: 0;
|
||
width: 340px; background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 8px;
|
||
box-shadow: 0 8px 28px var(--shadow); padding: 10px 12px;
|
||
}
|
||
.tip:hover .tipbox, .tip:focus-within .tipbox { display: block; }
|
||
.tipbox table { border-collapse: collapse; width: 100%; font-family: var(--st-mono); font-size: 11.5px; }
|
||
.tipbox th { text-align: left; font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: var(--st-faint); padding: 2px 6px 4px; border-bottom: 1px solid var(--st-line-soft); }
|
||
.tipbox td { padding: 3px 6px; color: var(--st-muted); }
|
||
.tipbox td:first-child { color: var(--st-faint); }
|
||
.tipbox .up { color: var(--st-ok); font-weight: 700; }
|
||
.tipbox .dn { color: var(--st-crit); font-weight: 700; }
|
||
.tipbox .ov { color: var(--st-warn); font-weight: 700; }
|
||
.tipbox .meta { margin-top: 6px; padding-top: 6px; border-top: 1px solid var(--st-line-soft); font-size: 11px; color: var(--st-faint); }
|
||
|
||
@media (max-width: 760px) {
|
||
.hhead { display: none; }
|
||
.hrow { grid-template-columns: minmax(0,1fr) auto auto; padding: 8px 12px; }
|
||
.hrow .num { text-align: left; }
|
||
.hrow .since { display: none; }
|
||
.lastalert { grid-column: 1 / -1; }
|
||
.tipbox { left: auto; right: 0; }
|
||
}
|
||
</style>
|
||
|
||
<body>
|
||
{% include 'nav.html' %}
|
||
|
||
<div class="st-toolbar">
|
||
<span class="brand">hbd<span class="tld">·live</span></span>
|
||
<span class="hintline">every host, one line — details on hover, history in the overview</span>
|
||
<span class="spacer"></span>
|
||
<input type="text" id="host-filter" placeholder="host filter (regex)">
|
||
<span class="live-dot reconnecting" id="ws-status">connecting…</span>
|
||
</div>
|
||
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||
</svg>
|
||
|
||
<div class="live-main">
|
||
<section class="st">
|
||
<div class="sec-head">
|
||
<h2>hosts<span class="colon">:</span></h2>
|
||
<span class="count" id="host-counts"></span>
|
||
<span class="sub">sorted by severity, then name</span>
|
||
</div>
|
||
<div class="list">
|
||
<div class="hhead">
|
||
<span>host</span><span>status</span><span class="num">latency</span>
|
||
<span>alerts</span><span>since</span><span>last alert</span>
|
||
</div>
|
||
<div id="host-rows"></div>
|
||
</div>
|
||
</section>
|
||
</div>
|
||
|
||
<script>
|
||
var HBD_VERSION = "{{ hbd_version }}";
|
||
var INITIAL_HOSTS = {{ hosts | tojson }};
|
||
|
||
var hostData = {}; // name -> latest stateinfo payload
|
||
var lastAlert = {}; // name -> {level, message, ts}
|
||
var hostFilterRe = null;
|
||
|
||
function escHtml(s) {
|
||
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');
|
||
}
|
||
|
||
// ---- derived per-host facts ----------------------------------------
|
||
var STATE_RANK = { down: 3, overdue: 2, up: 1 };
|
||
|
||
function worstConnection(data) {
|
||
var worst = null;
|
||
(data.connections || []).forEach(function (c) {
|
||
if (!c.state || c.state === 'unknown') return;
|
||
if (!worst || (STATE_RANK[c.state] || 0) > (STATE_RANK[worst.state] || 0)) worst = c;
|
||
});
|
||
return worst;
|
||
}
|
||
|
||
function severity(data) {
|
||
var w = worstConnection(data);
|
||
if ((data.alert_critical_unacked || 0) > 0 || (data.alert_critical_acked || 0) > 0) return 'crit';
|
||
if (w && w.state === 'down') return 'crit';
|
||
if ((data.alert_warning_unacked || 0) > 0 || (data.alert_warning_acked || 0) > 0) return 'warn';
|
||
if (w && w.state === 'overdue') return 'warn';
|
||
return 'ok';
|
||
}
|
||
var SEV_ORDER = { crit: 0, warn: 1, ok: 2 };
|
||
|
||
function relTime(ts) {
|
||
if (!ts) return '—';
|
||
var s = Math.max(0, Math.floor(Date.now() / 1000 - ts));
|
||
if (s < 60) return s + 's';
|
||
if (s < 3600) return Math.floor(s / 60) + 'm';
|
||
if (s < 86400) return Math.floor(s / 3600) + 'h ' + Math.floor((s % 3600) / 60) + 'm';
|
||
if (s < 86400 * 30) return Math.floor(s / 86400) + 'd ' + Math.floor((s % 86400) / 3600) + 'h';
|
||
return Math.floor(s / 86400) + 'd';
|
||
}
|
||
|
||
function fmtClock(ts) {
|
||
var d = new Date(ts * 1000);
|
||
function p(n) { return n < 10 ? '0' + n : '' + n; }
|
||
var t = p(d.getHours()) + ':' + p(d.getMinutes()) + ':' + p(d.getSeconds());
|
||
return (d.toDateString() === new Date().toDateString())
|
||
? t
|
||
: d.getFullYear() + '-' + p(d.getMonth() + 1) + '-' + p(d.getDate()) + ' ' + t;
|
||
}
|
||
|
||
// ---- host row rendering ---------------------------------------------
|
||
function statusChip(data) {
|
||
var w = worstConnection(data);
|
||
if (!w) return '<span class="chip outline">—</span>';
|
||
var cls = w.state === 'up' ? 'ok' : (w.state === 'down' ? 'crit' : 'warn');
|
||
return '<span class="chip ' + cls + '">' + escHtml(w.state) + '</span>';
|
||
}
|
||
|
||
function tipCard(data) {
|
||
var rows = '';
|
||
(data.connections || []).forEach(function (c) {
|
||
if (!c.state || c.state === 'unknown') return;
|
||
var cls = c.state === 'up' ? 'up' : (c.state === 'down' ? 'dn' : 'ov');
|
||
var fam = (c.afam || '').replace('IPv', 'v');
|
||
var rtt = (c.state === 'up' && c.rtts && c.rtts[0]) ? Math.round(c.rtts[0]) + ' ms' : '—';
|
||
rows += '<tr><td>' + escHtml(fam) + '</td><td>' + escHtml(c.addr || '—') + '</td>'
|
||
+ '<td class="' + cls + '">' + escHtml(c.state) + '</td>'
|
||
+ '<td>' + rtt + '</td><td>' + (c.statetime ? fmtClock(c.statetime) : '—') + '</td></tr>';
|
||
});
|
||
if (!rows) rows = '<tr><td colspan="5">no connections yet</td></tr>';
|
||
var meta = [];
|
||
meta.push('agent ' + (data.hbc_version ? 'v' + escHtml(String(data.hbc_version)) : 'unknown'));
|
||
if (data.dyn) meta.push('dyndns');
|
||
if (data.watched) meta.push('watched');
|
||
if (data.interval) meta.push('interval ' + escHtml(String(data.interval)) + ' s');
|
||
return '<span class="tipbox" role="tooltip">'
|
||
+ '<table><thead><tr><th></th><th>address</th><th>state</th><th>rtt</th><th>since</th></tr></thead>'
|
||
+ '<tbody>' + rows + '</tbody></table>'
|
||
+ '<div class="meta">' + meta.join(' · ') + '</div></span>';
|
||
}
|
||
|
||
function alertChips(data) {
|
||
var cu = data.alert_critical_unacked || 0, ca = data.alert_critical_acked || 0;
|
||
var wu = data.alert_warning_unacked || 0, wa = data.alert_warning_acked || 0;
|
||
var out = '';
|
||
if (cu) out += '<span class="chip crit"><b>' + cu + '</b> crit</span> ';
|
||
if (wu) out += '<span class="chip warn"><b>' + wu + '</b> warn</span> ';
|
||
var acked = ca + wa;
|
||
if (acked) out += '<span class="chip outline">' + acked + ' ack’d</span>';
|
||
return out;
|
||
}
|
||
|
||
function lastAlertHtml(name) {
|
||
var a = lastAlert[name];
|
||
if (!a) return '<span class="lastalert quiet">—</span>';
|
||
var lvl = (a.level || '').toUpperCase();
|
||
var quiet = (lvl !== 'WARNING' && lvl !== 'CRITICAL');
|
||
var lvlCls = lvl === 'CRITICAL' ? 'crit' : (lvl === 'WARNING' ? 'warn' : '');
|
||
var short = lvl === 'CRITICAL' ? 'CRIT' : (lvl === 'WARNING' ? 'WARN' : escHtml(lvl));
|
||
return '<span class="lastalert' + (quiet ? ' quiet' : '') + '" title="' + escHtml(a.message) + '">'
|
||
+ '<span class="lvl ' + lvlCls + '">' + short + '</span> '
|
||
+ escHtml(a.message)
|
||
+ (a.ts ? ' <span class="ago">· ' + relTime(a.ts) + ' ago</span>' : '')
|
||
+ '</span>';
|
||
}
|
||
|
||
function renderRow(data) {
|
||
var name = data.name;
|
||
var sev = severity(data);
|
||
var row = document.querySelector('.hrow[data-host="' + CSS.escape(name) + '"]');
|
||
var isNew = false;
|
||
if (!row) {
|
||
row = document.createElement('div');
|
||
row.dataset.host = name;
|
||
isNew = true;
|
||
}
|
||
row.className = 'hrow' + (sev !== 'ok' ? ' sev-' + sev : '');
|
||
row.dataset.sev = sev;
|
||
|
||
var w = worstConnection(data);
|
||
var up = (data.connections || []).find(function (c) { return c.state === 'up' && c.rtts && c.rtts.length; });
|
||
var lat = up ? Math.round(up.rtts[0]) + ' ms' : '—';
|
||
var since = w ? relTime(w.statetime) : '—';
|
||
var staleTag = '';
|
||
if (!data.hbc_version) staleTag = '<span class="stale" title="agent version unknown">agent?</span>';
|
||
else if (data.hbc_version !== HBD_VERSION) {
|
||
staleTag = '<span class="stale" title="agent v' + escHtml(String(data.hbc_version)) + ' ≠ server v' + HBD_VERSION + '">v' + escHtml(String(data.hbc_version)) + '</span>';
|
||
}
|
||
var dotCls = w ? (w.state === 'up' ? sev : (w.state === 'down' ? 'down' : 'overdue')) : 'off';
|
||
var flapTag = '';
|
||
var flapping = data.flapping || [];
|
||
if (flapping.length) {
|
||
var svcs = flapping.map(function (s) { return s || 'host'; }).join(', ');
|
||
flapTag = '<span class="flap" title="notifications suppressed: ' + escHtml(svcs) + '">flapping</span>';
|
||
}
|
||
|
||
row.innerHTML =
|
||
'<span class="host' + (data.watched ? ' watched' : '') + '">'
|
||
+ '<span class="dot ' + dotCls + '"></span>'
|
||
+ '<a href="/plugins#' + encodeURIComponent(name) + '">' + escHtml(name) + '</a>' + staleTag + flapTag + '</span>'
|
||
+ '<span class="tip" tabindex="0">' + statusChip(data) + tipCard(data) + '</span>'
|
||
+ '<span class="lat num">' + lat + '</span>'
|
||
+ '<span>' + alertChips(data) + '</span>'
|
||
+ '<span class="since">' + since + '</span>'
|
||
+ lastAlertHtml(name);
|
||
|
||
if (isNew) {
|
||
document.getElementById('host-rows').appendChild(row);
|
||
sortRows();
|
||
}
|
||
applyHostFilter(row);
|
||
}
|
||
|
||
function sortRows() {
|
||
var container = document.getElementById('host-rows');
|
||
var rows = [...container.children];
|
||
rows.sort(function (a, b) {
|
||
var d = (SEV_ORDER[a.dataset.sev] ?? 2) - (SEV_ORDER[b.dataset.sev] ?? 2);
|
||
return d !== 0 ? d : a.dataset.host.localeCompare(b.dataset.host);
|
||
});
|
||
rows.forEach(function (r) { container.appendChild(r); });
|
||
}
|
||
|
||
function updateCounts() {
|
||
var total = 0, up = 0, overdue = 0, down = 0;
|
||
Object.values(hostData).forEach(function (d) {
|
||
total++;
|
||
var w = worstConnection(d);
|
||
if (!w) return;
|
||
if (w.state === 'up') up++;
|
||
else if (w.state === 'overdue') overdue++;
|
||
else if (w.state === 'down') down++;
|
||
});
|
||
var parts = [total + ' hosts', up + ' up'];
|
||
if (overdue) parts.push(overdue + ' overdue');
|
||
if (down) parts.push(down + ' down');
|
||
document.getElementById('host-counts').textContent = parts.join(' · ');
|
||
}
|
||
|
||
function updateHost(data) {
|
||
var prev = hostData[data.name];
|
||
var prevSev = prev ? severity(prev) : null;
|
||
hostData[data.name] = data;
|
||
renderRow(data);
|
||
if (prevSev !== severity(data)) sortRows();
|
||
updateCounts();
|
||
}
|
||
|
||
// ---- host filter -----------------------------------------------------
|
||
function applyHostFilter(row) {
|
||
var rows = row ? [row] : [...document.querySelectorAll('.hrow')];
|
||
rows.forEach(function (r) {
|
||
r.style.display = (!hostFilterRe || hostFilterRe.test(r.dataset.host)) ? '' : 'none';
|
||
});
|
||
}
|
||
document.getElementById('host-filter').addEventListener('input', function () {
|
||
var v = this.value.trim();
|
||
try {
|
||
hostFilterRe = v ? new RegExp(v, 'i') : null;
|
||
this.style.borderColor = '';
|
||
} catch (_) {
|
||
hostFilterRe = null;
|
||
this.style.borderColor = 'var(--st-crit)';
|
||
}
|
||
applyHostFilter();
|
||
});
|
||
|
||
// ---- last alert tracking ---------------------------------------------
|
||
var ALERT_LEVELS = { WARNING: 1, CRITICAL: 1, RECOVER: 1 };
|
||
|
||
function noteAlert(host, level, message, ts) {
|
||
if (!host || !ALERT_LEVELS[(level || '').toUpperCase()]) return;
|
||
var cur = lastAlert[host];
|
||
if (cur && cur.ts && ts && ts < cur.ts) return; // keep the newest
|
||
lastAlert[host] = { level: level, message: message, ts: ts };
|
||
if (hostData[host]) renderRow(hostData[host]);
|
||
}
|
||
|
||
async function seedActiveAlerts() {
|
||
try {
|
||
var r = await fetch('/api/0/alerts');
|
||
if (!r.ok) return;
|
||
var data = await r.json();
|
||
(data.alerts || []).forEach(function (a) {
|
||
var metric = (a.metric_path.includes('.')
|
||
? a.metric_path.slice(a.metric_path.indexOf('.') + 1)
|
||
: a.metric_path).replace(/_status_code$/, '');
|
||
var msg = metric + ' = ' + a.last_value
|
||
+ (a.formatted_message ? ' ' + a.formatted_message
|
||
: (a.operator && a.threshold_value != null ? ' (' + a.operator + ' ' + a.threshold_value + ')' : ''));
|
||
noteAlert(a.hostname, a.level, msg, a.since);
|
||
});
|
||
} catch (_) { /* dashboard still works without the seed */ }
|
||
}
|
||
|
||
// ---- websocket -----------------------------------------------------------
|
||
function setWsStatus(ok) {
|
||
var el = document.getElementById('ws-status');
|
||
el.classList.toggle('reconnecting', !ok);
|
||
el.textContent = ok ? 'live' : 'reconnecting…';
|
||
}
|
||
|
||
function WS_Connect() {
|
||
if (!("WebSocket" in window)) {
|
||
console.log("WebSocket NOT supported by your Browser!");
|
||
return;
|
||
}
|
||
var ws_hbd = new WebSocket("{{ heartbeat_ws_url }}");
|
||
|
||
ws_hbd.onopen = function () {
|
||
setWsStatus(true);
|
||
ws_hbd.send("heartbeat_web");
|
||
};
|
||
|
||
ws_hbd.onerror = function (event) { console.log(event); };
|
||
|
||
ws_hbd.onmessage = function (event) {
|
||
var state = JSON.parse(event.data);
|
||
if (state.type === "host") {
|
||
updateHost(state.data);
|
||
} else if (state.type === "message") {
|
||
var msg = state.data;
|
||
noteAlert(msg.host, msg.level, msg.message, msg.ts);
|
||
}
|
||
};
|
||
|
||
ws_hbd.onclose = function () {
|
||
setWsStatus(false);
|
||
setTimeout(WS_Connect, 3000);
|
||
};
|
||
}
|
||
|
||
// ---- boot -----------------------------------------------------------------
|
||
INITIAL_HOSTS.forEach(function (h) { hostData[h.name] = h; });
|
||
Object.values(hostData).forEach(renderRow);
|
||
sortRows();
|
||
updateCounts();
|
||
seedActiveAlerts();
|
||
WS_Connect();
|
||
|
||
// refresh relative times once a minute
|
||
setInterval(function () {
|
||
Object.values(hostData).forEach(renderRow);
|
||
}, 60000);
|
||
</script>
|
||
</body>
|
||
</html>
|