display and acknowledge alerts

This commit is contained in:
Andreas Wrede
2026-04-03 06:35:45 -04:00
parent c5770006f7
commit 941f3ea4b0
6 changed files with 414 additions and 62 deletions
+17 -17
View File
@@ -99,7 +99,7 @@ hosts:
dyndns: true
weekend:
threshold_config: default
threshold_config: freebsd_server
watch: false
notification_channels: [pushover_standard]
dyndns: true
@@ -159,7 +159,7 @@ threshold_configs:
warning: 85.0
critical: 90.0
rtt:
warning: 30
warning: 50
critical: 250.0
@@ -170,9 +170,9 @@ threshold_configs:
warning: 80.0
critical: 90.0
memory_monitor:
percent:
warning: 3.0
critical: 95.0
memory_percent:
warning: 97.0
critical: 100.0
disk_monitor:
partitions:
/:
@@ -188,12 +188,12 @@ threshold_configs:
warning: WARNING
critical: CRITICAL
operator: "=="
UPS_load:
display: "{ups_output}"
ups_load:
display: "load to high: {ups_output}"
warning: 70
critical: 80
operator: ">="
UPS_status_code:
ups_status_code:
display: "{ups_output}"
warning: 1
critical: 2
@@ -204,7 +204,7 @@ threshold_configs:
critical: 2
operator: ">="
rtt:
warning: 30
warning: 50
critical: 250.0
truenas_server:
@@ -232,13 +232,13 @@ threshold_configs:
warning: WARNING
critical: CRITICAL
operator: "=="
UPS_load:
display: "{ups_output}"
warning: 70
critical: 80
operator: ">="
UPS_status_code:
display: "{ups_output}"
ups_load:
display: "load to high: {ups_output}"
WARNING: 70
CRITICAL: 80
OPERATOR: ">="
ups_status_code:
DISPLAY: "{ups_output}"
warning: 1
critical: 2
operator: ">="
@@ -248,7 +248,7 @@ threshold_configs:
critical: 2
operator: ">="
rtt:
warning: 100
warning: 120
critical: 250.0
+30
View File
@@ -307,6 +307,21 @@ class Host:
d["name"] = "<b>%s</b>" % d["name"]
d["dyn"] = str(self.dyn)
d["num"] = self.num
# Add alert counts
warning_count = 0
critical_count = 0
for metric_path, alert_state in self.alert_states.items():
# Import AlertLevel here to avoid circular imports
from .threshold import AlertLevel
if alert_state.level == AlertLevel.WARNING:
warning_count += 1
elif alert_state.level == AlertLevel.CRITICAL:
critical_count += 1
d["alert_warning_count"] = warning_count
d["alert_critical_count"] = critical_count
for c in ["IPv4", "IPv6"]:
if c in self.connections:
cs = self.connections[c].statedict()
@@ -363,6 +378,21 @@ class Host:
ddict[d] = cl
else:
ddict[d] = self.__dict__[d]
# Add alert counts (computed from alert_states)
warning_count = 0
critical_count = 0
if hasattr(self, 'alert_states'):
from .threshold import AlertLevel
for metric_path, alert_state in self.alert_states.items():
if alert_state.level == AlertLevel.WARNING:
warning_count += 1
elif alert_state.level == AlertLevel.CRITICAL:
critical_count += 1
ddict["alert_warning_count"] = warning_count
ddict["alert_critical_count"] = critical_count
return ddict
def jsons(self):
+44
View File
@@ -324,6 +324,49 @@ async def start(
"host_count": len(hbdclass.Host.hosts),
})
async def api_acknowledge_alert(request):
"""Acknowledge an alert to stop reminder notifications."""
try:
data = await request.json()
except Exception:
return web.json_response(
{"error": "Invalid JSON in request body"},
status=400
)
hostname = data.get("hostname")
metric_path = data.get("metric_path")
if not hostname or not metric_path:
return web.json_response(
{"error": "Missing required fields: hostname and metric_path"},
status=400
)
if hostname not in hbdclass.Host.hosts:
return web.json_response(
{"error": f"Host '{hostname}' not found"},
status=404
)
host = hbdclass.Host.hosts[hostname]
if metric_path not in host.alert_states:
return web.json_response(
{"error": f"Alert '{metric_path}' not found for host '{hostname}'"},
status=404
)
alert_state = host.alert_states[metric_path]
alert_state.acknowledge()
return web.json_response({
"success": True,
"hostname": hostname,
"metric_path": metric_path,
"acknowledged_at": alert_state.acknowledged_at,
})
# -------------------------------------------------------------------------
# UI Pages
# -------------------------------------------------------------------------
@@ -375,6 +418,7 @@ async def start(
web.get("/api/0/hosts/{hostname}/plugins/{plugin_name}", api_host_plugin_detail),
web.get("/api/0/hosts/{hostname}/alerts", api_host_alerts),
web.get("/api/0/alerts", api_all_alerts),
web.post("/api/0/alerts/acknowledge", api_acknowledge_alert),
web.get("/c", cmd),
web.get("/d", drop),
web.get("/n", register),
+112 -1
View File
@@ -154,6 +154,11 @@
transition: all 0.2s;
}
.alert-item.acknowledged {
opacity: 0.6;
background: #f0f0f0;
}
.alert-item:hover {
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
transform: translateX(5px);
@@ -238,6 +243,46 @@
font-size: 0.85em;
}
.alert-actions {
display: flex;
flex-direction: column;
gap: 8px;
margin-left: 15px;
}
.acknowledge-btn {
padding: 8px 16px;
background: #2196f3;
color: white;
border: none;
border-radius: 4px;
cursor: pointer;
font-size: 0.85em;
transition: all 0.2s;
white-space: nowrap;
}
.acknowledge-btn:hover {
background: #1976d2;
transform: scale(1.05);
}
.acknowledge-btn:disabled {
background: #ccc;
cursor: not-allowed;
transform: none;
}
.acknowledged-badge {
padding: 4px 8px;
background: #4caf50;
color: white;
border-radius: 4px;
font-size: 0.75em;
text-align: center;
white-space: nowrap;
}
.no-alerts {
text-align: center;
padding: 60px 20px;
@@ -396,6 +441,7 @@
function renderAlert(alert) {
const level = alert.level.toLowerCase();
const duration = getDuration(alert.since);
const acknowledged = alert.acknowledged || false;
// Use formatted message if available, otherwise build from individual fields
let valueText = `Value: <span class="alert-value">${formatValue(alert.last_value)}</span>`;
@@ -405,8 +451,26 @@
valueText += ` <span class="threshold-info">(threshold: ${alert.operator} ${formatValue(alert.threshold_value)})</span>`;
}
// Build actions section
let actionsHtml = '';
if (acknowledged) {
actionsHtml = `
<div class="alert-actions">
<div class="acknowledged-badge">✓ Acknowledged</div>
</div>
`;
} else {
actionsHtml = `
<div class="alert-actions">
<button class="acknowledge-btn" onclick="acknowledgeAlert('${alert.hostname}', '${alert.metric_path}', event)">
Acknowledge
</button>
</div>
`;
}
return `
<div class="alert-item ${level}">
<div class="alert-item ${level} ${acknowledged ? 'acknowledged' : ''}">
<div class="alert-main">
<div class="alert-header">
<span class="alert-level ${level}">${alert.level}</span>
@@ -418,6 +482,7 @@
<span class="alert-duration">Active for ${duration}</span>
</div>
</div>
${actionsHtml}
</div>
`;
}
@@ -464,6 +529,52 @@
renderAlerts(allAlerts);
}
async function acknowledgeAlert(hostname, metricPath, event) {
// Prevent event bubbling
if (event) {
event.stopPropagation();
}
// Disable the button
const button = event.target;
button.disabled = true;
button.textContent = 'Acknowledging...';
try {
const response = await fetch('/api/0/alerts/acknowledge', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
hostname: hostname,
metric_path: metricPath,
}),
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const result = await response.json();
// Update the alert in our local data
const alert = allAlerts.find(a => a.hostname === hostname && a.metric_path === metricPath);
if (alert) {
alert.acknowledged = true;
alert.acknowledged_at = result.acknowledged_at;
}
// Re-render alerts
renderAlerts(allAlerts);
} catch (error) {
alert(`Failed to acknowledge alert: ${error.message}`);
button.disabled = false;
button.textContent = 'Acknowledge';
}
}
// Auto-refresh every 15 seconds
setInterval(loadAlerts, 15000);
+188 -43
View File
@@ -3,10 +3,16 @@
{% include 'head.html' %}
<style>
body {
margin: 10px;
background: #f5f5f5;
overflow: hidden;
}
.nav {
background: #fff;
padding: 15px;
margin-bottom: 20px;
padding: 10px 15px;
margin-bottom: 10px;
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
border-radius: 4px;
}
@@ -16,6 +22,7 @@
text-decoration: none;
color: #0066cc;
font-weight: 500;
font-size: 0.9em;
}
.nav a:hover {
@@ -27,53 +34,140 @@
font-weight: bold;
}
.container {
max-width: 1600px;
margin: 0 auto;
max-height: calc(100vh - 120px);
overflow-y: auto;
padding-right: 10px;
}
h1 {
color: #333;
margin-bottom: 5px;
font-size: 1.5em;
}
h2 {
color: #333;
margin-bottom: 10px;
font-size: 1.2em;
padding: 10px 15px;
background: white;
border-radius: 6px;
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
}
.subtitle {
color: #666;
margin-bottom: 15px;
font-size: 0.9em;
}
.content {
display: flex;
flex-direction: column;
gap: 15px;
}
.table {
/* flex: 1; */
flex-grow: none;
.table-section {
background: white;
border-radius: 6px;
padding: 15px;
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
}
.log {
flex: 2;
flex-grow: 1;
.log-section {
background: white;
border-radius: 6px;
padding: 15px;
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
max-height: 400px;
overflow-y: auto;
}
#ntable {
border-collapse: collapse;
font-size: 95%;
/* width: 100%; */
width: 100%;
font-size: 0.9em;
}
#ntable td,
#ntable th {
border: 1px solid #ddd;
border: 1px solid #e0e0e0;
text-align: left;
padding: 0px;
padding: 8px 10px;
}
#ntable tr:nth-child(even) {
background-color: #f2f2f2;
background-color: #fafafa;
}
#ntable tr:hover {
background-color: #ddd;
background-color: #e3f2fd;
}
#ntable th {
padding-top: 12px;
padding-bottom: 12px;
background-color: #9d9d9d;
padding: 12px 10px;
background-color: #2196f3;
color: white;
font-weight: 600;
position: sticky;
top: 0;
z-index: 10;
}
#ntable
th:not(.sorttable_sorted):not(.sorttable_sorted_reverse):not(.sorttable_nosort):after {
content: " \2195";
content: " ";
opacity: 0.5;
}
/* Alert count column styling */
#ntable td.alert-warning {
color: #ff9800;
font-weight: bold;
text-align: center;
}
#ntable td.alert-critical {
color: #f44336;
font-weight: bold;
text-align: center;
}
/* Scrollbar styling */
.container::-webkit-scrollbar,
.log-section::-webkit-scrollbar {
width: 8px;
}
.container::-webkit-scrollbar-track,
.log-section::-webkit-scrollbar-track {
background: #f1f1f1;
border-radius: 4px;
}
.container::-webkit-scrollbar-thumb,
.log-section::-webkit-scrollbar-thumb {
background: #888;
border-radius: 4px;
}
.container::-webkit-scrollbar-thumb:hover,
.log-section::-webkit-scrollbar-thumb:hover {
background: #555;
}
/* Message styling */
#messages {
font-size: 0.85em;
line-height: 1.6;
}
#messages div {
padding: 5px 0;
border-bottom: 1px solid #f0f0f0;
}
/* Modal for connection status messages */
@@ -85,7 +179,7 @@
top: 0;
width: 100%;
height: 100%;
background-color: rgba(0, 0, 0, 0.4);
background-color: rgba(0, 0, 0, 0.5);
}
.connection-modal.show {
@@ -95,20 +189,35 @@
}
.connection-modal-content {
background-color: #f9f9f9;
padding: 20px;
border: 1px solid #888;
border-radius: 5px;
background-color: white;
padding: 30px 40px;
border-radius: 8px;
text-align: center;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
min-width: 300px;
}
.connection-modal-content p {
margin: 10px 0;
margin: 0;
font-size: 16px;
color: #333;
}
/* State indicators */
.state-up {
color: #4caf50;
font-weight: 600;
}
.state-down {
color: #f44336;
font-weight: 700;
}
.state-overdue {
color: #ff9800;
font-weight: 700;
}
</style>
<script type="text/javascript">
var cnt = 0;
@@ -130,6 +239,14 @@
function createRow(data) {
var row = document.createElement("tr");
var c_name = document.createElement("td");
var c_warning = document.createElement("td");
c_warning.style.textAlign = "center";
c_warning.style.color = "#ff9800";
c_warning.style.fontWeight = "bold";
var c_critical = document.createElement("td");
c_critical.style.textAlign = "center";
c_critical.style.color = "#f44336";
c_critical.style.fontWeight = "bold";
var c_ipv4addr = document.createElement("td");
var c_ipv4state = document.createElement("td");
var c_ipv4latency = document.createElement("td");
@@ -143,6 +260,8 @@
var c_ipv6statets = document.createElement("td");
c_ipv6statets.style.textAlign = "right";
row.appendChild(c_name);
row.appendChild(c_warning);
row.appendChild(c_critical);
row.appendChild(c_ipv4addr);
row.appendChild(c_ipv4state);
row.appendChild(c_ipv4latency);
@@ -156,6 +275,13 @@
} else {
c_name.innerHTML = data.name;
}
// Set alert counts
var warningCount = data.alert_warning_count || 0;
var criticalCount = data.alert_critical_count || 0;
c_warning.innerHTML = warningCount > 0 ? warningCount : "";
c_critical.innerHTML = criticalCount > 0 ? criticalCount : "";
c_ipv4addr.innerHTML = data.connections[0].addr;
c_ipv4state.innerHTML = data.connections[0].state;
if (data.connections.length > 1) {
@@ -179,27 +305,40 @@
setup();
}
// Update warning and critical counts
var warningCount = data.alert_warning_count || 0;
var criticalCount = data.alert_critical_count || 0;
name_idx[data.name].cells[1].innerHTML = warningCount > 0 ? warningCount : "";
name_idx[data.name].cells[2].innerHTML = criticalCount > 0 ? criticalCount : "";
for (var i = 0; i < data.connections.length; i++) {
name_idx[data.name].cells[2 + i * 4].innerHTML = data.connections[i].addr;
name_idx[data.name].cells[5 + i * 4].innerHTML = formatTS(
// Offset by 2 for the warning/critical count columns
name_idx[data.name].cells[3 + i * 4].innerHTML = data.connections[i].addr;
name_idx[data.name].cells[6 + i * 4].innerHTML = formatTS(
data.connections[i].statetime
);
if (data.connections[i].state == "up") {
state = "up";
state = '<span class="state-up">up</span>';
latency = Number.parseFloat(data.connections[i].rtts[0]).toFixed(2);
} else {
if (data.connections[i].state == "unknown") {
state = "";
latency = "";
name_idx[data.name].cells[2 + i * 4].innerHTML = "";
name_idx[data.name].cells[5 + i * 4].innerHTML = "";
name_idx[data.name].cells[3 + i * 4].innerHTML = "";
name_idx[data.name].cells[6 + i * 4].innerHTML = "";
} else if (data.connections[i].state == "down") {
state = '<span class="state-down">down</span>';
latency = "-";
} else if (data.connections[i].state == "overdue") {
state = '<span class="state-overdue">overdue</span>';
latency = "-";
} else {
state = "<b>" + data.connections[i].state + "</b>";
latency = "-";
}
}
name_idx[data.name].cells[3 + i * 4].innerHTML = state;
name_idx[data.name].cells[4 + i * 4].innerHTML = latency;
name_idx[data.name].cells[4 + i * 4].innerHTML = state;
name_idx[data.name].cells[5 + i * 4].innerHTML = latency;
}
}
@@ -231,7 +370,7 @@
update_table(state.data);
} else if (state.type == "message") {
var msgs = document.getElementById("messages");
msgs.insertAdjacentHTML("afterbegin", state.data + "<br>");
msgs.insertAdjacentHTML("afterbegin", "<div>" + state.data + "</div>");
}
cnt++;
};
@@ -264,20 +403,24 @@
{% include 'menu.html' %}
<div id="content" class="content" style="overflow: hidden">
<div id="table" class="table" style="overflow: hidden">
<!-- <h2>{{title}}</h2> -->
<div class="container">
<h1>{{ header }}</h1>
<p class="subtitle">Real-time host monitoring and event log</p>
<div class="table-section">
<table id="ntable" class="sortable">
<thead>
<tr>
<th>Name</th>
<th style="text-align: center" title="Warning Alerts">⚠️</th>
<th style="text-align: center" title="Critical Alerts">🔴</th>
<th>IPv4 Addr</th>
<th>State</th>
<th style="text-align: right">Latencey</th>
<th style="text-align: right">Latency</th>
<th style="text-align: right">Last State</th>
<th>IPv6 Addr</th>
<th>State</th>
<th style="text-align: right">Latencey</th>
<th style="text-align: right">Latency</th>
<th style="text-align: right">Last State</th>
</tr>
</thead>
@@ -285,6 +428,8 @@
{% for host in hosts %}
<tr>
<td>{{ host.name }}</td>
<td style="text-align: center; color: #ff9800; font-weight: bold;">{{ host.alert_warning_count if host.alert_warning_count > 0 else '' }}</td>
<td style="text-align: center; color: #f44336; font-weight: bold;">{{ host.alert_critical_count if host.alert_critical_count > 0 else '' }}</td>
{% for conn in host.connections %}
<td>{{ conn.addr if conn.addr else '' }}</td>
<td>{{ conn.state if conn.state else '' }}</td>
@@ -302,13 +447,13 @@
</tbody>
</table>
</div>
<div id="log" class="log" style="overflow: auto;">
<h2>Log of Events</h2>
<div id="messages">
</div>
<div class="log-section">
<h2>Log of Events</h2>
<div id="messages"></div>
</div>
</div>
{% include 'foot.html' %}
<!-- Connection status modal -->
+22
View File
@@ -56,6 +56,8 @@ class AlertState:
self.threshold_value = None # The threshold value that triggered alert
self.operator = None # The comparison operator (>, <, >=, etc.)
self.formatted_message = None # Formatted display message for UI
self.acknowledged = False # Whether alert has been acknowledged
self.acknowledged_at = None # Timestamp when acknowledged
def update(
self,
@@ -101,6 +103,11 @@ class AlertState:
self.level = level
self.since = now
self.notification_count = 0
# Reset acknowledgment on state change
if level != AlertLevel.OK:
# Only reset if changing to a different alert level
self.acknowledged = False
self.acknowledged_at = None
return True
return False
@@ -114,8 +121,13 @@ class AlertState:
"last_value": self.last_value,
"last_check": self.last_check,
"notification_count": self.notification_count,
"acknowledged": self.acknowledged,
}
# Include acknowledgment timestamp if acknowledged
if self.acknowledged_at is not None:
result["acknowledged_at"] = self.acknowledged_at
# Include threshold info if available
if self.threshold_value is not None:
result["threshold_value"] = self.threshold_value
@@ -126,6 +138,12 @@ class AlertState:
return result
def acknowledge(self):
"""Acknowledge this alert to stop reminder notifications."""
self.acknowledged = True
self.acknowledged_at = time.time()
logger.info("Alert acknowledged for %s", self.metric_path)
def __str__(self):
return self.to_dict().__str__()
@@ -1014,6 +1032,10 @@ class ThresholdChecker:
if alert_state.level == AlertLevel.OK:
return
# Skip reminders if alert has been acknowledged
if alert_state.acknowledged:
return
now = time.time()
# Check if we should re-notify