re-factor notifications, add sms and matrix as channels
This commit is contained in:
Andreas Wrede
@@ -0,0 +1,40 @@
|
||||
async def send_sms(hass, user, password, sender_did, call):
|
||||
"""Send SMS message using multipart form-data like MMS."""
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
recipient = call.data.get("recipient")
|
||||
message = call.data.get("message")
|
||||
|
||||
if not recipient or not message:
|
||||
_LOGGER.error("Recipient or message missing.")
|
||||
return
|
||||
|
||||
# Build form data dictionary
|
||||
form_data = {
|
||||
'api_username': str(user),
|
||||
'api_password': str(password),
|
||||
'did': str(sender_did),
|
||||
'dst': str(recipient),
|
||||
'message': str(message),
|
||||
'method': 'sendSMS'
|
||||
}
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
with aiohttp.MultipartWriter("form-data") as mp:
|
||||
for key, value in form_data.items():
|
||||
part = mp.append(value)
|
||||
part.set_content_disposition('form-data', name=key)
|
||||
|
||||
_LOGGER.error("voipms_sms: sending SMS: %s", mp)
|
||||
async with session.post(REST_ENDPOINT, data=mp) as response:
|
||||
response_text = await response.text()
|
||||
if response.status == 200:
|
||||
response_json = json.loads(response_text)
|
||||
if response_json['status'] == "success":
|
||||
_LOGGER.info("voipms_sms: SMS sent successfully: %s", response_text)
|
||||
else:
|
||||
_LOGGER.error("voipms_sms: SMS not sent: %s", response_text)
|
||||
else:
|
||||
_LOGGER.error("voipms_sms: Failed to send SMS. Status: %s, Response: %s", response.status, response_text)
|
||||
|
||||
|
||||
|
||||
+235
-473
@@ -2,532 +2,294 @@
|
||||
|
||||
## Overview
|
||||
|
||||
The Heartbeat Monitoring System includes a flexible notification system that can send alerts through multiple channels including Email, Pushover, Signal, and Mattermost. The system supports centralized channel definitions with per-host routing, allowing fine-grained control over notification delivery.
|
||||
Notifications are dispatched to the **owner and managers** of a host, each via their own configured notification channels. Channel definitions are global; users reference them by name. No users configured → no notifications sent.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Components
|
||||
```
|
||||
Alert event (udp.py / threshold.py)
|
||||
└─ notify.send_notification(host_name, Notification)
|
||||
├─ look up host.owner + host.managers
|
||||
├─ for each user → user.notification_channels
|
||||
└─ for each channel → _dispatch_to_channel (filtered by min_level)
|
||||
```
|
||||
|
||||
1. **Notification Channels** (`notification_channels` in config)
|
||||
- Centralized definitions of notification providers
|
||||
- Each channel has a type and type-specific credentials
|
||||
- Reusable across multiple hosts
|
||||
|
||||
2. **Channel Dispatcher** (`hbd/server/notify.py`)
|
||||
- `pushmsg_for_host(hostname, message)`: Main entry point for host-specific notifications
|
||||
- `_dispatch_to_channel(channel_name, channel_config, message)`: Routes to specific provider
|
||||
- Provider functions: `pushover()`, `pushsignal()`, `pushmattermost()`, `send_email()`
|
||||
|
||||
3. **Configuration Utilities** (`hbd/server/config.py`)
|
||||
- `get_notification_channels_for_host(config, hostname)`: Retrieves channel names for a host
|
||||
- `get_notification_channels_config(config, hostname)`: Retrieves full channel configurations
|
||||
- `get_channel_config(config, channel_name)`: Gets configuration for a specific channel
|
||||
|
||||
4. **Integration Points**
|
||||
- **Threshold alerts**: `threshold.py` calls `notify_mod.pushmsg_for_host()`
|
||||
- **Heartbeat events**: `udp.py` calls `notify_mod.pushmsg_for_host()` for boot/shutdown/overdue
|
||||
- **Custom alerts**: Any code can call `notify_mod.pushmsg_for_host(hostname, message)`
|
||||
Every notification carries:
|
||||
- **title** — `[LEVEL] hostname` (e.g. `[CRITICAL] webserver01`)
|
||||
- **body** — detail message (metric value, threshold, duration)
|
||||
- **url** — link to the plugin metrics page (`{base_url}/plugins#{hostname}`)
|
||||
- **level** — `RECOVER | WARNING | CRITICAL | INFO`
|
||||
|
||||
## Configuration
|
||||
|
||||
### Centralized Channel Definitions
|
||||
### Base URL
|
||||
|
||||
Define notification channels once in your configuration file:
|
||||
Set `base_url` so notification links point to your hbd instance:
|
||||
|
||||
```yaml
|
||||
base_url: https://hbd.example.com
|
||||
```
|
||||
|
||||
### Global channel definitions
|
||||
|
||||
Define channels once; reference them by name from user configs:
|
||||
|
||||
```yaml
|
||||
notification_channels:
|
||||
# Signal notifications
|
||||
signal_ops:
|
||||
type: signal
|
||||
cli_path: /usr/local/bin/signal-cli
|
||||
user: +1234567890 # Your Signal number
|
||||
recipient: +1234567890 # Recipient number
|
||||
|
||||
signal_oncall:
|
||||
type: signal
|
||||
cli_path: /usr/local/bin/signal-cli
|
||||
user: +1234567890
|
||||
recipient: +0987654321 # Different recipient
|
||||
pushover_ops:
|
||||
type: pushover
|
||||
token: your-app-token
|
||||
user: your-user-key
|
||||
min_level: WARNING # optional, default: WARNING
|
||||
|
||||
# Email notifications
|
||||
email_ops:
|
||||
type: email
|
||||
recipients:
|
||||
- ops@example.com
|
||||
- alerts@example.com
|
||||
sender: heartbeat@example.com
|
||||
recipients: [ops@example.com]
|
||||
sender: hbd@example.com
|
||||
smtp_server: smtp.example.com
|
||||
smtp_port: 587
|
||||
smtp_user: heartbeat@example.com
|
||||
smtp_password: your-smtp-password
|
||||
smtp_user: hbd@example.com
|
||||
smtp_password: secret
|
||||
min_level: WARNING
|
||||
|
||||
email_devteam:
|
||||
type: email
|
||||
recipients: [dev-alerts@example.com]
|
||||
sender: heartbeat-dev@example.com
|
||||
smtp_server: smtp.example.com
|
||||
smtp_port: 587
|
||||
smtp_user: heartbeat-dev@example.com
|
||||
smtp_password: your-smtp-password
|
||||
matrix_oncall:
|
||||
type: matrix
|
||||
homeserver: https://matrix.example.org
|
||||
access_token: syt_xxx
|
||||
room_id: "!abc:matrix.example.org"
|
||||
min_level: CRITICAL # only send critical alerts to this room
|
||||
|
||||
# Pushover notifications
|
||||
pushover_urgent:
|
||||
type: pushover
|
||||
token: your-pushover-app-token
|
||||
user: your-pushover-user-key
|
||||
sms_oncall:
|
||||
type: sms_voipms
|
||||
api_user: me@example.com
|
||||
api_password: secret
|
||||
did: "5551234567" # your voip.ms DID number
|
||||
dst: "5559876543" # destination number
|
||||
min_level: CRITICAL
|
||||
|
||||
pushover_normal:
|
||||
type: pushover
|
||||
token: your-pushover-app-token
|
||||
user: another-user-key
|
||||
|
||||
# Mattermost notifications
|
||||
mattermost_devops:
|
||||
type: mattermost
|
||||
host: mattermost.example.com
|
||||
token: your-webhook-token
|
||||
channel: devops-alerts
|
||||
username: heartbeat-bot
|
||||
icon: https://example.com/heartbeat-icon.png
|
||||
```
|
||||
|
||||
### Default Notification Channels
|
||||
|
||||
Specify default channels for hosts that don't have specific channel assignments:
|
||||
|
||||
```yaml
|
||||
default_notification_channels:
|
||||
- email_ops
|
||||
- mattermost_devops
|
||||
```
|
||||
|
||||
Hosts without `notification_channels` defined will use these defaults.
|
||||
|
||||
### Per-Host Channel Assignment
|
||||
|
||||
Assign specific channels to each host in the `hosts` section:
|
||||
|
||||
```yaml
|
||||
hosts:
|
||||
# Critical production web server - multiple channels for redundancy
|
||||
prod-web-01:
|
||||
threshold_config: high_sensitivity
|
||||
watch: true
|
||||
notification_channels:
|
||||
- signal_oncall # Immediate mobile notification
|
||||
- pushover_urgent # Secondary mobile notification
|
||||
- email_ops # Email for record keeping
|
||||
dyndns: false
|
||||
|
||||
# Database server - ops team notifications only
|
||||
prod-db-01:
|
||||
threshold_config: database
|
||||
watch: true
|
||||
notification_channels:
|
||||
- signal_ops
|
||||
- email_ops
|
||||
dyndns: false
|
||||
|
||||
# Development server - email only, no urgent notifications
|
||||
dev-server-01:
|
||||
threshold_config: low_sensitivity
|
||||
watch: false
|
||||
notification_channels:
|
||||
- email_devteam
|
||||
dyndns: false
|
||||
|
||||
# Test server - uses default_notification_channels
|
||||
test-server-01:
|
||||
threshold_config: default
|
||||
watch: false
|
||||
dyndns: false
|
||||
# No notification_channels specified = uses default_notification_channels
|
||||
```
|
||||
|
||||
## Channel Types
|
||||
|
||||
### Email
|
||||
|
||||
Sends notifications via SMTP.
|
||||
|
||||
**Configuration fields:**
|
||||
```yaml
|
||||
type: email
|
||||
recipients: [email1@example.com, email2@example.com] # Required: List of recipients
|
||||
sender: heartbeat@example.com # Required: From address
|
||||
smtp_server: smtp.example.com # Required: SMTP server hostname
|
||||
smtp_port: 587 # Optional: Default 587
|
||||
smtp_user: heartbeat@example.com # Optional: For authenticated SMTP
|
||||
smtp_password: your-password # Optional: For authenticated SMTP
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Supports multiple recipients
|
||||
- TLS/STARTTLS support on port 587
|
||||
- Authenticated and unauthenticated SMTP
|
||||
|
||||
**Example:**
|
||||
```yaml
|
||||
notification_channels:
|
||||
email_critical:
|
||||
type: email
|
||||
recipients: [admin@example.com, oncall@example.com]
|
||||
sender: alerts@example.com
|
||||
smtp_server: smtp.fastmail.com
|
||||
smtp_port: 587
|
||||
smtp_user: alerts@example.com
|
||||
smtp_password: app-specific-password
|
||||
```
|
||||
|
||||
### Pushover
|
||||
|
||||
Sends push notifications to mobile devices via Pushover API.
|
||||
|
||||
**Configuration fields:**
|
||||
```yaml
|
||||
type: pushover
|
||||
token: your-application-token # Required: Your Pushover app token
|
||||
user: your-user-key # Required: Recipient's user key
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Instant mobile push notifications
|
||||
- Works on iOS and Android
|
||||
- Supports delivery confirmations
|
||||
|
||||
**Setup:**
|
||||
1. Create a Pushover account at https://pushover.net
|
||||
2. Create an application to get your app token
|
||||
3. Note your user key from your account dashboard
|
||||
|
||||
**Example:**
|
||||
```yaml
|
||||
notification_channels:
|
||||
pushover_admin:
|
||||
type: pushover
|
||||
token: azGDORePK8gMaC0QOYAMyEEuzJnyUi
|
||||
user: uQiRzpo4DXghDmr9QzzfQu27cmVRsG
|
||||
```
|
||||
|
||||
### Signal
|
||||
|
||||
Sends notifications via Signal messenger using signal-cli.
|
||||
|
||||
**Configuration fields:**
|
||||
```yaml
|
||||
type: signal
|
||||
cli_path: /usr/local/bin/signal-cli # Optional: Path to signal-cli binary
|
||||
user: +1234567890 # Required: Your Signal phone number
|
||||
recipient: +0987654321 # Required: Recipient phone number
|
||||
```
|
||||
|
||||
**Prerequisites:**
|
||||
1. Install signal-cli: https://github.com/AsamK/signal-cli
|
||||
2. Register signal-cli with your phone number:
|
||||
```bash
|
||||
signal-cli -u +1234567890 register
|
||||
signal-cli -u +1234567890 verify CODE
|
||||
```
|
||||
3. Ensure signal-cli is in PATH or specify full path in config
|
||||
|
||||
**Features:**
|
||||
- End-to-end encrypted messaging
|
||||
- Works without phone being online
|
||||
- No API fees or rate limits
|
||||
|
||||
**Example:**
|
||||
```yaml
|
||||
notification_channels:
|
||||
signal_admin:
|
||||
signal_ops:
|
||||
type: signal
|
||||
cli_path: /usr/local/bin/signal-cli
|
||||
user: +12025551234
|
||||
recipient: +12025559999
|
||||
```
|
||||
|
||||
### Mattermost
|
||||
|
||||
Sends notifications to Mattermost team chat via incoming webhooks.
|
||||
|
||||
**Configuration fields:**
|
||||
```yaml
|
||||
type: mattermost
|
||||
host: mattermost.example.com # Required: Mattermost server hostname
|
||||
token: your-webhook-token # Required: Incoming webhook token
|
||||
channel: channel-name # Required: Target channel name
|
||||
username: heartbeat-bot # Optional: Bot display name
|
||||
icon: https://example.com/icon.png # Optional: Bot icon URL
|
||||
```
|
||||
|
||||
**Prerequisites:**
|
||||
1. Enable incoming webhooks in Mattermost
|
||||
2. Create an incoming webhook for your team
|
||||
3. Note the webhook token from the webhook URL
|
||||
|
||||
**Features:**
|
||||
- Team-wide visibility
|
||||
- Rich formatting support
|
||||
- Message threading
|
||||
|
||||
**Example:**
|
||||
```yaml
|
||||
notification_channels:
|
||||
mattermost_ops:
|
||||
mattermost_devops:
|
||||
type: mattermost
|
||||
host: chat.example.com
|
||||
token: abc123def456ghi789
|
||||
channel: infrastructure-alerts
|
||||
username: heartbeat-monitor
|
||||
icon: https://example.com/heartbeat-icon.png
|
||||
host: mattermost.example.com
|
||||
token: webhook-token
|
||||
channel: devops-alerts
|
||||
username: heartbeat-bot
|
||||
```
|
||||
|
||||
## Notification Events
|
||||
### Users with notification channels
|
||||
|
||||
The system sends notifications for various events:
|
||||
Each user lists which global channels they receive notifications on:
|
||||
|
||||
### Threshold Alerts
|
||||
```yaml
|
||||
users:
|
||||
alice:
|
||||
full_name: Alice Smith
|
||||
password: pbkdf2:sha256:...
|
||||
admin: true
|
||||
notification_channels: [pushover_ops, email_ops]
|
||||
|
||||
When monitored metrics exceed configured thresholds:
|
||||
|
||||
- **State changes**: OK → WARNING, WARNING → CRITICAL, CRITICAL → OK
|
||||
- **Format**: `{LEVEL}: {hostname} - {metric_path} = {value} {threshold_info}`
|
||||
- **Example**: `CRITICAL: prod-web-01 - cpu_monitor.cpu_percent = 95.2 (threshold: > 90.0)`
|
||||
- **Re-notifications**: Periodic reminders for ongoing alerts (default: hourly)
|
||||
|
||||
### Heartbeat Events
|
||||
|
||||
Host lifecycle events:
|
||||
|
||||
- **Host boot**: `{hostname} booted`
|
||||
- **Host shutdown**: `{hostname} {connection_type} shutdown`
|
||||
- **Host recovery**: `{hostname} {connection_type} is back`
|
||||
- **Connection issues**: `{hostname} {message}`
|
||||
- **Host overdue**: `{hostname} {connection_type} overdue`
|
||||
|
||||
Only hosts with `watch: true` send heartbeat event notifications.
|
||||
|
||||
### Custom Alerts
|
||||
|
||||
Application code can send custom notifications:
|
||||
|
||||
```python
|
||||
from hbd.server import notify as notify_mod
|
||||
|
||||
# Send to host-specific channels
|
||||
notify_mod.pushmsg_for_host("prod-web-01", "Custom alert message")
|
||||
|
||||
# Send using global config
|
||||
notify_mod.pushmsg_from_config("Global notification")
|
||||
|
||||
# Send to specific config
|
||||
notify_mod.pushmsg(custom_config_dict, "Targeted notification")
|
||||
bob:
|
||||
full_name: Bob Jones
|
||||
password: pbkdf2:sha256:...
|
||||
notification_channels: [sms_oncall, matrix_oncall]
|
||||
```
|
||||
|
||||
## Design Principles
|
||||
### Host access — owner and managers
|
||||
|
||||
The notification system follows these core principles:
|
||||
|
||||
- **Centralization**: Define notification providers once, reference them by name
|
||||
- **Flexibility**: Each host can use different channels for different notification needs
|
||||
- **Redundancy**: Critical hosts can specify multiple channels for failover
|
||||
- **Clarity**: Clean separation between channel definition and channel assignment
|
||||
- **Type Safety**: Provider-specific validation at configuration time
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Channel Organization
|
||||
|
||||
- **Create purpose-specific channels**: `email_ops`, `signal_oncall`, `pushover_urgent`
|
||||
- **Separate by team/role**: `email_devteam`, `signal_dbateam`, `mattermost_security`
|
||||
- **Use descriptive names**: Channel names appear in logs and debugging
|
||||
|
||||
### Redundancy
|
||||
|
||||
For critical hosts, use multiple notification channels:
|
||||
Notifications for a host go to its owner and all managers:
|
||||
|
||||
```yaml
|
||||
hosts:
|
||||
critical-db:
|
||||
notification_channels:
|
||||
- signal_oncall # Primary: Mobile alert
|
||||
- pushover_urgent # Backup: Different mobile platform
|
||||
- email_ops # Tertiary: Email for record-keeping
|
||||
webserver01:
|
||||
owner: alice # receives all notifications for this host
|
||||
managers: [bob] # also receives notifications
|
||||
threshold_config: default
|
||||
watch: true # bold in dashboard (cosmetic only)
|
||||
dyndns: false
|
||||
|
||||
dbserver01:
|
||||
owner: alice
|
||||
managers: [bob]
|
||||
threshold_config: database
|
||||
dyndns: false
|
||||
```
|
||||
|
||||
### Notification Fatigue Prevention
|
||||
`watch: true` only affects display (bold name in the live dashboard). Notifications are now controlled entirely by owner/managers.
|
||||
|
||||
- **Use `watch: false`** for non-critical hosts
|
||||
- **Configure appropriate thresholds** to avoid false positives
|
||||
- **Set different channels for different severities**
|
||||
- **Use `default_notification_channels`** for baseline, add more for critical systems
|
||||
## Channel Types
|
||||
|
||||
### Security
|
||||
### `min_level` filtering
|
||||
|
||||
- **Protect credentials**: Use file permissions to protect config files with passwords/tokens
|
||||
- **Rotate tokens**: Periodically rotate API tokens and passwords
|
||||
- **Use app-specific passwords**: For email, use app-specific passwords instead of main account password
|
||||
- **Separate accounts**: Consider separate notification accounts for different environments (prod vs dev)
|
||||
Every channel accepts an optional `min_level` field:
|
||||
|
||||
### Testing
|
||||
| Value | Channels receive |
|
||||
|---|---|
|
||||
| `WARNING` (default) | WARNING, CRITICAL, RECOVER |
|
||||
| `CRITICAL` | CRITICAL only (and RECOVER) |
|
||||
|
||||
Test notification channels before relying on them:
|
||||
`RECOVER` is always passed through — you don't want to miss a recovery.
|
||||
|
||||
### pushover
|
||||
|
||||
Sends push notifications via [Pushover](https://pushover.net). Includes title, body, and a clickable URL.
|
||||
|
||||
```yaml
|
||||
type: pushover
|
||||
token: your-app-token # Required: Pushover application token
|
||||
user: your-user-key # Required: Recipient's user key
|
||||
min_level: WARNING
|
||||
```
|
||||
|
||||
### email
|
||||
|
||||
Sends via SMTP. Subject = title, body = message + URL on final line.
|
||||
|
||||
```yaml
|
||||
type: email
|
||||
recipients: [ops@example.com, oncall@example.com]
|
||||
sender: hbd@example.com
|
||||
smtp_server: smtp.example.com
|
||||
smtp_port: 587 # 587 = STARTTLS (default), 465 = SSL
|
||||
smtp_user: hbd@example.com
|
||||
smtp_password: secret
|
||||
min_level: WARNING
|
||||
```
|
||||
|
||||
### matrix
|
||||
|
||||
Sends a formatted HTML message to a Matrix room via [matrix-nio](https://github.com/poljar/matrix-nio).
|
||||
|
||||
```yaml
|
||||
type: matrix
|
||||
homeserver: https://matrix.example.org
|
||||
access_token: syt_xxx # Bot account access token
|
||||
room_id: "!abc:matrix.example.org"
|
||||
min_level: WARNING
|
||||
```
|
||||
|
||||
**Setup:**
|
||||
1. Create a bot Matrix account
|
||||
2. Obtain its access token (Element → Settings → Help & About → Access Token)
|
||||
3. Invite the bot to the target room and note the room ID
|
||||
|
||||
### sms_voipms
|
||||
|
||||
Sends SMS via the [voip.ms REST API](https://voip.ms/api/v1/rest.php). Message is truncated to 160 characters.
|
||||
|
||||
```yaml
|
||||
type: sms_voipms
|
||||
api_user: me@example.com # voip.ms account email
|
||||
api_password: secret # voip.ms API password
|
||||
did: "5551234567" # Your voip.ms DID (sending number)
|
||||
dst: "5559876543" # Destination number
|
||||
min_level: CRITICAL
|
||||
```
|
||||
|
||||
### signal
|
||||
|
||||
Sends via [signal-cli](https://github.com/AsamK/signal-cli).
|
||||
|
||||
```yaml
|
||||
type: signal
|
||||
cli_path: /usr/local/bin/signal-cli
|
||||
user: +12025551234 # Your registered Signal number
|
||||
recipient: +12025559999 # Recipient number
|
||||
min_level: WARNING
|
||||
```
|
||||
|
||||
**Setup:**
|
||||
```bash
|
||||
# Test signal-cli directly
|
||||
signal-cli -u +1234567890 send -m "Test message" +0987654321
|
||||
|
||||
# Test SMTP
|
||||
echo "Test" | mail -s "Test Subject" admin@example.com
|
||||
|
||||
# Test through heartbeat system (Python REPL)
|
||||
from hbd.server import notify as notify_mod, config as config_mod
|
||||
cfg = config_mod.load_config(".hb.yaml")
|
||||
notify_mod.setup(cfg)
|
||||
notify_mod.pushmsg_for_host("test-host", "Test notification")
|
||||
signal-cli -u +12025551234 register
|
||||
signal-cli -u +12025551234 verify CODE
|
||||
```
|
||||
|
||||
### mattermost
|
||||
|
||||
Sends via Mattermost incoming webhook. Message is formatted as Markdown.
|
||||
|
||||
```yaml
|
||||
type: mattermost
|
||||
host: mattermost.example.com
|
||||
token: your-webhook-token
|
||||
channel: devops-alerts
|
||||
username: heartbeat-bot # Optional: display name
|
||||
icon: https://…/icon.png # Optional: bot icon URL
|
||||
min_level: WARNING
|
||||
```
|
||||
|
||||
## Notification events
|
||||
|
||||
| Source | Level | Title example | Body example |
|
||||
|---|---|---|---|
|
||||
| Host overdue | CRITICAL | `[CRITICAL] webserver01` | `IPv4 overdue` |
|
||||
| Host recover | RECOVER | `[RECOVER] webserver01` | `IPv4 back after being overdue for 5:23` |
|
||||
| Host boot | INFO | `[INFO] webserver01` | `webserver01 booted` |
|
||||
| Host shutdown | INFO | `[INFO] webserver01` | `IPv4 shutdown` |
|
||||
| Threshold breach | WARNING/CRITICAL | `[CRITICAL] webserver01` | `cpu_percent = 95.2 (threshold: > 90.0)` |
|
||||
| Threshold reminder | CRITICAL | `[REMINDER/CRITICAL] webserver01` | `REMINDER (CRITICAL): … ongoing for 3600s` |
|
||||
| Connection issue | WARNING | `[WARNING] webserver01` | `new address detected …` |
|
||||
|
||||
Reminder notifications (re-notify) are sent only for CRITICAL level alerts.
|
||||
|
||||
## API reference
|
||||
|
||||
### `send_notification(host_name, notif) -> dict`
|
||||
|
||||
Main entry point. Dispatches to owner + managers.
|
||||
|
||||
```python
|
||||
from hbd.server.notify import send_notification, Notification
|
||||
|
||||
send_notification(
|
||||
"webserver01",
|
||||
Notification(
|
||||
title="[CRITICAL] webserver01",
|
||||
body="cpu_percent = 95.2 (threshold: > 90.0)",
|
||||
level="CRITICAL",
|
||||
url="https://hbd.example.com/plugins#webserver01",
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
Returns `{channel_name: bool}` for each channel dispatched.
|
||||
|
||||
### `setup(cfg, loop=None)`
|
||||
|
||||
Called once at startup from `main.py`. Pass the running asyncio event loop so Matrix sends work correctly.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Notifications Not Sending
|
||||
**No notifications sent:**
|
||||
- Check that users are configured (`users:` section in yaml)
|
||||
- Check that the host has an `owner` or `managers` set
|
||||
- Check that users have `notification_channels` listed
|
||||
- Check that the channel names in user config match keys under `notification_channels:`
|
||||
|
||||
1. **Check logs**: Look for "Failed to send notification" errors
|
||||
2. **Verify host is watched**: Ensure `watch: true` in host definition
|
||||
3. **Check channel configuration**: Verify credentials and settings
|
||||
4. **Test channel directly**: Use command-line tools to test provider
|
||||
5. **Check network**: Ensure server can reach notification endpoints
|
||||
**min_level filtering too aggressive:**
|
||||
- Default is `WARNING` — both WARNING and CRITICAL are sent
|
||||
- Set `min_level: WARNING` explicitly if you were expecting warnings but set CRITICAL
|
||||
|
||||
### Signal Issues
|
||||
**Matrix sends time out:**
|
||||
- Verify the access token is valid and the bot is in the room
|
||||
- `matrix-nio` must be installed: `pip install matrix-nio`
|
||||
|
||||
- **signal-cli not found**: Specify full path in `cli_path`
|
||||
- **Not registered**: Run `signal-cli -u +NUMBER register` and verify
|
||||
- **Trust issues**: Run `signal-cli -u +NUMBER receive` to sync trust store
|
||||
- **Recipient not found**: Ensure recipient is in your Signal contacts
|
||||
**voip.ms SMS fails:**
|
||||
- Enable the API in your voip.ms account (Account → API)
|
||||
- Verify the DID is SMS-capable in your voip.ms account
|
||||
|
||||
### Email Issues
|
||||
**Signal not found:**
|
||||
- Specify full `cli_path`
|
||||
- Run `signal-cli -u +NUMBER receive` to sync trust store
|
||||
|
||||
- **Authentication failed**: Check SMTP username/password
|
||||
- **TLS errors**: Verify SMTP port (587 for STARTTLS, 465 for SSL)
|
||||
- **Relay denied**: Ensure SMTP server allows relay from your IP
|
||||
- **Timeout**: Check firewall rules for SMTP ports
|
||||
**Email authentication failed:**
|
||||
- Use app-specific passwords for Gmail/Fastmail
|
||||
- Verify port: 587 for STARTTLS, 465 for SSL
|
||||
|
||||
### Pushover Issues
|
||||
|
||||
- **Invalid token/user**: Verify token and user key from Pushover dashboard
|
||||
- **API rate limits**: Pushover has monthly message limits on free tier
|
||||
- **HTTP errors**: Check Pushover API status page
|
||||
|
||||
### Mattermost Issues
|
||||
|
||||
- **Webhook not found**: Verify webhook token and ensure webhook is enabled
|
||||
- **Channel not found**: Check channel name spelling and permissions
|
||||
- **Driver import error**: Install mattermostdriver: `pip install mattermostdriver`
|
||||
|
||||
## API Reference
|
||||
|
||||
### Main Functions
|
||||
|
||||
#### `pushmsg_for_host(hostname: str, msg: str, debug: int = 0) -> dict`
|
||||
|
||||
Send notification to host-specific channels.
|
||||
|
||||
**Parameters:**
|
||||
- `hostname`: Name of the host (used to look up notification channels)
|
||||
- `msg`: Message to send
|
||||
- `debug`: Debug level (0=no debug, 1+=debug output)
|
||||
|
||||
**Returns:** Dictionary of results per channel: `{"signal_ops": True, "email_ops": False}`
|
||||
|
||||
**Example:**
|
||||
```python
|
||||
from hbd.server import notify as notify_mod
|
||||
|
||||
notify_mod.pushmsg_for_host("prod-web-01", "Server CPU at 95%")
|
||||
```
|
||||
|
||||
**Behavior:**
|
||||
1. Looks up notification channels configured for the host
|
||||
2. If no host-specific channels, uses `default_notification_channels`
|
||||
3. Dispatches to each channel in parallel
|
||||
4. Returns dict of results keyed by channel name
|
||||
5. Logs success/failure for each channel
|
||||
|
||||
## Examples
|
||||
|
||||
### Complete Configuration Example
|
||||
|
||||
```yaml
|
||||
# Notification channel definitions
|
||||
notification_channels:
|
||||
signal_oncall:
|
||||
type: signal
|
||||
cli_path: /usr/local/bin/signal-cli
|
||||
user: +12025551234
|
||||
recipient: +12025555678
|
||||
|
||||
email_ops:
|
||||
type: email
|
||||
recipients: [ops@example.com, alerts@example.com]
|
||||
sender: heartbeat@example.com
|
||||
smtp_server: smtp.fastmail.com
|
||||
smtp_port: 587
|
||||
smtp_user: heartbeat@example.com
|
||||
smtp_password: app-password-here
|
||||
|
||||
# Default channels
|
||||
default_notification_channels: [email_ops]
|
||||
|
||||
# Host definitions with channel assignments
|
||||
hosts:
|
||||
prod-web-01:
|
||||
threshold_config: high_sensitivity
|
||||
watch: true
|
||||
notification_channels: [signal_oncall, email_ops]
|
||||
dyndns: false
|
||||
|
||||
dev-server-01:
|
||||
threshold_config: low_sensitivity
|
||||
watch: false
|
||||
notification_channels: [email_ops]
|
||||
dyndns: false
|
||||
```
|
||||
|
||||
### Multiple Environments Example
|
||||
|
||||
```yaml
|
||||
notification_channels:
|
||||
# Production channels
|
||||
signal_prod_oncall:
|
||||
type: signal
|
||||
user: +12025551234
|
||||
recipient: +12025551111 # On-call phone
|
||||
|
||||
email_prod_ops:
|
||||
type: email
|
||||
recipients: [prod-ops@example.com]
|
||||
sender: prod-heartbeat@example.com
|
||||
smtp_server: smtp.example.com
|
||||
|
||||
# Staging channels
|
||||
email_staging:
|
||||
type: email
|
||||
recipients: [staging-alerts@example.com]
|
||||
sender: staging-heartbeat@example.com
|
||||
smtp_server: smtp.example.com
|
||||
|
||||
# Development channels
|
||||
mattermost_dev:
|
||||
type: mattermost
|
||||
host: chat.example.com
|
||||
token: dev-webhook-token
|
||||
channel: dev-alerts
|
||||
|
||||
hosts:
|
||||
prod-api-01:
|
||||
notification_channels: [signal_prod_oncall, email_prod_ops]
|
||||
|
||||
staging-api-01:
|
||||
notification_channels: [email_staging]
|
||||
|
||||
dev-api-01:
|
||||
notification_channels: [mattermost_dev]
|
||||
```
|
||||
**Pushover `400` errors:**
|
||||
- Double-check `token` (app) and `user` (user key) — they are different values
|
||||
|
||||
@@ -47,6 +47,34 @@ def build_parser():
|
||||
help="Username (informational only, for display)",
|
||||
)
|
||||
|
||||
# --- notify ---
|
||||
notify_p = subparsers.add_parser(
|
||||
"notify",
|
||||
help="Send a test message via a configured notification channel",
|
||||
)
|
||||
notify_p.add_argument("-c", "--config", dest="configfile", help="Config file path (YAML)")
|
||||
notify_p.add_argument(
|
||||
"channel",
|
||||
help="Channel name as defined in notification_channels",
|
||||
)
|
||||
notify_p.add_argument(
|
||||
"message",
|
||||
nargs="?",
|
||||
default="Test notification from hbd",
|
||||
help="Message body (default: 'Test notification from hbd')",
|
||||
)
|
||||
notify_p.add_argument(
|
||||
"--level",
|
||||
default="WARNING",
|
||||
choices=["INFO", "WARNING", "CRITICAL", "RECOVER"],
|
||||
help="Notification level (default: WARNING)",
|
||||
)
|
||||
notify_p.add_argument(
|
||||
"--title",
|
||||
default=None,
|
||||
help="Notification title (default: '[LEVEL] test')",
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
@@ -75,6 +103,57 @@ def cmd_passwd(args):
|
||||
print(f" password: {hashed}")
|
||||
|
||||
|
||||
def cmd_notify(args):
|
||||
"""Send a test message via a single notification channel."""
|
||||
from .config import load_config
|
||||
from .notify import Notification, _dispatch_to_channel, setup
|
||||
|
||||
config = load_config(args.configfile)
|
||||
setup(config)
|
||||
|
||||
channels = config.get("notification_channels", {})
|
||||
if args.channel not in channels:
|
||||
available = ", ".join(channels.keys()) if channels else "(none)"
|
||||
print(f"Error: channel '{args.channel}' not found in notification_channels.", file=sys.stderr)
|
||||
print(f"Available channels: {available}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
channel_cfg = channels[args.channel]
|
||||
level = args.level.upper()
|
||||
title = args.title or f"[{level}] test"
|
||||
base_url = config.get("base_url", "").rstrip("/")
|
||||
|
||||
notif = Notification(
|
||||
title=title,
|
||||
body=args.message,
|
||||
level=level,
|
||||
url=f"{base_url}/plugins" if base_url else "",
|
||||
)
|
||||
|
||||
# Bypass min_level for explicit test sends; run async channels directly
|
||||
import asyncio
|
||||
ch_type = channel_cfg.get("type", "")
|
||||
print(f"Sending via {args.channel} ({ch_type}): {title} — {args.message}")
|
||||
|
||||
if ch_type in ("matrix", "sms_voipms"):
|
||||
from .notify import _send_matrix_async, _send_sms_voipms_async
|
||||
driver_async = _send_matrix_async if ch_type == "matrix" else _send_sms_voipms_async
|
||||
ok = asyncio.run(driver_async(channel_cfg, notif))
|
||||
else:
|
||||
from .notify import _DRIVERS
|
||||
driver = _DRIVERS.get(ch_type)
|
||||
if driver is None:
|
||||
print(f"Error: unknown channel type '{ch_type}'", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
ok = driver(channel_cfg, notif)
|
||||
|
||||
if ok:
|
||||
print("OK")
|
||||
else:
|
||||
print("FAILED — check logs for details", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
@@ -83,6 +162,10 @@ def main(argv=None):
|
||||
cmd_passwd(args)
|
||||
return
|
||||
|
||||
if args.command == "notify":
|
||||
cmd_notify(args)
|
||||
return
|
||||
|
||||
# Default: run the server (supports both `hbd serve ...` and `hbd ...`)
|
||||
config = load_config(args.configfile)
|
||||
|
||||
|
||||
+14
-118
@@ -22,7 +22,7 @@ SERVER_DEFAULTS = {
|
||||
"logfile": os.path.join(os.path.expanduser("~"), ".hb.log"),
|
||||
# Notification channels
|
||||
"notification_channels": {}, # Named channels with type and credentials
|
||||
"default_notification_channels": [], # Default channels if host doesn't specify
|
||||
"base_url": "", # Base URL for notification links (e.g. https://hbd.example.com)
|
||||
|
||||
# Monitoring settings
|
||||
"interval": 20, # Expected heartbeat interval (for server checks)
|
||||
@@ -34,8 +34,7 @@ SERVER_DEFAULTS = {
|
||||
"default_owner": None, # Username that owns hosts with no explicit owner
|
||||
|
||||
# Host management
|
||||
"hosts": {}, # New unified host definitions (optional)
|
||||
"watchhosts": [], # Hosts to monitor and notify about (legacy)
|
||||
"hosts": {}, # Unified host definitions
|
||||
"dyndnshosts": [], # Hosts with dynamic DNS (legacy)
|
||||
"drophosts": [], # Hosts to ignore
|
||||
"dyndomains": ["wrede.org"],
|
||||
@@ -216,34 +215,18 @@ class ReloadableConfig:
|
||||
|
||||
|
||||
def get_watchhosts(config):
|
||||
"""Extract watchhosts from config, supporting both new and legacy formats.
|
||||
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
"""Extract watched hostnames from config (hosts with watch: true).
|
||||
|
||||
Returns:
|
||||
List of hostnames to watch
|
||||
"""
|
||||
watchhosts = []
|
||||
|
||||
# New format: hosts section with watch attribute
|
||||
if "hosts" in config:
|
||||
hosts_config = config["hosts"]
|
||||
if isinstance(hosts_config, dict):
|
||||
for host_name, host_attrs in hosts_config.items():
|
||||
if isinstance(host_attrs, dict) and host_attrs.get("watch", False):
|
||||
watchhosts.append(host_name)
|
||||
|
||||
# Legacy format: watchhosts list
|
||||
if "watchhosts" in config:
|
||||
legacy_watchhosts = config.get("watchhosts", [])
|
||||
if isinstance(legacy_watchhosts, (list, set)):
|
||||
watchhosts.extend(legacy_watchhosts)
|
||||
elif isinstance(legacy_watchhosts, dict):
|
||||
# Old dict format: {"host1": {attrs}, "host2": {attrs}}
|
||||
watchhosts.extend(legacy_watchhosts.keys())
|
||||
|
||||
return list(set(watchhosts)) # Remove duplicates
|
||||
hosts_config = config.get("hosts", {})
|
||||
if isinstance(hosts_config, dict):
|
||||
for host_name, host_attrs in hosts_config.items():
|
||||
if isinstance(host_attrs, dict) and host_attrs.get("watch", False):
|
||||
watchhosts.append(host_name)
|
||||
return watchhosts
|
||||
|
||||
|
||||
def get_dyndnshosts(config):
|
||||
@@ -275,105 +258,18 @@ def get_dyndnshosts(config):
|
||||
|
||||
|
||||
def get_host_config(config, hostname):
|
||||
"""Get configuration for a specific host.
|
||||
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
hostname: Host name
|
||||
"""Get configuration for a specific host from the hosts section.
|
||||
|
||||
Returns:
|
||||
Dictionary with host attributes or empty dict
|
||||
"""
|
||||
if "hosts" in config:
|
||||
hosts_config = config.get("hosts", {})
|
||||
if isinstance(hosts_config, dict) and hostname in hosts_config:
|
||||
return hosts_config[hostname] if isinstance(hosts_config[hostname], dict) else {}
|
||||
|
||||
# Check legacy watchhosts for notification settings
|
||||
if "watchhosts" in config:
|
||||
watchhosts = config.get("watchhosts", {})
|
||||
if isinstance(watchhosts, dict) and hostname in watchhosts:
|
||||
legacy_attrs = watchhosts[hostname]
|
||||
if isinstance(legacy_attrs, dict):
|
||||
# Convert legacy format to new format
|
||||
return {
|
||||
"watch": True,
|
||||
"notify": legacy_attrs.get("notify"),
|
||||
"notify_src": legacy_attrs.get("src"),
|
||||
}
|
||||
|
||||
hosts_config = config.get("hosts", {})
|
||||
if isinstance(hosts_config, dict) and hostname in hosts_config:
|
||||
val = hosts_config[hostname]
|
||||
return val if isinstance(val, dict) else {}
|
||||
return {}
|
||||
|
||||
|
||||
def get_notification_channels_for_host(config, hostname):
|
||||
"""Get notification channels configured for a specific host.
|
||||
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
hostname: Host name
|
||||
|
||||
Returns:
|
||||
List of channel names to use for this host
|
||||
"""
|
||||
host_config = get_host_config(config, hostname)
|
||||
|
||||
# Check if host specifies notification channels
|
||||
channels = host_config.get("notification_channels", [])
|
||||
if channels:
|
||||
if isinstance(channels, str):
|
||||
return [channels]
|
||||
elif isinstance(channels, list):
|
||||
return channels
|
||||
|
||||
# Fall back to default channels
|
||||
default_channels = config.get("default_notification_channels", [])
|
||||
if default_channels:
|
||||
if isinstance(default_channels, str):
|
||||
return [default_channels]
|
||||
elif isinstance(default_channels, list):
|
||||
return default_channels
|
||||
|
||||
# No channels configured, return empty list (will use legacy global config)
|
||||
return []
|
||||
|
||||
|
||||
def get_channel_config(config, channel_name):
|
||||
"""Get configuration for a specific notification channel.
|
||||
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
channel_name: Name of the notification channel
|
||||
|
||||
Returns:
|
||||
Dictionary with channel configuration or None if not found
|
||||
"""
|
||||
channels = config.get("notification_channels", {})
|
||||
if isinstance(channels, dict) and channel_name in channels:
|
||||
return channels[channel_name]
|
||||
return None
|
||||
|
||||
|
||||
def get_notification_channels_config(config, hostname):
|
||||
"""Get list of notification channel configurations for a host.
|
||||
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
hostname: Host name
|
||||
|
||||
Returns:
|
||||
List of (channel_name, channel_config) tuples
|
||||
"""
|
||||
channel_names = get_notification_channels_for_host(config, hostname)
|
||||
|
||||
channels = []
|
||||
for channel_name in channel_names:
|
||||
channel_config = get_channel_config(config, channel_name)
|
||||
if channel_config and channel_config.get("type"):
|
||||
channels.append((channel_name, channel_config))
|
||||
|
||||
return channels
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User / host-access helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+3
-1
@@ -244,7 +244,9 @@ async def start(
|
||||
host = config.get("hb_host", "localhost")
|
||||
extra_scripts = config.get("http_extra_scripts", "")
|
||||
host = request.host # includes port if non-standard
|
||||
scheme = "wss" if request.secure else "ws"
|
||||
forwarded_proto = request.headers.get("X-Forwarded-Proto", "")
|
||||
is_secure = request.secure or forwarded_proto.lower() == "https"
|
||||
scheme = "wss" if is_secure else "ws"
|
||||
heartbeat_ws_url = f"{scheme}://{host}/ws"
|
||||
tmpl = env.get_template("live.html")
|
||||
body = tmpl.render(
|
||||
|
||||
+1
-1
@@ -162,7 +162,7 @@ async def _run_async(config, config_path=None):
|
||||
from . import journal as journal_mod
|
||||
from . import threshold as threshold_mod
|
||||
|
||||
notify_mod.setup(config)
|
||||
notify_mod.setup(config, loop=loop)
|
||||
|
||||
# Initialize message journal
|
||||
msg_journal = journal_mod.get_journal(config)
|
||||
|
||||
+367
-222
@@ -1,37 +1,99 @@
|
||||
"""Notification helpers: email, pushover, mattermost, signal and dispatcher."""
|
||||
"""Notification helpers: email, pushover, matrix, mattermost, signal, sms and dispatcher.
|
||||
|
||||
Channel types supported:
|
||||
pushover - Pushover app notifications
|
||||
email - SMTP email
|
||||
matrix - Matrix (via matrix-nio)
|
||||
mattermost - Mattermost webhook
|
||||
signal - Signal via signal-cli subprocess
|
||||
sms_voipms - SMS via voip.ms REST API
|
||||
|
||||
Each channel can specify ``min_level: WARNING|CRITICAL`` (default: WARNING).
|
||||
|
||||
Notifications are dispatched to the owner + managers of the host, each via
|
||||
their own ``notification_channels`` list. When no users are configured the
|
||||
server runs silently (no notifications sent).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Optional
|
||||
import http.client
|
||||
import urllib.parse
|
||||
import subprocess
|
||||
import smtplib
|
||||
import subprocess
|
||||
import time
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from . import data
|
||||
from . import ws as ws_mod
|
||||
from . import main as main_mod
|
||||
|
||||
DEFAULT_PUSHPROVIDERS = ["all", "pushover", "mattermost", "signal"]
|
||||
msg_to_websockets = ws_mod.broadcast
|
||||
|
||||
# module-level configuration set via setup()
|
||||
_config = {}
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
msg_to_websockets = ws_mod.broadcast
|
||||
|
||||
# Module-level state set via setup()
|
||||
_config: dict = {}
|
||||
_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
logf = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Level ordering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_LEVEL_ORDER = {"RECOVER": 0, "INFO": 0, "WARNING": 1, "CRITICAL": 2}
|
||||
|
||||
def _level_value(level: str) -> int:
|
||||
return _LEVEL_ORDER.get(level.upper(), 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Notification dataclass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class Notification:
|
||||
"""Structured notification payload."""
|
||||
title: str # e.g. "[CRITICAL] webserver01"
|
||||
body: str # detail message
|
||||
level: str # RECOVER | WARNING | CRITICAL | INFO
|
||||
url: str = "" # link to plugin metrics page
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Module setup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def setup(cfg: dict, loop: Optional[asyncio.AbstractEventLoop] = None):
|
||||
"""Initialize notifier from configuration dict and event loop."""
|
||||
global _config, _loop
|
||||
_config = dict(cfg)
|
||||
if loop is not None:
|
||||
_loop = loop
|
||||
|
||||
|
||||
def reload_config(cfg: dict):
|
||||
"""Reload notification configuration on SIGHUP."""
|
||||
global _config
|
||||
_config = dict(cfg)
|
||||
logger.info("Notification configuration reloaded")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Event log (websocket + file + in-memory)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def initlog(logfile):
|
||||
global logf
|
||||
try:
|
||||
logf = open(logfile, "a+")
|
||||
except Exception as e:
|
||||
import sys
|
||||
|
||||
print("cannot open logfile %s, using STDERR: %s" % (logfile, e))
|
||||
logf = sys.stderr
|
||||
return logf
|
||||
|
||||
|
||||
def closelog():
|
||||
global logf
|
||||
if logf and logf != sys.stderr:
|
||||
@@ -40,6 +102,7 @@ def closelog():
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def eventlog(host, lvl, m, service=None):
|
||||
ts = time.time()
|
||||
s = f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))} {lvl} "
|
||||
@@ -56,91 +119,29 @@ def eventlog(host, lvl, m, service=None):
|
||||
logger.warning("failed to write to logfile: %s", e)
|
||||
msg_to_websockets("message", s)
|
||||
|
||||
def setup(cfg: dict):
|
||||
"""Initialize notifier defaults from a configuration dict."""
|
||||
global _config
|
||||
_config = dict(cfg)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Low-level channel drivers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def reload_config(cfg: dict):
|
||||
"""Reload notification configuration.
|
||||
|
||||
This function updates the module-level notification configuration
|
||||
during runtime config reloads.
|
||||
|
||||
Args:
|
||||
cfg: New configuration dictionary
|
||||
"""
|
||||
global _config
|
||||
_config = dict(cfg)
|
||||
logger.info("Notification configuration reloaded")
|
||||
|
||||
|
||||
def send_email(toaddrs, smtpserver, sender, subject, body, debug=0):
|
||||
"""Send a plain email via SMTP. Returns True on success."""
|
||||
try:
|
||||
smtpport = _config.get("smtpport", 587)
|
||||
server = smtplib.SMTP(smtpserver, smtpport)
|
||||
if debug > 0:
|
||||
server.set_debuglevel(1)
|
||||
if smtpport == 587:
|
||||
server.starttls()
|
||||
server.ehlo()
|
||||
smtpuser = _config.get("smtpuser", None)
|
||||
smtppassword = _config.get("smtppassword", None)
|
||||
if smtpuser and smtppassword:
|
||||
server.login(smtpuser, smtppassword)
|
||||
server.sendmail(sender, toaddrs, body)
|
||||
except Exception as e:
|
||||
logger.warning("email send failed: %s", e)
|
||||
try:
|
||||
server.quit()
|
||||
except Exception:
|
||||
pass
|
||||
def _send_pushover(channel_cfg: dict, notif: Notification) -> bool:
|
||||
import http.client
|
||||
import urllib.parse
|
||||
token = channel_cfg.get("token", "")
|
||||
user = channel_cfg.get("user", "")
|
||||
if not token or not user:
|
||||
logger.warning("pushover: missing token or user")
|
||||
return False
|
||||
try:
|
||||
server.quit()
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def email(subject: str, msg: str, debug: int = 0) -> bool:
|
||||
"""Convenience wrapper exposed to the rest of the application.
|
||||
|
||||
Uses module-level configuration to supply recipient list, smtp server
|
||||
and sender address.
|
||||
"""
|
||||
toaddrs = _config.get("toemail")
|
||||
fromemail = _config.get("fromemail")
|
||||
smtpserver = _config.get("smtpserver")
|
||||
if not toaddrs or not fromemail or not smtpserver:
|
||||
logger.warning(
|
||||
"email config incomplete: toemail=%s, fromemail=%s, smtpserver=%s",
|
||||
toaddrs,
|
||||
fromemail,
|
||||
smtpserver,
|
||||
)
|
||||
return False
|
||||
date = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.localtime())
|
||||
body = "To: %s\nFrom: %s\nSubject: %s\nDate: %s\n\n%s" % (
|
||||
toaddrs[0] if toaddrs else "",
|
||||
fromemail,
|
||||
subject,
|
||||
date,
|
||||
msg,
|
||||
)
|
||||
return send_email(toaddrs, smtpserver, fromemail, subject, body, debug=debug)
|
||||
|
||||
|
||||
def pushover(token: str, user: str, msg: str, debug: int = 0) -> bool:
|
||||
"""Send message via Pushover API."""
|
||||
params: dict = {"token": token, "user": user, "title": notif.title, "message": notif.body}
|
||||
if notif.url:
|
||||
params["url"] = notif.url
|
||||
params["url_title"] = "Plugin metrics"
|
||||
conn = http.client.HTTPSConnection("api.pushover.net:443")
|
||||
try:
|
||||
conn.request(
|
||||
"POST",
|
||||
"/1/messages.json",
|
||||
urllib.parse.urlencode({"token": token, "user": user, "message": msg}),
|
||||
urllib.parse.urlencode(params),
|
||||
{"Content-type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
r = conn.getresponse()
|
||||
@@ -151,176 +152,320 @@ def pushover(token: str, user: str, msg: str, debug: int = 0) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def pushmattermost(
|
||||
host: str,
|
||||
token: str,
|
||||
channel: str,
|
||||
msg: str,
|
||||
username: str = "hbd",
|
||||
icon: Optional[str] = None,
|
||||
debug: int = 0,
|
||||
) -> bool:
|
||||
"""Send a message to Mattermost via simple webhook driver if available.
|
||||
def _send_email(channel_cfg: dict, notif: Notification) -> bool:
|
||||
recipients = channel_cfg.get("recipients", [])
|
||||
sender = channel_cfg.get("sender", "")
|
||||
smtp_server = channel_cfg.get("smtp_server", "")
|
||||
smtp_port = channel_cfg.get("smtp_port", 587)
|
||||
smtp_user = channel_cfg.get("smtp_user")
|
||||
smtp_password = channel_cfg.get("smtp_password")
|
||||
|
||||
This helper tries to import mattermostdriver.Driver and uses webhooks if present.
|
||||
If the import fails it returns False.
|
||||
"""
|
||||
if not recipients or not sender or not smtp_server:
|
||||
logger.warning("email: missing recipients, sender, or smtp_server")
|
||||
return False
|
||||
|
||||
date = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.localtime())
|
||||
body_text = notif.body
|
||||
if notif.url:
|
||||
body_text += f"\n\n{notif.url}"
|
||||
raw = "To: %s\nFrom: %s\nSubject: %s\nDate: %s\n\n%s" % (
|
||||
recipients[0] if isinstance(recipients, list) else recipients,
|
||||
sender,
|
||||
notif.title,
|
||||
date,
|
||||
body_text,
|
||||
)
|
||||
try:
|
||||
server = smtplib.SMTP(smtp_server, smtp_port)
|
||||
if smtp_port == 587:
|
||||
server.starttls()
|
||||
server.ehlo()
|
||||
if smtp_user and smtp_password:
|
||||
server.login(smtp_user, smtp_password)
|
||||
server.sendmail(sender, recipients, raw)
|
||||
server.quit()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("email send failed: %s", e)
|
||||
try:
|
||||
server.quit()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _send_mattermost(channel_cfg: dict, notif: Notification) -> bool:
|
||||
try:
|
||||
from mattermostdriver import Driver
|
||||
except Exception:
|
||||
except ImportError:
|
||||
logger.error("mattermostdriver not installed")
|
||||
return False
|
||||
host = channel_cfg.get("host", "")
|
||||
token = channel_cfg.get("token", "")
|
||||
channel = channel_cfg.get("channel", "")
|
||||
if not host or not token or not channel:
|
||||
logger.warning("mattermost: missing host, token, or channel")
|
||||
return False
|
||||
text = f"**{notif.title}**\n{notif.body}"
|
||||
if notif.url:
|
||||
text += f"\n[Plugin metrics]({notif.url})"
|
||||
ses = {"url": host, "scheme": "http", "basepath": "/api/v4", "port": 8065}
|
||||
mm = Driver(ses)
|
||||
payload = {"text": msg, "channel": channel, "username": username}
|
||||
payload: dict = {"text": text, "channel": channel, "username": channel_cfg.get("username", "hbd")}
|
||||
icon = channel_cfg.get("icon")
|
||||
if icon:
|
||||
payload["icon_url"] = icon
|
||||
try:
|
||||
rc = mm.webhooks.call_webhook(token, payload)
|
||||
logger.debug("mattermost rc: %s", rc)
|
||||
return bool(rc is None or rc == "")
|
||||
except Exception as e:
|
||||
logger.error("mattermost error: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def pushsignal(
|
||||
signal_cli_bin: str, user: str, recipient: str, msg: str, debug: int = 0
|
||||
) -> bool:
|
||||
"""Send a message via signal-cli (requires local installation).
|
||||
|
||||
Uses subprocess to call signal-cli. Returns True if the command succeeded.
|
||||
"""
|
||||
CLI = [signal_cli_bin, "-u", user, "send", "-m", msg, recipient]
|
||||
logger.debug("signal cli: %s", CLI)
|
||||
def _send_signal(channel_cfg: dict, notif: Notification) -> bool:
|
||||
cli = channel_cfg.get("cli_path", "/usr/local/bin/signal-cli")
|
||||
user = channel_cfg.get("user", "")
|
||||
recipient = channel_cfg.get("recipient", "")
|
||||
if not user or not recipient:
|
||||
logger.warning("signal: missing user or recipient")
|
||||
return False
|
||||
msg = f"{notif.title}\n{notif.body}"
|
||||
if notif.url:
|
||||
msg += f"\n{notif.url}"
|
||||
try:
|
||||
res = subprocess.run(CLI, capture_output=True)
|
||||
res = subprocess.run([cli, "-u", user, "send", "-m", msg, recipient], capture_output=True)
|
||||
if res.returncode != 0:
|
||||
logger.error("signal failed: %s".res.stderr.decode())
|
||||
logger.error("signal failed: %s", res.stderr.decode())
|
||||
return False
|
||||
logger.debug("signal sent: %s", res.stdout.decode())
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.exception("signal exception: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def _dispatch_to_channel(channel_name: str, channel_config: dict, msg: str, debug: int = 0) -> bool:
|
||||
"""Dispatch a message to a specific notification channel.
|
||||
async def _send_sms_voipms_async(channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Send SMS via voip.ms REST API using multipart form-data POST."""
|
||||
import json
|
||||
import aiohttp
|
||||
|
||||
Args:
|
||||
channel_name: Name of the channel (for logging)
|
||||
channel_config: Channel configuration dictionary with 'type' and type-specific fields
|
||||
msg: Message to send
|
||||
debug: Debug level
|
||||
api_user = channel_cfg.get("api_user", "")
|
||||
api_password = channel_cfg.get("api_password", "")
|
||||
did = channel_cfg.get("did", "")
|
||||
dst = channel_cfg.get("dst", "")
|
||||
if not api_user or not api_password or not did or not dst:
|
||||
logger.warning("sms_voipms: missing api_user, api_password, did, or dst")
|
||||
return False
|
||||
|
||||
Returns:
|
||||
True if notification sent successfully, False otherwise
|
||||
"""
|
||||
channel_type = channel_config.get("type")
|
||||
# SMS body: title + body, truncated to 160 chars
|
||||
text = f"{notif.title}: {notif.body}"
|
||||
if len(text) > 160:
|
||||
text = text[:157] + "..."
|
||||
|
||||
if channel_type == "pushover":
|
||||
return pushover(
|
||||
channel_config.get("token", ""),
|
||||
channel_config.get("user", ""),
|
||||
msg,
|
||||
debug=debug
|
||||
)
|
||||
form_data = {
|
||||
"api_username": api_user,
|
||||
"api_password": api_password,
|
||||
"method": "sendSMS",
|
||||
"did": did,
|
||||
"dst": dst,
|
||||
"message": text,
|
||||
}
|
||||
|
||||
elif channel_type == "email":
|
||||
# Build email from channel config
|
||||
recipients = channel_config.get("recipients", [])
|
||||
sender = channel_config.get("sender", "")
|
||||
smtp_server = channel_config.get("smtp_server", "")
|
||||
smtp_port = channel_config.get("smtp_port", 587)
|
||||
smtp_user = channel_config.get("smtp_user")
|
||||
smtp_password = channel_config.get("smtp_password")
|
||||
|
||||
if not recipients or not sender or not smtp_server:
|
||||
logger.warning(
|
||||
"Email channel '%s' missing required fields: recipients=%s, sender=%s, smtp_server=%s",
|
||||
channel_name, recipients, sender, smtp_server
|
||||
)
|
||||
return False
|
||||
|
||||
# Temporarily update _config for email() function
|
||||
old_config = dict(_config)
|
||||
_config["toemail"] = recipients
|
||||
_config["fromemail"] = sender
|
||||
_config["smtpserver"] = smtp_server
|
||||
_config["smtpport"] = smtp_port
|
||||
if smtp_user:
|
||||
_config["smtpuser"] = smtp_user
|
||||
if smtp_password:
|
||||
_config["smtppassword"] = smtp_password
|
||||
|
||||
result = email("Heartbeat notification", msg, debug=debug)
|
||||
|
||||
# Restore config
|
||||
_config.clear()
|
||||
_config.update(old_config)
|
||||
|
||||
return result
|
||||
|
||||
elif channel_type == "signal":
|
||||
return pushsignal(
|
||||
channel_config.get("cli_path", "/usr/local/bin/signal-cli"),
|
||||
channel_config.get("user", ""),
|
||||
channel_config.get("recipient", ""),
|
||||
msg,
|
||||
debug=debug
|
||||
)
|
||||
|
||||
elif channel_type == "mattermost":
|
||||
return pushmattermost(
|
||||
channel_config.get("host", ""),
|
||||
channel_config.get("token", ""),
|
||||
channel_config.get("channel", ""),
|
||||
msg,
|
||||
username=channel_config.get("username", "hbd"),
|
||||
icon=channel_config.get("icon"),
|
||||
debug=debug
|
||||
)
|
||||
|
||||
else:
|
||||
logger.warning("Unknown channel type '%s' for channel '%s'", channel_type, channel_name)
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
with aiohttp.MultipartWriter("form-data") as mp:
|
||||
for key, value in form_data.items():
|
||||
part = mp.append(value)
|
||||
part.set_content_disposition("form-data", name=key)
|
||||
async with session.post("https://voip.ms/api/v1/rest.php", data=mp) as resp:
|
||||
body = await resp.text()
|
||||
if resp.status != 200:
|
||||
logger.error("sms_voipms HTTP %s: %s", resp.status, body)
|
||||
return False
|
||||
result = json.loads(body)
|
||||
if result.get("status") == "success":
|
||||
return True
|
||||
logger.error("sms_voipms error: %s", result.get("status"))
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error("sms_voipms exception: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def pushmsg_for_host(hostname: str, msg: str, debug: int = 0) -> dict:
|
||||
"""Send notification for a specific host using its configured channels.
|
||||
def _send_sms_voipms(channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Dispatch voip.ms SMS send onto the shared event loop."""
|
||||
if _loop is None:
|
||||
logger.warning("sms_voipms: event loop not available")
|
||||
return False
|
||||
future = asyncio.run_coroutine_threadsafe(_send_sms_voipms_async(channel_cfg, notif), _loop)
|
||||
try:
|
||||
return future.result(timeout=15)
|
||||
except Exception as e:
|
||||
logger.error("sms_voipms send timed out or failed: %s", e)
|
||||
return False
|
||||
|
||||
This function looks up the host's notification channels from the config
|
||||
and sends the message to those channels.
|
||||
|
||||
Args:
|
||||
hostname: Name of the host to send notification for
|
||||
msg: Message to send
|
||||
debug: Debug level
|
||||
async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Send a Matrix message using matrix-nio."""
|
||||
try:
|
||||
from nio import AsyncClient, RoomMessageText # noqa: F401
|
||||
except ImportError:
|
||||
logger.error("matrix-nio not installed; pip install matrix-nio")
|
||||
return False
|
||||
|
||||
Returns:
|
||||
Dictionary of results per channel: {"channel_name": True/False}
|
||||
from nio import AsyncClient
|
||||
homeserver = channel_cfg.get("homeserver", "")
|
||||
access_token = channel_cfg.get("access_token", "")
|
||||
room_id = channel_cfg.get("room_id", "")
|
||||
if not homeserver or not access_token or not room_id:
|
||||
logger.warning("matrix: missing homeserver, access_token, or room_id")
|
||||
return False
|
||||
|
||||
text = f"{notif.title}\n{notif.body}"
|
||||
if notif.url:
|
||||
text += f"\n{notif.url}"
|
||||
html = f"<strong>{notif.title}</strong><br>{notif.body}"
|
||||
if notif.url:
|
||||
html += f'<br><a href="{notif.url}">Plugin metrics</a>'
|
||||
|
||||
client = AsyncClient(homeserver)
|
||||
client.access_token = access_token
|
||||
try:
|
||||
from nio import RoomSendResponse
|
||||
content = {
|
||||
"msgtype": "m.text",
|
||||
"body": text,
|
||||
"format": "org.matrix.custom.html",
|
||||
"formatted_body": html,
|
||||
}
|
||||
resp = await client.room_send(room_id, "m.room.message", content)
|
||||
if hasattr(resp, "event_id"):
|
||||
return True
|
||||
logger.error("matrix send failed: %s", resp)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error("matrix exception: %s", e)
|
||||
return False
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
def _send_matrix(channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Dispatch matrix send onto the shared event loop."""
|
||||
if _loop is None:
|
||||
logger.warning("matrix: event loop not available")
|
||||
return False
|
||||
future = asyncio.run_coroutine_threadsafe(_send_matrix_async(channel_cfg, notif), _loop)
|
||||
try:
|
||||
return future.result(timeout=15)
|
||||
except Exception as e:
|
||||
logger.error("matrix send timed out or failed: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Channel dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DRIVERS = {
|
||||
"pushover": _send_pushover,
|
||||
"email": _send_email,
|
||||
"mattermost": _send_mattermost,
|
||||
"signal": _send_signal,
|
||||
"sms_voipms": _send_sms_voipms,
|
||||
"matrix": _send_matrix,
|
||||
}
|
||||
|
||||
|
||||
def _dispatch_to_channel(channel_name: str, channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Send *notif* to a single named channel, honouring min_level."""
|
||||
min_level = channel_cfg.get("min_level", "WARNING").upper()
|
||||
if _level_value(notif.level) < _level_value(min_level):
|
||||
logger.debug(
|
||||
"channel '%s': skipping level %s (min_level=%s)", channel_name, notif.level, min_level
|
||||
)
|
||||
return True # not an error — filtered intentionally
|
||||
|
||||
ch_type = channel_cfg.get("type", "")
|
||||
driver = _DRIVERS.get(ch_type)
|
||||
if driver is None:
|
||||
logger.warning("unknown channel type '%s' for channel '%s'", ch_type, channel_name)
|
||||
return False
|
||||
return driver(channel_cfg, notif)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Central dispatch function
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_url(host_name: str) -> str:
|
||||
base_url = _config.get("base_url", "").rstrip("/")
|
||||
if not base_url:
|
||||
return ""
|
||||
return f"{base_url}/plugins#{host_name}"
|
||||
|
||||
|
||||
def send_notification(host_name: str, notif: Notification) -> dict:
|
||||
"""Dispatch *notif* to all managers/owner of *host_name*.
|
||||
|
||||
Looks up the host's owner + managers, resolves each user's
|
||||
notification_channels, and dispatches. Silently does nothing if
|
||||
no users are configured.
|
||||
|
||||
Returns a dict of {channel_name: bool} results.
|
||||
"""
|
||||
from . import config as config_mod
|
||||
from . import users as users_mod
|
||||
from . import hbdclass
|
||||
|
||||
# Get notification channels for this host
|
||||
channels = config_mod.get_notification_channels_config(_config, hostname)
|
||||
|
||||
if not channels:
|
||||
logger.warning("No notification channels configured for host '%s'", hostname)
|
||||
if not users_mod.users_enabled():
|
||||
return {}
|
||||
|
||||
# Dispatch to each channel
|
||||
results = {}
|
||||
for channel_name, channel_config in channels:
|
||||
try:
|
||||
success = _dispatch_to_channel(channel_name, channel_config, msg, debug=debug)
|
||||
results[channel_name] = success
|
||||
if success:
|
||||
logger.info("Notification sent to channel '%s': %s", channel_name, msg)
|
||||
else:
|
||||
logger.warning("Failed to send notification to channel '%s'", channel_name)
|
||||
except Exception as e:
|
||||
logger.error("Error sending to channel '%s': %s", channel_name, e)
|
||||
results[channel_name] = False
|
||||
# Collect recipient usernames: owner + managers
|
||||
host = hbdclass.Host.hosts.get(host_name)
|
||||
if host is None:
|
||||
logger.debug("send_notification: host '%s' not found", host_name)
|
||||
return {}
|
||||
|
||||
recipients: set[str] = set()
|
||||
owner = getattr(host, "owner", None)
|
||||
if owner:
|
||||
recipients.add(owner)
|
||||
for m in getattr(host, "managers", []):
|
||||
recipients.add(m)
|
||||
|
||||
if not recipients:
|
||||
logger.debug("send_notification: no owner/managers for '%s'", host_name)
|
||||
return {}
|
||||
|
||||
# Fill url if not already set
|
||||
if not notif.url:
|
||||
notif.url = _build_url(host_name)
|
||||
|
||||
global_channels: dict = _config.get("notification_channels", {})
|
||||
results: dict = {}
|
||||
|
||||
for username in recipients:
|
||||
user = users_mod.get_user(username)
|
||||
if user is None:
|
||||
logger.debug("send_notification: user '%s' not found", username)
|
||||
continue
|
||||
for channel_name in user.notification_channels:
|
||||
if channel_name in results:
|
||||
continue # already dispatched to this channel this notification
|
||||
channel_cfg = global_channels.get(channel_name)
|
||||
if not channel_cfg:
|
||||
logger.warning("channel '%s' not defined in notification_channels", channel_name)
|
||||
results[channel_name] = False
|
||||
continue
|
||||
try:
|
||||
ok = _dispatch_to_channel(channel_name, channel_cfg, notif)
|
||||
results[channel_name] = ok
|
||||
if ok:
|
||||
logger.info("notification sent to channel '%s': %s", channel_name, notif.title)
|
||||
else:
|
||||
logger.warning("failed to send notification to channel '%s'", channel_name)
|
||||
except Exception as e:
|
||||
logger.error("error sending to channel '%s': %s", channel_name, e)
|
||||
results[channel_name] = False
|
||||
|
||||
return results
|
||||
|
||||
@@ -81,7 +81,7 @@
|
||||
#ntable th {
|
||||
border: 1px solid #e0e0e0;
|
||||
text-align: left;
|
||||
padding: 8px 10px;
|
||||
padding: 2px 4px;
|
||||
}
|
||||
|
||||
#ntable tr:nth-child(even) {
|
||||
@@ -92,8 +92,24 @@
|
||||
background-color: #e3f2fd;
|
||||
}
|
||||
|
||||
#ntable tbody tr.row-warning {
|
||||
background-color: #fff8c5;
|
||||
}
|
||||
|
||||
#ntable tbody tr.row-critical {
|
||||
background-color: #fde8e8;
|
||||
}
|
||||
|
||||
#ntable tbody tr.row-warning:hover {
|
||||
background-color: #fff0a0;
|
||||
}
|
||||
|
||||
#ntable tbody tr.row-critical:hover {
|
||||
background-color: #f9c8c8;
|
||||
}
|
||||
|
||||
#ntable th {
|
||||
padding: 12px 10px;
|
||||
padding: 6px 8px;
|
||||
background-color: #2196f3;
|
||||
color: white;
|
||||
font-weight: 600;
|
||||
@@ -143,7 +159,7 @@
|
||||
/* Message styling */
|
||||
#messages {
|
||||
font-size: 0.85em;
|
||||
line-height: 1.6;
|
||||
line-height: 1.0;
|
||||
}
|
||||
|
||||
#messages div {
|
||||
@@ -217,6 +233,19 @@
|
||||
}
|
||||
}
|
||||
|
||||
function updateRowAlert(row, data) {
|
||||
var criticalUnacked = data.alert_critical_unacked || 0;
|
||||
var criticalAcked = data.alert_critical_acked || 0;
|
||||
var warningUnacked = data.alert_warning_unacked || 0;
|
||||
var warningAcked = data.alert_warning_acked || 0;
|
||||
row.classList.remove('row-warning', 'row-critical');
|
||||
if (criticalUnacked > 0 || criticalAcked > 0) {
|
||||
row.classList.add('row-critical');
|
||||
} else if (warningUnacked > 0 || warningAcked > 0) {
|
||||
row.classList.add('row-warning');
|
||||
}
|
||||
}
|
||||
|
||||
function createRow(data) {
|
||||
var row = document.createElement("tr");
|
||||
var c_name = document.createElement("td");
|
||||
@@ -284,12 +313,31 @@
|
||||
var table = document.getElementById("ntablebody"); // find table to append to
|
||||
table.appendChild(row); // append row to table
|
||||
name_idx[c_name] = row;
|
||||
updateRowAlert(row, data);
|
||||
}
|
||||
|
||||
function formatTS(ts) {
|
||||
const milliseconds = ts * 1000;
|
||||
const dateObject = new Date(milliseconds);
|
||||
return dateObject.toLocaleString("de-DE");
|
||||
const now = new Date();
|
||||
const d = new Date(ts * 1000);
|
||||
|
||||
const pad = n => String(n).padStart(2, '0');
|
||||
const timeStr = `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
|
||||
|
||||
// Same calendar day → show time only
|
||||
if (d.toDateString() === now.toDateString()) {
|
||||
return timeStr;
|
||||
}
|
||||
|
||||
// Within 8 days → show "-X d hh:mm:ss"
|
||||
const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
const dStart = new Date(d.getFullYear(), d.getMonth(), d.getDate());
|
||||
const diffDays = Math.round((todayStart - dStart) / 86400000);
|
||||
if (diffDays < 8) {
|
||||
return `-${diffDays}d ${timeStr}`;
|
||||
}
|
||||
|
||||
// Older → date only
|
||||
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
|
||||
}
|
||||
|
||||
function update_table(data) {
|
||||
@@ -345,6 +393,7 @@
|
||||
name_idx[data.name].cells[4 + i * 4].innerHTML = state;
|
||||
name_idx[data.name].cells[5 + i * 4].innerHTML = latency;
|
||||
}
|
||||
updateRowAlert(name_idx[data.name], data);
|
||||
}
|
||||
|
||||
function WS_Connect() {
|
||||
@@ -427,7 +476,7 @@
|
||||
</thead>
|
||||
<tbody id="ntablebody">
|
||||
{% for host in hosts %}
|
||||
<tr>
|
||||
<tr class="{% if host.alert_critical_unacked > 0 or host.alert_critical_acked > 0 %}row-critical{% elif host.alert_warning_unacked > 0 or host.alert_warning_acked > 0 %}row-warning{% endif %}">
|
||||
<td>{{ host.name }}</td>
|
||||
<td style="text-align: center; color: #ff9800; font-weight: bold;">
|
||||
{%- set warning_unacked = host.alert_warning_unacked -%}
|
||||
|
||||
+16
-4
@@ -1003,9 +1003,15 @@ class ThresholdChecker:
|
||||
value: Any,
|
||||
):
|
||||
"""Send notification and log to journal/eventlog."""
|
||||
# Send notification using host-specific channels
|
||||
try:
|
||||
notify_mod.pushmsg_for_host(host_name, f"{lvl}: {host_name} - {message}")
|
||||
notify_mod.send_notification(
|
||||
host_name,
|
||||
notify_mod.Notification(
|
||||
title=f"[{lvl}] {host_name}",
|
||||
body=message,
|
||||
level=lvl,
|
||||
),
|
||||
)
|
||||
logger.info("Notification sent: %s", message)
|
||||
except Exception as e:
|
||||
logger.error("Failed to send notification: %s", e)
|
||||
@@ -1137,9 +1143,15 @@ class ThresholdChecker:
|
||||
else:
|
||||
message = f"REMINDER ({alert_state.level.name}): {host_name} - {metric_path} = {value} (ongoing for {int(now - alert_state.since)}s)"
|
||||
|
||||
# Send re-notification using host-specific channels
|
||||
try:
|
||||
notify_mod.pushmsg_for_host(host_name, message)
|
||||
notify_mod.send_notification(
|
||||
host_name,
|
||||
notify_mod.Notification(
|
||||
title=f"[REMINDER/{alert_state.level.name}] {host_name}",
|
||||
body=message,
|
||||
level=alert_state.level.name,
|
||||
),
|
||||
)
|
||||
alert_state.last_notification = now
|
||||
alert_state.notification_count += 1
|
||||
logger.info("Re-notification sent: %s", message)
|
||||
|
||||
+26
-23
@@ -171,7 +171,7 @@ def dicttos(ID, d):
|
||||
DROPOVERDUE = 7 * 24 * 3600 # seconds before an overdue host becomes UNKNOWN
|
||||
|
||||
|
||||
def _make_timer_callbacks(uname, host, watchhosts, ctx):
|
||||
def _make_timer_callbacks(uname, host, ctx):
|
||||
"""Return (on_overdue, on_unknown) async callbacks for connection timer logic.
|
||||
|
||||
Captured values are bound at call time so callbacks are safe to use in loops.
|
||||
@@ -191,9 +191,11 @@ def _make_timer_callbacks(uname, host, watchhosts, ctx):
|
||||
now = time.time()
|
||||
connection.newstate(connection.__class__.OVERDUE, now, cfg.get("grace", 2))
|
||||
msg = f"{connection.afam} overdue"
|
||||
eventlog(uname, "CRITICAL" if uname in watchhosts else "WARNING", msg)
|
||||
if uname in watchhosts:
|
||||
notify_mod.pushmsg_for_host(uname, f"{uname} {msg}")
|
||||
eventlog(uname, "CRITICAL", msg)
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[CRITICAL] {uname}", body=msg, level="CRITICAL"),
|
||||
)
|
||||
if threshold_checker:
|
||||
threshold_checker.check_value(
|
||||
host_name=uname,
|
||||
@@ -218,8 +220,6 @@ def restore_connection_timers(hbdclass, ctx):
|
||||
now = time.time()
|
||||
cfg = ctx.get("config", {})
|
||||
grace = cfg.get("grace", 2)
|
||||
from . import config as config_mod
|
||||
watchhosts = config_mod.get_watchhosts(cfg)
|
||||
|
||||
restored = 0
|
||||
for uname, host in list(hbdclass.Host.hosts.items()):
|
||||
@@ -229,7 +229,7 @@ def restore_connection_timers(hbdclass, ctx):
|
||||
if state == hbdclass.Connection.DOWN:
|
||||
continue
|
||||
|
||||
on_overdue, on_unknown = _make_timer_callbacks(uname, host, watchhosts, ctx)
|
||||
on_overdue, on_unknown = _make_timer_callbacks(uname, host, ctx)
|
||||
|
||||
if state == hbdclass.Connection.UP and interval > 0:
|
||||
elapsed = now - conn.lastbeat
|
||||
@@ -322,9 +322,6 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
host = hbdcls.Host.hosts[uname]
|
||||
newh = False
|
||||
|
||||
# Get watchhosts once for use throughout message handling
|
||||
watchhosts = config_mod.get_watchhosts(cfg)
|
||||
|
||||
cid = msg.get("id", 0)
|
||||
try:
|
||||
rtt = float(msg.get("rtt"))
|
||||
@@ -390,8 +387,10 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
|
||||
if res:
|
||||
eventlog(uname, "WARNING", res)
|
||||
if uname in watchhosts:
|
||||
notify_mod.pushmsg_for_host(uname, "%s %s" % (host.name, res))
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[WARNING] {uname}", body=res, level="WARNING"),
|
||||
)
|
||||
|
||||
interval = int(msg.get("interval", 0) or 0)
|
||||
shutdown = msg.get("shutdown", 0)
|
||||
@@ -401,13 +400,12 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
|
||||
if boot:
|
||||
eventlog(uname, "INFO", "booted")
|
||||
if uname in watchhosts:
|
||||
m = "%s booted" % (host.name)
|
||||
notify_mod.pushmsg_for_host(uname, m)
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[INFO] {uname}", body=f"{host.name} booted", level="INFO"),
|
||||
)
|
||||
if message:
|
||||
eventlog(uname, "INFO", "msg: %s" % message, service=service)
|
||||
if uname in watchhosts:
|
||||
notify_mod.pushmsg_for_host(uname, message)
|
||||
|
||||
if conn.getstate() != hbdcls.Connection.UP:
|
||||
lasts = conn.state
|
||||
@@ -420,8 +418,10 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
else:
|
||||
m = "%s back after being %s for %s" % (conn.afam, lasts, dur(d))
|
||||
eventlog(uname, "RECOVER", m)
|
||||
if uname in watchhosts:
|
||||
notify_mod.pushmsg_for_host(uname, "%s %s is back" % (uname, conn.afam))
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[RECOVER] {uname}", body=m, level="RECOVER"),
|
||||
)
|
||||
|
||||
if boot or newh:
|
||||
host.upcount = host.doesack
|
||||
@@ -429,9 +429,12 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
host.upcount += 1
|
||||
|
||||
if shutdown:
|
||||
eventlog(uname, "INFO", "%s shutdown" % conn.afam)
|
||||
if uname in watchhosts:
|
||||
notify_mod.pushmsg_for_host(uname, "%s %s shutdown" % (uname, conn.afam))
|
||||
m = "%s shutdown" % conn.afam
|
||||
eventlog(uname, "INFO", m)
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[INFO] {uname}", body=m, level="INFO"),
|
||||
)
|
||||
conn.newstate(hbdcls.Connection.DOWN, now)
|
||||
|
||||
if interval > 0:
|
||||
@@ -442,7 +445,7 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
if interval > 0 and conn.getstate() != hbdcls.Connection.DOWN:
|
||||
grace = cfg.get("grace", 2)
|
||||
timeout_seconds = interval + grace
|
||||
on_overdue, _ = _make_timer_callbacks(uname, host, watchhosts, ctx)
|
||||
on_overdue, _ = _make_timer_callbacks(uname, host, ctx)
|
||||
conn.reset_overdue_timer(timeout_seconds, on_overdue)
|
||||
|
||||
# Check RTT thresholds using the threshold checker
|
||||
|
||||
@@ -31,6 +31,7 @@ server = [
|
||||
"mattermostdriver>=7.3.0",
|
||||
"aiohttp>=3.11",
|
||||
"Jinja2>=3.1.6",
|
||||
"matrix-nio>=0.24",
|
||||
]
|
||||
|
||||
# Install both client and server
|
||||
|
||||
Reference in New Issue
Block a user