re-factor notifications, add sms and matrix as channels

async_sms_send.py

@@ -0,0 +1,40 @@
+async def send_sms(hass, user, password, sender_did, call):
+    """Send SMS message using multipart form-data like MMS."""
+    _LOGGER = logging.getLogger(__name__)
+    recipient = call.data.get("recipient")
+    message = call.data.get("message")
+
+    if not recipient or not message:
+        _LOGGER.error("Recipient or message missing.")
+        return
+
+    # Build form data dictionary
+    form_data = {
+        'api_username': str(user),
+        'api_password': str(password),
+        'did': str(sender_did),
+        'dst': str(recipient),
+        'message': str(message),
+        'method': 'sendSMS'
+    }
+
+    async with aiohttp.ClientSession() as session:
+        with aiohttp.MultipartWriter("form-data") as mp:
+            for key, value in form_data.items():
+                part = mp.append(value)
+                part.set_content_disposition('form-data', name=key)
+
+            _LOGGER.error("voipms_sms: sending SMS: %s", mp)
+            async with session.post(REST_ENDPOINT, data=mp) as response:
+                response_text = await response.text()
+                if response.status == 200:
+                    response_json = json.loads(response_text)
+                    if response_json['status'] == "success": 
+                        _LOGGER.info("voipms_sms: SMS sent successfully: %s", response_text)
+                    else:
+                        _LOGGER.error("voipms_sms: SMS not sent: %s", response_text)
+                else:
+                    _LOGGER.error("voipms_sms: Failed to send SMS. Status: %s, Response: %s", response.status, response_text)
+
+
+

docs/NOTIFICATIONS.md

@@ -2,532 +2,294 @@
 
 ## Overview
 
-The Heartbeat Monitoring System includes a flexible notification system that can send alerts through multiple channels including Email, Pushover, Signal, and Mattermost. The system supports centralized channel definitions with per-host routing, allowing fine-grained control over notification delivery.
+Notifications are dispatched to the **owner and managers** of a host, each via their own configured notification channels. Channel definitions are global; users reference them by name. No users configured → no notifications sent.
 
 ## Architecture
 
-### Components
+```
+Alert event (udp.py / threshold.py)
+  └─ notify.send_notification(host_name, Notification)
+       ├─ look up host.owner + host.managers
+       ├─ for each user → user.notification_channels
+       └─ for each channel → _dispatch_to_channel (filtered by min_level)
+```
 
-1. **Notification Channels** (`notification_channels` in config)
+Every notification carries:
-   - Centralized definitions of notification providers
+- **title** — `[LEVEL] hostname` (e.g. `[CRITICAL] webserver01`)
-   - Each channel has a type and type-specific credentials
+- **body** — detail message (metric value, threshold, duration)
-   - Reusable across multiple hosts
+- **url** — link to the plugin metrics page (`{base_url}/plugins#{hostname}`)
-
+- **level** — `RECOVER | WARNING | CRITICAL | INFO`
-2. **Channel Dispatcher** (`hbd/server/notify.py`)
-   - `pushmsg_for_host(hostname, message)`: Main entry point for host-specific notifications
-   - `_dispatch_to_channel(channel_name, channel_config, message)`: Routes to specific provider
-   - Provider functions: `pushover()`, `pushsignal()`, `pushmattermost()`, `send_email()`
-
-3. **Configuration Utilities** (`hbd/server/config.py`)
-   - `get_notification_channels_for_host(config, hostname)`: Retrieves channel names for a host
-   - `get_notification_channels_config(config, hostname)`: Retrieves full channel configurations
-   - `get_channel_config(config, channel_name)`: Gets configuration for a specific channel
-
-4. **Integration Points**
-   - **Threshold alerts**: `threshold.py` calls `notify_mod.pushmsg_for_host()`
-   - **Heartbeat events**: `udp.py` calls `notify_mod.pushmsg_for_host()` for boot/shutdown/overdue
-   - **Custom alerts**: Any code can call `notify_mod.pushmsg_for_host(hostname, message)`
 
 ## Configuration
 
-### Centralized Channel Definitions
+### Base URL
 
-Define notification channels once in your configuration file:
+Set `base_url` so notification links point to your hbd instance:
+
+```yaml
+base_url: https://hbd.example.com
+```
+
+### Global channel definitions
+
+Define channels once; reference them by name from user configs:
 
 ```yaml
 notification_channels:
-  # Signal notifications
-  signal_ops:
-    type: signal
-    cli_path: /usr/local/bin/signal-cli
-    user: +1234567890        # Your Signal number
-    recipient: +1234567890   # Recipient number
 
-  signal_oncall:
+  pushover_ops:
-    type: signal
+    type: pushover
-    cli_path: /usr/local/bin/signal-cli
+    token: your-app-token
-    user: +1234567890
+    user: your-user-key
-    recipient: +0987654321   # Different recipient
+    min_level: WARNING        # optional, default: WARNING
 
-  # Email notifications
   email_ops:
     type: email
-    recipients:
+    recipients: [ops@example.com]
-      - ops@example.com
+    sender: hbd@example.com
-      - alerts@example.com
-    sender: heartbeat@example.com
     smtp_server: smtp.example.com
     smtp_port: 587
-    smtp_user: heartbeat@example.com
+    smtp_user: hbd@example.com
-    smtp_password: your-smtp-password
+    smtp_password: secret
+    min_level: WARNING
 
-  email_devteam:
+  matrix_oncall:
-    type: email
+    type: matrix
-    recipients: [dev-alerts@example.com]
+    homeserver: https://matrix.example.org
-    sender: heartbeat-dev@example.com
+    access_token: syt_xxx
-    smtp_server: smtp.example.com
+    room_id: "!abc:matrix.example.org"
-    smtp_port: 587
+    min_level: CRITICAL       # only send critical alerts to this room
-    smtp_user: heartbeat-dev@example.com
-    smtp_password: your-smtp-password
 
-  # Pushover notifications
+  sms_oncall:
-  pushover_urgent:
+    type: sms_voipms
-    type: pushover
+    api_user: me@example.com
-    token: your-pushover-app-token
+    api_password: secret
-    user: your-pushover-user-key
+    did: "5551234567"         # your voip.ms DID number
+    dst: "5559876543"         # destination number
+    min_level: CRITICAL
 
-  pushover_normal:
+  signal_ops:
-    type: pushover
-    token: your-pushover-app-token
-    user: another-user-key
-  
-  # Mattermost notifications
-  mattermost_devops:
-    type: mattermost
-    host: mattermost.example.com
-    token: your-webhook-token
-    channel: devops-alerts
-    username: heartbeat-bot
-    icon: https://example.com/heartbeat-icon.png
-```
-
-### Default Notification Channels
-
-Specify default channels for hosts that don't have specific channel assignments:
-
-```yaml
-default_notification_channels:
-  - email_ops
-  - mattermost_devops
-```
-
-Hosts without `notification_channels` defined will use these defaults.
-
-### Per-Host Channel Assignment
-
-Assign specific channels to each host in the `hosts` section:
-
-```yaml
-hosts:
-  # Critical production web server - multiple channels for redundancy
-  prod-web-01:
-    threshold_config: high_sensitivity
-    watch: true
-    notification_channels:
-      - signal_oncall        # Immediate mobile notification
-      - pushover_urgent      # Secondary mobile notification
-      - email_ops            # Email for record keeping
-    dyndns: false
-  
-  # Database server - ops team notifications only
-  prod-db-01:
-    threshold_config: database
-    watch: true
-    notification_channels:
-      - signal_ops
-      - email_ops
-    dyndns: false
-  
-  # Development server - email only, no urgent notifications
-  dev-server-01:
-    threshold_config: low_sensitivity
-    watch: false
-    notification_channels:
-      - email_devteam
-    dyndns: false
-  
-  # Test server - uses default_notification_channels
-  test-server-01:
-    threshold_config: default
-    watch: false
-    dyndns: false
-    # No notification_channels specified = uses default_notification_channels
-```
-
-## Channel Types
-
-### Email
-
-Sends notifications via SMTP.
-
-**Configuration fields:**
-```yaml
-type: email
-recipients: [email1@example.com, email2@example.com]  # Required: List of recipients
-sender: heartbeat@example.com                         # Required: From address
-smtp_server: smtp.example.com                         # Required: SMTP server hostname
-smtp_port: 587                                        # Optional: Default 587
-smtp_user: heartbeat@example.com                      # Optional: For authenticated SMTP
-smtp_password: your-password                          # Optional: For authenticated SMTP
-```
-
-**Features:**
-- Supports multiple recipients
-- TLS/STARTTLS support on port 587
-- Authenticated and unauthenticated SMTP
-
-**Example:**
-```yaml
-notification_channels:
-  email_critical:
-    type: email
-    recipients: [admin@example.com, oncall@example.com]
-    sender: alerts@example.com
-    smtp_server: smtp.fastmail.com
-    smtp_port: 587
-    smtp_user: alerts@example.com
-    smtp_password: app-specific-password
-```
-
-### Pushover
-
-Sends push notifications to mobile devices via Pushover API.
-
-**Configuration fields:**
-```yaml
-type: pushover
-token: your-application-token    # Required: Your Pushover app token
-user: your-user-key              # Required: Recipient's user key
-```
-
-**Features:**
-- Instant mobile push notifications
-- Works on iOS and Android
-- Supports delivery confirmations
-
-**Setup:**
-1. Create a Pushover account at https://pushover.net
-2. Create an application to get your app token
-3. Note your user key from your account dashboard
-
-**Example:**
-```yaml
-notification_channels:
-  pushover_admin:
-    type: pushover
-    token: azGDORePK8gMaC0QOYAMyEEuzJnyUi
-    user: uQiRzpo4DXghDmr9QzzfQu27cmVRsG
-```
-
-### Signal
-
-Sends notifications via Signal messenger using signal-cli.
-
-**Configuration fields:**
-```yaml
-type: signal
-cli_path: /usr/local/bin/signal-cli    # Optional: Path to signal-cli binary
-user: +1234567890                       # Required: Your Signal phone number
-recipient: +0987654321                  # Required: Recipient phone number
-```
-
-**Prerequisites:**
-1. Install signal-cli: https://github.com/AsamK/signal-cli
-2. Register signal-cli with your phone number:
-   ```bash
-   signal-cli -u +1234567890 register
-   signal-cli -u +1234567890 verify CODE
-   ```
-3. Ensure signal-cli is in PATH or specify full path in config
-
-**Features:**
-- End-to-end encrypted messaging
-- Works without phone being online
-- No API fees or rate limits
-
-**Example:**
-```yaml
-notification_channels:
-  signal_admin:
     type: signal
     cli_path: /usr/local/bin/signal-cli
     user: +12025551234
     recipient: +12025559999
-```
 
-### Mattermost
+  mattermost_devops:
-
-Sends notifications to Mattermost team chat via incoming webhooks.
-
-**Configuration fields:**
-```yaml
     type: mattermost
-host: mattermost.example.com           # Required: Mattermost server hostname
+    host: mattermost.example.com
-token: your-webhook-token               # Required: Incoming webhook token
+    token: webhook-token
-channel: channel-name                   # Required: Target channel name
+    channel: devops-alerts
-username: heartbeat-bot                 # Optional: Bot display name
+    username: heartbeat-bot
-icon: https://example.com/icon.png      # Optional: Bot icon URL
 ```
 
-**Prerequisites:**
+### Users with notification channels
-1. Enable incoming webhooks in Mattermost
-2. Create an incoming webhook for your team
-3. Note the webhook token from the webhook URL
 
-**Features:**
+Each user lists which global channels they receive notifications on:
-- Team-wide visibility
-- Rich formatting support
-- Message threading
 
-**Example:**
 ```yaml
-notification_channels:
+users:
-  mattermost_ops:
+  alice:
-    type: mattermost
+    full_name: Alice Smith
-    host: chat.example.com
+    password: pbkdf2:sha256:...
-    token: abc123def456ghi789
+    admin: true
-    channel: infrastructure-alerts
+    notification_channels: [pushover_ops, email_ops]
-    username: heartbeat-monitor
+
-    icon: https://example.com/heartbeat-icon.png
+  bob:
+    full_name: Bob Jones
+    password: pbkdf2:sha256:...
+    notification_channels: [sms_oncall, matrix_oncall]
 ```
 
-## Notification Events
+### Host access — owner and managers
 
-The system sends notifications for various events:
+Notifications for a host go to its owner and all managers:
-
-### Threshold Alerts
-
-When monitored metrics exceed configured thresholds:
-
-- **State changes**: OK → WARNING, WARNING → CRITICAL, CRITICAL → OK
-- **Format**: `{LEVEL}: {hostname} - {metric_path} = {value} {threshold_info}`
-- **Example**: `CRITICAL: prod-web-01 - cpu_monitor.cpu_percent = 95.2 (threshold: > 90.0)`
-- **Re-notifications**: Periodic reminders for ongoing alerts (default: hourly)
-
-### Heartbeat Events
-
-Host lifecycle events:
-
-- **Host boot**: `{hostname} booted`
-- **Host shutdown**: `{hostname} {connection_type} shutdown`
-- **Host recovery**: `{hostname} {connection_type} is back`
-- **Connection issues**: `{hostname} {message}`
-- **Host overdue**: `{hostname} {connection_type} overdue`
-
-Only hosts with `watch: true` send heartbeat event notifications.
-
-### Custom Alerts
-
-Application code can send custom notifications:
-
-```python
-from hbd.server import notify as notify_mod
-
-# Send to host-specific channels
-notify_mod.pushmsg_for_host("prod-web-01", "Custom alert message")
-
-# Send using global config
-notify_mod.pushmsg_from_config("Global notification")
-
-# Send to specific config
-notify_mod.pushmsg(custom_config_dict, "Targeted notification")
-```
-
-## Design Principles
-
-The notification system follows these core principles:
-
-- **Centralization**: Define notification providers once, reference them by name
-- **Flexibility**: Each host can use different channels for different notification needs
-- **Redundancy**: Critical hosts can specify multiple channels for failover
-- **Clarity**: Clean separation between channel definition and channel assignment
-- **Type Safety**: Provider-specific validation at configuration time
-
-## Best Practices
-
-### Channel Organization
-
-- **Create purpose-specific channels**: `email_ops`, `signal_oncall`, `pushover_urgent`
-- **Separate by team/role**: `email_devteam`, `signal_dbateam`, `mattermost_security`
-- **Use descriptive names**: Channel names appear in logs and debugging
-
-### Redundancy
-
-For critical hosts, use multiple notification channels:
 
 ```yaml
 hosts:
-  critical-db:
+  webserver01:
-    notification_channels:
+    owner: alice             # receives all notifications for this host
-      - signal_oncall      # Primary: Mobile alert
+    managers: [bob]          # also receives notifications
-      - pushover_urgent    # Backup: Different mobile platform
+    threshold_config: default
-      - email_ops          # Tertiary: Email for record-keeping
+    watch: true              # bold in dashboard (cosmetic only)
+    dyndns: false
+
+  dbserver01:
+    owner: alice
+    managers: [bob]
+    threshold_config: database
+    dyndns: false
 ```
 
-### Notification Fatigue Prevention
+`watch: true` only affects display (bold name in the live dashboard). Notifications are now controlled entirely by owner/managers.
 
-- **Use `watch: false`** for non-critical hosts
+## Channel Types
-- **Configure appropriate thresholds** to avoid false positives
-- **Set different channels for different severities**
-- **Use `default_notification_channels`** for baseline, add more for critical systems
 
-### Security
+### `min_level` filtering
 
-- **Protect credentials**: Use file permissions to protect config files with passwords/tokens
+Every channel accepts an optional `min_level` field:
-- **Rotate tokens**: Periodically rotate API tokens and passwords
-- **Use app-specific passwords**: For email, use app-specific passwords instead of main account password
-- **Separate accounts**: Consider separate notification accounts for different environments (prod vs dev)
 
-### Testing
+| Value | Channels receive |
+|---|---|
+| `WARNING` (default) | WARNING, CRITICAL, RECOVER |
+| `CRITICAL` | CRITICAL only (and RECOVER) |
 
-Test notification channels before relying on them:
+`RECOVER` is always passed through — you don't want to miss a recovery.
 
+### pushover
+
+Sends push notifications via [Pushover](https://pushover.net). Includes title, body, and a clickable URL.
+
+```yaml
+type: pushover
+token: your-app-token     # Required: Pushover application token
+user: your-user-key       # Required: Recipient's user key
+min_level: WARNING
+```
+
+### email
+
+Sends via SMTP. Subject = title, body = message + URL on final line.
+
+```yaml
+type: email
+recipients: [ops@example.com, oncall@example.com]
+sender: hbd@example.com
+smtp_server: smtp.example.com
+smtp_port: 587             # 587 = STARTTLS (default), 465 = SSL
+smtp_user: hbd@example.com
+smtp_password: secret
+min_level: WARNING
+```
+
+### matrix
+
+Sends a formatted HTML message to a Matrix room via [matrix-nio](https://github.com/poljar/matrix-nio).
+
+```yaml
+type: matrix
+homeserver: https://matrix.example.org
+access_token: syt_xxx      # Bot account access token
+room_id: "!abc:matrix.example.org"
+min_level: WARNING
+```
+
+**Setup:**
+1. Create a bot Matrix account
+2. Obtain its access token (Element → Settings → Help & About → Access Token)
+3. Invite the bot to the target room and note the room ID
+
+### sms_voipms
+
+Sends SMS via the [voip.ms REST API](https://voip.ms/api/v1/rest.php). Message is truncated to 160 characters.
+
+```yaml
+type: sms_voipms
+api_user: me@example.com   # voip.ms account email
+api_password: secret       # voip.ms API password
+did: "5551234567"          # Your voip.ms DID (sending number)
+dst: "5559876543"          # Destination number
+min_level: CRITICAL
+```
+
+### signal
+
+Sends via [signal-cli](https://github.com/AsamK/signal-cli).
+
+```yaml
+type: signal
+cli_path: /usr/local/bin/signal-cli
+user: +12025551234         # Your registered Signal number
+recipient: +12025559999    # Recipient number
+min_level: WARNING
+```
+
+**Setup:**
 ```bash
-# Test signal-cli directly
+signal-cli -u +12025551234 register
-signal-cli -u +1234567890 send -m "Test message" +0987654321
+signal-cli -u +12025551234 verify CODE
-
-# Test SMTP
-echo "Test" | mail -s "Test Subject" admin@example.com
-
-# Test through heartbeat system (Python REPL)
-from hbd.server import notify as notify_mod, config as config_mod
-cfg = config_mod.load_config(".hb.yaml")
-notify_mod.setup(cfg)
-notify_mod.pushmsg_for_host("test-host", "Test notification")
 ```
 
+### mattermost
+
+Sends via Mattermost incoming webhook. Message is formatted as Markdown.
+
+```yaml
+type: mattermost
+host: mattermost.example.com
+token: your-webhook-token
+channel: devops-alerts
+username: heartbeat-bot    # Optional: display name
+icon: https://…/icon.png   # Optional: bot icon URL
+min_level: WARNING
+```
+
+## Notification events
+
+| Source | Level | Title example | Body example |
+|---|---|---|---|
+| Host overdue | CRITICAL | `[CRITICAL] webserver01` | `IPv4 overdue` |
+| Host recover | RECOVER | `[RECOVER] webserver01` | `IPv4 back after being overdue for 5:23` |
+| Host boot | INFO | `[INFO] webserver01` | `webserver01 booted` |
+| Host shutdown | INFO | `[INFO] webserver01` | `IPv4 shutdown` |
+| Threshold breach | WARNING/CRITICAL | `[CRITICAL] webserver01` | `cpu_percent = 95.2 (threshold: > 90.0)` |
+| Threshold reminder | CRITICAL | `[REMINDER/CRITICAL] webserver01` | `REMINDER (CRITICAL): … ongoing for 3600s` |
+| Connection issue | WARNING | `[WARNING] webserver01` | `new address detected …` |
+
+Reminder notifications (re-notify) are sent only for CRITICAL level alerts.
+
+## API reference
+
+### `send_notification(host_name, notif) -> dict`
+
+Main entry point. Dispatches to owner + managers.
+
+```python
+from hbd.server.notify import send_notification, Notification
+
+send_notification(
+    "webserver01",
+    Notification(
+        title="[CRITICAL] webserver01",
+        body="cpu_percent = 95.2 (threshold: > 90.0)",
+        level="CRITICAL",
+        url="https://hbd.example.com/plugins#webserver01",
+    ),
+)
+```
+
+Returns `{channel_name: bool}` for each channel dispatched.
+
+### `setup(cfg, loop=None)`
+
+Called once at startup from `main.py`. Pass the running asyncio event loop so Matrix sends work correctly.
+
 ## Troubleshooting
 
-### Notifications Not Sending
+**No notifications sent:**
+- Check that users are configured (`users:` section in yaml)
+- Check that the host has an `owner` or `managers` set
+- Check that users have `notification_channels` listed
+- Check that the channel names in user config match keys under `notification_channels:`
 
-1. **Check logs**: Look for "Failed to send notification" errors
+**min_level filtering too aggressive:**
-2. **Verify host is watched**: Ensure `watch: true` in host definition
+- Default is `WARNING` — both WARNING and CRITICAL are sent
-3. **Check channel configuration**: Verify credentials and settings
+- Set `min_level: WARNING` explicitly if you were expecting warnings but set CRITICAL
-4. **Test channel directly**: Use command-line tools to test provider
-5. **Check network**: Ensure server can reach notification endpoints
 
-### Signal Issues
+**Matrix sends time out:**
+- Verify the access token is valid and the bot is in the room
+- `matrix-nio` must be installed: `pip install matrix-nio`
 
-- **signal-cli not found**: Specify full path in `cli_path`
+**voip.ms SMS fails:**
-- **Not registered**: Run `signal-cli -u +NUMBER register` and verify
+- Enable the API in your voip.ms account (Account → API)
-- **Trust issues**: Run `signal-cli -u +NUMBER receive` to sync trust store
+- Verify the DID is SMS-capable in your voip.ms account
-- **Recipient not found**: Ensure recipient is in your Signal contacts
 
-### Email Issues
+**Signal not found:**
+- Specify full `cli_path`
+- Run `signal-cli -u +NUMBER receive` to sync trust store
 
-- **Authentication failed**: Check SMTP username/password
+**Email authentication failed:**
-- **TLS errors**: Verify SMTP port (587 for STARTTLS, 465 for SSL)
+- Use app-specific passwords for Gmail/Fastmail
-- **Relay denied**: Ensure SMTP server allows relay from your IP
+- Verify port: 587 for STARTTLS, 465 for SSL
-- **Timeout**: Check firewall rules for SMTP ports
 
-### Pushover Issues
+**Pushover `400` errors:**
-
+- Double-check `token` (app) and `user` (user key) — they are different values
-- **Invalid token/user**: Verify token and user key from Pushover dashboard
-- **API rate limits**: Pushover has monthly message limits on free tier
-- **HTTP errors**: Check Pushover API status page
-
-### Mattermost Issues
-
-- **Webhook not found**: Verify webhook token and ensure webhook is enabled
-- **Channel not found**: Check channel name spelling and permissions
-- **Driver import error**: Install mattermostdriver: `pip install mattermostdriver`
-
-## API Reference
-
-### Main Functions
-
-#### `pushmsg_for_host(hostname: str, msg: str, debug: int = 0) -> dict`
-
-Send notification to host-specific channels.
-
-**Parameters:**
-- `hostname`: Name of the host (used to look up notification channels)
-- `msg`: Message to send
-- `debug`: Debug level (0=no debug, 1+=debug output)
-
-**Returns:** Dictionary of results per channel: `{"signal_ops": True, "email_ops": False}`
-
-**Example:**
-```python
-from hbd.server import notify as notify_mod
-
-notify_mod.pushmsg_for_host("prod-web-01", "Server CPU at 95%")
-```
-
-**Behavior:**
-1. Looks up notification channels configured for the host
-2. If no host-specific channels, uses `default_notification_channels`
-3. Dispatches to each channel in parallel
-4. Returns dict of results keyed by channel name
-5. Logs success/failure for each channel
-
-## Examples
-
-### Complete Configuration Example
-
-```yaml
-# Notification channel definitions
-notification_channels:
-  signal_oncall:
-    type: signal
-    cli_path: /usr/local/bin/signal-cli
-    user: +12025551234
-    recipient: +12025555678
-  
-  email_ops:
-    type: email
-    recipients: [ops@example.com, alerts@example.com]
-    sender: heartbeat@example.com
-    smtp_server: smtp.fastmail.com
-    smtp_port: 587
-    smtp_user: heartbeat@example.com
-    smtp_password: app-password-here
-
-# Default channels
-default_notification_channels: [email_ops]
-
-# Host definitions with channel assignments
-hosts:
-  prod-web-01:
-    threshold_config: high_sensitivity
-    watch: true
-    notification_channels: [signal_oncall, email_ops]
-    dyndns: false
-  
-  dev-server-01:
-    threshold_config: low_sensitivity
-    watch: false
-    notification_channels: [email_ops]
-    dyndns: false
-```
-
-### Multiple Environments Example
-
-```yaml
-notification_channels:
-  # Production channels
-  signal_prod_oncall:
-    type: signal
-    user: +12025551234
-    recipient: +12025551111  # On-call phone
-  
-  email_prod_ops:
-    type: email
-    recipients: [prod-ops@example.com]
-    sender: prod-heartbeat@example.com
-    smtp_server: smtp.example.com
-  
-  # Staging channels
-  email_staging:
-    type: email
-    recipients: [staging-alerts@example.com]
-    sender: staging-heartbeat@example.com
-    smtp_server: smtp.example.com
-  
-  # Development channels
-  mattermost_dev:
-    type: mattermost
-    host: chat.example.com
-    token: dev-webhook-token
-    channel: dev-alerts
-
-hosts:
-  prod-api-01:
-    notification_channels: [signal_prod_oncall, email_prod_ops]
-  
-  staging-api-01:
-    notification_channels: [email_staging]
-  
-  dev-api-01:
-    notification_channels: [mattermost_dev]
-```

hbd/server/cli.py

@@ -47,6 +47,34 @@ def build_parser():
         help="Username (informational only, for display)",
     )
 
+    # --- notify ---
+    notify_p = subparsers.add_parser(
+        "notify",
+        help="Send a test message via a configured notification channel",
+    )
+    notify_p.add_argument("-c", "--config", dest="configfile", help="Config file path (YAML)")
+    notify_p.add_argument(
+        "channel",
+        help="Channel name as defined in notification_channels",
+    )
+    notify_p.add_argument(
+        "message",
+        nargs="?",
+        default="Test notification from hbd",
+        help="Message body (default: 'Test notification from hbd')",
+    )
+    notify_p.add_argument(
+        "--level",
+        default="WARNING",
+        choices=["INFO", "WARNING", "CRITICAL", "RECOVER"],
+        help="Notification level (default: WARNING)",
+    )
+    notify_p.add_argument(
+        "--title",
+        default=None,
+        help="Notification title (default: '[LEVEL] test')",
+    )
+
     return parser
 
 
@@ -75,6 +103,57 @@ def cmd_passwd(args):
     print(f"  password: {hashed}")
 
 
+def cmd_notify(args):
+    """Send a test message via a single notification channel."""
+    from .config import load_config
+    from .notify import Notification, _dispatch_to_channel, setup
+
+    config = load_config(args.configfile)
+    setup(config)
+
+    channels = config.get("notification_channels", {})
+    if args.channel not in channels:
+        available = ", ".join(channels.keys()) if channels else "(none)"
+        print(f"Error: channel '{args.channel}' not found in notification_channels.", file=sys.stderr)
+        print(f"Available channels: {available}", file=sys.stderr)
+        sys.exit(1)
+
+    channel_cfg = channels[args.channel]
+    level = args.level.upper()
+    title = args.title or f"[{level}] test"
+    base_url = config.get("base_url", "").rstrip("/")
+
+    notif = Notification(
+        title=title,
+        body=args.message,
+        level=level,
+        url=f"{base_url}/plugins" if base_url else "",
+    )
+
+    # Bypass min_level for explicit test sends; run async channels directly
+    import asyncio
+    ch_type = channel_cfg.get("type", "")
+    print(f"Sending via {args.channel} ({ch_type}): {title} — {args.message}")
+
+    if ch_type in ("matrix", "sms_voipms"):
+        from .notify import _send_matrix_async, _send_sms_voipms_async
+        driver_async = _send_matrix_async if ch_type == "matrix" else _send_sms_voipms_async
+        ok = asyncio.run(driver_async(channel_cfg, notif))
+    else:
+        from .notify import _DRIVERS
+        driver = _DRIVERS.get(ch_type)
+        if driver is None:
+            print(f"Error: unknown channel type '{ch_type}'", file=sys.stderr)
+            sys.exit(1)
+        ok = driver(channel_cfg, notif)
+
+    if ok:
+        print("OK")
+    else:
+        print("FAILED — check logs for details", file=sys.stderr)
+        sys.exit(1)
+
+
 def main(argv=None):
     parser = build_parser()
     args = parser.parse_args(argv)
@@ -83,6 +162,10 @@ def main(argv=None):
         cmd_passwd(args)
         return
 
+    if args.command == "notify":
+        cmd_notify(args)
+        return
+
     # Default: run the server (supports both `hbd serve ...` and `hbd ...`)
     config = load_config(args.configfile)
 

hbd/server/config.py

@@ -22,7 +22,7 @@ SERVER_DEFAULTS = {
     "logfile": os.path.join(os.path.expanduser("~"), ".hb.log"),
     # Notification channels
     "notification_channels": {},  # Named channels with type and credentials
-    "default_notification_channels": [],  # Default channels if host doesn't specify
+    "base_url": "",               # Base URL for notification links (e.g. https://hbd.example.com)
 
     # Monitoring settings
     "interval": 20,             # Expected heartbeat interval (for server checks)
@@ -34,8 +34,7 @@ SERVER_DEFAULTS = {
     "default_owner": None,      # Username that owns hosts with no explicit owner
 
     # Host management
-    "hosts": {},                # New unified host definitions (optional)
+    "hosts": {},                # Unified host definitions
-    "watchhosts": [],           # Hosts to monitor and notify about (legacy)
     "dyndnshosts": [],          # Hosts with dynamic DNS (legacy)
     "drophosts": [],            # Hosts to ignore
     "dyndomains": ["wrede.org"],
@@ -216,34 +215,18 @@ class ReloadableConfig:
 
 
 def get_watchhosts(config):
-    """Extract watchhosts from config, supporting both new and legacy formats.
+    """Extract watched hostnames from config (hosts with watch: true).
-    
-    Args:
-        config: Configuration dictionary
 
     Returns:
         List of hostnames to watch
     """
     watchhosts = []
-    
+    hosts_config = config.get("hosts", {})
-    # New format: hosts section with watch attribute
-    if "hosts" in config:
-        hosts_config = config["hosts"]
     if isinstance(hosts_config, dict):
         for host_name, host_attrs in hosts_config.items():
             if isinstance(host_attrs, dict) and host_attrs.get("watch", False):
                 watchhosts.append(host_name)
-    
+    return watchhosts
-    # Legacy format: watchhosts list
-    if "watchhosts" in config:
-        legacy_watchhosts = config.get("watchhosts", [])
-        if isinstance(legacy_watchhosts, (list, set)):
-            watchhosts.extend(legacy_watchhosts)
-        elif isinstance(legacy_watchhosts, dict):
-            # Old dict format: {"host1": {attrs}, "host2": {attrs}}
-            watchhosts.extend(legacy_watchhosts.keys())
-    
-    return list(set(watchhosts))  # Remove duplicates
 
 
 def get_dyndnshosts(config):
@@ -275,105 +258,18 @@ def get_dyndnshosts(config):
 
 
 def get_host_config(config, hostname):
-    """Get configuration for a specific host.
+    """Get configuration for a specific host from the hosts section.
-    
-    Args:
-        config: Configuration dictionary
-        hostname: Host name
 
     Returns:
         Dictionary with host attributes or empty dict
     """
-    if "hosts" in config:
     hosts_config = config.get("hosts", {})
     if isinstance(hosts_config, dict) and hostname in hosts_config:
-            return hosts_config[hostname] if isinstance(hosts_config[hostname], dict) else {}
+        val = hosts_config[hostname]
-    
+        return val if isinstance(val, dict) else {}
-    # Check legacy watchhosts for notification settings
-    if "watchhosts" in config:
-        watchhosts = config.get("watchhosts", {})
-        if isinstance(watchhosts, dict) and hostname in watchhosts:
-            legacy_attrs = watchhosts[hostname]
-            if isinstance(legacy_attrs, dict):
-                # Convert legacy format to new format
-                return {
-                    "watch": True,
-                    "notify": legacy_attrs.get("notify"),
-                    "notify_src": legacy_attrs.get("src"),
-                }
-    
     return {}
 
 
-def get_notification_channels_for_host(config, hostname):
-    """Get notification channels configured for a specific host.
-    
-    Args:
-        config: Configuration dictionary
-        hostname: Host name
-        
-    Returns:
-        List of channel names to use for this host
-    """
-    host_config = get_host_config(config, hostname)
-    
-    # Check if host specifies notification channels
-    channels = host_config.get("notification_channels", [])
-    if channels:
-        if isinstance(channels, str):
-            return [channels]
-        elif isinstance(channels, list):
-            return channels
-    
-    # Fall back to default channels
-    default_channels = config.get("default_notification_channels", [])
-    if default_channels:
-        if isinstance(default_channels, str):
-            return [default_channels]
-        elif isinstance(default_channels, list):
-            return default_channels
-    
-    # No channels configured, return empty list (will use legacy global config)
-    return []
-
-
-def get_channel_config(config, channel_name):
-    """Get configuration for a specific notification channel.
-    
-    Args:
-        config: Configuration dictionary
-        channel_name: Name of the notification channel
-        
-    Returns:
-        Dictionary with channel configuration or None if not found
-    """
-    channels = config.get("notification_channels", {})
-    if isinstance(channels, dict) and channel_name in channels:
-        return channels[channel_name]
-    return None
-
-
-def get_notification_channels_config(config, hostname):
-    """Get list of notification channel configurations for a host.
-
-    Args:
-        config: Configuration dictionary
-        hostname: Host name
-
-    Returns:
-        List of (channel_name, channel_config) tuples
-    """
-    channel_names = get_notification_channels_for_host(config, hostname)
-
-    channels = []
-    for channel_name in channel_names:
-        channel_config = get_channel_config(config, channel_name)
-        if channel_config and channel_config.get("type"):
-            channels.append((channel_name, channel_config))
-
-    return channels
-
-
 # ---------------------------------------------------------------------------
 # User / host-access helpers
 # ---------------------------------------------------------------------------

hbd/server/http.py

@@ -244,7 +244,9 @@ async def start(
         host = config.get("hb_host", "localhost")
         extra_scripts = config.get("http_extra_scripts", "")
         host = request.host  # includes port if non-standard
-        scheme = "wss" if request.secure else "ws"
+        forwarded_proto = request.headers.get("X-Forwarded-Proto", "")
+        is_secure = request.secure or forwarded_proto.lower() == "https"
+        scheme = "wss" if is_secure else "ws"
         heartbeat_ws_url = f"{scheme}://{host}/ws"
         tmpl = env.get_template("live.html")
         body = tmpl.render(

hbd/server/main.py

@@ -162,7 +162,7 @@ async def _run_async(config, config_path=None):
     from . import journal as journal_mod
     from . import threshold as threshold_mod
 
-    notify_mod.setup(config)
+    notify_mod.setup(config, loop=loop)
     
     # Initialize message journal
     msg_journal = journal_mod.get_journal(config)

hbd/server/notify.py

@@ -1,37 +1,103 @@
-"""Notification helpers: email, pushover, mattermost, signal and dispatcher."""
+"""Notification helpers: email, pushover, matrix, mattermost, signal, sms and dispatcher.
 
+Channel types supported:
+  pushover      - Pushover app notifications
+  email         - SMTP email
+  matrix        - Matrix (via matrix-nio)
+  mattermost    - Mattermost webhook
+  signal        - Signal via signal-cli subprocess
+  sms_voipms    - SMS via voip.ms REST API
+
+Each channel can specify ``min_level: WARNING|CRITICAL`` (default: WARNING).
+
+Notifications are dispatched to the owner + managers of the host, each via
+their own ``notification_channels`` list.  When no users are configured the
+server runs silently (no notifications sent).
+"""
+
+import asyncio
 import logging
-from typing import Optional
-import http.client
-import urllib.parse
-import subprocess
 import smtplib
+import subprocess
 import time
 import sys
+from dataclasses import dataclass, field
+from typing import Optional
+
 from . import data
 from . import ws as ws_mod
-from . import main as main_mod
 
-DEFAULT_PUSHPROVIDERS = ["all", "pushover", "mattermost", "signal"]
-msg_to_websockets = ws_mod.broadcast
-
-# module-level configuration set via setup()
-_config = {}
 logger = logging.getLogger(__name__)
 
+msg_to_websockets = ws_mod.broadcast
+
+# Module-level state set via setup()
+_config: dict = {}
+_loop: Optional[asyncio.AbstractEventLoop] = None
+
+# Tracks which channels fired a WARNING/CRITICAL per host.
+# {host_name: set of channel_names}  — used to route RECOVER to the same channels.
+_alerted_channels: dict = {}
+
 logf = None
 
+
+# ---------------------------------------------------------------------------
+# Level ordering
+# ---------------------------------------------------------------------------
+
+_LEVEL_ORDER = {"RECOVER": 0, "INFO": 0, "WARNING": 1, "CRITICAL": 2}
+
+def _level_value(level: str) -> int:
+    return _LEVEL_ORDER.get(level.upper(), 0)
+
+
+# ---------------------------------------------------------------------------
+# Notification dataclass
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Notification:
+    """Structured notification payload."""
+    title: str          # e.g. "[CRITICAL] webserver01"
+    body: str           # detail message
+    level: str          # RECOVER | WARNING | CRITICAL | INFO
+    url: str = ""       # link to plugin metrics page
+
+
+# ---------------------------------------------------------------------------
+# Module setup
+# ---------------------------------------------------------------------------
+
+def setup(cfg: dict, loop: Optional[asyncio.AbstractEventLoop] = None):
+    """Initialize notifier from configuration dict and event loop."""
+    global _config, _loop
+    _config = dict(cfg)
+    if loop is not None:
+        _loop = loop
+
+
+def reload_config(cfg: dict):
+    """Reload notification configuration on SIGHUP."""
+    global _config
+    _config = dict(cfg)
+    logger.info("Notification configuration reloaded")
+
+
+# ---------------------------------------------------------------------------
+# Event log (websocket + file + in-memory)
+# ---------------------------------------------------------------------------
+
 def initlog(logfile):
     global logf
     try:
         logf = open(logfile, "a+")
     except Exception as e:
-        import sys
-
         print("cannot open logfile %s, using STDERR: %s" % (logfile, e))
         logf = sys.stderr
     return logf
 
+
 def closelog():
     global logf
     if logf and logf != sys.stderr:
@@ -40,6 +106,7 @@ def closelog():
         except Exception:
             pass
 
+
 def eventlog(host, lvl, m, service=None):
     ts = time.time()
     s = f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))} {lvl} "
@@ -56,91 +123,29 @@ def eventlog(host, lvl, m, service=None):
             logger.warning("failed to write to logfile: %s", e)
     msg_to_websockets("message", s)
 
-def setup(cfg: dict):
-    """Initialize notifier defaults from a configuration dict."""
-    global _config
-    _config = dict(cfg)
 
+# ---------------------------------------------------------------------------
+# Low-level channel drivers
+# ---------------------------------------------------------------------------
 
-def reload_config(cfg: dict):
+def _send_pushover(channel_cfg: dict, notif: Notification) -> bool:
-    """Reload notification configuration.
+    import http.client
-    
+    import urllib.parse
-    This function updates the module-level notification configuration
+    token = channel_cfg.get("token", "")
-    during runtime config reloads.
+    user = channel_cfg.get("user", "")
-    
+    if not token or not user:
-    Args:
+        logger.warning("pushover: missing token or user")
-        cfg: New configuration dictionary
-    """
-    global _config
-    _config = dict(cfg)
-    logger.info("Notification configuration reloaded")
-
-
-def send_email(toaddrs, smtpserver, sender, subject, body, debug=0):
-    """Send a plain email via SMTP. Returns True on success."""
-    try:
-        smtpport = _config.get("smtpport", 587)
-        server = smtplib.SMTP(smtpserver, smtpport)
-        if debug > 0:
-            server.set_debuglevel(1)
-        if smtpport == 587:
-            server.starttls()
-            server.ehlo()
-            smtpuser = _config.get("smtpuser", None)
-            smtppassword = _config.get("smtppassword", None)
-            if smtpuser and smtppassword:
-                server.login(smtpuser, smtppassword)
-        server.sendmail(sender, toaddrs, body)
-    except Exception as e:
-        logger.warning("email send failed: %s", e)
-        try:
-            server.quit()
-        except Exception:
-            pass
         return False
-    try:
+    params: dict = {"token": token, "user": user, "title": notif.title, "message": notif.body}
-        server.quit()
+    if notif.url:
-    except Exception:
+        params["url"] = notif.url
-        pass
+        params["url_title"] = "Plugin metrics"
-    return True
-
-
-def email(subject: str, msg: str, debug: int = 0) -> bool:
-    """Convenience wrapper exposed to the rest of the application.
-
-    Uses module-level configuration to supply recipient list, smtp server
-    and sender address.
-    """
-    toaddrs = _config.get("toemail")
-    fromemail = _config.get("fromemail")
-    smtpserver = _config.get("smtpserver")
-    if not toaddrs or not fromemail or not smtpserver:
-        logger.warning(
-            "email config incomplete: toemail=%s, fromemail=%s, smtpserver=%s",
-            toaddrs,
-            fromemail,
-            smtpserver,
-        )
-        return False
-    date = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.localtime())
-    body = "To: %s\nFrom: %s\nSubject: %s\nDate: %s\n\n%s" % (
-        toaddrs[0] if toaddrs else "",
-        fromemail,
-        subject,
-        date,
-        msg,
-    )
-    return send_email(toaddrs, smtpserver, fromemail, subject, body, debug=debug)
-
-
-def pushover(token: str, user: str, msg: str, debug: int = 0) -> bool:
-    """Send message via Pushover API."""
     conn = http.client.HTTPSConnection("api.pushover.net:443")
     try:
         conn.request(
             "POST",
             "/1/messages.json",
-            urllib.parse.urlencode({"token": token, "user": user, "message": msg}),
+            urllib.parse.urlencode(params),
             {"Content-type": "application/x-www-form-urlencoded"},
         )
         r = conn.getresponse()
@@ -151,176 +156,346 @@ def pushover(token: str, user: str, msg: str, debug: int = 0) -> bool:
         return False
 
 
-def pushmattermost(
+def _send_email(channel_cfg: dict, notif: Notification) -> bool:
-    host: str,
+    recipients = channel_cfg.get("recipients", [])
-    token: str,
+    sender = channel_cfg.get("sender", "")
-    channel: str,
+    smtp_server = channel_cfg.get("smtp_server", "")
-    msg: str,
+    smtp_port = channel_cfg.get("smtp_port", 587)
-    username: str = "hbd",
+    smtp_user = channel_cfg.get("smtp_user")
-    icon: Optional[str] = None,
+    smtp_password = channel_cfg.get("smtp_password")
-    debug: int = 0,
-) -> bool:
-    """Send a message to Mattermost via simple webhook driver if available.
 
-    This helper tries to import mattermostdriver.Driver and uses webhooks if present.
+    if not recipients or not sender or not smtp_server:
-    If the import fails it returns False.
+        logger.warning("email: missing recipients, sender, or smtp_server")
-    """
+        return False
+
+    date = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.localtime())
+    body_text = notif.body
+    if notif.url:
+        body_text += f"\n\n{notif.url}"
+    raw = "To: %s\nFrom: %s\nSubject: %s\nDate: %s\n\n%s" % (
+        recipients[0] if isinstance(recipients, list) else recipients,
+        sender,
+        notif.title,
+        date,
+        body_text,
+    )
+    try:
+        server = smtplib.SMTP(smtp_server, smtp_port)
+        if smtp_port == 587:
+            server.starttls()
+            server.ehlo()
+            if smtp_user and smtp_password:
+                server.login(smtp_user, smtp_password)
+        server.sendmail(sender, recipients, raw)
+        server.quit()
+        return True
+    except Exception as e:
+        logger.warning("email send failed: %s", e)
+        try:
+            server.quit()
+        except Exception:
+            pass
+        return False
+
+
+def _send_mattermost(channel_cfg: dict, notif: Notification) -> bool:
     try:
         from mattermostdriver import Driver
-    except Exception:
+    except ImportError:
+        logger.error("mattermostdriver not installed")
         return False
+    host = channel_cfg.get("host", "")
+    token = channel_cfg.get("token", "")
+    channel = channel_cfg.get("channel", "")
+    if not host or not token or not channel:
+        logger.warning("mattermost: missing host, token, or channel")
+        return False
+    text = f"**{notif.title}**\n{notif.body}"
+    if notif.url:
+        text += f"\n[Plugin metrics]({notif.url})"
     ses = {"url": host, "scheme": "http", "basepath": "/api/v4", "port": 8065}
     mm = Driver(ses)
-    payload = {"text": msg, "channel": channel, "username": username}
+    payload: dict = {"text": text, "channel": channel, "username": channel_cfg.get("username", "hbd")}
+    icon = channel_cfg.get("icon")
     if icon:
         payload["icon_url"] = icon
     try:
         rc = mm.webhooks.call_webhook(token, payload)
-        logger.debug("mattermost rc: %s", rc)
         return bool(rc is None or rc == "")
     except Exception as e:
         logger.error("mattermost error: %s", e)
         return False
 
 
-def pushsignal(
+def _send_signal(channel_cfg: dict, notif: Notification) -> bool:
-    signal_cli_bin: str, user: str, recipient: str, msg: str, debug: int = 0
+    cli = channel_cfg.get("cli_path", "/usr/local/bin/signal-cli")
-) -> bool:
+    user = channel_cfg.get("user", "")
-    """Send a message via signal-cli (requires local installation).
+    recipient = channel_cfg.get("recipient", "")
-
+    if not user or not recipient:
-    Uses subprocess to call signal-cli. Returns True if the command succeeded.
+        logger.warning("signal: missing user or recipient")
-    """
+        return False
-    CLI = [signal_cli_bin, "-u", user, "send", "-m", msg, recipient]
+    msg = f"{notif.title}\n{notif.body}"
-    logger.debug("signal cli: %s", CLI)
+    if notif.url:
-    try:
+        msg += f"\n{notif.url}"
-        res = subprocess.run(CLI, capture_output=True)
+    try:
-        if res.returncode != 0:
+        res = subprocess.run([cli, "-u", user, "send", "-m", msg, recipient], capture_output=True)
-            logger.error("signal failed: %s".res.stderr.decode())
+        if res.returncode != 0:
+            logger.error("signal failed: %s", res.stderr.decode())
             return False
-        logger.debug("signal sent: %s", res.stdout.decode())
         return True
     except Exception as e:
         logger.exception("signal exception: %s", e)
         return False
 
 
-def _dispatch_to_channel(channel_name: str, channel_config: dict, msg: str, debug: int = 0) -> bool:
+async def _send_sms_voipms_async(channel_cfg: dict, notif: Notification) -> bool:
-    """Dispatch a message to a specific notification channel.
+    """Send SMS via voip.ms REST API using multipart form-data POST."""
+    import json
+    import aiohttp
 
-    Args:
+    api_user = channel_cfg.get("api_user", "")
-        channel_name: Name of the channel (for logging)
+    api_password = channel_cfg.get("api_password", "")
-        channel_config: Channel configuration dictionary with 'type' and type-specific fields
+    did = channel_cfg.get("did", "")
-        msg: Message to send
+    dst = channel_cfg.get("dst", "")
-        debug: Debug level
+    if not api_user or not api_password or not did or not dst:
-        
+        logger.warning("sms_voipms: missing api_user, api_password, did, or dst")
-    Returns:
-        True if notification sent successfully, False otherwise
-    """
-    channel_type = channel_config.get("type")
-    
-    if channel_type == "pushover":
-        return pushover(
-            channel_config.get("token", ""),
-            channel_config.get("user", ""),
-            msg,
-            debug=debug
-        )
-    
-    elif channel_type == "email":
-        # Build email from channel config
-        recipients = channel_config.get("recipients", [])
-        sender = channel_config.get("sender", "")
-        smtp_server = channel_config.get("smtp_server", "")
-        smtp_port = channel_config.get("smtp_port", 587)
-        smtp_user = channel_config.get("smtp_user")
-        smtp_password = channel_config.get("smtp_password")
-        
-        if not recipients or not sender or not smtp_server:
-            logger.warning(
-                "Email channel '%s' missing required fields: recipients=%s, sender=%s, smtp_server=%s",
-                channel_name, recipients, sender, smtp_server
-            )
         return False
 
-        # Temporarily update _config for email() function
+    # SMS body: title + body, truncated to 160 chars
-        old_config = dict(_config)
+    text = f"{notif.title}: {notif.body}"
-        _config["toemail"] = recipients
+    if len(text) > 160:
-        _config["fromemail"] = sender
+        text = text[:157] + "..."
-        _config["smtpserver"] = smtp_server
-        _config["smtpport"] = smtp_port
-        if smtp_user:
-            _config["smtpuser"] = smtp_user
-        if smtp_password:
-            _config["smtppassword"] = smtp_password
 
-        result = email("Heartbeat notification", msg, debug=debug)
+    form_data = {
+        "api_username": api_user,
+        "api_password": api_password,
+        "method": "sendSMS",
+        "did": did,
+        "dst": dst,
+        "message": text,
+    }
 
-        # Restore config
+    try:
-        _config.clear()
+        async with aiohttp.ClientSession() as session:
-        _config.update(old_config)
+            with aiohttp.MultipartWriter("form-data") as mp:
-        
+                for key, value in form_data.items():
-        return result
+                    part = mp.append(value)
-    
+                    part.set_content_disposition("form-data", name=key)
-    elif channel_type == "signal":
+            async with session.post("https://voip.ms/api/v1/rest.php", data=mp) as resp:
-        return pushsignal(
+                body = await resp.text()
-            channel_config.get("cli_path", "/usr/local/bin/signal-cli"),
+                if resp.status != 200:
-            channel_config.get("user", ""),
+                    logger.error("sms_voipms HTTP %s: %s", resp.status, body)
-            channel_config.get("recipient", ""),
+                    return False
-            msg,
+                result = json.loads(body)
-            debug=debug
+                if result.get("status") == "success":
-        )
+                    return True
-    
+                logger.error("sms_voipms error: %s", result.get("status"))
-    elif channel_type == "mattermost":
+                return False
-        return pushmattermost(
+    except Exception as e:
-            channel_config.get("host", ""),
+        logger.error("sms_voipms exception: %s", e)
-            channel_config.get("token", ""),
-            channel_config.get("channel", ""),
-            msg,
-            username=channel_config.get("username", "hbd"),
-            icon=channel_config.get("icon"),
-            debug=debug
-        )
-    
-    else:
-        logger.warning("Unknown channel type '%s' for channel '%s'", channel_type, channel_name)
         return False
 
 
-def pushmsg_for_host(hostname: str, msg: str, debug: int = 0) -> dict:
+def _send_sms_voipms(channel_cfg: dict, notif: Notification) -> bool:
-    """Send notification for a specific host using its configured channels.
+    """Dispatch voip.ms SMS send onto the shared event loop."""
+    if _loop is None:
+        logger.warning("sms_voipms: event loop not available")
+        return False
+    future = asyncio.run_coroutine_threadsafe(_send_sms_voipms_async(channel_cfg, notif), _loop)
+    try:
+        return future.result(timeout=15)
+    except Exception as e:
+        logger.error("sms_voipms send timed out or failed: %s", e)
+        return False
 
-    This function looks up the host's notification channels from the config
-    and sends the message to those channels.
 
-    Args:
+async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
-        hostname: Name of the host to send notification for
+    """Send a Matrix message using matrix-nio."""
-        msg: Message to send
+    try:
-        debug: Debug level
+        from nio import AsyncClient, RoomMessageText  # noqa: F401
+    except ImportError:
+        logger.error("matrix-nio not installed; pip install matrix-nio")
+        return False
 
-    Returns:
+    from nio import AsyncClient
-        Dictionary of results per channel: {"channel_name": True/False}
+    homeserver = channel_cfg.get("homeserver", "")
+    access_token = channel_cfg.get("access_token", "")
+    room_id = channel_cfg.get("room_id", "")
+    if not homeserver or not access_token or not room_id:
+        logger.warning("matrix: missing homeserver, access_token, or room_id")
+        return False
+
+    text = f"{notif.title}\n{notif.body}"
+    if notif.url:
+        text += f"\n{notif.url}"
+    html = f"<strong>{notif.title}</strong><br>{notif.body}"
+    if notif.url:
+        html += f'<br><a href="{notif.url}">Plugin metrics</a>'
+
+    client = AsyncClient(homeserver)
+    client.access_token = access_token
+    try:
+        from nio import RoomSendResponse
+        content = {
+            "msgtype": "m.text",
+            "body": text,
+            "format": "org.matrix.custom.html",
+            "formatted_body": html,
+        }
+        resp = await client.room_send(room_id, "m.room.message", content)
+        if hasattr(resp, "event_id"):
+            return True
+        logger.error("matrix send failed: %s", resp)
+        return False
+    except Exception as e:
+        logger.error("matrix exception: %s", e)
+        return False
+    finally:
+        await client.close()
+
+
+def _send_matrix(channel_cfg: dict, notif: Notification) -> bool:
+    """Dispatch matrix send onto the shared event loop."""
+    if _loop is None:
+        logger.warning("matrix: event loop not available")
+        return False
+    future = asyncio.run_coroutine_threadsafe(_send_matrix_async(channel_cfg, notif), _loop)
+    try:
+        return future.result(timeout=15)
+    except Exception as e:
+        logger.error("matrix send timed out or failed: %s", e)
+        return False
+
+
+# ---------------------------------------------------------------------------
+# Channel dispatcher
+# ---------------------------------------------------------------------------
+
+_DRIVERS = {
+    "pushover": _send_pushover,
+    "email": _send_email,
+    "mattermost": _send_mattermost,
+    "signal": _send_signal,
+    "sms_voipms": _send_sms_voipms,
+    "matrix": _send_matrix,
+}
+
+
+def _dispatch_to_channel(channel_name: str, channel_cfg: dict, notif: Notification) -> bool:
+    """Send *notif* to a single named channel, honouring min_level."""
+    min_level = channel_cfg.get("min_level", "WARNING").upper()
+    if _level_value(notif.level) < _level_value(min_level):
+        logger.debug(
+            "channel '%s': skipping level %s (min_level=%s)", channel_name, notif.level, min_level
+        )
+        return True  # not an error — filtered intentionally
+
+    ch_type = channel_cfg.get("type", "")
+    driver = _DRIVERS.get(ch_type)
+    if driver is None:
+        logger.warning("unknown channel type '%s' for channel '%s'", ch_type, channel_name)
+        return False
+    return driver(channel_cfg, notif)
+
+
+# ---------------------------------------------------------------------------
+# Central dispatch function
+# ---------------------------------------------------------------------------
+
+def _build_url(host_name: str) -> str:
+    base_url = _config.get("base_url", "").rstrip("/")
+    if not base_url:
+        return ""
+    return f"{base_url}/plugins#{host_name}"
+
+
+def send_notification(host_name: str, notif: Notification) -> dict:
+    """Dispatch *notif* to all managers/owner of *host_name*.
+
+    Looks up the host's owner + managers, resolves each user's
+    notification_channels, and dispatches.  Silently does nothing if
+    no users are configured.
+
+    Returns a dict of {channel_name: bool} results.
     """
-    from . import config as config_mod
+    from . import users as users_mod
+    from . import hbdclass
 
-    # Get notification channels for this host
+    if not users_mod.users_enabled():
-    channels = config_mod.get_notification_channels_config(_config, hostname)
-    
-    if not channels:
-        logger.warning("No notification channels configured for host '%s'", hostname)
         return {}
 
-    # Dispatch to each channel
+    # Collect recipient usernames: owner + managers
-    results = {}
+    host = hbdclass.Host.hosts.get(host_name)
-    for channel_name, channel_config in channels:
+    if host is None:
+        logger.debug("send_notification: host '%s' not found", host_name)
+        return {}
+
+    recipients: set[str] = set()
+    owner = getattr(host, "owner", None)
+    if owner:
+        recipients.add(owner)
+    for m in getattr(host, "managers", []):
+        recipients.add(m)
+
+    if not recipients:
+        logger.debug("send_notification: no owner/managers for '%s'", host_name)
+        return {}
+
+    # Fill url if not already set
+    if not notif.url:
+        notif.url = _build_url(host_name)
+
+    global_channels: dict = _config.get("notification_channels", {})
+    results: dict = {}
+    level = notif.level.upper()
+    is_alert = level in ("WARNING", "CRITICAL")
+    is_recover = level in ("RECOVER",)
+
+    # For RECOVER: send to every channel that previously fired an alert for this host,
+    # regardless of that channel's min_level.
+    if is_recover and host_name in _alerted_channels:
+        for channel_name in list(_alerted_channels[host_name]):
+            channel_cfg = global_channels.get(channel_name)
+            if not channel_cfg:
+                continue
             try:
-            success = _dispatch_to_channel(channel_name, channel_config, msg, debug=debug)
+                ch_type = channel_cfg.get("type", "")
-            results[channel_name] = success
+                driver = _DRIVERS.get(ch_type)
-            if success:
+                if driver:
-                logger.info("Notification sent to channel '%s': %s", channel_name, msg)
+                    ok = driver(channel_cfg, notif)
-            else:
+                    results[channel_name] = ok
-                logger.warning("Failed to send notification to channel '%s'", channel_name)
+                    if ok:
+                        logger.info("recover sent to channel '%s': %s", channel_name, notif.title)
             except Exception as e:
-            logger.error("Error sending to channel '%s': %s", channel_name, e)
+                logger.error("error sending recover to channel '%s': %s", channel_name, e)
+        # Clear the alerted set once recovery is delivered
+        del _alerted_channels[host_name]
+        return results
+
+    for username in recipients:
+        user = users_mod.get_user(username)
+        if user is None:
+            logger.debug("send_notification: user '%s' not found", username)
+            continue
+        for channel_name in user.notification_channels:
+            if channel_name in results:
+                continue  # already dispatched to this channel this notification
+            channel_cfg = global_channels.get(channel_name)
+            if not channel_cfg:
+                logger.warning("channel '%s' not defined in notification_channels", channel_name)
+                results[channel_name] = False
+                continue
+            try:
+                ok = _dispatch_to_channel(channel_name, channel_cfg, notif)
+                results[channel_name] = ok
+                if ok:
+                    logger.info("notification sent to channel '%s': %s", channel_name, notif.title)
+                    if is_alert:
+                        _alerted_channels.setdefault(host_name, set()).add(channel_name)
+                else:
+                    logger.warning("failed to send notification to channel '%s'", channel_name)
+            except Exception as e:
+                logger.error("error sending to channel '%s': %s", channel_name, e)
                 results[channel_name] = False
 
     return results

hbd/server/templates/live.html

@@ -81,7 +81,7 @@
     #ntable th {
       border: 1px solid #e0e0e0;
       text-align: left;
-      padding: 8px 10px;
+      padding: 2px 4px;
     }
 
     #ntable tr:nth-child(even) {
@@ -92,8 +92,24 @@
       background-color: #e3f2fd;
     }
 
+    #ntable tbody tr.row-warning {
+      background-color: #fff8c5;
+    }
+
+    #ntable tbody tr.row-critical {
+      background-color: #fde8e8;
+    }
+
+    #ntable tbody tr.row-warning:hover {
+      background-color: #fff0a0;
+    }
+
+    #ntable tbody tr.row-critical:hover {
+      background-color: #f9c8c8;
+    }
+
     #ntable th {
-      padding: 12px 10px;
+      padding: 6px 8px;
       background-color: #2196f3;
       color: white;
       font-weight: 600;
@@ -143,7 +159,7 @@
     /* Message styling */
     #messages {
       font-size: 0.85em;
-      line-height: 1.6;
+      line-height: 1.0;
     }
 
     #messages div {
@@ -217,6 +233,19 @@
       }
     }
 
+    function updateRowAlert(row, data) {
+      var criticalUnacked = data.alert_critical_unacked || 0;
+      var criticalAcked = data.alert_critical_acked || 0;
+      var warningUnacked = data.alert_warning_unacked || 0;
+      var warningAcked = data.alert_warning_acked || 0;
+      row.classList.remove('row-warning', 'row-critical');
+      if (criticalUnacked > 0 || criticalAcked > 0) {
+        row.classList.add('row-critical');
+      } else if (warningUnacked > 0 || warningAcked > 0) {
+        row.classList.add('row-warning');
+      }
+    }
+
     function createRow(data) {
       var row = document.createElement("tr");
       var c_name = document.createElement("td");
@@ -284,12 +313,31 @@
       var table = document.getElementById("ntablebody"); // find table to append to
       table.appendChild(row); // append row to table
       name_idx[c_name] = row;
+      updateRowAlert(row, data);
     }
 
     function formatTS(ts) {
-      const milliseconds = ts * 1000;
+      const now = new Date();
-      const dateObject = new Date(milliseconds);
+      const d = new Date(ts * 1000);
-      return dateObject.toLocaleString("de-DE");
+
+      const pad = n => String(n).padStart(2, '0');
+      const timeStr = `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
+
+      // Same calendar day → show time only
+      if (d.toDateString() === now.toDateString()) {
+        return timeStr;
+      }
+
+      // Within 8 days → show "-X d hh:mm:ss"
+      const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate());
+      const dStart = new Date(d.getFullYear(), d.getMonth(), d.getDate());
+      const diffDays = Math.round((todayStart - dStart) / 86400000);
+      if (diffDays < 8) {
+        return `-${diffDays}d ${timeStr}`;
+      }
+
+      // Older → date only
+      return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
     }
 
     function update_table(data) {
@@ -345,6 +393,7 @@
         name_idx[data.name].cells[4 + i * 4].innerHTML = state;
         name_idx[data.name].cells[5 + i * 4].innerHTML = latency;
       }
+      updateRowAlert(name_idx[data.name], data);
     }
 
     function WS_Connect() {
@@ -427,7 +476,7 @@
           </thead>
           <tbody id="ntablebody">
             {% for host in hosts %}
-            <tr>
+            <tr class="{% if host.alert_critical_unacked > 0 or host.alert_critical_acked > 0 %}row-critical{% elif host.alert_warning_unacked > 0 or host.alert_warning_acked > 0 %}row-warning{% endif %}">
               <td>{{ host.name }}</td>
               <td style="text-align: center; color: #ff9800; font-weight: bold;">
                 {%- set warning_unacked = host.alert_warning_unacked -%}

hbd/server/threshold.py

@@ -1003,9 +1003,15 @@ class ThresholdChecker:
         value: Any,
     ):
         """Send notification and log to journal/eventlog."""
-        # Send notification using host-specific channels
         try:
-            notify_mod.pushmsg_for_host(host_name, f"{lvl}: {host_name} - {message}")
+            notify_mod.send_notification(
+                host_name,
+                notify_mod.Notification(
+                    title=f"[{lvl}] {host_name}",
+                    body=message,
+                    level=lvl,
+                ),
+            )
             logger.info("Notification sent: %s", message)
         except Exception as e:
             logger.error("Failed to send notification: %s", e)
@@ -1137,9 +1143,15 @@ class ThresholdChecker:
             else:
                 message = f"REMINDER ({alert_state.level.name}): {host_name} - {metric_path} = {value} (ongoing for {int(now - alert_state.since)}s)"
             
-            # Send re-notification using host-specific channels
             try:
-                notify_mod.pushmsg_for_host(host_name, message)
+                notify_mod.send_notification(
+                    host_name,
+                    notify_mod.Notification(
+                        title=f"[REMINDER/{alert_state.level.name}] {host_name}",
+                        body=message,
+                        level=alert_state.level.name,
+                    ),
+                )
                 alert_state.last_notification = now
                 alert_state.notification_count += 1
                 logger.info("Re-notification sent: %s", message)

hbd/server/udp.py

@@ -171,7 +171,7 @@ def dicttos(ID, d):
 DROPOVERDUE = 7 * 24 * 3600  # seconds before an overdue host becomes UNKNOWN
 
 
-def _make_timer_callbacks(uname, host, watchhosts, ctx):
+def _make_timer_callbacks(uname, host, ctx):
     """Return (on_overdue, on_unknown) async callbacks for connection timer logic.
 
     Captured values are bound at call time so callbacks are safe to use in loops.
@@ -191,9 +191,11 @@ def _make_timer_callbacks(uname, host, watchhosts, ctx):
         now = time.time()
         connection.newstate(connection.__class__.OVERDUE, now, cfg.get("grace", 2))
         msg = f"{connection.afam} overdue"
-        eventlog(uname, "CRITICAL" if uname in watchhosts else "WARNING", msg)
+        eventlog(uname, "CRITICAL", msg)
-        if uname in watchhosts:
+        notify_mod.send_notification(
-            notify_mod.pushmsg_for_host(uname, f"{uname} {msg}")
+            uname,
+            notify_mod.Notification(title=f"[CRITICAL] {uname}", body=msg, level="CRITICAL"),
+        )
         if threshold_checker:
             threshold_checker.check_value(
                 host_name=uname,
@@ -218,8 +220,6 @@ def restore_connection_timers(hbdclass, ctx):
     now = time.time()
     cfg = ctx.get("config", {})
     grace = cfg.get("grace", 2)
-    from . import config as config_mod
-    watchhosts = config_mod.get_watchhosts(cfg)
 
     restored = 0
     for uname, host in list(hbdclass.Host.hosts.items()):
@@ -229,7 +229,7 @@ def restore_connection_timers(hbdclass, ctx):
             if state == hbdclass.Connection.DOWN:
                 continue
 
-            on_overdue, on_unknown = _make_timer_callbacks(uname, host, watchhosts, ctx)
+            on_overdue, on_unknown = _make_timer_callbacks(uname, host, ctx)
 
             if state == hbdclass.Connection.UP and interval > 0:
                 elapsed = now - conn.lastbeat
@@ -322,9 +322,6 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
         host = hbdcls.Host.hosts[uname]
         newh = False
     
-    # Get watchhosts once for use throughout message handling
-    watchhosts = config_mod.get_watchhosts(cfg)
-
     cid = msg.get("id", 0)
     try:
         rtt = float(msg.get("rtt"))
@@ -390,8 +387,10 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
 
     if res:
         eventlog(uname, "WARNING", res)
-        if uname in watchhosts:
+        notify_mod.send_notification(
-            notify_mod.pushmsg_for_host(uname, "%s %s" % (host.name, res))
+            uname,
+            notify_mod.Notification(title=f"[WARNING] {uname}", body=res, level="WARNING"),
+        )
 
     interval = int(msg.get("interval", 0) or 0)
     shutdown = msg.get("shutdown", 0)
@@ -401,13 +400,12 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
 
     if boot:
         eventlog(uname, "INFO", "booted")
-        if uname in watchhosts:
+        notify_mod.send_notification(
-            m = "%s booted" % (host.name)
+            uname,
-            notify_mod.pushmsg_for_host(uname, m)
+            notify_mod.Notification(title=f"[INFO] {uname}", body=f"{host.name} booted", level="INFO"),
+        )
     if message:
         eventlog(uname, "INFO", "msg: %s" % message, service=service)
-        if uname in watchhosts:
-            notify_mod.pushmsg_for_host(uname, message)
 
     if conn.getstate() != hbdcls.Connection.UP:
         lasts = conn.state
@@ -420,8 +418,10 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
             else:
                 m = "%s back after being %s for %s" % (conn.afam, lasts, dur(d))
             eventlog(uname, "RECOVER", m)
-            if uname in watchhosts:
+            notify_mod.send_notification(
-                notify_mod.pushmsg_for_host(uname, "%s %s is back" % (uname, conn.afam))
+                uname,
+                notify_mod.Notification(title=f"[RECOVER] {uname}", body=m, level="RECOVER"),
+            )
 
     if boot or newh:
         host.upcount = host.doesack
@@ -429,9 +429,12 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
         host.upcount += 1
 
     if shutdown:
-        eventlog(uname, "INFO", "%s shutdown" % conn.afam)
+        m = "%s shutdown" % conn.afam
-        if uname in watchhosts:
+        eventlog(uname, "INFO", m)
-            notify_mod.pushmsg_for_host(uname, "%s %s shutdown" % (uname, conn.afam))
+        notify_mod.send_notification(
+            uname,
+            notify_mod.Notification(title=f"[INFO] {uname}", body=m, level="INFO"),
+        )
         conn.newstate(hbdcls.Connection.DOWN, now)
 
     if interval > 0:
@@ -442,7 +445,7 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
     if interval > 0 and conn.getstate() != hbdcls.Connection.DOWN:
         grace = cfg.get("grace", 2)
         timeout_seconds = interval + grace
-        on_overdue, _ = _make_timer_callbacks(uname, host, watchhosts, ctx)
+        on_overdue, _ = _make_timer_callbacks(uname, host, ctx)
         conn.reset_overdue_timer(timeout_seconds, on_overdue)
     
     # Check RTT thresholds using the threshold checker

pyproject.toml

@@ -31,6 +31,7 @@ server = [
   "mattermostdriver>=7.3.0",
   "aiohttp>=3.11",
   "Jinja2>=3.1.6",
+  "matrix-nio>=0.24",
 ]
 
 # Install both client and server