Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e8d9c4a958 | ||
|
|
328f248c43 | ||
|
|
747a8c27c9 | ||
|
|
5243f0a834 | ||
|
|
3c6f7fc868 | ||
|
|
adbd2f9b46 | ||
|
|
248b17edca | ||
|
|
3639464ba6 | ||
|
|
997279dce8 | ||
|
|
00d6ba607e | ||
|
|
cd5de7f205 | ||
|
|
95355374f8 | ||
|
|
8122c8d7dc | ||
|
|
3970f4cdde | ||
|
|
c3736f47ee | ||
|
|
2f1cad9156 | ||
|
|
1898bc52e5 | ||
|
|
c61a571995 | ||
|
|
9e8d68e712 | ||
|
|
ad19d2596a | ||
|
|
82ba69d827 | ||
|
|
9519169b47 | ||
|
|
f838920f7c | ||
|
|
45d11a66b5 | ||
|
|
a42d783c0c | ||
|
|
18e33656c0 | ||
|
|
4ac5a34a43 | ||
|
|
6d39c5be47 | ||
|
|
cdb2183989 | ||
|
|
41c78e8e34 | ||
|
|
acf1eb8804 | ||
|
|
49b631f02d | ||
|
|
8204737ea6 | ||
|
|
5b2fc8fd46 | ||
|
|
70c3e93684 | ||
|
|
52968201fa | ||
|
|
a7b698e6b9 | ||
|
|
136b10239f | ||
|
|
1e46f33648 | ||
|
|
4c3a63e645 | ||
|
|
1f4dd7cb1f | ||
|
|
ae3f2fc70f | ||
|
|
4414967bdc | ||
|
|
e3b0e5041f | ||
|
|
92982c0ca7 | ||
|
|
00fdbcaa5b | ||
|
|
600e68b509 | ||
|
|
b26e853a1c | ||
|
|
36adad5b3b | ||
|
|
7591268a89 | ||
|
|
7afc06a329 | ||
|
|
a3f303e6ba | ||
|
|
16c2922bea | ||
|
|
0343eb4a24 | ||
|
|
fb5570155b | ||
|
|
8d268adf70 | ||
|
|
5d9f161706 | ||
|
|
067feacd59 | ||
|
|
c564fafe73 | ||
|
|
8eeae48964 | ||
|
|
880dc0e33f | ||
|
|
f06b2ef9e8 | ||
|
|
447b9574a3 | ||
|
|
e98cca22f7 | ||
|
|
5fe069a1a2 | ||
|
|
ca2868a888 | ||
|
|
ddb1b2b875 | ||
|
|
39c4d45bd7 | ||
|
|
07fefab861 | ||
|
|
6b4524b30d | ||
|
|
a149a1e285 | ||
|
|
dd939ab86e | ||
|
|
5c6f462081 | ||
|
|
98b50dbbf4 | ||
|
|
960822918a | ||
|
|
a376ebba8d | ||
|
|
f603ef9bd2 | ||
|
|
731545dd0e | ||
|
|
356c77b0b8 | ||
|
|
6282077fe0 | ||
|
|
ddd857173b | ||
|
|
f46f725d12 | ||
|
|
3da6976b53 | ||
|
|
3a0c48e32b | ||
|
|
cf6e19704f | ||
|
|
b0addd7c67 | ||
|
|
32680d34a4 | ||
|
|
a7abdcb5c5 | ||
|
|
7bab15ae52 | ||
|
|
e0443293e9 | ||
|
|
39670f4e63 | ||
|
|
2e88ee2269 | ||
|
|
2ef7d473c3 | ||
|
|
862a9cdea0 | ||
|
|
9351938b15 | ||
|
|
b6ef2fe065 | ||
|
|
d5d2f066b3 | ||
|
|
d9563392c3 | ||
|
|
5f090b9d96 | ||
|
|
3cc1d92eb4 | ||
|
|
2ddba203df | ||
|
|
8a1f412d1d | ||
|
|
40c44f53f1 | ||
|
|
a6fe8546a8 | ||
|
|
e56660454d | ||
|
|
9cbf0ecb13 | ||
|
|
313bbd37ac | ||
|
|
f7320644f3 | ||
|
|
76e11b92f2 | ||
|
|
d39c0da5fe | ||
|
|
832b9d04d8 | ||
|
|
44d5f15a67 | ||
|
|
37b8e35a26 | ||
|
|
fa317a3b78 | ||
|
|
8729fe7038 | ||
|
|
f4231dd5f3 | ||
|
|
c47576637f | ||
|
|
2b9523ec28 | ||
|
|
610ad0af30 | ||
|
|
69b5b410ed | ||
|
|
8b2b0fd9d0 | ||
|
|
756b2323be | ||
|
|
6e7156b42d | ||
|
|
928035df50 | ||
|
|
0f90be659e | ||
|
|
4160e34a96 | ||
|
|
6430d2ddf3 | ||
|
|
4b87a90e76 | ||
|
|
450814daca | ||
|
|
e7786ac5da | ||
|
|
fed71d97d6 | ||
|
|
ba96da9622 | ||
|
|
7f17ddc2ff | ||
|
|
7750c5a303 | ||
|
|
e58530df7d | ||
|
|
fe7143759c | ||
|
|
236b40cfe4 | ||
|
|
4e5bafd26c | ||
|
|
817ae064af | ||
|
|
a00282913b | ||
|
|
d699a29fa9 | ||
|
|
4ce7eacfdd | ||
|
|
1cefc2676e | ||
|
|
668a135e53 | ||
|
|
59e256a042 | ||
|
|
708508157f | ||
|
|
f67fa9baff | ||
|
|
588eb2a792 | ||
|
|
b907343e36 | ||
|
|
e50a3996ae | ||
|
|
e1056a0365 | ||
|
|
1dbe0f8e64 | ||
|
|
12e8812070 | ||
|
|
9b5d8ac9b1 | ||
|
|
500d256d76 | ||
|
|
a7a45bf8c3 | ||
|
|
3e9b052f71 | ||
|
|
7444262985 | ||
|
|
3401cc0dbb | ||
|
|
ab0132a38d | ||
|
|
9e389736f8 | ||
|
|
b64a2a9313 | ||
|
|
a52744a448 | ||
|
|
5e2b04b811 | ||
|
|
8e07b09d7e | ||
|
|
653e018e4f | ||
|
|
c7326da7d9 | ||
|
|
0426a75d8c | ||
|
|
539f25d877 | ||
|
|
3e3099fc6d | ||
|
|
c9f15a3f1c | ||
|
|
6e396ad760 | ||
|
|
2800de0b4a | ||
|
|
15f7e6a64d | ||
|
|
9768d13b88 | ||
|
|
8640d731aa | ||
|
|
de81751e59 | ||
|
|
60c692cefc | ||
|
|
9a0baf3c78 | ||
|
|
55bdb9593a | ||
|
|
2009626fb4 | ||
|
|
18769afd37 | ||
|
|
31db5cf35e | ||
|
|
326f53f23d | ||
|
|
4f9bc8c868 | ||
|
|
259b4a3594 | ||
|
|
8646f68957 | ||
|
|
a4a6c1e3d9 | ||
|
|
0e8250362e | ||
|
|
2f5da9fc5e | ||
|
|
87aeec5999 | ||
|
|
f24500a6b5 | ||
|
|
a7bb183222 | ||
|
|
8207cd7b5f | ||
|
|
11f1eefa8c | ||
|
|
62f496e9f8 | ||
|
|
aef9e7769b | ||
|
|
58c2b9d996 | ||
|
|
2e8bcb630d | ||
|
|
338711181b | ||
|
|
43487f17e7 | ||
|
|
40205bf5c7 | ||
|
|
b95f1a5bb7 | ||
|
|
12f7eb722b | ||
|
|
217bba1b76 | ||
|
|
967e05ed74 | ||
|
|
c20245b0ab | ||
|
|
b9db0c552e | ||
|
|
05045bafa2 | ||
|
|
39f1b5de30 | ||
|
|
b06de6fdd3 | ||
|
|
940d0af35e | ||
|
|
d6d31aa2e3 | ||
|
|
76edfe7577 | ||
|
|
d190029728 | ||
|
|
b8307e7a9d | ||
|
|
a2fdf091f5 | ||
|
|
1914e6f28e | ||
|
|
82cbce9615 | ||
|
|
dbb779b013 | ||
|
|
ca908ee967 | ||
|
|
73c697b6c5 | ||
|
|
3e2357380b | ||
|
|
cc4a103bae | ||
|
|
53fb10fdf5 | ||
|
|
2df2ad18c9 | ||
|
|
b81a0d2a6c | ||
|
|
1a19088cfe | ||
|
|
172f6e950f | ||
|
|
4349ae217a | ||
|
|
b3aa7b585f | ||
|
|
88a3c09b51 | ||
|
|
0504402a8a | ||
|
|
ca58c18802 | ||
|
|
1ddc4b8132 | ||
|
|
5e1720ed32 | ||
|
|
77f127fe60 | ||
|
|
54fbd8d73d | ||
|
|
7ab17e26e2 | ||
|
|
28f5fa951c | ||
|
|
37f1c58969 | ||
|
|
f006077a71 | ||
|
|
d9fc8d632f | ||
|
|
f640574e4f | ||
|
|
9a19424279 | ||
|
|
ca8ba84e65 | ||
|
|
f3d08d1c9e | ||
|
|
1e4263b793 | ||
|
|
e931acb9f5 | ||
|
|
018409e71d | ||
|
|
1824f637b4 | ||
|
|
a534c06b26 | ||
|
|
d7b5c97a4e | ||
|
|
ae447ac4a6 | ||
|
|
d44ce3d124 | ||
|
|
b1985d0eb2 | ||
|
|
de778f680f | ||
|
|
d7b368c7c6 | ||
|
|
e790663f9f | ||
|
|
475319e248 | ||
|
|
ca5ef384a8 | ||
|
|
c93dbdc0f4 | ||
|
|
3a546a1e5c | ||
|
|
74c89d098c | ||
|
|
3301dbfe34 | ||
|
|
d00d903e7d | ||
|
|
babb5d61aa | ||
|
|
11d1c718b3 | ||
|
|
a99b6b54c7 | ||
|
|
8da3d550eb | ||
|
|
a76d0fc840 | ||
|
|
94cbb31c48 | ||
|
|
ae60844a8a | ||
|
|
49fa310361 | ||
|
|
28e2180f7b | ||
|
|
ce0590f015 | ||
|
|
f50acca509 | ||
|
|
72fc82b91f | ||
|
|
46f8c32c0b | ||
|
|
691f62aa69 | ||
|
|
cffc9805f9 | ||
|
|
917d6a401b | ||
|
|
2bd3a9beb6 | ||
|
|
5523c60866 | ||
|
|
ab37ac7194 | ||
|
|
f811a19d80 | ||
|
|
6239825f43 | ||
|
|
b56245bb23 | ||
|
|
331c4e804d | ||
|
|
9fd945a481 | ||
|
|
26df08eeff | ||
|
|
5819dd6b25 | ||
|
|
6fb67f8615 | ||
|
|
e70ae6f176 | ||
|
|
a77f6d380c | ||
|
|
6aae2a1dab | ||
|
|
85ee0e1040 | ||
|
|
c4f09e9ced | ||
|
|
64710fd4cd | ||
|
|
1f5e7465a3 | ||
|
|
b290b21e23 | ||
|
|
65c4267847 | ||
|
|
462a445235 | ||
|
|
368e178f93 | ||
|
|
6905bf266a | ||
|
|
b6dcce4f35 | ||
|
|
e6436fc236 | ||
|
|
c5ce41762e | ||
|
|
26ca0c095f | ||
|
|
1eecd67594 | ||
|
|
caf3c2c0ac | ||
|
|
9af4006097 | ||
|
|
ddf7067d13 | ||
|
|
505353a8a8 | ||
|
|
0402d33c71 | ||
|
|
7d8ca5d8db | ||
|
|
56037a036d | ||
|
|
65ceb31d8d | ||
|
|
1c9b6c1ca9 | ||
|
|
d7e6b478e1 | ||
|
|
535dbda47d | ||
|
|
c9567dddae | ||
|
|
b5963badd6 | ||
|
|
a76a39b4a0 | ||
|
|
94e1597978 | ||
|
|
c9c2ed772f | ||
|
|
aeb78dcb8e | ||
|
|
77b337e4dd | ||
|
|
293461f3f6 |
@@ -10,36 +10,48 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# - name: Set up Python
|
||||
# uses: actions/setup-python@v5
|
||||
# with:
|
||||
# python-version: '3.11'
|
||||
- name: Set up Python
|
||||
# Use a generic run step for FreeBSD if actions/setup-python
|
||||
# fails in restricted environments.
|
||||
run: |
|
||||
python3 --version
|
||||
python3 -m ensurepip --upgrade
|
||||
|
||||
- name: Install build tools
|
||||
run: |
|
||||
python3 -m pip install --upgrade pip
|
||||
python3 -m pip install build twine
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install --upgrade pip
|
||||
.venv/bin/pip install build twine
|
||||
|
||||
- name: Build package
|
||||
run: python3 -m build
|
||||
run: .venv/bin/python -m build
|
||||
|
||||
- name: Extract version from tag
|
||||
id: get_version
|
||||
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
||||
- name: Generate changelog
|
||||
id: changelog
|
||||
run: |
|
||||
PREV_TAG=$(git tag --sort=-version:refname | grep -m 1 -v "^${GITHUB_REF#refs/tags/}$")
|
||||
if [ -n "$PREV_TAG" ]; then
|
||||
CHANGELOG=$(git log --pretty=format:"- %s" "${PREV_TAG}..HEAD")
|
||||
else
|
||||
CHANGELOG="Initial release"
|
||||
fi
|
||||
# Write multiline to output
|
||||
{
|
||||
echo "CHANGELOG<<EOF"
|
||||
echo "$CHANGELOG"
|
||||
echo "EOF"
|
||||
} >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Upload to Gitea PyPI registry
|
||||
env:
|
||||
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
|
||||
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
|
||||
run: |
|
||||
python3 -m twine upload --repository-url https://git.wrede.ca/api/packages/andreas/pypi dist/*
|
||||
.venv/bin/python3 -m twine upload --repository-url https://git.wrede.ca/api/packages/andreas/pypi dist/*
|
||||
|
||||
- name: Create release
|
||||
uses: actions/gitea-release-action@v1
|
||||
@@ -48,4 +60,4 @@ jobs:
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
title: "Release ${{ steps.get_version.outputs.VERSION }}"
|
||||
body: "Release version ${{ steps.get_version.outputs.VERSION }}"
|
||||
body: "${{ steps.changelog.outputs.CHANGELOG }}"
|
||||
|
||||
@@ -5,6 +5,7 @@ __pycache__/
|
||||
*.pyo
|
||||
.flake8
|
||||
.venv/
|
||||
.continue/
|
||||
test/
|
||||
build/
|
||||
dist/
|
||||
@@ -12,3 +13,7 @@ dist/
|
||||
ssl/
|
||||
uv.lock
|
||||
.hb.yaml
|
||||
.superpowers/
|
||||
rndc-key
|
||||
docs/superpowers/
|
||||
graphify-out/
|
||||
|
||||
+589
@@ -0,0 +1,589 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to this project are documented here, organized by release.
|
||||
|
||||
## [5.4.5]
|
||||
|
||||
### Added
|
||||
- `hbc_mini.py` and `hbc_mini.c` (the dependency-free clients) get the same per-protocol connectivity probe as `hbc`: `p4`/`p6` flags on every heartbeat and a `connectivity` metric (presence, own address, gateway, RTT) via `network_monitor`, with gateway lookup on Linux, FreeBSD/NetBSD/DragonFly (both clients) and macOS/Windows (`hbc_mini.py` only)
|
||||
- `scripts/c/test_probe.c` and a `make test` target — the C client's first automated test suite
|
||||
|
||||
### Fixed
|
||||
- `hbc_mini.c`: the clean-shutdown heartbeat could be delayed or lost behind an in-flight connectivity probe on SIGTERM; it is now sent before the probe thread is joined
|
||||
- `hbc_mini.c`: a connectivity probe that fails to start, or is configured with a non-IP-literal target, now logs a warning instead of failing silently
|
||||
- `hbc_mini.py`: removed an unused import
|
||||
|
||||
---
|
||||
|
||||
## [5.4.4]
|
||||
|
||||
### Added
|
||||
- client reports per-protocol (IPv4/IPv6) internet presence: a connectivity probe (route check every ~5s, ICMP echo every ~60s, debounced) sends `p4`/`p6` flags on every heartbeat and a `connectivity` metric (presence, own address, gateway, RTT) via `network_monitor`
|
||||
- server monitors only the protocols a client reports as present; a protocol reported absent is shown as `absent` and raises no alert
|
||||
- host status (`up`, `simplex`, `overdue`, `down`, `unknown`) computed on the server: a host is overdue only when every present protocol is overdue; `simplex` (both present, one silent) has the same severity as `up` and is logged without notification
|
||||
- live and plugins pages show the host status, `simplex`/`absent` protocols and a Connectivity table
|
||||
- `hbc` config keys under `network_monitor`: `probe_enabled`, `probe_v4`, `probe_v6`, `probe_interval`
|
||||
|
||||
### Changed
|
||||
- overdue and recovery alerts are raised per host, not per protocol: a single CRITICAL when the last present protocol goes overdue; a protocol coming back while another stayed up is logged as INFO instead of a RECOVER notification (also applies to clients that do not report presence)
|
||||
|
||||
### Fixed
|
||||
- never park the last present family as absent, so a host that reports no protocol as present still alerts when it dies
|
||||
- keep the connectivity alert for a host overdue past the drop window across an `hbd` restart
|
||||
- `default_gateway` never raises; a protocol whose probe cannot run stays unknown rather than absent
|
||||
|
||||
---
|
||||
|
||||
## [5.4.3]
|
||||
|
||||
### Fixed
|
||||
- fix RTT y-axis label clipping, add x-axis tickmarks, enlarge charts
|
||||
|
||||
---
|
||||
|
||||
## [5.4.2]
|
||||
|
||||
### Fixed
|
||||
- preserve chart history across reconnects, show gaps for missing data
|
||||
|
||||
---
|
||||
|
||||
## [5.4.1]
|
||||
|
||||
### Added
|
||||
- live-refresh Connectivity section and RTT charts every 30s
|
||||
- render RTT history chart per connection in Connectivity section
|
||||
- record RTT history per heartbeat for charting
|
||||
|
||||
### Fixed
|
||||
- correct empty-state copy on Host Overview page
|
||||
- stop synthetic rtt_* keys from masking real plugin-data checks
|
||||
- use the latest RTT sample, not the oldest, in host info API
|
||||
- hide synthetic rtt history keys from plugin accordion list
|
||||
- remove extra blank line to pass flake8 E303 check
|
||||
- clear flap state when a host is dropped
|
||||
- correct plugin import path in docs, wire up orphaned footer template
|
||||
|
||||
---
|
||||
|
||||
## [5.4.0]
|
||||
|
||||
### Added
|
||||
- gate remote command execution behind allow_remote_command
|
||||
- flapping detection suppresses notification storms
|
||||
- remove log section from live dashboard (moved to /log)
|
||||
- dedicated /log page with journal-backed history and nav item
|
||||
- GET /api/0/log — paged, filtered events journal API
|
||||
- eventlog writes to dedicated events journal; init/backfill/close wiring
|
||||
- events journal read path with filters and backward pagination
|
||||
- events journal write primitives (log_event, backfill, get_events_journal)
|
||||
- Live Dashboard redesign — 6 columns, hover details, last-alert column
|
||||
- extend the record-row design system to Host Overview, Alerts, and About
|
||||
- unified record-row redesign of the settings page
|
||||
|
||||
### Fixed
|
||||
- harden events log — non-dict ring entries, off-loop journal reads, no default-config singleton
|
||||
- skip journal scheduling when the event loop is not running
|
||||
- dedup log rows between API seed and websocket tail
|
||||
- journal startup/shutdown lifecycle events (fired outside journal lifetime)
|
||||
- drop typing names not yet used from journal.py import
|
||||
- settings toolbar sticks below the site nav instead of scrolling away
|
||||
- settings page not scrollable — restore html/body overflow override
|
||||
|
||||
---
|
||||
|
||||
## [5.3.12]
|
||||
|
||||
### Added
|
||||
- retire per-channel private flag on profile page
|
||||
- settings UI renders per-user edit rights and owner controls
|
||||
- settings page accessible to all authenticated users
|
||||
- per-user filtering of settings sections
|
||||
- scoped non-admin saves through POST /api/0/config
|
||||
- threshold form payload carries per-config owner
|
||||
- channel ownership rule — owner-presence means private, admin promote/demote
|
||||
- scoped threshold-config merge for non-admin saves
|
||||
- scoped host merge for non-admin config saves
|
||||
- ownership visibility helpers for config entities
|
||||
|
||||
### Fixed
|
||||
- users loop clobbered requesting username in settings filtering
|
||||
- type annotations for mypy parity with master
|
||||
- don't re-announce boot/shutdown on SIGHUP restart
|
||||
|
||||
---
|
||||
|
||||
## [5.3.11]
|
||||
|
||||
### Added
|
||||
- add Windows hbc client with PyInstaller spec and NSSM install script
|
||||
- clear alerts for individual plugin metrics that disappear between samples
|
||||
- show alerts for all hosts on Alerts page, not just watched
|
||||
|
||||
### Fixed
|
||||
- declare plugin interval so stale data waits two full intervals
|
||||
- cap event buffer and replay only recent messages on dashboard connect
|
||||
- strip plugin timers when pickling Host to prevent save failure
|
||||
- correct zero-safe pathconf checks and connectivity prefix match
|
||||
- address security vulnerabilities from audit
|
||||
- don't purge connectivity/rtt alerts in purge_stale_alerts
|
||||
- restore connectivity alerts for overdue/unknown/down hosts on startup
|
||||
- clear plugin data and timers on connection UP transition
|
||||
- restore host link from Dashboard to Host Overview
|
||||
- don't set stale timer until two plugin samples establish real interval
|
||||
|
||||
---
|
||||
|
||||
## [5.3.10]
|
||||
|
||||
### Added
|
||||
- clear stale plugin data and persist OAuth users to config
|
||||
- auto-scale CPU history graph Y axis
|
||||
- add CPU usage history graph to CPU Monitor section
|
||||
|
||||
### Fixed
|
||||
- remove bak file in bumpminor.sh
|
||||
|
||||
---
|
||||
|
||||
## [5.3.9]
|
||||
|
||||
### Added
|
||||
- auto-update CHANGELOG and README in bumpminor.sh
|
||||
|
||||
---
|
||||
|
||||
## [5.3.8]
|
||||
|
||||
### Added
|
||||
- Wiki home page with overview and getting started guide
|
||||
|
||||
### Fixed
|
||||
- Release workflow: use `GITHUB_REF`/`GITHUB_OUTPUT` (Gitea Actions uses GitHub-compatible variable names)
|
||||
- Release workflow: replace `head -1` with `grep -m 1` to avoid SIGPIPE (exit 141) in changelog step
|
||||
|
||||
---
|
||||
|
||||
## [5.3.7]
|
||||
|
||||
### Added
|
||||
- Dark mode with light/dark/auto theme setting
|
||||
- UNKNOWN level filter in Log of Events
|
||||
- Per-metric grace period input in threshold settings
|
||||
- Replace Dynamic DNS YAML editor with a web form
|
||||
- Sort hosts, thresholds, and channels alphabetically on settings page
|
||||
- Suppress alerts for unwatched hosts
|
||||
|
||||
### Fixed
|
||||
- Preserve log message order when replaying history on connect
|
||||
|
||||
---
|
||||
|
||||
## [5.3.6]
|
||||
|
||||
### Added
|
||||
- MIT license
|
||||
|
||||
### Fixed
|
||||
- Correct ZFS pool status threshold operator and add per-metric grace
|
||||
- Normalize email and domain fields
|
||||
- Move dependencies back under `[project]` in pyproject.toml
|
||||
|
||||
---
|
||||
|
||||
## [5.3.4]
|
||||
|
||||
### Fixed
|
||||
- Run full reload after HTTP config publish, not just `config.reload()`
|
||||
|
||||
---
|
||||
|
||||
## [5.3.3]
|
||||
|
||||
### Added
|
||||
- Replace YAML threshold editor with a form-based UI
|
||||
- Replace multi-select fields with dual-panel picker on settings page
|
||||
- Nav bar button to publish pending config changes
|
||||
- Host, level, and message filters in Log of Events
|
||||
|
||||
### Fixed
|
||||
- Remove container max-width; stop stretching inputs on settings page
|
||||
|
||||
### Removed
|
||||
- Legacy `dyndnshosts`/`drophosts` config keys
|
||||
|
||||
---
|
||||
|
||||
## [5.3.2]
|
||||
|
||||
### Added
|
||||
- Retry DNS resolution indefinitely; add `-4`/`-6` address-family flags to `hbc` and `hbc_mini`
|
||||
- Replace YAML hosts editor with form-based CRUD table
|
||||
- Replace YAML notification channel editor with form-based UI
|
||||
|
||||
### Fixed
|
||||
- Support list-valued `threshold_config` in hosts table
|
||||
- Derive hosts threshold config list from config file keys
|
||||
- Replace channel checkboxes in Users table with multi-select
|
||||
- Support plugin-level `enabled: false` in threshold config
|
||||
- Always populate glance strip for all hosts on page load
|
||||
- Fetch host info on initial page load
|
||||
|
||||
---
|
||||
|
||||
## [5.3.1]
|
||||
|
||||
### Added
|
||||
- Host info section in Host Overview (fetched and rendered on card expand)
|
||||
- `GET /api/0/hosts/{hostname}/info` endpoint
|
||||
- Show suffix-matched metric coverage in host info threshold table
|
||||
- Move `hbc_version` and `hbc_type` out of `os_info` into the host info section
|
||||
|
||||
### Fixed
|
||||
- Correct `THRESHOLD_DEFAULTS` metric keys and add missing defaults
|
||||
|
||||
---
|
||||
|
||||
## [5.3.0]
|
||||
|
||||
### Added
|
||||
- Profile page self-service: change identity, password, and notification channels
|
||||
- Settings page editor with form sections, YAML editors, stage/publish/rollback workflow
|
||||
- Config read API: `GET /api/0/config`, `/section/{name}`, `/backups`
|
||||
- Config write API: `POST /api/0/config`, `POST /api/0/config/rollback`
|
||||
- `configio` module for comment-preserving YAML round-trip writes
|
||||
- Multi-provider OAuth2 login page and generic provider routes
|
||||
- Log login/logout events to the event log with auth source
|
||||
|
||||
### Fixed
|
||||
- ZFS monitor alerts dropped on restart with wildcard pool thresholds
|
||||
- Preserve OAuth users across config reload
|
||||
- Config API error handling, consistent 403 messages, deduplicated key lists
|
||||
- Validate password body type; coerce `notification_channels` to strings in profile API
|
||||
- Preserve OAuth `client_secret` on roundtrip; harden rollback path validation
|
||||
|
||||
---
|
||||
|
||||
## [5.2.6]
|
||||
|
||||
### Added
|
||||
- Alerts host-filter field with URL query parameter and notify URL
|
||||
- Optional logo on Gitea OAuth login button
|
||||
|
||||
### Fixed
|
||||
- Show human-readable duration in re-notification messages
|
||||
|
||||
---
|
||||
|
||||
## [5.2.5]
|
||||
|
||||
### Added
|
||||
- Alert CRITICAL on degraded or suspended ZFS pools (ONLINE=OK, DEGRADED=WARNING, all else=CRITICAL)
|
||||
- Sign in with Gitea button on login page with OAuth2 redirect/callback routes
|
||||
- OAuth2 CSRF state management
|
||||
- Host owner shown in glance strip for admin users
|
||||
- C port of `hbc_mini` (single-file client in `scripts/c/`)
|
||||
|
||||
### Fixed
|
||||
- Use `base_url` config for OAuth redirect URI to handle reverse proxy deployments
|
||||
- Preserve OAuth users across config reload
|
||||
- Escape HTML in login page error display
|
||||
|
||||
---
|
||||
|
||||
## [5.2.4]
|
||||
|
||||
### Added
|
||||
- `hbc`/`hbc_mini`: `owner` config field included in `os_info`; server applies to host record
|
||||
- Server requests InfoPlugin refresh when a host has no plugin data
|
||||
- Event log stores structured dicts; filter by user
|
||||
|
||||
### Fixed
|
||||
- Strip `_status_code` suffix from displayed metric names in threshold alerts
|
||||
- Use plain URL in Mattermost plugin metrics link
|
||||
- Fall back to `default_owner` when `os_info` has no owner
|
||||
|
||||
---
|
||||
|
||||
## [5.2.3]
|
||||
|
||||
### Added
|
||||
- `hbc`/`hbc_mini`: log name and version at startup
|
||||
- Show metric name inline with hostname in alerts and notifications
|
||||
|
||||
### Fixed
|
||||
- Send shutdown message only if a boot message was previously sent; suppress both on restart
|
||||
|
||||
---
|
||||
|
||||
## [5.2.2]
|
||||
|
||||
### Fixed
|
||||
- Retry connection on network error instead of permanently dropping it
|
||||
- Silence `aiohttp.access` log; strip plugin prefix in alerts UI
|
||||
|
||||
---
|
||||
|
||||
## [5.2.1]
|
||||
|
||||
### Fixed
|
||||
- Threshold and logging improvements
|
||||
|
||||
---
|
||||
|
||||
## [5.2.0]
|
||||
|
||||
### Added
|
||||
- `nagios` operator for direct exit-code severity mapping
|
||||
|
||||
### Fixed
|
||||
- Always show `THRESHOLD_DEFAULTS` in Settings threshold config
|
||||
|
||||
---
|
||||
|
||||
## [5.1.21]
|
||||
|
||||
### Added
|
||||
- `nagios_runner` improvements and alerts page fixes
|
||||
|
||||
---
|
||||
|
||||
## [5.1.20]
|
||||
|
||||
### Added
|
||||
- Generic threshold matching for `nagios_runner` with `{check_name}` display support
|
||||
|
||||
### Fixed
|
||||
- Reduce default hysteresis from 10% to 2%
|
||||
- Show recovery threshold in alerts UI
|
||||
|
||||
---
|
||||
|
||||
## [5.1.19]
|
||||
|
||||
### Added
|
||||
- Exclude ZFS ARC from `memory_percent`
|
||||
- Add `uptime_seconds` to `cpu_monitor`
|
||||
|
||||
### Fixed
|
||||
- Send boot/shutdown message on the first open connection, not blindly on the first in list
|
||||
|
||||
---
|
||||
|
||||
## [5.1.18]
|
||||
|
||||
### Added
|
||||
- Fetch-based Update/Delete buttons with toast notifications on Host Overview
|
||||
|
||||
### Fixed
|
||||
- Settings thresholds show correct per-config metrics; miscellaneous `hbc` fixes
|
||||
|
||||
---
|
||||
|
||||
## [5.1.17]
|
||||
|
||||
### Added
|
||||
- Owner Update/Delete buttons on Host Overview; purge stale alerts on reload
|
||||
- Retry `AsyncConnection.open()` indefinitely; drop IPv6 only on early startup failure
|
||||
- Alert pie chart in the nav bar
|
||||
|
||||
### Fixed
|
||||
- Make Alerts page scrollable
|
||||
|
||||
---
|
||||
|
||||
## [5.1.16]
|
||||
|
||||
### Added
|
||||
- Generic `ping_monitor` thresholds; round RTT to nearest ms
|
||||
|
||||
---
|
||||
|
||||
## [5.1.15]
|
||||
|
||||
### Added
|
||||
- Link hostnames in Live Dashboard to Host Overview
|
||||
- Threshold Configurations section on settings page
|
||||
|
||||
### Fixed
|
||||
- Suppress notifications on alert de-escalation (e.g. CRITICAL→WARNING)
|
||||
- Suppress recover messages for down durations under 4 seconds
|
||||
|
||||
---
|
||||
|
||||
## [5.1.14]
|
||||
|
||||
### Added
|
||||
- ZFS pool renderer in Host Overview
|
||||
|
||||
---
|
||||
|
||||
## [5.1.13]
|
||||
|
||||
### Added
|
||||
- ZFS monitor plugin
|
||||
- Host-level watch flag to suppress notifications
|
||||
- Filter Live Dashboard and Host Overview by owner/manager
|
||||
- Composable `threshold_config` list for per-host threshold layering
|
||||
- Restart on SIGHUP in `hbc` and `hbc_mini`
|
||||
|
||||
### Fixed
|
||||
- Mask `api_password` and `access_token` in settings page
|
||||
|
||||
---
|
||||
|
||||
## [5.1.12]
|
||||
|
||||
Internal release — no user-visible changes.
|
||||
|
||||
---
|
||||
|
||||
## [5.1.11]
|
||||
|
||||
### Fixed
|
||||
- Install under Docker
|
||||
- Clean up install script
|
||||
|
||||
---
|
||||
|
||||
## [5.1.10]
|
||||
|
||||
### Fixed
|
||||
- Synchronize version in `hbc_mini`
|
||||
- Install script no longer overwrites itself
|
||||
|
||||
---
|
||||
|
||||
## [5.1.9]
|
||||
|
||||
### Added
|
||||
- Install `hbc_mini` via package or install script
|
||||
|
||||
---
|
||||
|
||||
## [5.1.8]
|
||||
|
||||
### Added
|
||||
- Track `hbc` type and version
|
||||
|
||||
### Fixed
|
||||
- Nav bar position
|
||||
|
||||
---
|
||||
|
||||
## [5.1.7]
|
||||
|
||||
### Added
|
||||
- `hbc_mini`: single-file heartbeat client
|
||||
|
||||
### Fixed
|
||||
- Drop dead connections on protocol error
|
||||
|
||||
---
|
||||
|
||||
## [5.1.6]
|
||||
|
||||
### Fixed
|
||||
- Simplify event log usage; fix argument handling
|
||||
|
||||
---
|
||||
|
||||
## [5.1.5]
|
||||
|
||||
### Added
|
||||
- Update `hbc` via `hb_install.sh` instead of code patching
|
||||
|
||||
---
|
||||
|
||||
## [5.1.4]
|
||||
|
||||
### Added
|
||||
- Redesign Plugin Metrics page as Host Overview
|
||||
|
||||
---
|
||||
|
||||
## [5.1.3]
|
||||
|
||||
### Added
|
||||
- Validate absolute command paths at `nagios_runner` init
|
||||
- Async subprocess in `nagios_runner` with stderr capture and signal handling
|
||||
- `skip_reason` field on `Plugin`; surface in `PluginLoader` init messaging
|
||||
|
||||
### Fixed
|
||||
- Use `shlex.split()` for `nagios_runner` path validation to handle quoted paths
|
||||
- Reconfigure logging to syslog after `daemonize()`
|
||||
|
||||
---
|
||||
|
||||
## [5.1.2]
|
||||
|
||||
### Fixed
|
||||
- Plugin config lookup shadowed by `CLIENT_DEFAULTS` plugins key
|
||||
- Apply grace period to all threshold alerts before logging/notifying
|
||||
- RECOVER routing: use consistent level name and route via alerted channel
|
||||
- Early reminder notifications and lost recovery notifications
|
||||
- Non-alerting of overdue hosts
|
||||
|
||||
### Added
|
||||
- Swiss clock widget in the UI
|
||||
|
||||
---
|
||||
|
||||
## [5.1.1]
|
||||
|
||||
### Added
|
||||
- SMS and Matrix notification channels
|
||||
- CLI commands `stop`, `restart`, and `reload` for `hbd`
|
||||
- WebSocket endpoint at `http://.../ws`
|
||||
- Mobile HTML pages
|
||||
|
||||
### Fixed
|
||||
- Profile not updating
|
||||
- Sortable columns in tables
|
||||
|
||||
---
|
||||
|
||||
## [5.1.0]
|
||||
|
||||
### Added
|
||||
- Ping monitor plugin
|
||||
- Persist state to pickle file; restart timers on server restart
|
||||
- SIGHUP config reload for `hbd`
|
||||
- Renotify on CRITICAL only; persistent user sessions
|
||||
- RTT count threshold
|
||||
|
||||
### Fixed
|
||||
- Bogus notification on new clients
|
||||
- Show "overdue" in alerts instead of null
|
||||
|
||||
---
|
||||
|
||||
## [5.0.12]
|
||||
|
||||
### Added
|
||||
- User management and settings page
|
||||
|
||||
---
|
||||
|
||||
## [5.0.10]
|
||||
|
||||
### Added
|
||||
- Publish package to Gitea PyPI registry
|
||||
|
||||
---
|
||||
|
||||
## [5.0.9]
|
||||
|
||||
### Added
|
||||
- Use `SO_TIMESTAMP` for RTT measurement (Linux, FreeBSD, macOS)
|
||||
- Persist state to pickle file; restart timers on restart
|
||||
|
||||
---
|
||||
|
||||
## [5.0.6]
|
||||
|
||||
### Added
|
||||
- Major codebase refactoring: restructured into client/server components
|
||||
- Per-client threshold configuration
|
||||
- Display and acknowledge alerts in the UI
|
||||
- Proper `hbc` termination; `hbd` config reloadable at runtime
|
||||
@@ -0,0 +1,97 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Working principles
|
||||
|
||||
1. Don't assume. Don't hide confusion. Surface tradeoffs.
|
||||
2. Minimum code that solves the problem. Nothing speculative.
|
||||
3. Touch only what you must. Clean up only your own mess.
|
||||
4. Define success criteria. Loop until verified.
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Run tests
|
||||
pytest -q # all tests
|
||||
pytest tests/test_threshold.py # single file
|
||||
pytest -k test_name # single test
|
||||
|
||||
# Lint and type check
|
||||
tox -e lint # flake8 over hbd/ and tests/
|
||||
tox -e mypy # mypy over hbd/
|
||||
|
||||
# Install for development
|
||||
pip install -e ".[all,dev]"
|
||||
|
||||
# Run server
|
||||
python -m hbd.server.cli serve -c .hb.yaml -f -v
|
||||
|
||||
# Generate a password hash
|
||||
hbd passwd <username>
|
||||
```
|
||||
|
||||
Tests live in `tests/` (pytest, imported as a package). A second directory `test/` contains only TLS fixtures (`.pem` files), not test code.
|
||||
|
||||
Line length: 111 characters (`black`, `flake8`, both configured).
|
||||
|
||||
## Architecture
|
||||
|
||||
The system has two components: `hbc` (client) and `hbd` (server), both in the `hbd` Python package.
|
||||
|
||||
```
|
||||
hbd/
|
||||
common/ # proto.py (encode/decode), utils.py
|
||||
client/ # hbc: plugins/, main.py, config.py, plugin.py
|
||||
server/ # hbd: all server modules
|
||||
```
|
||||
|
||||
### Server module map
|
||||
|
||||
| Module | Role |
|
||||
|---|---|
|
||||
| `cli.py` | Argument parsing, entry point, daemon startup |
|
||||
| `main.py` | Asyncio runtime: starts UDP, HTTP, WebSocket; pickle save/load |
|
||||
| `hbdclass.py` | `Host` and `Connection` domain objects; DNS queue |
|
||||
| `udp.py` | UDP datagram listener; processes `HTB` and `PLG` messages; sets/resets overdue timers |
|
||||
| `http.py` | aiohttp web server; all REST API routes (`/api/0/`) and page routes |
|
||||
| `ws.py` | WebSocket broadcast to connected dashboard clients |
|
||||
| `notify.py` | Notification dispatch (Pushover, email, Mattermost, Matrix, Signal, SMS); `eventlog()` is the single choke-point for all alert events |
|
||||
| `threshold.py` | Threshold evaluation against plugin metrics; state transitions (OK/WARNING/CRITICAL/UNKNOWN) |
|
||||
| `monitor.py` | Timer cleanup on shutdown (reachability is event-driven via timers in `udp.py`, not polled) |
|
||||
| `data.py` | Shared in-memory message ring buffer |
|
||||
| `journal.py` | JSONL message journal with size-based rotation |
|
||||
| `users.py` | Session management, password hashing (PBKDF2), role checks |
|
||||
| `config.py` | Config loading and defaults |
|
||||
| `configio.py` | Config file read/write via `ruamel.yaml` (preserves comments) |
|
||||
| `settings.py` | Settings sections for the web UI settings page |
|
||||
| `dns.py` | `nsupdate` integration for dynamic DNS |
|
||||
| `oauth.py` | OAuth2 login (Gitea) |
|
||||
|
||||
### Key data flows
|
||||
|
||||
**Heartbeat received:** `udp.py` decodes the `HTB` datagram → updates `Connection` state in `hbdclass.py` → resets overdue asyncio timer → broadcasts via `ws.py` → `notify.py` fires connectivity alerts on state change.
|
||||
|
||||
**Plugin data received:** `udp.py` decodes `PLG` datagram → stores on `Host` → `threshold.py` evaluates against configured thresholds → `notify.py` fires threshold alerts on state transitions.
|
||||
|
||||
**State persistence:** `Host.hosts` dict + `data.msgs` ring + active sessions are pickled every 5 minutes and on clean shutdown. Asyncio timers are stripped before pickling (`Connection.__getstate__`).
|
||||
|
||||
**Config reload:** SIGHUP → `configio.py` re-reads YAML → live-updates hosts, thresholds, users, notification channels. Port/cert/pickle/journal changes require a full restart.
|
||||
|
||||
### Client plugin system
|
||||
|
||||
Plugins in `hbd/client/plugins/` subclass `InfoPlugin` (collected once, on demand) or `MonitorPlugin` (periodic). `initialize()` returns `False` to self-disable. Data is sent as `PLG` UDP messages.
|
||||
|
||||
`hbc_mini.py` (scripts/) and `hbc_mini.c` (scripts/c/) are standalone single-file clients with no external dependencies.
|
||||
|
||||
### Protocol
|
||||
|
||||
All UDP messages: `!<ID>: <zlib-compressed key=value payload>`. Encoding in `hbd/common/proto.py`. Lists/dicts encoded as JSON with `@` prefix; booleans as `1`/`0`.
|
||||
|
||||
### Web UI
|
||||
|
||||
Jinja2 templates in `hbd/server/templates/`. Static assets in `hbd/server/static/`. Live pages (`/live`, `/plugins`) use WebSocket connections for real-time push.
|
||||
|
||||
### CI
|
||||
|
||||
Gitea Actions workflow at `.gitea/workflows/release.yml`.
|
||||
@@ -0,0 +1,210 @@
|
||||
# Heartbeat
|
||||
|
||||
Heartbeat is a lightweight host monitoring system built around a simple idea: each machine you want to monitor runs a small client (`hbc`) that sends a UDP "heartbeat" packet to a central server (`hbd`) on a regular interval. If a heartbeat stops arriving, you get notified. Alongside reachability, clients can ship system metrics — CPU, memory, disk, network — and the server will alert you when any of those cross a threshold.
|
||||
|
||||
## How it works
|
||||
|
||||
```
|
||||
[ monitored host ] [ your server ]
|
||||
┌─────────────┐ UDP 50003 ┌────────────────────────┐
|
||||
│ hbc │ ────────────> │ hbd │
|
||||
│ │ │ host state tracking │
|
||||
│ plugins: │ <──────────── │ threshold alerting │
|
||||
│ cpu, mem, │ ACK / CMD │ notifications │
|
||||
│ disk, ... │ │ web dashboard + API │
|
||||
└─────────────┘ └────────────────────────┘
|
||||
```
|
||||
|
||||
- **hbd** — the server daemon. Tracks which hosts are alive, evaluates metric thresholds, fires notifications, serves the web dashboard and REST API.
|
||||
- **hbc** — the client. Sends heartbeats and plugin data over UDP. Runs on any Linux/BSD/macOS host.
|
||||
- **hbc_mini** — a zero-dependency single-file alternative (`hbc_mini.py` or `hbc_mini.c`) for hosts where you can't install Python packages.
|
||||
|
||||
Notifications can go to Pushover, email, Mattermost, Matrix, Signal, or VoIP.ms SMS. The dashboard shows host connectivity, RTT graphs, active alerts, and per-host plugin metrics in real time via WebSocket.
|
||||
|
||||
---
|
||||
|
||||
## Getting started
|
||||
|
||||
This tutorial sets up a server on one machine and a client on a second machine. You'll end up with a working dashboard and your first host being monitored.
|
||||
|
||||
### 1. Install the server
|
||||
|
||||
On the machine that will run `hbd`:
|
||||
|
||||
```bash
|
||||
git clone https://git.wrede.ca/andreas/heartbeat.git
|
||||
cd heartbeat
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install .
|
||||
```
|
||||
|
||||
Verify the install:
|
||||
|
||||
```bash
|
||||
hbd --help
|
||||
```
|
||||
|
||||
### 2. Create a server config
|
||||
|
||||
Create `~/.hb.yaml`:
|
||||
|
||||
```yaml
|
||||
hb_port: 50003 # UDP port — clients send heartbeats here
|
||||
hbd_port: 50004 # HTTP port — web dashboard and API
|
||||
ws_port: 50005 # WebSocket port — live dashboard updates
|
||||
|
||||
interval: 20 # Expected heartbeat interval (seconds)
|
||||
grace: 2 # Seconds of slack before a host is considered overdue
|
||||
|
||||
pickfile: ~/.hb.pick
|
||||
pidfile: ~/.hb.pid
|
||||
logfile: ~/.hb.log
|
||||
```
|
||||
|
||||
That's enough to get started. No hosts, no users, no notifications needed yet — the server will accept any client that connects.
|
||||
|
||||
### 3. Start the server
|
||||
|
||||
```bash
|
||||
hbd serve -c ~/.hb.yaml -f -v
|
||||
```
|
||||
|
||||
`-f` keeps it in the foreground so you can watch the log. You should see:
|
||||
|
||||
```
|
||||
Heartbeat daemon starting on UDP :50003, HTTP :50004, WS :50005
|
||||
```
|
||||
|
||||
Open `http://your-server:50004/live` in a browser. The dashboard is empty for now.
|
||||
|
||||
### 4. Install the client on a host to monitor
|
||||
|
||||
On the machine you want to monitor (must be able to reach the server on UDP 50003):
|
||||
|
||||
```bash
|
||||
pip install hbd # or: copy scripts/hbc_mini.py if you can't install packages
|
||||
```
|
||||
|
||||
#### Quick start — no config file
|
||||
|
||||
```bash
|
||||
hbc your-server.example.com
|
||||
```
|
||||
|
||||
Within a few seconds the server log will show the host checking in, and it will appear on the dashboard.
|
||||
|
||||
#### With a config file
|
||||
|
||||
Create `~/.hbc.yaml` on the client host:
|
||||
|
||||
```yaml
|
||||
hb_port: 50003
|
||||
interval: 10 # Send a heartbeat every 10 seconds
|
||||
|
||||
plugins:
|
||||
cpu_monitor:
|
||||
interval: 60
|
||||
memory_monitor:
|
||||
interval: 60
|
||||
disk_monitor:
|
||||
interval: 60
|
||||
```
|
||||
|
||||
Then start the client:
|
||||
|
||||
```bash
|
||||
hbc -c ~/.hbc.yaml your-server.example.com
|
||||
```
|
||||
|
||||
Send a boot message at startup so the server logs when the host came up:
|
||||
|
||||
```bash
|
||||
hbc -b -c ~/.hbc.yaml your-server.example.com
|
||||
```
|
||||
|
||||
Run as a daemon (logs go to syslog):
|
||||
|
||||
```bash
|
||||
hbc -d -b -c ~/.hbc.yaml your-server.example.com
|
||||
```
|
||||
|
||||
### 5. View the dashboard
|
||||
|
||||
Open `http://your-server:50004/live`. You'll see the monitored host, its last heartbeat time, and RTT. Click the host name to see plugin metrics.
|
||||
|
||||
Navigate to `/plugins/<hostname>` for CPU, memory, and disk graphs.
|
||||
|
||||
### 6. Add a notification channel (optional)
|
||||
|
||||
Edit `~/.hb.yaml` on the server:
|
||||
|
||||
```yaml
|
||||
notification_channels:
|
||||
pushover_ops:
|
||||
type: pushover
|
||||
token: YOUR_APP_TOKEN
|
||||
user: YOUR_USER_KEY
|
||||
|
||||
users:
|
||||
alice:
|
||||
password: pbkdf2:sha256:... # generate: hbd passwd alice
|
||||
admin: true
|
||||
notification_channels: [pushover_ops]
|
||||
|
||||
default_owner: alice
|
||||
```
|
||||
|
||||
Generate the password hash:
|
||||
|
||||
```bash
|
||||
hbd passwd alice
|
||||
```
|
||||
|
||||
Paste the output into the config, then reload:
|
||||
|
||||
```bash
|
||||
hbd reload
|
||||
```
|
||||
|
||||
Test the channel:
|
||||
|
||||
```bash
|
||||
hbd notify
|
||||
```
|
||||
|
||||
### 7. Set a threshold alert (optional)
|
||||
|
||||
Add to `~/.hb.yaml`:
|
||||
|
||||
```yaml
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
cpu_percent:
|
||||
warning: 80.0
|
||||
critical: 90.0
|
||||
disk_monitor:
|
||||
partitions:
|
||||
/:
|
||||
percent:
|
||||
warning: 80.0
|
||||
critical: 90.0
|
||||
```
|
||||
|
||||
Reload: `hbd reload`. The server will now alert when a monitored host crosses these values.
|
||||
|
||||
---
|
||||
|
||||
## What's next
|
||||
|
||||
| Topic | Where to look |
|
||||
|---|---|
|
||||
| Full server config reference | [README — Server](https://git.wrede.ca/andreas/heartbeat/src/branch/master/README.md#server-hbd) |
|
||||
| Client options and all plugins | [README — Client](https://git.wrede.ca/andreas/heartbeat/src/branch/master/README.md#client-hbc) |
|
||||
| Threshold alerting details | [THRESHOLD_ALERTING.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/THRESHOLD_ALERTING.md) |
|
||||
| Notification channels | [NOTIFICATIONS.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/NOTIFICATIONS.md) |
|
||||
| User accounts and roles | [USERS.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/USERS.md) |
|
||||
| Writing a custom plugin | [PLUGIN_DEVELOPMENT.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/PLUGIN_DEVELOPMENT.md) |
|
||||
| Nagios check integration | [NAGIOS_INTEGRATION.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/NAGIOS_INTEGRATION.md) |
|
||||
| REST API | [HTTP_API.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/HTTP_API.md) |
|
||||
| Zero-dependency client | [README — hbc_mini](https://git.wrede.ca/andreas/heartbeat/src/branch/master/README.md#hbc_mini--zero-dependency-client) |
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# MIT License
|
||||
|
||||
Copyright (c) 2002 - 2026 Andreas Wrede
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
-234
@@ -1,234 +0,0 @@
|
||||
# HBD/HBC Separation Refactoring
|
||||
|
||||
## Overview
|
||||
|
||||
The heartbeat monitoring system has been refactored into a modular package structure with separate client and server components. This allows users to install only what they need and provides clear separation of concerns.
|
||||
|
||||
## New Package Structure
|
||||
|
||||
```
|
||||
hbd/
|
||||
├── __init__.py # Main package (minimal)
|
||||
├── client/ # HBC - System monitoring client
|
||||
│ ├── __init__.py
|
||||
│ ├── main.py # Entry point (was hbc.py)
|
||||
│ ├── config.py # Client-specific configuration
|
||||
│ ├── plugin.py # Plugin framework
|
||||
│ ├── threshold.py # Threshold checking
|
||||
│ └── plugins/ # Monitoring plugins
|
||||
│ ├── cpu_monitor.py
|
||||
│ ├── disk_monitor.py
|
||||
│ ├── memory_monitor.py
|
||||
│ ├── network_monitor.py
|
||||
│ ├── filesystem_info.py
|
||||
│ ├── os_info.py
|
||||
│ └── nagios_runner.py
|
||||
├── server/ # HBD - Heartbeat daemon/server
|
||||
│ ├── __init__.py
|
||||
│ ├── main.py # Server runtime (was server.py)
|
||||
│ ├── cli.py # Command-line interface
|
||||
│ ├── config.py # Server-specific configuration
|
||||
│ ├── http.py # HTTP/REST API
|
||||
│ ├── ws.py # WebSocket server
|
||||
│ ├── udp.py # UDP heartbeat listener
|
||||
│ ├── dns.py # DNS update functionality
|
||||
│ ├── notify.py # Notification handlers
|
||||
│ ├── monitor.py # Host monitoring
|
||||
│ ├── hbdclass.py # Host class definitions
|
||||
│ ├── journal.py # Message journaling
|
||||
│ ├── templates/ # Jinja2 web templates
|
||||
│ └── static/ # Web UI assets
|
||||
└── common/ # Shared utilities
|
||||
├── __init__.py
|
||||
├── proto.py # Protocol encoding/decoding
|
||||
└── utils.py # Common utilities
|
||||
|
||||
## Configuration Files
|
||||
|
||||
### Client Configuration (hbd/client/config.py)
|
||||
|
||||
Client-specific defaults:
|
||||
- `hb_port`: Port where hbd servers listen (default: 50003)
|
||||
- `interval`: Heartbeat interval in seconds (default: 10)
|
||||
- `plugins`: Per-plugin configuration
|
||||
- `thresholds`: Threshold configuration for monitoring
|
||||
|
||||
### Server Configuration (hbd/server/config.py)
|
||||
|
||||
Server-specific defaults:
|
||||
- `hb_port`: Port to listen for heartbeats (default: 50003)
|
||||
- `hbd_port`: HTTP API port (default: 50004)
|
||||
- `ws_port`: WebSocket port (default: 50005)
|
||||
- `logfile`: Log file path
|
||||
- `pushsrv`, `pushover_token`, etc.: Notification settings
|
||||
- `watchhosts`, `dyndnshosts`: Host monitoring
|
||||
- `smtpserver`, etc.: Email settings
|
||||
- `journal_*`: Message journaling settings
|
||||
|
||||
## Installation Options
|
||||
|
||||
### Install Core Only (minimal, PyYAML only)
|
||||
```bash
|
||||
pip install hbd
|
||||
```
|
||||
|
||||
### Install Client Only (for monitoring)
|
||||
```bash
|
||||
pip install hbd[client]
|
||||
# Installs: PyYAML, psutil
|
||||
```
|
||||
|
||||
### Install Server Only (for daemon)
|
||||
```bash
|
||||
pip install hbd[server]
|
||||
# Installs: PyYAML, websockets, mattermostdriver, aiohttp, Jinja2
|
||||
```
|
||||
|
||||
### Install Everything
|
||||
```bash
|
||||
pip install hbd[all]
|
||||
# Installs all dependencies for both client and server
|
||||
```
|
||||
|
||||
### Development Installation
|
||||
```bash
|
||||
pip install -e ".[dev]"
|
||||
# Includes all dependencies plus testing/linting tools
|
||||
```
|
||||
|
||||
## Command-Line Interfaces
|
||||
|
||||
### HBC (Client)
|
||||
```bash
|
||||
hbc [options] host1 [host2 ...]
|
||||
|
||||
# Entry point: hbd.client.main:main
|
||||
# Location: hbd/client/main.py
|
||||
```
|
||||
|
||||
### HBD (Server)
|
||||
```bash
|
||||
hbd [options]
|
||||
|
||||
# Entry point: hbd.server.cli:main
|
||||
# Location: hbd/server/cli.py → hbd/server/main.py
|
||||
```
|
||||
|
||||
## Import Changes
|
||||
|
||||
### Client Code
|
||||
```python
|
||||
# Old imports
|
||||
from .config import load_config
|
||||
from .proto import dicttos, stodict
|
||||
from .plugin import PluginRegistry
|
||||
|
||||
# New imports
|
||||
from .config import load_config # Still in client/
|
||||
from ..common.proto import dicttos # Moved to common/
|
||||
from .plugin import PluginRegistry # Still in client/
|
||||
```
|
||||
|
||||
### Server Code
|
||||
```python
|
||||
# Old imports
|
||||
from .config import load_config
|
||||
from .proto import stodict
|
||||
from .threshold import AlertLevel
|
||||
|
||||
# New imports
|
||||
from .config import load_config # Server-specific config
|
||||
from ..common.proto import stodict # Moved to common/
|
||||
from ..client.threshold import AlertLevel # Client module
|
||||
```
|
||||
|
||||
### Plugin Code
|
||||
```python
|
||||
# Old import
|
||||
from hbd.plugin import MonitorPlugin
|
||||
|
||||
# New import
|
||||
from hbd.client.plugin import MonitorPlugin
|
||||
```
|
||||
|
||||
## Benefits
|
||||
|
||||
1. **Modular Installation**: Install only what you need
|
||||
- Client-only systems don't need web server dependencies
|
||||
- Server-only systems don't need psutil
|
||||
|
||||
2. **Clearer Architecture**: Explicit separation of concerns
|
||||
- Client: System monitoring and data collection
|
||||
- Server: Heartbeat reception, web UI, notifications
|
||||
- Common: Shared protocol and utilities
|
||||
|
||||
3. **Independent Evolution**: Client and server can evolve separately
|
||||
- Different release cycles possible
|
||||
- Clear API boundaries via common/
|
||||
|
||||
4. **Smaller Footprint**: Reduced dependency installation
|
||||
- Client: ~1 dependency (psutil)
|
||||
- Server: ~4 dependencies (websockets, aiohttp, Jinja2, mattermostdriver)
|
||||
|
||||
## Migration Guide
|
||||
|
||||
### For Existing Installations
|
||||
|
||||
1. **Reinstall the package**:
|
||||
```bash
|
||||
pip install -e ".[all]" # For development
|
||||
# or
|
||||
pip install hbd[all] # For production
|
||||
```
|
||||
|
||||
2. **Configuration files remain unchanged**:
|
||||
- Both client and server read from `~/.hb.yaml`
|
||||
- All existing config keys are supported in both configs
|
||||
- Server has additional keys (journal, websocket, email, etc.)
|
||||
- Client has minimal keys (interval, plugins, thresholds)
|
||||
|
||||
3. **Commands remain the same**:
|
||||
- `hbc` command works identically
|
||||
- `hbd` command works identically
|
||||
|
||||
### For New Deployments
|
||||
|
||||
1. **Client-only system** (monitoring host):
|
||||
```bash
|
||||
pip install hbd[client]
|
||||
hbc server1.example.com server2.example.com
|
||||
```
|
||||
|
||||
2. **Server-only system** (monitoring daemon):
|
||||
```bash
|
||||
pip install hbd[server]
|
||||
hbd -c /etc/hbd.yaml -f
|
||||
```
|
||||
|
||||
3. **Combined system** (dev/test):
|
||||
```bash
|
||||
pip install hbd[all]
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
All imports and entry points have been tested and validated:
|
||||
- ✅ Package imports work correctly
|
||||
- ✅ `hbc` command entry point functional
|
||||
- ✅ `hbd` command entry point functional
|
||||
- ✅ Optional dependencies properly configured
|
||||
- ✅ All internal imports updated
|
||||
|
||||
## Files Archived
|
||||
|
||||
The following files were renamed to avoid conflicts:
|
||||
- `hbd/config.py` → `hbd/config.py.old` (split into client/server configs)
|
||||
- `hbd/hbc_old.py` → `hbd/hbc_old.py.bak` (backup file)
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Test client functionality with a monitoring host
|
||||
2. Test server functionality with web UI and notifications
|
||||
3. Update documentation (README.md) with new structure
|
||||
4. Consider publishing to PyPI with new structure
|
||||
5. Update any deployment scripts/Dockerfiles to use optional dependencies
|
||||
@@ -0,0 +1,66 @@
|
||||
# Dark Mode
|
||||
|
||||
Every page in the Heartbeat web UI supports light mode, dark mode, and automatic (follows the OS/browser setting). Each user picks their preference independently; it is stored in the browser and takes effect immediately without a page reload.
|
||||
|
||||
---
|
||||
|
||||
## Choosing a theme
|
||||
|
||||
Open your profile page (`/profile`) and scroll to the **Appearance** section. Click one of the three buttons:
|
||||
|
||||
| Button | Behaviour |
|
||||
|--------|-----------|
|
||||
| **Auto** | Follows the OS or browser dark-mode preference. Updates live if the system setting changes. |
|
||||
| **Light** | Always light, regardless of system setting. |
|
||||
| **Dark** | Always dark, regardless of system setting. |
|
||||
|
||||
The preference is stored in `localStorage` under the key `hbd_theme` and applies to the current browser only. Clearing browser storage resets it to **Auto**.
|
||||
|
||||
---
|
||||
|
||||
## Implementation notes
|
||||
|
||||
### No flash of unstyled content
|
||||
|
||||
A small synchronous `<script>` runs at the very top of `<head>`, before any CSS is parsed, and sets `data-theme="dark"` on `<html>` when the stored preference (or the system setting in auto mode) calls for dark. Because it runs before paint, there is no visible flicker on page load.
|
||||
|
||||
### CSS custom properties
|
||||
|
||||
All colours are expressed as CSS custom properties defined in `head.html`:
|
||||
|
||||
```
|
||||
:root — light-mode values (default)
|
||||
html[data-theme="dark"] — dark-mode overrides
|
||||
```
|
||||
|
||||
Key variables:
|
||||
|
||||
| Variable | Purpose |
|
||||
|----------|---------|
|
||||
| `--bg` | Page background |
|
||||
| `--surface` | Card / panel background |
|
||||
| `--surface-2` / `--surface-3` | Slightly lighter/darker surfaces (table rows, hover states) |
|
||||
| `--text` / `--text-sec` / `--text-muted` | Primary, secondary, muted text |
|
||||
| `--border` / `--border-2`…`4` | Border shades from prominent to faint |
|
||||
| `--link` | Hyperlink and interactive-element colour |
|
||||
| `--nav-bg` | Navigation bar background |
|
||||
| `--input-bg` / `--input-border` | Form control colours |
|
||||
| `--shadow` / `--shadow-sm` | Box-shadow alphas |
|
||||
|
||||
A single global rule in `head.html` themes all `<input>`, `<select>`, and `<textarea>` elements across every page at once:
|
||||
|
||||
```css
|
||||
html[data-theme="dark"] input:not([type=checkbox]):not([type=radio]),
|
||||
html[data-theme="dark"] select,
|
||||
html[data-theme="dark"] textarea { … }
|
||||
```
|
||||
|
||||
Each page template adds its own `html[data-theme="dark"]` block for page-specific elements (cards, tables, badges, etc.).
|
||||
|
||||
### Auto-mode live updates
|
||||
|
||||
A `matchMedia` change listener in `head.html` updates `data-theme` whenever the OS preference changes, so users in **Auto** mode see the theme switch without reloading.
|
||||
|
||||
### Semantic colours are unchanged
|
||||
|
||||
Alert colours (red for critical, orange for warning, green for ok) and status indicators are intentionally left as fixed values — they are semantic signals, not surface colours, and look correct on both light and dark backgrounds.
|
||||
@@ -53,6 +53,17 @@ See [User Management](USERS.md) for full authentication documentation.
|
||||
|--------|------|-------------|------|
|
||||
| `GET` | `/api/0/users` | List all users | Admin |
|
||||
| `GET` | `/api/0/users/me` | Own profile | Authenticated |
|
||||
| `PUT` | `/api/0/users/me` | Update own profile | Authenticated |
|
||||
|
||||
### Notification Channels
|
||||
|
||||
| Method | Path | Description | Role |
|
||||
|--------|------|-------------|------|
|
||||
| `GET` | `/api/0/notification_channel_types` | Channel type schemas | Authenticated |
|
||||
| `GET` | `/api/0/notification_channels` | List visible channels | Authenticated |
|
||||
| `POST` | `/api/0/notification_channels` | Create a channel | Authenticated |
|
||||
| `PUT` | `/api/0/notification_channels/{name}` | Update a channel | Owner or Admin |
|
||||
| `DELETE` | `/api/0/notification_channels/{name}` | Delete a channel | Owner or Admin |
|
||||
|
||||
### Host Management
|
||||
|
||||
@@ -203,6 +214,101 @@ Changes take effect immediately but are not written back to the config file. Upd
|
||||
|
||||
---
|
||||
|
||||
---
|
||||
|
||||
### Notification Channel Endpoints
|
||||
|
||||
Channels are visible to all users by default. Channels marked `private: true` are only visible to their owner. Admins see all channels.
|
||||
|
||||
#### GET /api/0/notification_channel_types
|
||||
Return the schema for every supported notifier type. Used by the web UI to dynamically render the channel creation form.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"pushover": {
|
||||
"label": "Pushover",
|
||||
"fields": [
|
||||
{"key": "token", "label": "App token", "type": "secret", "required": true},
|
||||
{"key": "user", "label": "User key", "type": "secret", "required": true},
|
||||
{"key": "sound", "label": "Sound", "type": "text", "required": false}
|
||||
]
|
||||
},
|
||||
"email": { "label": "E-mail", "fields": [ ... ] },
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### GET /api/0/notification_channels
|
||||
List channels visible to the current user (public channels + own private channels). Admins receive all channels.
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "pushover_ops",
|
||||
"type": "pushover",
|
||||
"type_label": "Pushover",
|
||||
"owner": null,
|
||||
"private": false,
|
||||
"min_level": "WARNING",
|
||||
"fields": [
|
||||
{"key": "token", "label": "App token", "value": "•••", "sensitive": true},
|
||||
{"key": "user", "label": "User key", "value": "•••", "sensitive": true}
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Sensitive fields (`type: "secret"`) are always returned as `"•••"`.
|
||||
|
||||
---
|
||||
|
||||
#### POST /api/0/notification_channels
|
||||
Create a new channel. The creating user becomes the channel's `owner`.
|
||||
|
||||
**Request body:**
|
||||
```json
|
||||
{
|
||||
"name": "my_pushover",
|
||||
"type": "pushover",
|
||||
"token": "app-token",
|
||||
"user": "user-key",
|
||||
"min_level": "WARNING",
|
||||
"private": true
|
||||
}
|
||||
```
|
||||
|
||||
**Response:** `{"ok": true, "name": "my_pushover"}`
|
||||
|
||||
**Status codes:** `200 OK`, `400` (missing required field or unknown type), `409` (name already exists)
|
||||
|
||||
---
|
||||
|
||||
#### PUT /api/0/notification_channels/{name}
|
||||
Update an existing channel. Only the channel owner or an admin may update it.
|
||||
|
||||
Secret fields sent as `"•••"` are preserved from the existing config (same pattern as OAuth secrets in the admin config editor).
|
||||
|
||||
**Request body:** same shape as POST, `name` ignored (taken from URL).
|
||||
|
||||
**Response:** `{"ok": true}`
|
||||
|
||||
**Status codes:** `200 OK`, `403 Forbidden`, `404 Not Found`
|
||||
|
||||
---
|
||||
|
||||
#### DELETE /api/0/notification_channels/{name}
|
||||
Delete a channel. Only the channel owner or an admin may delete it.
|
||||
|
||||
**Response:** `{"ok": true}`
|
||||
|
||||
**Status codes:** `200 OK`, `403 Forbidden`, `404 Not Found`
|
||||
|
||||
---
|
||||
|
||||
### Alert Endpoints
|
||||
|
||||
#### GET /api/0/hosts/{hostname}/alerts
|
||||
|
||||
@@ -406,6 +406,21 @@ Potential improvements for future versions:
|
||||
- Journal file encryption
|
||||
- Signed journal entries
|
||||
|
||||
## Events journal
|
||||
|
||||
Alert/connectivity events shown on the `/log` page are written to a second,
|
||||
dedicated journal (one JSON object per line, same rotation mechanism):
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `events_journal_file` | `events.journal` | Filename inside `journal_dir` |
|
||||
| `events_journal_max_size` | 10 MB | Rotation threshold |
|
||||
| `events_journal_max_backups` | 10 | Rotated files kept |
|
||||
|
||||
`journal_dir` and `journal_enabled` are shared with the message journal.
|
||||
On startup, if the events journal file is empty, it is seeded once from the
|
||||
pickled in-memory message ring so history survives the upgrade.
|
||||
|
||||
## See Also
|
||||
|
||||
- [Configuration Guide](../hbd/config.py) - Full configuration options
|
||||
|
||||
@@ -104,11 +104,6 @@ The `nagios_runner` plugin collects:
|
||||
- `{name}_{metric}_min` - Minimum value (if present)
|
||||
- `{name}_{metric}_max` - Maximum value (if present)
|
||||
|
||||
**Overall:**
|
||||
- `overall_status` - Worst status from all commands
|
||||
- `overall_status_code` - Worst status code
|
||||
- `plugin_count` - Number of Nagios plugins executed
|
||||
|
||||
## Configuration Options
|
||||
|
||||
```yaml
|
||||
|
||||
+78
-7
@@ -9,6 +9,7 @@ Notifications are dispatched to the **owner and managers** of a host, each via t
|
||||
```
|
||||
Alert event (udp.py / threshold.py)
|
||||
└─ notify.send_notification(host_name, Notification)
|
||||
├─ flap.observe(host, service, level) → pass | trip | suppress
|
||||
├─ look up host.owner + host.managers
|
||||
├─ for each user → user.notification_channels
|
||||
└─ for each channel → _dispatch_to_channel (filtered by min_level)
|
||||
@@ -19,6 +20,7 @@ Every notification carries:
|
||||
- **body** — detail message (metric value, threshold, duration)
|
||||
- **url** — link to the plugin metrics page (`{base_url}/plugins#{hostname}`)
|
||||
- **level** — `RECOVER | WARNING | CRITICAL | INFO`
|
||||
- **service** — flap-detection key within the host (empty = the host itself)
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -30,9 +32,18 @@ Set `base_url` so notification links point to your hbd instance:
|
||||
base_url: https://hbd.example.com
|
||||
```
|
||||
|
||||
### Global channel definitions
|
||||
### Channel definitions
|
||||
|
||||
Define channels once; reference them by name from user configs:
|
||||
Channels are defined under `notification_channels`. Each entry specifies a delivery type and its credentials. Ownership is the single visibility signal:
|
||||
|
||||
| Field | Default | Description |
|
||||
|---|---|---|
|
||||
| `owner` | *(absent)* | Owning username. Present = private to that user; absent = global. |
|
||||
| `min_level` | `WARNING` | Minimum alert level this channel receives. |
|
||||
|
||||
(The former `private` flag is retired; leftover `private` keys are ignored and dropped on the next edit.)
|
||||
|
||||
**Global channels** (no `owner`; set in the config file or by an admin) can be selected by all users but edited only by admins:
|
||||
|
||||
```yaml
|
||||
notification_channels:
|
||||
@@ -41,7 +52,7 @@ notification_channels:
|
||||
type: pushover
|
||||
token: your-app-token
|
||||
user: your-user-key
|
||||
min_level: WARNING # optional, default: WARNING
|
||||
min_level: WARNING
|
||||
|
||||
email_ops:
|
||||
type: email
|
||||
@@ -58,14 +69,14 @@ notification_channels:
|
||||
homeserver: https://matrix.example.org
|
||||
access_token: syt_xxx
|
||||
room_id: "!abc:matrix.example.org"
|
||||
min_level: CRITICAL # only send critical alerts to this room
|
||||
min_level: CRITICAL
|
||||
|
||||
sms_oncall:
|
||||
type: sms_voipms
|
||||
api_user: me@example.com
|
||||
api_password: secret
|
||||
did: "5551234567" # your voip.ms DID number
|
||||
dst: "5559876543" # destination number
|
||||
did: "5551234567"
|
||||
dst: "5559876543"
|
||||
min_level: CRITICAL
|
||||
|
||||
signal_ops:
|
||||
@@ -82,9 +93,31 @@ notification_channels:
|
||||
username: heartbeat-bot
|
||||
```
|
||||
|
||||
**User-created channels** are written by authenticated users through the API, their profile page, or the settings page. They carry an `owner` field and are private to that user:
|
||||
|
||||
```yaml
|
||||
notification_channels:
|
||||
|
||||
alice_personal:
|
||||
type: pushover
|
||||
token: personal-token
|
||||
user: personal-key
|
||||
owner: alice # private to alice
|
||||
```
|
||||
|
||||
### Channel visibility
|
||||
|
||||
| Channel | Who can see / select it | Who can edit it |
|
||||
|---|---|---|
|
||||
| No `owner` (global) | All users | Admins |
|
||||
| `owner` set (private) | Only the `owner` | The owner |
|
||||
| Any channel | Admins always see everything | Admins |
|
||||
|
||||
Admins can **promote** a private channel to global by clearing its owner on the settings page, or **demote** a global channel by assigning an owner.
|
||||
|
||||
### Users with notification channels
|
||||
|
||||
Each user lists which global channels they receive notifications on:
|
||||
Each user lists which channels they receive notifications on. Users can manage their own selection from the profile page:
|
||||
|
||||
```yaml
|
||||
users:
|
||||
@@ -237,6 +270,43 @@ min_level: WARNING
|
||||
|
||||
Reminder notifications (re-notify) are sent only for CRITICAL level alerts.
|
||||
|
||||
## Flap detection
|
||||
|
||||
A check that toggles between OK and alerting produces a notification per swing. Flap
|
||||
detection silences it after the first few.
|
||||
|
||||
A **`(host, service)`** pair is flapping once it exceeds `flap_count` WARNING/CRITICAL
|
||||
notifications within `flap_interval` minutes. Threshold alerts key on their metric path,
|
||||
so a flapping disk check does not silence an unrelated CPU alert; connectivity, boot and
|
||||
shutdown events key on the host itself.
|
||||
|
||||
```yaml
|
||||
flap_count: 5 # notifications within the window that trip flapping (0 disables)
|
||||
flap_interval: 10 # minutes — both the counting window and the quiet window
|
||||
```
|
||||
|
||||
Lifecycle:
|
||||
|
||||
| Event | Effect |
|
||||
|---|---|
|
||||
| Alerts 1..`flap_count` within the window | Delivered normally |
|
||||
| Alert `flap_count + 1` | Delivered with ` Now flapping!! No more messages!` appended to the body |
|
||||
| Every notification after that | Dropped — including RECOVER and INFO |
|
||||
| RECOVER while flapping | Dropped, and starts the `flap_interval` quiet window |
|
||||
| WARNING/CRITICAL during the quiet window | Restarts the quiet window; still flapping |
|
||||
| Quiet window elapses | Flapping ends **silently** — no notification |
|
||||
|
||||
Only outbound notifications are suppressed. `notify.eventlog` keeps recording every event,
|
||||
so the journal and the `/log` page retain the full history of the flap.
|
||||
|
||||
Flapping pairs appear in each host's `stateinfo()` under `flapping` (a list of service
|
||||
keys; `""` means the host itself) and render as an amber **flapping** badge next to the
|
||||
host name on the live dashboard, with the affected services in its tooltip.
|
||||
|
||||
State lives at module level in `hbd/server/flap.py` and is never pickled — a server restart
|
||||
starts every check with a clean slate. Hosts with `watch: false` never reach the
|
||||
notification path, so they never flap.
|
||||
|
||||
## API reference
|
||||
|
||||
### `send_notification(host_name, notif) -> dict`
|
||||
@@ -270,6 +340,7 @@ Called once at startup from `main.py`. Pass the running asyncio event loop so Ma
|
||||
- Check that the host has an `owner` or `managers` set
|
||||
- Check that users have `notification_channels` listed
|
||||
- Check that the channel names in user config match keys under `notification_channels:`
|
||||
- If a user can't select a channel, check whether it has an `owner` other than that user
|
||||
|
||||
**min_level filtering too aggressive:**
|
||||
- Default is `WARNING` — both WARNING and CRITICAL are sent
|
||||
|
||||
+33
-10
@@ -8,6 +8,7 @@ This guide explains how to create custom plugins for the Heartbeat monitoring sy
|
||||
- [Plugin Types](#plugin-types)
|
||||
- [Creating a Plugin](#creating-a-plugin)
|
||||
- [Plugin Lifecycle](#plugin-lifecycle)
|
||||
- [Server-initiated InfoPlugin refresh](#server-initiated-infoplugin-refresh)
|
||||
- [Configuration](#configuration)
|
||||
- [Best Practices](#best-practices)
|
||||
- [Examples](#examples)
|
||||
@@ -20,7 +21,7 @@ Heartbeat's plugin system is designed to be simple yet powerful. Plugins are Pyt
|
||||
### Key Concepts
|
||||
|
||||
- **Plugin Registry**: Central registry that manages all loaded plugins
|
||||
- **Plugin Loader**: Automatically discovers and loads plugins from the `hbd/plugins/` directory
|
||||
- **Plugin Loader**: Automatically discovers and loads plugins from the `hbd/client/plugins/` directory
|
||||
- **Plugin Types**: InfoPlugin (static data) and MonitorPlugin (periodic metrics)
|
||||
- **Async/Await**: All plugin methods are async for non-blocking operation
|
||||
|
||||
@@ -63,7 +64,7 @@ Decide whether your plugin collects static information (InfoPlugin) or dynamic m
|
||||
|
||||
### Step 2: Create Plugin File
|
||||
|
||||
Create a new Python file in `hbd/plugins/` directory:
|
||||
Create a new Python file in `hbd/client/plugins/` directory:
|
||||
|
||||
```python
|
||||
"""
|
||||
@@ -81,7 +82,7 @@ try:
|
||||
except ImportError:
|
||||
psutil = None
|
||||
|
||||
from hbd.plugin import MonitorPlugin # or InfoPlugin
|
||||
from hbd.client.plugin import MonitorPlugin # or InfoPlugin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -192,7 +193,7 @@ from pathlib import Path
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from hbd.plugins.my_awesome_plugin import MyAwesomePlugin
|
||||
from hbd.client.plugins.my_awesome_plugin import MyAwesomePlugin
|
||||
|
||||
async def test():
|
||||
# Create plugin instance
|
||||
@@ -223,7 +224,7 @@ Understanding the plugin lifecycle helps you implement plugins correctly:
|
||||
|
||||
```
|
||||
1. Plugin Discovery
|
||||
└─> Loader scans hbd/plugins/ directory
|
||||
└─> Loader scans hbd/client/plugins/ directory
|
||||
└─> Finds Python files (except those starting with _)
|
||||
└─> Imports modules
|
||||
|
||||
@@ -250,6 +251,28 @@ Understanding the plugin lifecycle helps you implement plugins correctly:
|
||||
└─> Plugin releases resources, closes connections
|
||||
```
|
||||
|
||||
## Server-initiated InfoPlugin refresh
|
||||
|
||||
When a heartbeat packet arrives from a host the server has no plugin data for (e.g. after a server restart), the server sets `request_update = 1` in the ACK reply. The client detects this flag and immediately re-runs all InfoPlugins — clearing their cached results first — then resends the data as PLG messages.
|
||||
|
||||
This means InfoPlugin data will always reach the server as soon as possible without requiring a client restart. No action is needed from plugin authors: the framework handles cache invalidation and re-collection automatically.
|
||||
|
||||
The lifecycle for this case looks like:
|
||||
|
||||
```
|
||||
Server restarts, host reconnects
|
||||
└─> hbd receives HTB with no existing plugin_data for host
|
||||
└─> hbd sets request_update=1 in ACK
|
||||
|
||||
Client receives ACK
|
||||
└─> Detects request_update flag
|
||||
└─> Clears _cache on every registered InfoPlugin
|
||||
└─> Calls collect() on each InfoPlugin
|
||||
└─> Sends fresh PLG messages to server
|
||||
```
|
||||
|
||||
If you write an `InfoPlugin` with side effects in `_collect_info()` (opening connections, writing files, etc.), be aware it may be called more than once per client session when this mechanism triggers.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Plugin-Specific Configuration
|
||||
@@ -355,7 +378,7 @@ Document your plugin thoroughly:
|
||||
### Example 1: Simple InfoPlugin
|
||||
|
||||
```python
|
||||
from hbd.plugin import InfoPlugin
|
||||
from hbd.client.plugin import InfoPlugin
|
||||
import platform
|
||||
|
||||
class SimpleInfoPlugin(InfoPlugin):
|
||||
@@ -383,7 +406,7 @@ plugin = SimpleInfoPlugin
|
||||
### Example 2: MonitorPlugin with State
|
||||
|
||||
```python
|
||||
from hbd.plugin import MonitorPlugin
|
||||
from hbd.client.plugin import MonitorPlugin
|
||||
import time
|
||||
|
||||
class CounterPlugin(MonitorPlugin):
|
||||
@@ -419,7 +442,7 @@ plugin = CounterPlugin
|
||||
### Example 3: Plugin with External Command
|
||||
|
||||
```python
|
||||
from hbd.plugin import MonitorPlugin
|
||||
from hbd.client.plugin import MonitorPlugin
|
||||
import asyncio
|
||||
|
||||
class CommandPlugin(MonitorPlugin):
|
||||
@@ -538,7 +561,7 @@ python -m hbd.hbc -c test_config.yaml --verbose
|
||||
|
||||
## Further Reading
|
||||
|
||||
- [Plugin Framework Source](../hbd/plugin.py) - Core plugin implementation
|
||||
- [Built-in Plugins](../hbd/plugins/) - Examples of working plugins
|
||||
- [Plugin Framework Source](../hbd/client/plugin.py) - Core plugin implementation
|
||||
- [Built-in Plugins](../hbd/client/plugins/) - Examples of working plugins
|
||||
- [Nagios Integration](NAGIOS_INTEGRATION.md) - Running external plugins
|
||||
- [Configuration Guide](../hbd/config_example.yaml) - Full configuration reference
|
||||
|
||||
+224
-65
@@ -256,6 +256,56 @@ disk_monitor:
|
||||
operator: "<"
|
||||
```
|
||||
|
||||
### ZFS Monitor
|
||||
|
||||
ZFS pool health is checked automatically for every pool. A pool in any state
|
||||
other than `ONLINE` (e.g. `DEGRADED`, `SUSPENDED`, `FAULTED`, `UNAVAIL`) raises
|
||||
a **CRITICAL** alert by default — no configuration required.
|
||||
|
||||
The default threshold is equivalent to:
|
||||
|
||||
```yaml
|
||||
zfs_monitor:
|
||||
pools:
|
||||
'*':
|
||||
status:
|
||||
warning: 1
|
||||
critical: 2
|
||||
operator: ">"
|
||||
hysteresis: 0.0
|
||||
display: "ZFS pool {pool_name} is {health}"
|
||||
```
|
||||
|
||||
`'*'` matches every pool on the host. The notification message includes the pool
|
||||
name and its current health string, e.g. `ZFS pool tank is DEGRADED`.
|
||||
|
||||
**Override for specific pools** — named pool entries take priority over `'*'`:
|
||||
|
||||
```yaml
|
||||
zfs_monitor:
|
||||
pools:
|
||||
# Suppress health alerts for a scratch pool (not mission-critical)
|
||||
scratch:
|
||||
status:
|
||||
enabled: false
|
||||
|
||||
# Capacity threshold for a specific pool
|
||||
tank:
|
||||
capacity:
|
||||
warning: 75.0
|
||||
critical: 90.0
|
||||
operator: ">"
|
||||
hysteresis: 0.05
|
||||
```
|
||||
|
||||
**Alert state paths** follow the pattern `zfs_monitor.<pool_name>.status`,
|
||||
so acknowledgements and silences target individual pools:
|
||||
|
||||
```
|
||||
zfs_monitor.tank.status
|
||||
zfs_monitor.backup.status
|
||||
```
|
||||
|
||||
### Network Monitor
|
||||
|
||||
```yaml
|
||||
@@ -814,34 +864,32 @@ Planned features:
|
||||
|
||||
## Multi-Threshold Configuration
|
||||
|
||||
**New in version 2.0**: Support for multiple named threshold configurations with per-host mapping.
|
||||
Support for multiple named threshold configurations with per-host mapping and composable layering.
|
||||
|
||||
### Overview
|
||||
|
||||
The multi-threshold feature allows you to:
|
||||
- Define multiple sets of threshold configurations
|
||||
- Map different hosts to different threshold sets
|
||||
- Define multiple named threshold configurations
|
||||
- Assign one or more configurations to each host
|
||||
- Compose configurations by layering — each named config's overrides are applied in order on top of the defaults
|
||||
- Use different sensitivity levels for different environments
|
||||
- Maintain a default configuration for unmapped hosts
|
||||
|
||||
### Configuration Structure
|
||||
|
||||
Named configurations are defined under `threshold_configs`. Each host selects which ones to use via `threshold_config` in the `hosts` section (a string for a single config, or a list to layer multiple):
|
||||
|
||||
```yaml
|
||||
# Optional: Set the default configuration name (defaults to "default")
|
||||
# Optional: set the default configuration name (defaults to "default")
|
||||
default_threshold_config: "default"
|
||||
|
||||
# Define multiple named threshold configurations
|
||||
threshold_configs:
|
||||
# Configuration name 1
|
||||
default:
|
||||
thresholds:
|
||||
# Standard threshold definitions
|
||||
cpu_monitor:
|
||||
cpu_percent:
|
||||
warning: 80.0
|
||||
critical: 90.0
|
||||
|
||||
# Configuration name 2
|
||||
high_sensitivity:
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
@@ -849,7 +897,6 @@ threshold_configs:
|
||||
warning: 60.0
|
||||
critical: 75.0
|
||||
|
||||
# Configuration name 3
|
||||
low_sensitivity:
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
@@ -857,14 +904,77 @@ threshold_configs:
|
||||
warning: 90.0
|
||||
critical: 95.0
|
||||
|
||||
# Map specific hosts to specific configurations
|
||||
host_threshold_mapping:
|
||||
prod-web-01: high_sensitivity
|
||||
prod-web-02: high_sensitivity
|
||||
dev-server-01: low_sensitivity
|
||||
# Unmapped hosts use default_threshold_config
|
||||
hosts:
|
||||
prod-web-01:
|
||||
threshold_config: high_sensitivity # single config
|
||||
|
||||
dev-server-01:
|
||||
threshold_config: low_sensitivity
|
||||
|
||||
# Hosts with no threshold_config use default_threshold_config
|
||||
```
|
||||
|
||||
### Composable Configurations (list form)
|
||||
|
||||
`threshold_config` can be a list. Configs are applied **left to right**: the defaults are the base, then each named config's overrides are layered on top. Later entries in the list win on any metric they define.
|
||||
|
||||
```yaml
|
||||
threshold_configs:
|
||||
default:
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
cpu_percent: {warning: 80, critical: 90}
|
||||
memory_monitor:
|
||||
memory_percent: {warning: 85, critical: 95}
|
||||
disk_monitor:
|
||||
partitions:
|
||||
/:
|
||||
percent: {warning: 80, critical: 90}
|
||||
|
||||
# Tighter CPU limits for busy servers
|
||||
high_cpu_load:
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
cpu_percent: {warning: 60, critical: 75}
|
||||
|
||||
# Tighter disk limits for data-heavy servers
|
||||
busy_disk:
|
||||
thresholds:
|
||||
disk_monitor:
|
||||
partitions:
|
||||
/:
|
||||
percent: {warning: 70, critical: 85}
|
||||
|
||||
hosts:
|
||||
# Gets default thresholds only
|
||||
web-01:
|
||||
threshold_config: default
|
||||
|
||||
# Gets tighter CPU limits, default memory and disk
|
||||
build-server:
|
||||
threshold_config: high_cpu_load
|
||||
|
||||
# Layers both: tighter CPU AND tighter disk, default memory
|
||||
db-01:
|
||||
threshold_config: [high_cpu_load, busy_disk]
|
||||
|
||||
# Three layers: busy_disk overrides high_cpu_load if they conflict
|
||||
storage-01:
|
||||
threshold_config: [default, high_cpu_load, busy_disk]
|
||||
```
|
||||
|
||||
**How layering works:**
|
||||
|
||||
Starting from the `default` thresholds:
|
||||
|
||||
| Layer | Applied config | Effect |
|
||||
|-------|---------------|--------|
|
||||
| Base | `default` | all default thresholds |
|
||||
| +1 | `high_cpu_load` | cpu_percent overridden to 60/75 |
|
||||
| +2 | `busy_disk` | disk percent overridden to 70/85; cpu_percent stays at 60/75 |
|
||||
|
||||
Each named config only overrides the metrics it explicitly defines. Metrics not mentioned in a config inherit from the layers beneath.
|
||||
|
||||
### Use Cases
|
||||
|
||||
#### 1. Environment-Based Thresholds
|
||||
@@ -887,11 +997,15 @@ threshold_configs:
|
||||
warning: 90.0 # More relaxed for dev
|
||||
critical: 98.0
|
||||
|
||||
host_threshold_mapping:
|
||||
prod-web-01: production
|
||||
prod-web-02: production
|
||||
dev-web-01: development
|
||||
dev-web-02: development
|
||||
hosts:
|
||||
prod-web-01:
|
||||
threshold_config: production
|
||||
prod-web-02:
|
||||
threshold_config: production
|
||||
dev-web-01:
|
||||
threshold_config: development
|
||||
dev-web-02:
|
||||
threshold_config: development
|
||||
```
|
||||
|
||||
#### 2. Server Role-Based Thresholds
|
||||
@@ -914,7 +1028,7 @@ threshold_configs:
|
||||
warning: 70.0
|
||||
critical: 85.0
|
||||
memory_monitor:
|
||||
percent:
|
||||
memory_percent:
|
||||
warning: 90.0 # Databases can use high memory
|
||||
critical: 97.0
|
||||
disk_monitor:
|
||||
@@ -927,17 +1041,23 @@ threshold_configs:
|
||||
cache:
|
||||
thresholds:
|
||||
memory_monitor:
|
||||
percent:
|
||||
memory_percent:
|
||||
warning: 95.0 # Redis/Memcached can use very high memory
|
||||
critical: 99.0
|
||||
|
||||
host_threshold_mapping:
|
||||
web-01: webserver
|
||||
web-02: webserver
|
||||
db-01: database
|
||||
db-02: database
|
||||
redis-01: cache
|
||||
memcached-01: cache
|
||||
hosts:
|
||||
web-01:
|
||||
threshold_config: webserver
|
||||
web-02:
|
||||
threshold_config: webserver
|
||||
db-01:
|
||||
threshold_config: database
|
||||
db-02:
|
||||
threshold_config: database
|
||||
redis-01:
|
||||
threshold_config: cache
|
||||
memcached-01:
|
||||
threshold_config: cache
|
||||
```
|
||||
|
||||
#### 3. Sensitivity Levels
|
||||
@@ -952,7 +1072,7 @@ threshold_configs:
|
||||
partitions:
|
||||
/:
|
||||
percent:
|
||||
warning: 70.0 # Very sensitive
|
||||
warning: 70.0
|
||||
critical: 80.0
|
||||
hysteresis: 0.15
|
||||
|
||||
@@ -976,52 +1096,91 @@ threshold_configs:
|
||||
critical: 98.0
|
||||
hysteresis: 0.05
|
||||
|
||||
host_threshold_mapping:
|
||||
payment-gateway: critical
|
||||
auth-server: critical
|
||||
web-01: standard
|
||||
web-02: standard
|
||||
test-server: relaxed
|
||||
hosts:
|
||||
payment-gateway:
|
||||
threshold_config: critical
|
||||
auth-server:
|
||||
threshold_config: critical
|
||||
web-01:
|
||||
threshold_config: standard
|
||||
web-02:
|
||||
threshold_config: standard
|
||||
test-server:
|
||||
threshold_config: relaxed
|
||||
```
|
||||
|
||||
### Backward Compatibility
|
||||
#### 4. Composable Profiles
|
||||
|
||||
The legacy single threshold configuration is fully supported:
|
||||
Build host-specific thresholds by combining small, focused configs:
|
||||
|
||||
```yaml
|
||||
# Old format - still works
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
cpu_percent:
|
||||
warning: 80.0
|
||||
critical: 90.0
|
||||
```
|
||||
|
||||
This is equivalent to:
|
||||
|
||||
```yaml
|
||||
# New format
|
||||
threshold_configs:
|
||||
# Baseline — everything at default levels
|
||||
default:
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
cpu_percent:
|
||||
warning: 80.0
|
||||
critical: 90.0
|
||||
```
|
||||
cpu_percent: {warning: 80, critical: 90}
|
||||
memory_monitor:
|
||||
memory_percent: {warning: 85, critical: 95}
|
||||
|
||||
# Overlay: tighter CPU only
|
||||
tight_cpu:
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
cpu_percent: {warning: 60, critical: 75}
|
||||
|
||||
# Overlay: tighter memory only
|
||||
tight_memory:
|
||||
thresholds:
|
||||
memory_monitor:
|
||||
memory_percent: {warning: 70, critical: 85}
|
||||
|
||||
# Overlay: extra disk partition for database servers
|
||||
db_disk:
|
||||
thresholds:
|
||||
disk_monitor:
|
||||
partitions:
|
||||
/var/lib/postgresql:
|
||||
percent: {warning: 75, critical: 88}
|
||||
|
||||
hosts:
|
||||
# Plain web server
|
||||
web-01:
|
||||
threshold_config: default
|
||||
|
||||
# Build server: tight CPU, default memory and disk
|
||||
build-01:
|
||||
threshold_config: tight_cpu
|
||||
|
||||
# Database: tight CPU + tight memory + extra disk partition
|
||||
db-01:
|
||||
threshold_config: [tight_cpu, tight_memory, db_disk]
|
||||
|
||||
# Replica database: tight memory + extra disk, normal CPU
|
||||
db-02:
|
||||
threshold_config: [tight_memory, db_disk]
|
||||
```
|
||||
### Configuration Priority
|
||||
|
||||
1. **Host-specific mapping**: If host is in `host_threshold_mapping`, use that config
|
||||
2. **Default config**: Use `default_threshold_config`
|
||||
3. **First alphabetically**: If default not found, use first config alphabetically
|
||||
4. **Legacy fallback**: If `threshold_configs` not present, use `thresholds`
|
||||
1. **Host `threshold_config` (list)**: Layer each named config's overrides left-to-right on top of the defaults
|
||||
2. **Host `threshold_config` (string)**: Use that single named config directly
|
||||
3. **`host_threshold_mapping`** (legacy): Same as above, string only
|
||||
4. **`default_threshold_config`**: Used for hosts with no mapping
|
||||
5. **First alphabetically**: If the default config is not found, use the first config alphabetically
|
||||
6. **Legacy `thresholds` section**: Used when `threshold_configs` is absent entirely
|
||||
|
||||
### Example: Complete Multi-Threshold Setup
|
||||
### Backward Compatibility
|
||||
|
||||
See `hbd/config_multi_threshold_example.yaml` for a complete example with:
|
||||
- 4 named configurations (default, high_sensitivity, low_sensitivity, database)
|
||||
- Host-to-config mappings for production, development, and test systems
|
||||
- Specialized database server thresholds
|
||||
- Custom display messages with plugin data
|
||||
The legacy `host_threshold_mapping` top-level key and the flat `thresholds` section are still fully supported:
|
||||
|
||||
```yaml
|
||||
# Still works — equivalent to hosts: {prod-web-01: {threshold_config: high_sensitivity}}
|
||||
host_threshold_mapping:
|
||||
prod-web-01: high_sensitivity
|
||||
|
||||
# Still works — equivalent to threshold_configs: {default: {thresholds: ...}}
|
||||
thresholds:
|
||||
cpu_monitor:
|
||||
cpu_percent: {warning: 80, critical: 90}
|
||||
```
|
||||
|
||||
|
||||
+54
-1
@@ -19,6 +19,15 @@ Users are defined in the server config file. Each host can have an **owner**, ze
|
||||
|
||||
`admin` is a flag on the user, not a per-host role. An admin user has owner-level access on every host without being listed as owner/manager/monitor.
|
||||
|
||||
### Settings page access
|
||||
|
||||
All authenticated users may open `/settings`. Admins see every section; non-admins see only **Notification Channels**, **Hosts**, and **Threshold Configurations**, filtered to global items plus what they own or manage:
|
||||
|
||||
- **Owners** may add hosts, delete their hosts, edit host settings, and change access lists (managers/monitors, ownership transfer).
|
||||
- **Managers** may edit host settings (watch, dyndns, channel and threshold assignments) but not access lists, and may not delete hosts.
|
||||
- Anyone may create private notification channels and threshold configs (owned by them) and assign them — or global ones — to their hosts.
|
||||
- Admins promote a private channel/threshold config to global by clearing its owner, or demote by assigning one.
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
@@ -36,7 +45,7 @@ users:
|
||||
bob:
|
||||
full_name: Bob Smith
|
||||
password: pbkdf2:sha256:...
|
||||
notification_channels: [pushover_standard]
|
||||
notification_channels: [pushover_standard] # channels bob has selected
|
||||
|
||||
carol:
|
||||
full_name: Carol Jones
|
||||
@@ -46,6 +55,24 @@ default_owner: andreas # owns hosts with no explicit owner
|
||||
# falls back to the first admin user if omitted
|
||||
```
|
||||
|
||||
### Client-declared host ownership
|
||||
|
||||
A host can declare its own owner directly in the hbc or hbc_mini client configuration. This is useful for hosts that are not listed in the server config, or during initial setup before a server-side config entry has been created.
|
||||
|
||||
**`~/.hbc.yaml`** (hbc):
|
||||
```yaml
|
||||
owner: andreas
|
||||
```
|
||||
|
||||
**`~/.hbc.json`** (hbc_mini):
|
||||
```json
|
||||
{ "owner": "andreas" }
|
||||
```
|
||||
|
||||
When set, the value is included in the `os_info` plugin data sent to the server. The server applies it as `host.owner` the first time `os_info` arrives, provided no owner has been configured server-side for that host. Server-configured ownership always takes precedence.
|
||||
|
||||
---
|
||||
|
||||
### Assigning roles to hosts
|
||||
|
||||
```yaml
|
||||
@@ -170,6 +197,32 @@ Return the currently authenticated user's profile.
|
||||
|
||||
---
|
||||
|
||||
#### PUT /api/0/users/me
|
||||
Update the current user's profile. All fields are optional — send only what you want to change.
|
||||
|
||||
**Update display name and avatar:**
|
||||
```json
|
||||
{ "full_name": "Carol Jones", "avatar": "/avatars/carol.png" }
|
||||
```
|
||||
|
||||
**Change notification channel selection:**
|
||||
```json
|
||||
{ "notification_channels": ["pushover_ops", "email_ops"] }
|
||||
```
|
||||
Only channels visible to the user (global + own) are accepted; others are silently dropped.
|
||||
|
||||
**Change password:**
|
||||
```json
|
||||
{ "password": { "current": "oldpass", "new": "newpass" } }
|
||||
```
|
||||
Requires the correct current password. New password is hashed before storage.
|
||||
|
||||
**Response:** `{"ok": true}`
|
||||
|
||||
**Status codes:** `200 OK`, `400` (missing/invalid field), `401` (unauthenticated), `403` (wrong current password)
|
||||
|
||||
---
|
||||
|
||||
### Host Access
|
||||
|
||||
#### GET /api/0/hosts/{hostname}/access
|
||||
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
Plan the following changes, ask questions to clarify before implementing
|
||||
|
||||
Re-factor the notification system:
|
||||
- use available libraries for pushover, matrix, email and sms notifications.
|
||||
- notifications have a title/subject: alert_type (recover/warning/critical), a body (info from threshold check) and a link to the host plugin metrix page
|
||||
- define a list of notification channels for each user
|
||||
- notifications are dispatched to users that are listed as managers for the host
|
||||
|
||||
|
||||
|
||||
1 - correct
|
||||
2 - for now channels are defined globaly
|
||||
3 - matrix-nio)sounds good, homeserver URL, access token, room ID per channel?
|
||||
4 - use the REST api provided by https://voip.ms/api/v1/rest.php
|
||||
5 - The page does not exist yet, point at the host tab in the /plugins
|
||||
6 - per-channel minimum severity is a good idea, go fo it
|
||||
7 - yes
|
||||
|
||||
1 - use base_url, there might not have been any incoming requests yet
|
||||
2 - use same asyncio loop for matrix-nio
|
||||
3 - for now, just silently do nothing
|
||||
+1
-1
@@ -14,4 +14,4 @@ Install options:
|
||||
"""
|
||||
|
||||
__all__ = ["__version__"]
|
||||
__version__ = "5.1.2"
|
||||
__version__ = "5.4.5"
|
||||
|
||||
@@ -16,6 +16,12 @@ CLIENT_DEFAULTS = {
|
||||
"hb_port": 50003, # Port where hbd servers listen
|
||||
"interval": 10, # Heartbeat interval in seconds
|
||||
|
||||
# Host identity
|
||||
"owner": None, # Optional username to set as this host's owner on the server
|
||||
|
||||
# Security
|
||||
"allow_remote_command": False, # Execute shell commands received in CMD packets from the server
|
||||
|
||||
# Runtime flags
|
||||
"foreground": False,
|
||||
"verbose": False,
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Per-protocol internet presence probe.
|
||||
|
||||
Tier 1 (every tier1_interval seconds, sends no packets): a UDP connect() to the probe
|
||||
address reveals whether a route exists and which source address would be used.
|
||||
Tier 2 (every probe_interval seconds, sooner after a failure or a change of address or
|
||||
gateway): one ICMP echo to the probe address. Absence needs fail_limit consecutive
|
||||
failures (or no route at all); presence needs one success.
|
||||
|
||||
The result is read by the heartbeat sender (p4/p6 flags on every HTB) and by the
|
||||
network_monitor plugin (connectivity metric) via the shared instance.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from typing import Any, Awaitable, Callable, Dict, Optional, Set, Tuple
|
||||
|
||||
from hbd.client import gateway as gateway_mod
|
||||
from hbd.client.pingutil import PingUnavailable, run_ping
|
||||
|
||||
logger = logging.getLogger("hbc.connectivity")
|
||||
|
||||
FAMILIES = ("ipv4", "ipv6")
|
||||
FLAG_KEYS = {"ipv4": "p4", "ipv6": "p6"}
|
||||
DEFAULT_PROBES = {"ipv4": "1.1.1.1", "ipv6": "2606:4700:4700::1111"}
|
||||
|
||||
|
||||
class ProbeUnavailable(Exception):
|
||||
"""The address family cannot be probed on this host (e.g. no IPv6 support)."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class FamilyState:
|
||||
present: Optional[bool] = None # None = unknown (probe cannot run yet/at all)
|
||||
addr: Optional[str] = None
|
||||
gateway: Optional[str] = None
|
||||
rtt: Optional[float] = None
|
||||
|
||||
|
||||
def local_address(family: str, target: str) -> Optional[str]:
|
||||
"""Source address the kernel would use to reach target, or None if there is no route.
|
||||
|
||||
A UDP connect() sends nothing. Raises ProbeUnavailable if the family is unsupported.
|
||||
"""
|
||||
af = socket.AF_INET if family == "ipv4" else socket.AF_INET6
|
||||
try:
|
||||
sock = socket.socket(af, socket.SOCK_DGRAM)
|
||||
except OSError as e:
|
||||
raise ProbeUnavailable(str(e)) from e
|
||||
try:
|
||||
sock.connect((target, 9))
|
||||
return str(sock.getsockname()[0])
|
||||
except OSError:
|
||||
return None
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
async def ping_rtt(target: str, family: str) -> Optional[float]:
|
||||
"""One ICMP echo; returns RTT in ms or None if no reply."""
|
||||
result = await run_ping(target, 1, 2, 4 if family == "ipv4" else 6)
|
||||
rtt = result["rtt_avg"]
|
||||
return None if rtt == float("inf") else rtt
|
||||
|
||||
|
||||
class Connectivity:
|
||||
def __init__(
|
||||
self,
|
||||
probes: Optional[Dict[str, str]] = None,
|
||||
probe_interval: float = 60.0,
|
||||
retry_interval: float = 10.0,
|
||||
tier1_interval: float = 5.0,
|
||||
fail_limit: int = 2,
|
||||
local_address_fn: Callable[[str, str], Optional[str]] = local_address,
|
||||
ping_fn: Callable[[str, str], Awaitable[Optional[float]]] = ping_rtt,
|
||||
gateway_fn: Callable[[str], Awaitable[Optional[str]]] = gateway_mod.default_gateway,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
):
|
||||
self.probes = {**DEFAULT_PROBES, **(probes or {})}
|
||||
self.probe_interval = probe_interval
|
||||
self.retry_interval = retry_interval
|
||||
self.tier1_interval = tier1_interval
|
||||
self.fail_limit = fail_limit
|
||||
self._local_address = local_address_fn
|
||||
self._ping = ping_fn
|
||||
self._gateway_fn = gateway_fn
|
||||
self._clock = clock
|
||||
# Reading the gateway is a cheap /proc read on Linux but a subprocess elsewhere,
|
||||
# so elsewhere it is refreshed only when a tier-2 probe is due.
|
||||
self._cheap_gateway = sys.platform.startswith("linux")
|
||||
self.states: Dict[str, FamilyState] = {f: FamilyState() for f in FAMILIES}
|
||||
self._fails: Dict[str, int] = {f: 0 for f in FAMILIES}
|
||||
self._next_ping: Dict[str, float] = {f: 0.0 for f in FAMILIES}
|
||||
self._last_key: Dict[str, Optional[Tuple[Optional[str], Optional[str]]]] = {
|
||||
f: None for f in FAMILIES}
|
||||
self._ping_disabled: Set[str] = set()
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, cfg: Optional[Dict[str, Any]]) -> "Connectivity":
|
||||
cfg = cfg or {}
|
||||
probes = {}
|
||||
if cfg.get("probe_v4"):
|
||||
probes["ipv4"] = str(cfg["probe_v4"])
|
||||
if cfg.get("probe_v6"):
|
||||
probes["ipv6"] = str(cfg["probe_v6"])
|
||||
return cls(probes=probes, probe_interval=float(cfg.get("probe_interval", 60)))
|
||||
|
||||
# ---- readers (cheap; safe to call on every heartbeat) --------------------------
|
||||
def snapshot(self) -> Dict[str, FamilyState]:
|
||||
return {f: replace(s) for f, s in self.states.items()}
|
||||
|
||||
def flags(self) -> Dict[str, bool]:
|
||||
"""HTB flags for families whose presence is known."""
|
||||
return {FLAG_KEYS[f]: s.present for f, s in self.states.items() if s.present is not None}
|
||||
|
||||
def as_metric(self) -> Dict[str, Dict[str, Any]]:
|
||||
return {f: asdict(s) for f, s in self.states.items()}
|
||||
|
||||
# ---- probing -------------------------------------------------------------------
|
||||
async def refresh(self) -> None:
|
||||
for fam in FAMILIES:
|
||||
try:
|
||||
await self._refresh_family(fam)
|
||||
except Exception:
|
||||
logger.exception("connectivity probe failed for %s", fam)
|
||||
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
while not stop.is_set():
|
||||
await self.refresh()
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=self.tier1_interval)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
async def _gateway(self, fam: str) -> Optional[str]:
|
||||
try:
|
||||
return await self._gateway_fn(fam)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
async def _refresh_family(self, fam: str) -> None:
|
||||
st = self.states[fam]
|
||||
try:
|
||||
addr = self._local_address(fam, self.probes[fam])
|
||||
except ProbeUnavailable:
|
||||
st.present = None
|
||||
return
|
||||
|
||||
if addr is None: # tier 1: no route => absent immediately
|
||||
self._fails[fam] = 0
|
||||
self._next_ping[fam] = 0.0
|
||||
self._last_key[fam] = None
|
||||
st.present, st.addr, st.gateway, st.rtt = False, None, None, None
|
||||
return
|
||||
|
||||
now = self._clock()
|
||||
gw = await self._gateway(fam) if self._cheap_gateway else st.gateway
|
||||
key = (addr, gw)
|
||||
due = key != self._last_key[fam] or now >= self._next_ping[fam]
|
||||
if due and not self._cheap_gateway:
|
||||
gw = await self._gateway(fam)
|
||||
key = (addr, gw)
|
||||
st.addr, st.gateway = addr, gw
|
||||
self._last_key[fam] = key
|
||||
if fam in self._ping_disabled:
|
||||
st.present = None # cannot probe: unknown, never absent, while a route exists
|
||||
elif due:
|
||||
await self._tier2(fam, st, now)
|
||||
|
||||
async def _tier2(self, fam: str, st: FamilyState, now: float) -> None:
|
||||
try:
|
||||
rtt = await self._ping(self.probes[fam], fam)
|
||||
except PingUnavailable:
|
||||
logger.warning("ping unavailable; %s presence stays unknown", fam)
|
||||
self._ping_disabled.add(fam)
|
||||
st.present = None
|
||||
return
|
||||
except (asyncio.TimeoutError, OSError):
|
||||
rtt = None
|
||||
if rtt is None:
|
||||
self._fails[fam] += 1
|
||||
self._next_ping[fam] = now + self.retry_interval
|
||||
if self._fails[fam] >= self.fail_limit:
|
||||
st.present, st.rtt = False, None
|
||||
else:
|
||||
self._fails[fam] = 0
|
||||
self._next_ping[fam] = now + self.probe_interval
|
||||
st.present, st.rtt = True, rtt
|
||||
|
||||
|
||||
# ---- shared instance (heartbeat sender and network_monitor plugin) -----------------
|
||||
_shared: Optional[Connectivity] = None
|
||||
|
||||
|
||||
def set_shared(conn: Optional[Connectivity]) -> None:
|
||||
global _shared
|
||||
_shared = conn
|
||||
|
||||
|
||||
def get_shared() -> Optional[Connectivity]:
|
||||
return _shared
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Best-effort default-gateway lookup (informational only; never affects presence)."""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
_RTF_GATEWAY = 0x2
|
||||
_RTF_REJECT = 0x200
|
||||
|
||||
|
||||
def parse_proc_route_v4(text: str) -> Optional[str]:
|
||||
"""Default gateway from /proc/net/route (lowest metric wins)."""
|
||||
best: Optional[Tuple[int, str]] = None
|
||||
for line in text.splitlines()[1:]:
|
||||
f = line.split()
|
||||
if len(f) < 8:
|
||||
continue
|
||||
_iface, dest, gw, flags = f[0], f[1], f[2], f[3] # noqa: F841
|
||||
if dest != "00000000" or not int(flags, 16) & _RTF_GATEWAY:
|
||||
continue
|
||||
metric = int(f[6])
|
||||
if best is None or metric < best[0]:
|
||||
best = (metric, socket.inet_ntoa(struct.pack("<L", int(gw, 16))))
|
||||
return best[1] if best else None
|
||||
|
||||
|
||||
def parse_proc_route_v6(text: str) -> Optional[str]:
|
||||
"""Default gateway from /proc/net/ipv6_route (lowest metric wins)."""
|
||||
best: Optional[Tuple[int, str]] = None
|
||||
for line in text.splitlines():
|
||||
f = line.split()
|
||||
if len(f) < 10:
|
||||
continue
|
||||
dest, plen, nexthop, metric, flags, iface = f[0], f[1], f[4], f[5], f[8], f[9]
|
||||
if dest != "0" * 32 or plen != "00" or nexthop == "0" * 32 or iface == "lo":
|
||||
continue
|
||||
fl = int(flags, 16)
|
||||
if not fl & _RTF_GATEWAY or fl & _RTF_REJECT:
|
||||
continue
|
||||
m = int(metric, 16)
|
||||
if best is None or m < best[0]:
|
||||
best = (m, socket.inet_ntop(socket.AF_INET6, bytes.fromhex(nexthop)))
|
||||
return best[1] if best else None
|
||||
|
||||
|
||||
def parse_macos_route(text: str) -> Optional[str]:
|
||||
"""Gateway from `route -n get [-inet6] default`."""
|
||||
m = re.search(r"^\s*gateway:\s*(\S+)", text, re.MULTILINE)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def parse_windows_route(text: str, family: str) -> Optional[str]:
|
||||
"""Default gateway from `route print -4` / `route print -6` (lowest metric wins)."""
|
||||
best: Optional[Tuple[int, str]] = None
|
||||
for line in text.splitlines():
|
||||
if family == "ipv4":
|
||||
f = line.split()
|
||||
if len(f) >= 5 and f[0] == "0.0.0.0" and f[1] == "0.0.0.0" and f[2] != "On-link":
|
||||
cand = (int(f[4]), f[2])
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
m = re.match(r"^\s*\d+\s+(\d+)\s+::/0\s+(\S+)\s*$", line)
|
||||
if not m or m.group(2) == "On-link":
|
||||
continue
|
||||
cand = (int(m.group(1)), m.group(2))
|
||||
if best is None or cand[0] < best[0]:
|
||||
best = cand
|
||||
return best[1] if best else None
|
||||
|
||||
|
||||
async def _run(cmd: List[str]) -> str:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
out, _ = await asyncio.wait_for(proc.communicate(), timeout=5)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
raise
|
||||
return out.decode(errors="replace")
|
||||
|
||||
|
||||
async def default_gateway(family: str) -> Optional[str]:
|
||||
"""Return the default gateway address for "ipv4"/"ipv6", or None on any failure."""
|
||||
v4 = family == "ipv4"
|
||||
try:
|
||||
if sys.platform.startswith("linux"):
|
||||
with open("/proc/net/route" if v4 else "/proc/net/ipv6_route") as fh:
|
||||
text = fh.read()
|
||||
return parse_proc_route_v4(text) if v4 else parse_proc_route_v6(text)
|
||||
if sys.platform == "darwin":
|
||||
cmd = ["route", "-n", "get"] + ([] if v4 else ["-inet6"]) + ["default"]
|
||||
return parse_macos_route(await _run(cmd))
|
||||
if sys.platform == "win32":
|
||||
return parse_windows_route(await _run(["route", "print", "-4" if v4 else "-6"]), family)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
+245
-104
@@ -14,16 +14,18 @@ import signal
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from hashlib import md5
|
||||
from logging.handlers import SysLogHandler
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
# Import protocol and config
|
||||
from .config import load_config
|
||||
from ..common.proto import dicttos, stodict
|
||||
from .. import __version__
|
||||
|
||||
# Import plugin system
|
||||
from .plugin import PluginRegistry, PluginLoader, InfoPlugin, MonitorPlugin
|
||||
from .connectivity import Connectivity, set_shared
|
||||
|
||||
# Constants
|
||||
PORT = 50003
|
||||
@@ -36,6 +38,10 @@ dorestart = False
|
||||
shutdown_event: Optional[asyncio.Event] = None
|
||||
active_tasks: List[asyncio.Task] = []
|
||||
|
||||
# Set from config in async_main. Off by default: a CMD packet is an unauthenticated
|
||||
# UDP datagram, so executing one must be opted into per host.
|
||||
allow_remote_command = False
|
||||
|
||||
|
||||
class AsyncConnection:
|
||||
"""Async UDP connection to a heartbeat server."""
|
||||
@@ -55,6 +61,10 @@ class AsyncConnection:
|
||||
|
||||
self.transport: Optional[asyncio.DatagramTransport] = None
|
||||
self.protocol: Optional[asyncio.DatagramProtocol] = None
|
||||
self._dead = False
|
||||
self._ever_opened = False
|
||||
self._open_fail_count = 0 # consecutive failures before first success
|
||||
self.request_info_event: asyncio.Event = asyncio.Event()
|
||||
|
||||
self.logger = logging.getLogger(f"hbc.conn.{addr}")
|
||||
|
||||
@@ -72,6 +82,7 @@ class AsyncConnection:
|
||||
lambda: HeartbeatProtocol(self),
|
||||
family=self.af
|
||||
)
|
||||
self._ever_opened = True
|
||||
self.logger.debug(f"Opened connection to {self.addr}:{self.port}")
|
||||
return True
|
||||
except Exception as e:
|
||||
@@ -92,6 +103,9 @@ class AsyncConnection:
|
||||
msg: Message dictionary
|
||||
msg_id: Message ID (HTB, PLG, etc.)
|
||||
"""
|
||||
if self._dead:
|
||||
return
|
||||
|
||||
if not self.transport:
|
||||
await self.open()
|
||||
|
||||
@@ -130,6 +144,9 @@ class AsyncConnection:
|
||||
|
||||
self.ackcount += 1
|
||||
self.logger.debug(f"ACK received, RTT: {rtt:.1f}ms")
|
||||
if msg.get("request_update"):
|
||||
self.logger.info("server requested plugin info refresh")
|
||||
self.request_info_event.set()
|
||||
|
||||
|
||||
class HeartbeatProtocol(asyncio.DatagramProtocol):
|
||||
@@ -165,12 +182,13 @@ class HeartbeatProtocol(asyncio.DatagramProtocol):
|
||||
self.logger.error(f"Error processing datagram: {e}", exc_info=True)
|
||||
|
||||
def error_received(self, exc):
|
||||
"""Handle protocol errors."""
|
||||
self.logger.error(f"Protocol error: {exc}")
|
||||
"""Handle protocol errors — close transport so the heartbeat sender retries."""
|
||||
self.logger.warning(f"Protocol error on {self.connection.addr}: {exc} — will retry")
|
||||
self.connection.close()
|
||||
|
||||
|
||||
async def handle_command(conn: AsyncConnection, msg: dict):
|
||||
"""Execute a command received from server."""
|
||||
"""Execute a command received from server, if allow_remote_command is set."""
|
||||
import subprocess
|
||||
|
||||
cmd = msg.get("cmd", "")
|
||||
@@ -178,6 +196,15 @@ async def handle_command(conn: AsyncConnection, msg: dict):
|
||||
return
|
||||
|
||||
logger = logging.getLogger("hbc.command")
|
||||
|
||||
if not allow_remote_command:
|
||||
logger.warning(f"Refused command (allow_remote_command is false): {cmd}")
|
||||
await conn.sendto({
|
||||
"service": "command",
|
||||
"msg": "Refused: allow_remote_command is false",
|
||||
})
|
||||
return
|
||||
|
||||
logger.info(f"Executing command: {cmd}")
|
||||
|
||||
try:
|
||||
@@ -203,48 +230,45 @@ async def handle_command(conn: AsyncConnection, msg: dict):
|
||||
await conn.sendto(response)
|
||||
|
||||
|
||||
async def handle_update(conn: AsyncConnection, msg: dict):
|
||||
"""Handle self-update from server."""
|
||||
import codecs
|
||||
async def handle_update(conn: AsyncConnection, _msg: dict): # pyright: ignore[reportUnusedParameter]
|
||||
"""Handle self-update by running hb_install.sh."""
|
||||
import shutil
|
||||
|
||||
logger = logging.getLogger("hbc.update")
|
||||
|
||||
try:
|
||||
code = codecs.decode(msg["code"], "base64").decode()
|
||||
csum = msg["csum"]
|
||||
except Exception as e:
|
||||
error = f"Missing code/csum: {e}"
|
||||
installer = shutil.which("hb_install.sh")
|
||||
if installer is None:
|
||||
candidate = Path(sys.argv[0]).parent / "hb_install.sh"
|
||||
if candidate.exists():
|
||||
installer = str(candidate)
|
||||
|
||||
if installer is None:
|
||||
error = "hb_install.sh not found in PATH or alongside hbc"
|
||||
logger.error(error)
|
||||
await conn.sendto({"service": "update", "msg": error})
|
||||
return
|
||||
|
||||
# Verify checksum
|
||||
m = md5()
|
||||
m.update(code.encode())
|
||||
if m.hexdigest() != csum:
|
||||
error = "Checksum mismatch"
|
||||
logger.info(f"Running installer: {installer}")
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
installer, "client",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
out, _ = await asyncio.wait_for(proc.communicate(), timeout=120)
|
||||
except asyncio.TimeoutError:
|
||||
error = "Installer timed out"
|
||||
logger.error(error)
|
||||
await conn.sendto({"service": "update", "msg": error})
|
||||
return
|
||||
except Exception as e:
|
||||
error = f"Installer failed: {e}"
|
||||
logger.error(error)
|
||||
await conn.sendto({"service": "update", "msg": error})
|
||||
return
|
||||
|
||||
# Backup current file
|
||||
fn = sys.argv[0]
|
||||
ofn = f"{fn}.sav"
|
||||
try:
|
||||
shutil.copy2(fn, ofn)
|
||||
except Exception as e:
|
||||
error = f"Backup failed: {e}"
|
||||
logger.error(error)
|
||||
await conn.sendto({"service": "update", "msg": error})
|
||||
return
|
||||
|
||||
# Write new code
|
||||
try:
|
||||
with open(fn, "w") as fh:
|
||||
fh.write(code)
|
||||
except Exception as e:
|
||||
error = f"Write failed: {e}"
|
||||
if proc.returncode != 0:
|
||||
error = f"Installer exited {proc.returncode}: {out.decode().strip()}"
|
||||
logger.error(error)
|
||||
await conn.sendto({"service": "update", "msg": error})
|
||||
return
|
||||
@@ -258,37 +282,76 @@ async def handle_update(conn: AsyncConnection, msg: dict):
|
||||
stop()
|
||||
|
||||
|
||||
async def heartbeat_sender(conn: AsyncConnection, interval: int):
|
||||
"""Send periodic heartbeats.
|
||||
def build_heartbeat_msg(conn, interval: int, connectivity: Optional[Connectivity] = None) -> dict:
|
||||
"""HTB payload; carries p4/p6 presence flags when the probe knows them."""
|
||||
msg = {"acks": conn.ackcount, "rtt": conn.rtts[-1], "interval": interval}
|
||||
if connectivity is not None:
|
||||
msg.update(connectivity.flags())
|
||||
return msg
|
||||
|
||||
|
||||
async def heartbeat_sender(conn: AsyncConnection, interval: int,
|
||||
connectivity: Optional[Connectivity] = None):
|
||||
"""Send periodic heartbeats, retrying the connection if it is not open.
|
||||
|
||||
IPv6 connections that fail to open before their first successful send are
|
||||
dropped after IPV6_EARLY_FAIL_LIMIT attempts so that a network without IPv6
|
||||
does not keep a dead sender alive. IPv4 connections are retried indefinitely.
|
||||
|
||||
Args:
|
||||
conn: Connection to send on
|
||||
interval: Heartbeat interval in seconds
|
||||
connectivity: Optional probe whose presence flags are attached to every heartbeat
|
||||
"""
|
||||
logger = logging.getLogger("hbc.heartbeat")
|
||||
IPV6_EARLY_FAIL_LIMIT = 3
|
||||
|
||||
while running and not conn._dead:
|
||||
# Ensure transport is open before attempting to send.
|
||||
if not conn.transport:
|
||||
opened = await conn.open()
|
||||
if opened:
|
||||
conn._open_fail_count = 0
|
||||
else:
|
||||
conn._open_fail_count += 1
|
||||
# Drop an IPv6 connection that has never come up within the
|
||||
# first few attempts — it is likely unavailable on this network.
|
||||
if (not conn._ever_opened
|
||||
and conn.af == socket.AF_INET6
|
||||
and conn._open_fail_count >= IPV6_EARLY_FAIL_LIMIT):
|
||||
logger.warning(
|
||||
f"IPv6 connection to {conn.addr} unreachable after "
|
||||
f"{conn._open_fail_count} attempts, disabling"
|
||||
)
|
||||
conn._dead = True
|
||||
break
|
||||
# Retry after the normal interval; IPv4 retries forever.
|
||||
try:
|
||||
if shutdown_event:
|
||||
await asyncio.wait_for(shutdown_event.wait(), timeout=interval)
|
||||
break
|
||||
else:
|
||||
await asyncio.sleep(interval)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
continue
|
||||
|
||||
while running:
|
||||
try:
|
||||
msg = {
|
||||
"acks": conn.ackcount,
|
||||
"rtt": conn.rtts[-1],
|
||||
"interval": interval
|
||||
}
|
||||
msg = build_heartbeat_msg(conn, interval, connectivity)
|
||||
await conn.sendto(msg, "HTB")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending heartbeat: {e}", exc_info=True)
|
||||
except asyncio.CancelledError:
|
||||
logger.debug("Heartbeat sender cancelled")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending heartbeat: {e}", exc_info=True)
|
||||
|
||||
# Wait for next interval or shutdown event
|
||||
try:
|
||||
if shutdown_event:
|
||||
await asyncio.wait_for(
|
||||
shutdown_event.wait(),
|
||||
timeout=interval
|
||||
)
|
||||
await asyncio.wait_for(shutdown_event.wait(), timeout=interval)
|
||||
break
|
||||
else:
|
||||
await asyncio.sleep(interval)
|
||||
@@ -299,6 +362,27 @@ async def heartbeat_sender(conn: AsyncConnection, interval: int):
|
||||
raise
|
||||
|
||||
|
||||
async def _info_plugin_refresh_loop(conn: AsyncConnection, info_plugins: List):
|
||||
"""Wait for server requests to re-send InfoPlugin data."""
|
||||
logger = logging.getLogger("hbc.plugins")
|
||||
while running:
|
||||
await conn.request_info_event.wait()
|
||||
if not running:
|
||||
break
|
||||
conn.request_info_event.clear()
|
||||
logger.info("refreshing InfoPlugins on server request")
|
||||
for plugin in info_plugins:
|
||||
plugin._cache = None
|
||||
try:
|
||||
data = await plugin.collect()
|
||||
if data:
|
||||
await conn.sendto(
|
||||
{"plugin": plugin.name, **data, "_interval": plugin.interval}, "PLG")
|
||||
logger.info(f"Resent {plugin.name} data")
|
||||
except Exception as e:
|
||||
logger.error(f"Error re-collecting {plugin.name}: {e}", exc_info=True)
|
||||
|
||||
|
||||
async def plugin_collector(conn: AsyncConnection, registry: PluginRegistry):
|
||||
"""Collect and send plugin data.
|
||||
|
||||
@@ -314,8 +398,8 @@ async def plugin_collector(conn: AsyncConnection, registry: PluginRegistry):
|
||||
try:
|
||||
data = await plugin.collect()
|
||||
if data:
|
||||
# Create PLG message with plugin name
|
||||
plugin_msg = {"plugin": plugin.name, **data}
|
||||
# Create PLG message with plugin name and declared interval
|
||||
plugin_msg = {"plugin": plugin.name, **data, "_interval": plugin.interval}
|
||||
await conn.sendto(plugin_msg, "PLG")
|
||||
logger.info(f"Sent {plugin.name} data")
|
||||
except Exception as e:
|
||||
@@ -330,24 +414,21 @@ async def plugin_collector(conn: AsyncConnection, registry: PluginRegistry):
|
||||
for plugin in monitor_plugins:
|
||||
by_interval[plugin.interval].append(plugin)
|
||||
|
||||
# Create tasks for each interval
|
||||
tasks = []
|
||||
# Create tasks for each interval; always include the info-refresh watcher
|
||||
tasks = [asyncio.create_task(_info_plugin_refresh_loop(conn, info_plugins))]
|
||||
for interval, plugins in by_interval.items():
|
||||
task = asyncio.create_task(
|
||||
tasks.append(asyncio.create_task(
|
||||
plugin_collector_interval(conn, plugins, interval)
|
||||
)
|
||||
tasks.append(task)
|
||||
))
|
||||
|
||||
# Wait for all tasks
|
||||
if tasks:
|
||||
try:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
except asyncio.CancelledError:
|
||||
logger.debug("Plugin collector cancelled, cancelling sub-tasks")
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
raise
|
||||
try:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
except asyncio.CancelledError:
|
||||
logger.debug("Plugin collector cancelled, cancelling sub-tasks")
|
||||
for task in tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
raise
|
||||
|
||||
|
||||
async def plugin_collector_interval(
|
||||
@@ -370,7 +451,7 @@ async def plugin_collector_interval(
|
||||
data = await plugin.collect()
|
||||
if data:
|
||||
# Don't use encode_plugin_data - create dict directly
|
||||
plugin_msg = {"plugin": plugin.name, **data}
|
||||
plugin_msg = {"plugin": plugin.name, **data, "_interval": plugin.interval}
|
||||
await conn.sendto(plugin_msg, "PLG")
|
||||
logger.debug(f"Sent {plugin.name} data")
|
||||
except asyncio.CancelledError:
|
||||
@@ -424,16 +505,14 @@ async def cleanup(connections: List[AsyncConnection]):
|
||||
logger = logging.getLogger("hbc.cleanup")
|
||||
logger.info("Cleaning up connections")
|
||||
|
||||
for conn in connections:
|
||||
target = next((c for c in connections if c.transport), connections[0] if connections else None)
|
||||
# A SIGHUP restart is not a host shutdown, so don't announce one.
|
||||
if target and send_shutdown and not dorestart:
|
||||
try:
|
||||
msg = {
|
||||
"shutdown": 1,
|
||||
"acks": conn.ackcount
|
||||
}
|
||||
await conn.sendto(msg)
|
||||
await target.sendto({"shutdown": 1, "acks": target.ackcount})
|
||||
except Exception as e:
|
||||
logger.error(f"Error sending shutdown: {e}")
|
||||
|
||||
for conn in connections:
|
||||
conn.close()
|
||||
|
||||
# Give messages time to send
|
||||
@@ -442,11 +521,12 @@ async def cleanup(connections: List[AsyncConnection]):
|
||||
|
||||
async def async_main(args, config):
|
||||
"""Async main function."""
|
||||
global running, shutdown_event, active_tasks
|
||||
global running, shutdown_event, active_tasks, send_shutdown, allow_remote_command
|
||||
|
||||
# Create shutdown event
|
||||
shutdown_event = asyncio.Event()
|
||||
active_tasks = []
|
||||
allow_remote_command = bool(config.get("allow_remote_command", False))
|
||||
|
||||
logger = logging.getLogger("hbc.main")
|
||||
|
||||
@@ -459,47 +539,63 @@ async def async_main(args, config):
|
||||
hb_port = config.get("hb_port", PORT)
|
||||
interval = config.get("interval", INTERVAL)
|
||||
|
||||
logger.info(f"Starting hbc for {iam} -> {hb_hosts}")
|
||||
logger.info(f"Port: {hb_port}, Interval: {interval}s")
|
||||
logger.info(f"hbc {__version__} on {iam} -> {hb_hosts} port={hb_port}, interval={interval}s")
|
||||
if allow_remote_command:
|
||||
logger.warning("allow_remote_command is true — CMD packets from the server will be executed")
|
||||
|
||||
af_filter = (socket.AF_INET if getattr(args, "ipv4_only", False)
|
||||
else socket.AF_INET6 if getattr(args, "ipv6_only", False)
|
||||
else 0)
|
||||
|
||||
# Create connections
|
||||
connections = []
|
||||
conn_id = 1
|
||||
_retry_delay = 5
|
||||
|
||||
for host in hb_hosts:
|
||||
try:
|
||||
addrs = socket.getaddrinfo(host, hb_port, 0, 0, socket.SOL_UDP)
|
||||
except socket.gaierror as e:
|
||||
logger.error(f"Cannot resolve {host}: {e}")
|
||||
continue
|
||||
|
||||
for addr_info in addrs:
|
||||
af = addr_info[0]
|
||||
addr = addr_info[4][0]
|
||||
|
||||
conn = AsyncConnection(conn_id, addr, hb_port, af, iam)
|
||||
if await conn.open():
|
||||
while running and not connections:
|
||||
for host in hb_hosts:
|
||||
try:
|
||||
addrs = socket.getaddrinfo(host, hb_port, af_filter, 0, socket.SOL_UDP)
|
||||
except socket.gaierror as e:
|
||||
logger.warning(f"Cannot resolve {host}: {e} — retrying in {_retry_delay}s")
|
||||
continue
|
||||
for addr_info in addrs:
|
||||
af = addr_info[0]
|
||||
addr = addr_info[4][0]
|
||||
conn = AsyncConnection(conn_id, addr, hb_port, af, iam)
|
||||
if not await conn.open():
|
||||
logger.warning(f"Initial open to {addr} failed, heartbeat sender will retry")
|
||||
connections.append(conn)
|
||||
conn_id += 1
|
||||
if not connections:
|
||||
try:
|
||||
if shutdown_event:
|
||||
await asyncio.wait_for(shutdown_event.wait(), timeout=_retry_delay)
|
||||
else:
|
||||
await asyncio.sleep(_retry_delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
_retry_delay = min(_retry_delay * 2, 60)
|
||||
|
||||
if not connections:
|
||||
logger.error("No connections established")
|
||||
return 1
|
||||
|
||||
logger.info(f"Created {len(connections)} connections")
|
||||
|
||||
# Send boot/message if requested
|
||||
send_shutdown = False
|
||||
if args.boot or args.message:
|
||||
boot_msg = {}
|
||||
if args.boot:
|
||||
boot_msg["boot"] = 1
|
||||
send_shutdown = True
|
||||
if args.message:
|
||||
boot_msg["service"] = "service"
|
||||
boot_msg["msg"] = args.message
|
||||
|
||||
boot_msg["acks"] = 0
|
||||
for conn in connections:
|
||||
await conn.sendto(boot_msg)
|
||||
target = next((c for c in connections if c.transport), connections[0])
|
||||
await target.sendto(boot_msg)
|
||||
|
||||
if args.message and not args.daemon:
|
||||
# Message-only mode
|
||||
@@ -522,10 +618,26 @@ async def async_main(args, config):
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, stop)
|
||||
|
||||
def _sighup():
|
||||
global dorestart
|
||||
dorestart = True
|
||||
stop()
|
||||
|
||||
loop.add_signal_handler(signal.SIGHUP, _sighup)
|
||||
|
||||
# Start async tasks
|
||||
# Per-protocol internet presence probe (shared with the network_monitor plugin)
|
||||
net_cfg = dict((config.get("plugins") or {}).get("network_monitor")
|
||||
or config.get("network_monitor") or {})
|
||||
connectivity = None
|
||||
if net_cfg.get("probe_enabled", True):
|
||||
connectivity = Connectivity.from_config(net_cfg)
|
||||
set_shared(connectivity)
|
||||
active_tasks.append(asyncio.create_task(connectivity.run(shutdown_event)))
|
||||
|
||||
# Heartbeat senders (one per connection)
|
||||
for conn in connections:
|
||||
task = asyncio.create_task(heartbeat_sender(conn, interval))
|
||||
task = asyncio.create_task(heartbeat_sender(conn, interval, connectivity))
|
||||
active_tasks.append(task)
|
||||
|
||||
# Plugin collector (uses all connections, but we'll use first one)
|
||||
@@ -586,6 +698,36 @@ def daemonize(
|
||||
os.dup2(se.fileno(), sys.stderr.fileno())
|
||||
|
||||
|
||||
def _reconfigure_logging_for_daemon(log_level: int) -> None:
|
||||
"""Replace StreamHandlers (now writing to /dev/null) with a SysLogHandler."""
|
||||
root = logging.getLogger()
|
||||
for handler in root.handlers[:]:
|
||||
root.removeHandler(handler)
|
||||
handler.close()
|
||||
|
||||
use_udp_fallback = not os.path.exists("/dev/log")
|
||||
|
||||
if use_udp_fallback:
|
||||
syslog_handler = SysLogHandler(
|
||||
address=("localhost", 514),
|
||||
facility=SysLogHandler.LOG_DAEMON,
|
||||
)
|
||||
else:
|
||||
syslog_handler = SysLogHandler(
|
||||
address="/dev/log",
|
||||
facility=SysLogHandler.LOG_DAEMON,
|
||||
)
|
||||
|
||||
syslog_handler.setFormatter(
|
||||
logging.Formatter("hbc[%(process)d]: %(name)s %(levelname)s: %(message)s")
|
||||
)
|
||||
root.addHandler(syslog_handler)
|
||||
root.setLevel(log_level)
|
||||
|
||||
if use_udp_fallback:
|
||||
logging.warning("/dev/log not found, using syslog UDP localhost:514")
|
||||
|
||||
|
||||
def build_parser():
|
||||
"""Build argument parser."""
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -629,6 +771,9 @@ def build_parser():
|
||||
default=0,
|
||||
help="Increase debug level"
|
||||
)
|
||||
af_group = parser.add_mutually_exclusive_group()
|
||||
af_group.add_argument("-4", dest="ipv4_only", action="store_true", help="Use IPv4 only")
|
||||
af_group.add_argument("-6", dest="ipv6_only", action="store_true", help="Use IPv6 only")
|
||||
parser.add_argument(
|
||||
"hosts",
|
||||
nargs="+",
|
||||
@@ -662,17 +807,10 @@ def main(argv=None):
|
||||
|
||||
# Daemonize if requested
|
||||
if args.daemon:
|
||||
print("Daemonizing...")
|
||||
import syslog
|
||||
syslog.openlog("hbc", syslog.LOG_PID, syslog.LOG_DAEMON)
|
||||
syslog.syslog(syslog.LOG_INFO, f"Starting heartbeat to {', '.join(args.hosts)}")
|
||||
logging.info("Daemonizing...")
|
||||
daemonize()
|
||||
|
||||
# Reconfigure logging for syslog
|
||||
logging.basicConfig(
|
||||
level=log_level,
|
||||
format="hbc[%(process)d]: %(name)s %(levelname)s: %(message)s"
|
||||
)
|
||||
_reconfigure_logging_for_daemon(log_level)
|
||||
logging.info(f"hbc starting, sending heartbeat to {', '.join(args.hosts)}")
|
||||
|
||||
# Run async main
|
||||
try:
|
||||
@@ -687,7 +825,10 @@ def main(argv=None):
|
||||
# Handle restart
|
||||
if dorestart:
|
||||
logging.info("Restarting...")
|
||||
os.execv(sys.argv[0], sys.argv)
|
||||
# Drop -b/--boot so the re-exec'd process doesn't re-announce a boot;
|
||||
# a SIGHUP restart is not a host reboot.
|
||||
restart_argv = [sys.argv[0]] + [a for a in sys.argv[1:] if a not in ("-b", "--boot")]
|
||||
os.execv(restart_argv[0], restart_argv)
|
||||
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""System ping helpers shared by ping_monitor and the connectivity probe."""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import sys
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
_INF = float("inf")
|
||||
|
||||
|
||||
class PingUnavailable(Exception):
|
||||
"""The system ping binary could not be executed."""
|
||||
|
||||
|
||||
def ping_command(host: str, count: int, timeout: int, family: Optional[int] = None) -> List[str]:
|
||||
"""Build the platform ping command. family is 4, 6 or None (let ping decide)."""
|
||||
if sys.platform == "win32":
|
||||
cmd = ["ping", "-n", str(count), "-w", str(timeout * 1000)]
|
||||
if family:
|
||||
cmd.append(f"-{family}")
|
||||
return cmd + [host]
|
||||
if sys.platform == "darwin" and family:
|
||||
if family == 6:
|
||||
return ["ping6", "-c", str(count), host]
|
||||
return ["ping", "-c", str(count), "-W", str(timeout * 1000), host]
|
||||
cmd = ["ping", "-c", str(count), "-W", str(timeout)]
|
||||
if family:
|
||||
cmd.append(f"-{family}")
|
||||
return cmd + [host]
|
||||
|
||||
|
||||
def parse_ping_output(output: str) -> Dict[str, float]:
|
||||
"""Parse rtt_min/avg/max/loss from ping output (Linux, macOS, BSD)."""
|
||||
loss = 100.0
|
||||
loss_match = re.search(r"(\d+(?:\.\d+)?)\s*%\s*packet\s*loss", output)
|
||||
if loss_match:
|
||||
loss = float(loss_match.group(1))
|
||||
|
||||
# Linux: "rtt min/avg/max/mdev = x/x/x/x ms"; macOS: "round-trip min/avg/max/stddev = ..."
|
||||
rtt_match = re.search(
|
||||
r"(?:rtt|round-trip)\s+min/avg/max/\S+\s*=\s*([\d.]+)/([\d.]+)/([\d.]+)",
|
||||
output,
|
||||
)
|
||||
if rtt_match:
|
||||
return {
|
||||
"rtt_min": float(rtt_match.group(1)),
|
||||
"rtt_avg": float(rtt_match.group(2)),
|
||||
"rtt_max": float(rtt_match.group(3)),
|
||||
"loss": loss,
|
||||
}
|
||||
return {"rtt_min": _INF, "rtt_avg": _INF, "rtt_max": _INF, "loss": loss}
|
||||
|
||||
|
||||
async def run_ping(host: str, count: int, timeout: int, family: Optional[int] = None) -> Dict[str, float]:
|
||||
"""Run ping and return parsed metrics.
|
||||
|
||||
Raises PingUnavailable if the ping binary is missing and asyncio.TimeoutError
|
||||
(after killing the process) if it hangs.
|
||||
"""
|
||||
cmd = ping_command(host, count, timeout, family)
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
except FileNotFoundError as e:
|
||||
raise PingUnavailable(str(e)) from e
|
||||
try:
|
||||
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout * count + 2)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
raise
|
||||
return parse_ping_output(stdout.decode(errors="replace"))
|
||||
+14
-4
@@ -29,6 +29,7 @@ class Plugin(ABC):
|
||||
description: Human-readable description
|
||||
interval: Collection interval in seconds (0 for InfoPlugin = collect once)
|
||||
enabled: Whether plugin is active (can be disabled via config)
|
||||
skip_reason: Set by plugin before returning False from initialize(); causes loader to log INFO instead of WARNING.
|
||||
"""
|
||||
|
||||
name: str = ""
|
||||
@@ -46,6 +47,7 @@ class Plugin(ABC):
|
||||
self.config = config or {}
|
||||
self.logger = logging.getLogger(f"plugin.{self.name}")
|
||||
self._initialized = False
|
||||
self.skip_reason: Optional[str] = None
|
||||
|
||||
@abstractmethod
|
||||
async def initialize(self) -> bool:
|
||||
@@ -362,16 +364,24 @@ class PluginLoader:
|
||||
|
||||
# Instantiate plugin with config — check plugins subdict first,
|
||||
# then top-level keys (e.g. nagios_runner: ... at root of config).
|
||||
plugin_instance_config = plugins_subconfig.get(obj.name) or raw_config.get(obj.name, {})
|
||||
plugin_instance_config = dict(plugins_subconfig.get(obj.name) or raw_config.get(obj.name) or {})
|
||||
# Propagate top-level owner so os_info (and any future plugin) can report it.
|
||||
if "owner" in raw_config and "owner" not in plugin_instance_config:
|
||||
plugin_instance_config["owner"] = raw_config["owner"]
|
||||
plugin = obj(config=plugin_instance_config)
|
||||
|
||||
# Initialize plugin
|
||||
try:
|
||||
initialized = await plugin.initialize()
|
||||
if not initialized:
|
||||
self.logger.warning(
|
||||
f"Plugin {plugin.name} failed initialization, skipping"
|
||||
)
|
||||
if plugin.skip_reason:
|
||||
self.logger.info(
|
||||
f"Plugin {plugin.name} skipped: {plugin.skip_reason}"
|
||||
)
|
||||
else:
|
||||
self.logger.warning(
|
||||
f"Plugin {plugin.name} failed initialization, skipping"
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
self.logger.error(
|
||||
|
||||
@@ -119,6 +119,13 @@ class CPUMonitorPlugin(MonitorPlugin):
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Could not get CPU times: {e}")
|
||||
|
||||
# Uptime in seconds
|
||||
try:
|
||||
import time
|
||||
data["uptime_seconds"] = int(time.time() - self.psutil.boot_time())
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Could not get uptime: {e}")
|
||||
|
||||
self.logger.debug(
|
||||
f"Collected CPU metrics: {data.get('cpu_percent', 'N/A')}% usage"
|
||||
)
|
||||
|
||||
@@ -127,7 +127,7 @@ class FilesystemInfoPlugin(InfoPlugin):
|
||||
try:
|
||||
# Maximum filename length
|
||||
max_name = os.pathconf(partition.mountpoint, 'PC_NAME_MAX')
|
||||
if max_name:
|
||||
if max_name is not None:
|
||||
fs_info['maxfile'] = max_name
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
@@ -135,7 +135,7 @@ class FilesystemInfoPlugin(InfoPlugin):
|
||||
try:
|
||||
# Maximum path length
|
||||
max_path = os.pathconf(partition.mountpoint, 'PC_PATH_MAX')
|
||||
if max_path:
|
||||
if max_path is not None:
|
||||
fs_info['maxpath'] = max_path
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
@@ -14,6 +14,24 @@ except ImportError:
|
||||
|
||||
from hbd.client.plugin import MonitorPlugin
|
||||
|
||||
|
||||
def _zfs_arc_bytes() -> int:
|
||||
"""Return current ZFS ARC size in bytes, or 0 if ZFS is not present.
|
||||
|
||||
ZFS ARC is reclaimable but is not included in MemAvailable by the Linux
|
||||
kernel (it is not in SReclaimable), so it would otherwise be counted as
|
||||
used memory.
|
||||
"""
|
||||
try:
|
||||
with open("/proc/spl/kstat/zfs/arcstats") as fh:
|
||||
for line in fh:
|
||||
parts = line.split()
|
||||
if len(parts) >= 3 and parts[0] == "size":
|
||||
return int(parts[2])
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
return 0
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -101,11 +119,21 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
||||
|
||||
# Virtual (physical) memory statistics
|
||||
vmem = psutil.virtual_memory()
|
||||
|
||||
# psutil's available already excludes page cache / file buffers
|
||||
# (uses MemAvailable on Linux). Add ZFS ARC on top because the kernel
|
||||
# does not include it in SReclaimable / MemAvailable even though it is
|
||||
# reclaimable.
|
||||
arc_bytes = _zfs_arc_bytes()
|
||||
available = min(vmem.available + arc_bytes, vmem.total)
|
||||
used = vmem.total - available
|
||||
percent = round(used / vmem.total * 100, 1) if vmem.total else 0.0
|
||||
|
||||
metrics['memory_total'] = vmem.total
|
||||
metrics['memory_available'] = vmem.available
|
||||
metrics['memory_used'] = vmem.used
|
||||
metrics['memory_available'] = available
|
||||
metrics['memory_used'] = used
|
||||
metrics['memory_free'] = vmem.free
|
||||
metrics['memory_percent'] = vmem.percent
|
||||
metrics['memory_percent'] = percent
|
||||
|
||||
# Platform-specific memory details
|
||||
if hasattr(vmem, 'active'):
|
||||
|
||||
@@ -21,24 +21,23 @@ nagios_runner:
|
||||
```
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import shlex
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from hbd.client.plugin import MonitorPlugin
|
||||
|
||||
|
||||
# Nagios exit codes
|
||||
NAGIOS_OK = 0
|
||||
NAGIOS_WARNING = 1
|
||||
NAGIOS_CRITICAL = 2
|
||||
NAGIOS_UNKNOWN = 3
|
||||
|
||||
STATUS_NAMES = {
|
||||
NAGIOS_OK: "OK",
|
||||
NAGIOS_WARNING: "WARNING",
|
||||
NAGIOS_CRITICAL: "CRITICAL",
|
||||
NAGIOS_UNKNOWN: "UNKNOWN"
|
||||
0: "OK",
|
||||
1: "WARNING",
|
||||
2: "CRITICAL",
|
||||
3: "UNKNOWN",
|
||||
}
|
||||
|
||||
|
||||
@@ -52,7 +51,6 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
||||
interval: Collection interval in seconds (default: 300)
|
||||
commands: List of command definitions with 'name' and 'command' keys
|
||||
timeout: Command execution timeout in seconds (default: 30)
|
||||
shell: Whether to execute commands via shell (default: True)
|
||||
|
||||
Example:
|
||||
nagios_runner:
|
||||
@@ -76,15 +74,8 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
||||
# Extract configuration
|
||||
self.commands: List[Dict[str, str]] = config.get("commands", []) if config else []
|
||||
self.timeout: int = config.get("timeout", 30) if config else 30
|
||||
self.shell: bool = config.get("shell", True) if config else True
|
||||
self.interval = config.get("interval", 300) if config else 300
|
||||
|
||||
# Validate commands
|
||||
if not self.commands:
|
||||
self.logger.info(
|
||||
"No Nagios commands configured. Add 'nagios_runner.commands' to config."
|
||||
)
|
||||
|
||||
async def initialize(self) -> bool:
|
||||
"""Initialize the Nagios runner plugin.
|
||||
|
||||
@@ -94,7 +85,7 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
||||
self.logger.info(f"Initializing {self.name} plugin")
|
||||
|
||||
if not self.commands:
|
||||
self.logger.info("No Nagios commands configured")
|
||||
self.skip_reason = "no commands configured (add nagios_runner.commands to config)"
|
||||
return False
|
||||
|
||||
self.logger.info(f"Configured to run {len(self.commands)} Nagios plugin(s)")
|
||||
@@ -102,6 +93,29 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
||||
name = cmd_config.get("name", "unnamed")
|
||||
self.logger.info(f" - {name}: {cmd_config.get('command', 'N/A')}")
|
||||
|
||||
# Validate absolute command paths early
|
||||
for cmd_config in self.commands:
|
||||
name = cmd_config.get("name", "unnamed")
|
||||
command = cmd_config.get("command", "")
|
||||
if not command:
|
||||
continue
|
||||
try:
|
||||
tokens = shlex.split(command)
|
||||
except ValueError:
|
||||
continue # malformed command string; skip validation
|
||||
if not tokens:
|
||||
continue
|
||||
exe = tokens[0]
|
||||
if os.path.isabs(exe):
|
||||
if not os.path.isfile(exe):
|
||||
self.logger.warning(
|
||||
f"Command '{name}': executable not found: {exe}"
|
||||
)
|
||||
elif not os.access(exe, os.X_OK):
|
||||
self.logger.warning(
|
||||
f"Command '{name}': executable not executable: {exe}"
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||
@@ -112,9 +126,6 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
||||
"""
|
||||
results = {}
|
||||
|
||||
# Track overall status (worst status wins)
|
||||
worst_status = NAGIOS_OK
|
||||
|
||||
for cmd_config in self.commands:
|
||||
name = cmd_config.get("name")
|
||||
command = cmd_config.get("command")
|
||||
@@ -132,16 +143,12 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
||||
results[f"{name}_status_code"] = status_code
|
||||
results[f"{name}_output"] = output
|
||||
|
||||
# Track worst status
|
||||
if status_code > worst_status:
|
||||
worst_status = status_code
|
||||
|
||||
# Parse and add performance data
|
||||
if perfdata:
|
||||
for metric_name, metric_value in perfdata.items():
|
||||
results[f"{name}_{metric_name}"] = metric_value
|
||||
|
||||
self.logger.debug(
|
||||
self.logger.info(
|
||||
f"Executed {name}: {STATUS_NAMES.get(status_code, 'UNKNOWN')} - {output[:50]}"
|
||||
)
|
||||
|
||||
@@ -150,12 +157,6 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
||||
results[f"{name}_status"] = "ERROR"
|
||||
results[f"{name}_status_code"] = NAGIOS_UNKNOWN
|
||||
results[f"{name}_output"] = str(e)
|
||||
worst_status = NAGIOS_UNKNOWN
|
||||
|
||||
# Add overall status
|
||||
results["overall_status"] = STATUS_NAMES.get(worst_status, "UNKNOWN")
|
||||
results["overall_status_code"] = worst_status
|
||||
results["plugin_count"] = len(self.commands)
|
||||
|
||||
return results
|
||||
|
||||
@@ -163,46 +164,49 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
||||
self,
|
||||
command: str
|
||||
) -> Tuple[int, str, Dict[str, Any]]:
|
||||
"""Execute a Nagios plugin and parse its output.
|
||||
|
||||
Args:
|
||||
command: Command string to execute
|
||||
|
||||
Returns:
|
||||
Tuple of (status_code, output_message, performance_data_dict)
|
||||
"""
|
||||
"""Execute a Nagios plugin and parse its output."""
|
||||
try:
|
||||
# Run command
|
||||
result = subprocess.run(
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
command,
|
||||
shell=self.shell,
|
||||
capture_output=True,
|
||||
timeout=self.timeout,
|
||||
text=True
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
||||
proc.communicate(), timeout=self.timeout
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.communicate()
|
||||
self.logger.error(f"Command timed out: {command}")
|
||||
return NAGIOS_UNKNOWN, f"Command timed out after {self.timeout}s", {}
|
||||
|
||||
status_code = result.returncode
|
||||
output = result.stdout.strip()
|
||||
status_code = proc.returncode
|
||||
|
||||
if status_code < 0:
|
||||
return NAGIOS_UNKNOWN, f"Process killed by signal {-status_code}", {}
|
||||
|
||||
# Nagios plugins can return codes > 3, treat as UNKNOWN
|
||||
if status_code > 3:
|
||||
status_code = NAGIOS_UNKNOWN
|
||||
|
||||
# Parse performance data
|
||||
perfdata = self._parse_perfdata(output)
|
||||
stdout = stdout_bytes.decode(errors="replace").strip()
|
||||
stderr = stderr_bytes.decode(errors="replace").strip()
|
||||
|
||||
# Extract just the status message (before the pipe if present)
|
||||
if '|' in output:
|
||||
output_msg = output.split('|')[0].strip()
|
||||
# Parse perfdata from stdout before mixing in stderr
|
||||
perfdata = self._parse_perfdata(stdout)
|
||||
|
||||
# Build status message
|
||||
status_part = stdout.split('|')[0].strip() if '|' in stdout else stdout
|
||||
|
||||
if not stdout and stderr:
|
||||
output_msg = stderr
|
||||
elif stdout and stderr:
|
||||
output_msg = f"{status_part} [stderr: {stderr}]"
|
||||
else:
|
||||
output_msg = output
|
||||
output_msg = status_part
|
||||
|
||||
return status_code, output_msg, perfdata
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
self.logger.error(f"Command timed out: {command}")
|
||||
return NAGIOS_UNKNOWN, f"Command timed out after {self.timeout}s", {}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error executing command: {e}")
|
||||
return NAGIOS_UNKNOWN, f"Execution error: {str(e)}", {}
|
||||
|
||||
@@ -13,6 +13,7 @@ except ImportError:
|
||||
psutil = None
|
||||
|
||||
from hbd.client.plugin import MonitorPlugin
|
||||
from hbd.client.connectivity import get_shared
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -32,6 +33,10 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
||||
interfaces: List of interfaces to monitor (default: all)
|
||||
include_connections: Include connection statistics (default: True)
|
||||
include_addresses: Include interface addresses (default: False)
|
||||
probe_enabled: Run the per-protocol internet presence probe (default: True)
|
||||
probe_v4: IPv4 address to ping (default: 1.1.1.1)
|
||||
probe_v6: IPv6 address to ping (default: 2606:4700:4700::1111)
|
||||
probe_interval: Seconds between ICMP probes while healthy (default: 60)
|
||||
"""
|
||||
|
||||
name = "network_monitor"
|
||||
@@ -103,6 +108,7 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
||||
- etc.
|
||||
- addresses: Interface address information (if include_addresses)
|
||||
- Dict keyed by interface name with address details
|
||||
- connectivity: per-protocol presence {ipv4, ipv6} -> {present, addr, gateway, rtt}
|
||||
"""
|
||||
if psutil is None:
|
||||
logger.error("psutil not available")
|
||||
@@ -229,6 +235,11 @@ class NetworkMonitorPlugin(MonitorPlugin):
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not collect interface stats: {e}")
|
||||
|
||||
# Per-protocol internet presence, address and gateway (from the shared probe)
|
||||
shared = get_shared()
|
||||
if shared is not None:
|
||||
metrics['connectivity'] = shared.as_metric()
|
||||
|
||||
return metrics
|
||||
|
||||
async def cleanup(self):
|
||||
|
||||
@@ -60,7 +60,11 @@ class OSInfoPlugin(InfoPlugin):
|
||||
"python_version": platform.python_version(),
|
||||
"python_implementation": platform.python_implementation(),
|
||||
"hbc_version": hbc_version,
|
||||
"hbc_type": "full",
|
||||
}
|
||||
if self.config.get("owner"):
|
||||
self.logger.debug(f"Adding owner from config: {self.config['owner']}")
|
||||
data["owner"] = self.config["owner"]
|
||||
|
||||
# Add Linux-specific distribution info
|
||||
if platform.system() == "Linux":
|
||||
|
||||
@@ -13,12 +13,8 @@ plugins:
|
||||
count: 3 # ICMP packets per ping run (default 3)
|
||||
timeout: 5 # seconds before a host is considered unreachable (default 5)
|
||||
hosts:
|
||||
8.8.8.8:
|
||||
warning: 20.0 # ms
|
||||
critical: 100.0 # ms
|
||||
192.168.1.1:
|
||||
warning: 5.0
|
||||
critical: 20.0
|
||||
- 8.8.8.8
|
||||
- 192.168.1.1
|
||||
```
|
||||
|
||||
Reported metrics per host (metric key uses the hostname with dots/colons replaced
|
||||
@@ -44,9 +40,9 @@ threshold_configs:
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from hbd.client.pingutil import PingUnavailable, run_ping
|
||||
from hbd.client.plugin import MonitorPlugin
|
||||
|
||||
|
||||
@@ -88,51 +84,13 @@ class PingMonitorPlugin(MonitorPlugin):
|
||||
|
||||
async def _ping(self, host: str) -> Dict[str, float]:
|
||||
"""Run a system ping command and return rtt_min/avg/max/loss."""
|
||||
if sys.platform == "win32":
|
||||
cmd = ["ping", "-n", str(self.count), "-w", str(self.timeout * 1000), host]
|
||||
else:
|
||||
cmd = ["ping", "-c", str(self.count), "-W", str(self.timeout), host]
|
||||
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, _ = await asyncio.wait_for(
|
||||
proc.communicate(),
|
||||
timeout=self.timeout * self.count + 2,
|
||||
)
|
||||
output = stdout.decode(errors="replace")
|
||||
except (asyncio.TimeoutError, FileNotFoundError, OSError) as e:
|
||||
return await run_ping(host, self.count, self.timeout)
|
||||
except (asyncio.TimeoutError, PingUnavailable, OSError) as e:
|
||||
self.logger.warning("ping_monitor: ping failed for %s: %s", host, e)
|
||||
return {"rtt_min": float("inf"), "rtt_avg": float("inf"),
|
||||
"rtt_max": float("inf"), "loss": 100.0}
|
||||
|
||||
# Parse packet loss
|
||||
loss = 100.0
|
||||
loss_match = re.search(r"(\d+(?:\.\d+)?)\s*%\s*packet\s*loss", output)
|
||||
if loss_match:
|
||||
loss = float(loss_match.group(1))
|
||||
|
||||
# Parse rtt min/avg/max — Linux: "rtt min/avg/max/mdev = x/x/x/x ms"
|
||||
# macOS: "round-trip min/avg/max/stddev = x/x/x/x ms"
|
||||
rtt_match = re.search(
|
||||
r"(?:rtt|round-trip)\s+min/avg/max/\S+\s*=\s*([\d.]+)/([\d.]+)/([\d.]+)",
|
||||
output,
|
||||
)
|
||||
if rtt_match:
|
||||
return {
|
||||
"rtt_min": float(rtt_match.group(1)),
|
||||
"rtt_avg": float(rtt_match.group(2)),
|
||||
"rtt_max": float(rtt_match.group(3)),
|
||||
"loss": loss,
|
||||
}
|
||||
|
||||
# Host unreachable or all packets lost
|
||||
return {"rtt_min": float("inf"), "rtt_avg": float("inf"),
|
||||
"rtt_max": float("inf"), "loss": loss}
|
||||
|
||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||
data: Dict[str, Any] = {}
|
||||
tasks = {host: asyncio.create_task(self._ping(host)) for host in self.hosts}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
ZFS pool monitoring plugin for Heartbeat.
|
||||
|
||||
Collects per-pool health, capacity, and cumulative I/O statistics via zpool(8).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import shutil
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from hbd.client.plugin import MonitorPlugin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _int(s: str) -> Optional[int]:
|
||||
try:
|
||||
return int(s.strip().rstrip("KMGTkBkmgt%x"))
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
def _float(s: str) -> Optional[float]:
|
||||
try:
|
||||
return float(s.strip().rstrip("%x"))
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
class ZFSMonitorPlugin(MonitorPlugin):
|
||||
"""Monitor ZFS pool health, capacity, and I/O statistics.
|
||||
|
||||
Collects per pool:
|
||||
- health: ONLINE, DEGRADED, FAULTED, etc.
|
||||
- size / alloc / free: total, allocated and free bytes
|
||||
- capacity: percentage used (0-100)
|
||||
- frag: fragmentation percentage
|
||||
- dedup: deduplication ratio
|
||||
- read_ops / write_ops: cumulative I/O operations since last boot/clear
|
||||
- read_bw / write_bw: cumulative bytes transferred since last boot/clear
|
||||
|
||||
Configuration:
|
||||
interval: collection interval in seconds (default: 300)
|
||||
pools: list of pool names to monitor (default: all)
|
||||
"""
|
||||
|
||||
name = "zfs_monitor"
|
||||
description = "ZFS pool health, capacity, and I/O statistics"
|
||||
interval = 300
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
super().__init__(config)
|
||||
self.interval = self.config.get("interval", 300)
|
||||
self._pools_filter: Optional[List[str]] = self.config.get("pools", None)
|
||||
|
||||
async def initialize(self) -> bool:
|
||||
if not shutil.which("zpool"):
|
||||
self.skip_reason = "zpool not found"
|
||||
return False
|
||||
logger.info("ZFS monitor initialized (interval: %ds)", self.interval)
|
||||
return True
|
||||
|
||||
async def _run(self, *args: str) -> List[str]:
|
||||
"""Run a command and return its stdout lines, or [] on error."""
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=15)
|
||||
return stdout.decode(errors="replace").splitlines()
|
||||
except (FileNotFoundError, asyncio.TimeoutError) as exc:
|
||||
logger.warning("zfs_monitor: %s: %s", args[0], exc)
|
||||
return []
|
||||
|
||||
async def _zpool_list(self) -> Dict[str, Dict]:
|
||||
"""Return per-pool health and capacity from `zpool list`."""
|
||||
lines = await self._run(
|
||||
"zpool", "list", "-H", "-p",
|
||||
"-o", "name,health,size,alloc,free,cap,frag,dedup",
|
||||
)
|
||||
pools: Dict[str, Dict] = {}
|
||||
for line in lines:
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 8:
|
||||
continue
|
||||
name = parts[0].strip()
|
||||
if self._pools_filter and name not in self._pools_filter:
|
||||
continue
|
||||
health = parts[1].strip()
|
||||
if health == "ONLINE":
|
||||
status = 0
|
||||
elif health in ("DEGRADED", "ONLINE with errors"):
|
||||
status = 1
|
||||
elif health in ("FAULTED", "OFFLINE", "UNAVAIL"):
|
||||
status = 2
|
||||
else:
|
||||
status = 3 # unknown status
|
||||
pools[name] = {
|
||||
"health": health,
|
||||
"status": status,
|
||||
"size": _int(parts[2]),
|
||||
"alloc": _int(parts[3]),
|
||||
"free": _int(parts[4]),
|
||||
"capacity": _float(parts[5]),
|
||||
"frag": _float(parts[6]),
|
||||
"dedup": _float(parts[7]),
|
||||
}
|
||||
return pools
|
||||
|
||||
async def _zpool_iostat(self) -> Dict[str, Dict]:
|
||||
"""Return per-pool cumulative I/O counters from `zpool iostat`."""
|
||||
lines = await self._run("zpool", "iostat", "-H", "-p")
|
||||
io: Dict[str, Dict] = {}
|
||||
for line in lines:
|
||||
parts = line.split("\t")
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
name = parts[0].strip()
|
||||
if not name or name.startswith(" "):
|
||||
continue
|
||||
io[name] = {
|
||||
"read_ops": _int(parts[3]),
|
||||
"write_ops": _int(parts[4]),
|
||||
"read_bw": _int(parts[5]),
|
||||
"write_bw": _int(parts[6]),
|
||||
}
|
||||
return io
|
||||
|
||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||
pools, io = await asyncio.gather(self._zpool_list(), self._zpool_iostat())
|
||||
for name, stats in io.items():
|
||||
if name in pools:
|
||||
pools[name].update(stats)
|
||||
return {"pools": pools}
|
||||
|
||||
|
||||
plugin = ZFSMonitorPlugin
|
||||
@@ -134,6 +134,31 @@ thresholds:
|
||||
hysteresis: 0.1
|
||||
enabled: true
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# ZFS Monitor Thresholds
|
||||
# ----------------------------------------------------------------------------
|
||||
zfs_monitor:
|
||||
# Pool health check — built-in default; shown here for reference/override.
|
||||
# status is 0 (ONLINE) or 1 (DEGRADED) or 2 (SUSPENDED, FAULTED, UNAVAIL…).
|
||||
# Use '*' to apply the same rule to every pool, or name a specific pool.
|
||||
pools:
|
||||
'*':
|
||||
status:
|
||||
warning: 1 # Alert WARNING when pool is DEGRADED
|
||||
critical: 2 # Alert CRITICAL when pool is SUSPENDED/FAULTED/UNAVAIL
|
||||
operator: ">="
|
||||
hysteresis: 0.0 # No hysteresis — a degraded pool is always alerting
|
||||
grace: 0 # Fire immediately — don't wait for a second collection
|
||||
display: "ZFS pool {pool_name} is {health}"
|
||||
|
||||
# Per-pool capacity thresholds (optional; add pools you care about)
|
||||
# tank:
|
||||
# capacity:
|
||||
# warning: 75.0 # Warn at 75% used
|
||||
# critical: 90.0 # Critical at 90% used
|
||||
# operator: ">"
|
||||
# hysteresis: 0.05
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Network Monitor Thresholds
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
+5
-6
@@ -144,17 +144,16 @@ def cmd_notify(args):
|
||||
url=f"{base_url}/plugins" if base_url else "",
|
||||
)
|
||||
|
||||
# Bypass min_level for explicit test sends; run async channels directly
|
||||
import asyncio
|
||||
from .notify import _send_matrix_async, _send_sms_voipms_async, _DRIVERS
|
||||
ch_type = channel_cfg.get("type", "")
|
||||
print(f"Sending via {args.channel} ({ch_type}): {title} — {args.message}")
|
||||
|
||||
if ch_type in ("matrix", "sms_voipms"):
|
||||
from .notify import _send_matrix_async, _send_sms_voipms_async
|
||||
driver_async = _send_matrix_async if ch_type == "matrix" else _send_sms_voipms_async
|
||||
ok = asyncio.run(driver_async(channel_cfg, notif))
|
||||
if ch_type == "matrix":
|
||||
ok = asyncio.run(_send_matrix_async(channel_cfg, notif))
|
||||
elif ch_type == "sms_voipms":
|
||||
ok = asyncio.run(_send_sms_voipms_async(channel_cfg, notif))
|
||||
else:
|
||||
from .notify import _DRIVERS
|
||||
driver = _DRIVERS.get(ch_type)
|
||||
if driver is None:
|
||||
print(f"Error: unknown channel type '{ch_type}'", file=sys.stderr)
|
||||
|
||||
+51
-35
@@ -27,21 +27,26 @@ SERVER_DEFAULTS = {
|
||||
|
||||
# Monitoring settings
|
||||
"interval": 20, # Expected heartbeat interval (for server checks)
|
||||
"grace": 2, # Grace multiplier (interval * grace = timeout)
|
||||
"grace": 2, # Grace period (extra seconds before notifying after a missed heartbeat)
|
||||
"threshold_renotify_interval": 3600, # Seconds between threshold re-notifications
|
||||
|
||||
# Flap detection (0 in either key disables it)
|
||||
"flap_count": 5, # Warning/critical notifications within flap_interval that trip flapping
|
||||
"flap_interval": 10, # Minutes: the counting window, and the quiet window after an OK
|
||||
|
||||
# User management
|
||||
"users": {}, # username -> {full_name, avatar, password, admin, notification_channels}
|
||||
"default_owner": None, # Username that owns hosts with no explicit owner
|
||||
|
||||
# OAuth2 providers
|
||||
"oauth": {}, # oauth.gitea.{url,client_id,client_secret}
|
||||
|
||||
# Host management
|
||||
"hosts": {}, # Unified host definitions
|
||||
"dyndnshosts": [], # Hosts with dynamic DNS (legacy)
|
||||
"drophosts": [], # Hosts to ignore
|
||||
"dyndomains": ["wrede.org"],
|
||||
"dyndomains": ["example.org"], # Domains to update via nsupdate when a host with dyndns: true is updated
|
||||
|
||||
# DNS updates
|
||||
"nsupdate_bin": "/usr/bin/nsupdate",
|
||||
"nsupdate_bin": "/usr/bin/nsupdate", # Path to nsupdate binary
|
||||
|
||||
# WebSocket settings
|
||||
"ws_port": 50005,
|
||||
@@ -76,9 +81,13 @@ THRESHOLD_DEFAULTS = {
|
||||
}
|
||||
},
|
||||
'memory_monitor': {
|
||||
'percent': {
|
||||
'memory_percent': {
|
||||
'warning': 85.0,
|
||||
'critical': 95.0
|
||||
},
|
||||
'swap_percent': {
|
||||
'warning': 40.0,
|
||||
'critical': 75.0
|
||||
}
|
||||
},
|
||||
'disk_monitor': {
|
||||
@@ -95,7 +104,31 @@ THRESHOLD_DEFAULTS = {
|
||||
'warning': 200,
|
||||
'critical': 250.0,
|
||||
'count': 3 # Optional: number of consecutive breaches before alerting
|
||||
}
|
||||
},
|
||||
'nagios_runner': {
|
||||
'status_code': {
|
||||
'display': '{check_name} {output}',
|
||||
'operator': "nagios"
|
||||
}
|
||||
},
|
||||
'zfs_monitor': {
|
||||
'pools': {
|
||||
'*': {
|
||||
'status': {
|
||||
'warning': 1,
|
||||
'critical': 2,
|
||||
'operator': '>=',
|
||||
'hysteresis': 0.0,
|
||||
'grace': 0,
|
||||
'display': 'ZFS pool {pool_name} is {health}'
|
||||
},
|
||||
'capacity': {
|
||||
'warning': 80.0,
|
||||
'critical': 90.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,43 +252,26 @@ def get_watchhosts(config):
|
||||
"""Extract watched hostnames from config (hosts with watch: true).
|
||||
|
||||
Returns:
|
||||
List of hostnames to watch
|
||||
# List of hostnames to watch
|
||||
"""
|
||||
watchhosts = []
|
||||
hosts_config = config.get("hosts", {})
|
||||
if isinstance(hosts_config, dict):
|
||||
for host_name, host_attrs in hosts_config.items():
|
||||
if isinstance(host_attrs, dict) and host_attrs.get("watch", False):
|
||||
if isinstance(host_attrs, dict) and host_attrs.get("watch", True):
|
||||
watchhosts.append(host_name)
|
||||
return watchhosts
|
||||
|
||||
|
||||
def get_dyndnshosts(config):
|
||||
"""Extract dyndnshosts from config, supporting both new and legacy formats.
|
||||
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
|
||||
Returns:
|
||||
List of hostnames with dynamic DNS
|
||||
"""
|
||||
dyndnshosts = []
|
||||
|
||||
# New format: hosts section with dyndns attribute
|
||||
if "hosts" in config:
|
||||
hosts_config = config["hosts"]
|
||||
if isinstance(hosts_config, dict):
|
||||
for host_name, host_attrs in hosts_config.items():
|
||||
if isinstance(host_attrs, dict) and host_attrs.get("dyndns", False):
|
||||
dyndnshosts.append(host_name)
|
||||
|
||||
# Legacy format: dyndnshosts list/set
|
||||
if "dyndnshosts" in config:
|
||||
legacy_dyndnshosts = config.get("dyndnshosts", [])
|
||||
if isinstance(legacy_dyndnshosts, (list, set)):
|
||||
dyndnshosts.extend(legacy_dyndnshosts)
|
||||
|
||||
return list(set(dyndnshosts)) # Remove duplicates
|
||||
"""Return hostnames that have a dyndns setting in the hosts section."""
|
||||
hosts_config = config.get("hosts", {})
|
||||
if not isinstance(hosts_config, dict):
|
||||
return []
|
||||
return [
|
||||
name for name, attrs in hosts_config.items()
|
||||
if isinstance(attrs, dict) and attrs.get("dyndns")
|
||||
]
|
||||
|
||||
|
||||
def get_host_config(config, hostname):
|
||||
@@ -303,7 +319,7 @@ def get_host_access(config, hostname) -> dict:
|
||||
"""
|
||||
host_cfg = get_host_config(config, hostname)
|
||||
|
||||
owner = host_cfg.get("owner") or get_default_owner(config)
|
||||
owner = host_cfg.get("owner") # or get_default_owner(config)
|
||||
|
||||
managers = host_cfg.get("managers", [])
|
||||
if isinstance(managers, str):
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Ownership rules for config-file entities (hosts, channels, threshold configs).
|
||||
|
||||
Rule: an entry whose config dict has a non-empty ``owner`` is private to that
|
||||
owner — visible, usable, and editable only by the owner (and admins). An
|
||||
entry with no ``owner`` is global: usable by everyone, editable by admins only.
|
||||
|
||||
The scoped-merge helpers implement the non-admin save path for
|
||||
``POST /api/0/config``: the caller's visible subset of a section is replaced
|
||||
by the submitted payload; everything else is preserved untouched.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
class ScopedMergeError(Exception):
|
||||
"""A non-admin payload violated an ownership rule; message names the entry."""
|
||||
|
||||
|
||||
def is_global(cfg: Any) -> bool:
|
||||
"""True when *cfg* has no owner (usable by everyone)."""
|
||||
return not (isinstance(cfg, dict) and cfg.get("owner"))
|
||||
|
||||
|
||||
def user_can_use(cfg: Any, username: str) -> bool:
|
||||
"""True when *username* may use/see this entry (global or own)."""
|
||||
return is_global(cfg) or cfg.get("owner") == username
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, str):
|
||||
return [value]
|
||||
return list(value)
|
||||
|
||||
|
||||
def user_hosts(hosts_cfg: Any, username: str) -> Dict[str, Any]:
|
||||
"""Subset of *hosts_cfg* where *username* is owner or manager."""
|
||||
result = {}
|
||||
for name, cfg in (hosts_cfg or {}).items():
|
||||
if not isinstance(cfg, dict):
|
||||
continue
|
||||
if cfg.get("owner") == username or username in _as_list(cfg.get("managers")):
|
||||
result[name] = cfg
|
||||
return result
|
||||
|
||||
|
||||
def user_channels(channels_cfg: Any, username: str) -> Dict[str, Any]:
|
||||
"""Subset of channels usable by *username*: global + own."""
|
||||
return {
|
||||
name: cfg for name, cfg in (channels_cfg or {}).items()
|
||||
if isinstance(cfg, dict) and user_can_use(cfg, username)
|
||||
}
|
||||
|
||||
|
||||
def user_threshold_configs(threshold_cfgs: Any, username: str) -> Dict[str, Any]:
|
||||
"""Subset of threshold configs usable by *username*: global + own."""
|
||||
return {
|
||||
name: cfg for name, cfg in (threshold_cfgs or {}).items()
|
||||
if isinstance(cfg, dict) and user_can_use(cfg, username)
|
||||
}
|
||||
|
||||
|
||||
_HOST_SETTING_KEYS = ("watch", "dyndns", "notification_channels", "threshold_config")
|
||||
_HOST_ACCESS_KEYS = ("owner", "managers", "monitors")
|
||||
_HOST_ALLOWED_KEYS = frozenset(_HOST_SETTING_KEYS) | frozenset(_HOST_ACCESS_KEYS)
|
||||
|
||||
|
||||
def _check_added_assignments(host_name: str, old_cfg: Any, new_cfg: dict,
|
||||
key: str, usable: set) -> None:
|
||||
"""Every value under *key* not already on the host must be in *usable*."""
|
||||
old_vals = set(_as_list((old_cfg or {}).get(key)))
|
||||
for v in _as_list(new_cfg.get(key)):
|
||||
if v not in old_vals and v not in usable:
|
||||
raise ScopedMergeError(
|
||||
f"host {host_name!r}: {key} entry {v!r} is not available to you")
|
||||
|
||||
|
||||
def merge_hosts_scoped(existing: Any, payload: Any, username: str,
|
||||
channels_cfg: Any, threshold_cfgs: Any) -> Dict[str, Any]:
|
||||
"""Return a new hosts section with *username*'s visible subset replaced by *payload*.
|
||||
|
||||
Hosts the user cannot see are preserved untouched. Visible hosts missing
|
||||
from the payload are deleted (owners only). Managers may change setting
|
||||
keys but not access keys and may not delete. New hosts get their owner
|
||||
forced to *username*. Newly added channel/threshold assignments must be
|
||||
global or owned by the user. Violations raise ScopedMergeError.
|
||||
"""
|
||||
existing = existing or {}
|
||||
payload = payload or {}
|
||||
visible = user_hosts(existing, username)
|
||||
usable_channels = set(user_channels(channels_cfg, username))
|
||||
usable_tcs = set(user_threshold_configs(threshold_cfgs, username)) | {"default"}
|
||||
|
||||
result: Dict[str, Any] = {n: c for n, c in existing.items() if n not in visible}
|
||||
|
||||
for name, entry in payload.items():
|
||||
if not isinstance(entry, dict):
|
||||
raise ScopedMergeError(f"host {name!r}: invalid entry")
|
||||
unknown = set(entry) - _HOST_ALLOWED_KEYS
|
||||
if unknown:
|
||||
raise ScopedMergeError(
|
||||
f"host {name!r}: fields not permitted: {', '.join(sorted(unknown))}")
|
||||
if name in existing and name not in visible:
|
||||
raise ScopedMergeError(f"host {name!r}: you are not an owner or manager")
|
||||
|
||||
old = visible.get(name)
|
||||
if old is None:
|
||||
new_cfg = dict(entry)
|
||||
new_cfg["owner"] = username
|
||||
else:
|
||||
new_cfg = dict(old)
|
||||
for key in _HOST_SETTING_KEYS:
|
||||
if key in entry:
|
||||
new_cfg[key] = entry[key]
|
||||
else:
|
||||
new_cfg.pop(key, None)
|
||||
if old.get("owner") == username:
|
||||
for key in _HOST_ACCESS_KEYS:
|
||||
if key in entry:
|
||||
new_cfg[key] = entry[key]
|
||||
else:
|
||||
new_cfg.pop(key, None)
|
||||
else:
|
||||
if "owner" in entry and (entry.get("owner") or None) != (old.get("owner") or None):
|
||||
raise ScopedMergeError(
|
||||
f"host {name!r}: only the owner may change ownership")
|
||||
for key in ("managers", "monitors"):
|
||||
if key in entry and set(_as_list(entry[key])) != set(_as_list(old.get(key))):
|
||||
raise ScopedMergeError(
|
||||
f"host {name!r}: only the owner may change {key}")
|
||||
|
||||
_check_added_assignments(name, old, new_cfg, "notification_channels", usable_channels)
|
||||
_check_added_assignments(name, old, new_cfg, "threshold_config", usable_tcs)
|
||||
result[name] = new_cfg
|
||||
|
||||
for name, old in visible.items():
|
||||
if name not in payload and old.get("owner") != username:
|
||||
raise ScopedMergeError(f"host {name!r}: only the owner may delete a host")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def merge_threshold_configs_scoped(existing: Any, payload: Any,
|
||||
username: str) -> Dict[str, Any]:
|
||||
"""Return a new threshold_configs section with the user's own configs
|
||||
replaced by *payload*.
|
||||
|
||||
Global and foreign-owned configs are preserved and may not appear in the
|
||||
payload ('default' included). Own configs missing from the payload are
|
||||
deleted. Every payload entry gets its owner forced to *username*.
|
||||
"""
|
||||
existing = existing or {}
|
||||
payload = payload or {}
|
||||
result: Dict[str, Any] = {
|
||||
n: c for n, c in existing.items()
|
||||
if not (isinstance(c, dict) and c.get("owner") == username)
|
||||
}
|
||||
for name, entry in payload.items():
|
||||
if name == "default":
|
||||
raise ScopedMergeError("threshold config 'default' is global and admin-managed")
|
||||
old = existing.get(name)
|
||||
if old is not None and (not isinstance(old, dict) or old.get("owner") != username):
|
||||
raise ScopedMergeError(f"threshold config {name!r}: not owned by you")
|
||||
new_cfg = dict(entry) if isinstance(entry, dict) else {}
|
||||
new_cfg["owner"] = username
|
||||
result[name] = new_cfg
|
||||
return result
|
||||
@@ -0,0 +1,137 @@
|
||||
"""YAML round-trip read/write for .hb.yaml, with backup and atomic writes."""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import threading
|
||||
from datetime import datetime
|
||||
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
_write_lock = threading.Lock()
|
||||
|
||||
|
||||
def _make_yaml() -> YAML:
|
||||
y = YAML()
|
||||
y.preserve_quotes = True
|
||||
return y
|
||||
|
||||
# Top-level keys managed by the 'server' logical section
|
||||
_SERVER_KEYS = [
|
||||
"hbd_port", "hbd_host", "ws_port", "wss_port", "hb_port",
|
||||
"interval", "grace", "base_url", "threshold_renotify_interval",
|
||||
"flap_count", "flap_interval",
|
||||
"logfile", "pidfile", "pickfile", "journal_enabled", "journal_dir",
|
||||
"journal_max_size", "journal_max_backups", "default_owner",
|
||||
"default_threshold_config",
|
||||
]
|
||||
|
||||
# Top-level keys managed by the 'dns' logical section
|
||||
_DNS_KEYS = ["nsupdate_bin", "rndc_key", "dyndomains"]
|
||||
|
||||
|
||||
def read_roundtrip(path: str):
|
||||
"""Load .hb.yaml with ruamel.yaml, preserving comments and ordering."""
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return _make_yaml().load(f)
|
||||
|
||||
|
||||
def write_config(path: str, data) -> None:
|
||||
"""Backup current file then atomically write data.
|
||||
|
||||
Backup naming: {path}.bak.YYYYMMDD-HHMMSS
|
||||
Rotation: keep the 10 most recent backups, delete older ones.
|
||||
Atomic write: write to {path}.tmp then os.replace({path}.tmp, path).
|
||||
Acquires _write_lock for the full backup+write sequence.
|
||||
"""
|
||||
with _write_lock:
|
||||
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
backup_path = f"{path}.bak.{ts}"
|
||||
n = 0
|
||||
while os.path.exists(backup_path):
|
||||
n += 1
|
||||
backup_path = f"{path}.bak.{ts}-{n}"
|
||||
orig_mode = None
|
||||
if os.path.exists(path):
|
||||
orig_mode = os.stat(path).st_mode
|
||||
with open(path, "rb") as src, open(backup_path, "wb") as dst:
|
||||
dst.write(src.read())
|
||||
os.chmod(backup_path, orig_mode)
|
||||
backups = sorted(glob.glob(f"{path}.bak.*"), reverse=True)
|
||||
for old in backups[10:]:
|
||||
os.unlink(old)
|
||||
tmp = f"{path}.tmp"
|
||||
try:
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
_make_yaml().dump(data, f)
|
||||
if orig_mode is not None:
|
||||
os.chmod(tmp, orig_mode)
|
||||
os.replace(tmp, path)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def list_backups(path: str) -> list:
|
||||
"""Return backup paths sorted newest-first."""
|
||||
return sorted(glob.glob(f"{path}.bak.*"), reverse=True)
|
||||
|
||||
|
||||
def apply_structured_section(data, section: str, values: dict) -> None:
|
||||
"""Merge a dict of scalar/list values into data for the named logical section.
|
||||
|
||||
For 'server': updates each known key individually, preserving comments on
|
||||
unchanged keys. For 'users': replaces the entire users dict.
|
||||
"""
|
||||
if section == "server":
|
||||
for key in _SERVER_KEYS:
|
||||
if key in values:
|
||||
data[key] = values[key]
|
||||
elif section == "dns":
|
||||
for key in _DNS_KEYS:
|
||||
if key in values:
|
||||
data[key] = values[key]
|
||||
else:
|
||||
data.pop(key, None)
|
||||
elif section == "users":
|
||||
data["users"] = values
|
||||
elif section == "hosts":
|
||||
data["hosts"] = values
|
||||
else:
|
||||
raise ValueError(f"Unknown structured section: {section!r}")
|
||||
|
||||
|
||||
def apply_channel(data, name: str, channel_cfg: dict) -> None:
|
||||
"""Insert or replace a single notification channel entry, preserving others."""
|
||||
if not data.get("notification_channels"):
|
||||
data["notification_channels"] = {}
|
||||
data["notification_channels"][name] = channel_cfg
|
||||
|
||||
|
||||
def delete_channel(data, name: str) -> None:
|
||||
"""Remove a notification channel by name. No-op if not found."""
|
||||
nc = data.get("notification_channels") or {}
|
||||
nc.pop(name, None)
|
||||
|
||||
|
||||
def apply_yaml_section(data, section: str, yaml_text: str) -> None:
|
||||
"""Replace the named logical section by parsing yaml_text."""
|
||||
parsed = _make_yaml().load(yaml_text)
|
||||
if section == "notification_channels":
|
||||
data["notification_channels"] = parsed
|
||||
elif section == "thresholds":
|
||||
data["threshold_configs"] = parsed
|
||||
elif section == "hosts":
|
||||
data["hosts"] = parsed
|
||||
elif section == "dns":
|
||||
if parsed:
|
||||
for key in _DNS_KEYS:
|
||||
if key in parsed:
|
||||
data[key] = parsed[key]
|
||||
else:
|
||||
for key in _DNS_KEYS:
|
||||
data.pop(key, None)
|
||||
else:
|
||||
raise ValueError(f"Unknown YAML section: {section!r}")
|
||||
+18
-15
@@ -4,6 +4,9 @@ from __future__ import annotations
|
||||
from subprocess import Popen, PIPE, STDOUT
|
||||
from typing import Optional
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_nsupdate_payload(
|
||||
@@ -123,7 +126,6 @@ async def dns_update_worker(
|
||||
pass
|
||||
continue
|
||||
|
||||
m = f"changed address to {addr}"
|
||||
for dyndomain in cfg.get("dyndomains", []):
|
||||
err = await loop.run_in_executor(
|
||||
None,
|
||||
@@ -135,28 +137,29 @@ async def dns_update_worker(
|
||||
cfg.get("rndc_key", "/etc/dhcpc/rndc-key"),
|
||||
)
|
||||
if err:
|
||||
m += f", DNS update failed: {err}"
|
||||
m = f"DNS update failed for {addr} ({dyndomain}): {err}"
|
||||
logger.error("DNS update failed for %s: %s", name, err)
|
||||
if log:
|
||||
try:
|
||||
await loop.run_in_executor(None, log, name, "ERROR", m)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
m += ", DNS updated."
|
||||
m = f"DNS updated {name}.dy.{dyndomain} → {addr}"
|
||||
if log:
|
||||
try:
|
||||
await loop.run_in_executor(None, log, name, "INFO", m)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not cfg.get("dyndomains"):
|
||||
logger.warning("DNS update triggered for %s but no dyndomains configured", name)
|
||||
|
||||
try:
|
||||
dnsq.task_done()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if log:
|
||||
try:
|
||||
await loop.run_in_executor(None, log, name, m)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if log:
|
||||
try:
|
||||
await loop.run_in_executor(None, log, None, "dns_update_worker exiting")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def start_dns_worker(
|
||||
hbdclass,
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Flap detection: silence checks that toggle faster than they are useful.
|
||||
|
||||
A ``(host, service)`` pair is *flapping* once it exceeds ``flap_count``
|
||||
WARNING/CRITICAL notifications within ``flap_interval`` minutes. The
|
||||
notification that trips the threshold is delivered with ``FLAP_MARKER``
|
||||
appended; every notification for that pair afterwards is dropped. The state
|
||||
clears silently ``flap_interval`` minutes after a RECOVER, provided no further
|
||||
WARNING/CRITICAL arrived in the meantime.
|
||||
|
||||
Only outbound notifications are affected — ``notify.eventlog`` keeps recording
|
||||
every event, so the journal and ``/log`` retain the full history of the flap.
|
||||
|
||||
State lives here at module level rather than on ``Host`` so it is never
|
||||
pickled: a restart starts every check with a clean slate.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FLAP_MARKER = "Now flapping!! No more messages!"
|
||||
|
||||
# Actions returned by observe()
|
||||
PASS = "pass" # deliver unchanged
|
||||
TRIP = "trip" # deliver with FLAP_MARKER appended
|
||||
SUPPRESS = "suppress" # drop
|
||||
|
||||
_count = 0 # flap_count: alerts within the window needed to trip
|
||||
_window = 0.0 # flap_interval in seconds
|
||||
|
||||
# {(host, service): {"events": [ts, ...], "flapping": bool, "ok_since": float|None}}
|
||||
_state: dict = {}
|
||||
|
||||
|
||||
def setup(cfg) -> None:
|
||||
"""Read flap_count / flap_interval from *cfg* (also called on reload)."""
|
||||
global _count, _window
|
||||
_count = int(cfg.get("flap_count", 0) or 0)
|
||||
_window = float(cfg.get("flap_interval", 0) or 0) * 60.0
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
return _count > 0 and _window > 0
|
||||
|
||||
|
||||
def _label(key) -> str:
|
||||
host, service = key
|
||||
return f"{host}/{service}" if service else host
|
||||
|
||||
|
||||
def _sweep(st: dict, now: float) -> bool:
|
||||
"""Clear the flapping flag once the post-RECOVER quiet window has elapsed."""
|
||||
if st["flapping"] and st["ok_since"] is not None and now - st["ok_since"] >= _window:
|
||||
st["flapping"] = False
|
||||
st["ok_since"] = None
|
||||
st["events"].clear()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def observe(host: str, service: str, level: str) -> str:
|
||||
"""Record a notification for *host*/*service* and return what to do with it.
|
||||
|
||||
Returns PASS, TRIP or SUPPRESS.
|
||||
"""
|
||||
if not _enabled():
|
||||
return PASS
|
||||
|
||||
now = time.time()
|
||||
key = (host or "", service or "")
|
||||
level = (level or "").upper()
|
||||
st = _state.get(key)
|
||||
if st is not None and _sweep(st, now):
|
||||
logger.info("flapping cleared for %s", _label(key))
|
||||
|
||||
if level in ("WARNING", "CRITICAL"):
|
||||
if st is None:
|
||||
st = _state[key] = {"events": [], "flapping": False, "ok_since": None}
|
||||
# An alert inside the quiet window means it never settled.
|
||||
st["ok_since"] = None
|
||||
st["events"] = [t for t in st["events"] if now - t < _window]
|
||||
st["events"].append(now)
|
||||
if st["flapping"]:
|
||||
return SUPPRESS
|
||||
if len(st["events"]) > _count:
|
||||
st["flapping"] = True
|
||||
logger.info(
|
||||
"flapping detected for %s (%d alerts in %.0f min)",
|
||||
_label(key), len(st["events"]), _window / 60,
|
||||
)
|
||||
return TRIP
|
||||
return PASS
|
||||
|
||||
if st is None or not st["flapping"]:
|
||||
return PASS
|
||||
if level == "RECOVER":
|
||||
# Starts the quiet window; the state ends silently when it elapses.
|
||||
st["ok_since"] = now
|
||||
return SUPPRESS
|
||||
|
||||
|
||||
def clear_host(host: str) -> None:
|
||||
"""Discard all flap state for *host* (called when a host is dropped).
|
||||
|
||||
A dropped host may be mid-flap with no RECOVER ever received, in which
|
||||
case ``ok_since`` stays ``None`` and ``_sweep`` can never clear it on its
|
||||
own — the state would otherwise persist forever.
|
||||
"""
|
||||
for key in [k for k in _state if k[0] == host]:
|
||||
del _state[key]
|
||||
|
||||
|
||||
def flapping_services(host: str) -> list:
|
||||
"""Return the services of *host* that are currently flapping.
|
||||
|
||||
An empty string in the result means the host itself (connectivity, boot,
|
||||
shutdown) rather than a named service.
|
||||
"""
|
||||
if not _enabled():
|
||||
return []
|
||||
now = time.time()
|
||||
flapping = []
|
||||
for (h, service), st in _state.items():
|
||||
if h != host:
|
||||
continue
|
||||
_sweep(st, now)
|
||||
if st["flapping"]:
|
||||
flapping.append(service)
|
||||
return sorted(flapping)
|
||||
+98
-3
@@ -29,6 +29,7 @@ class Connection:
|
||||
UP = "up"
|
||||
DOWN = "down"
|
||||
OVERDUE = "overdue"
|
||||
ABSENT = "absent"
|
||||
|
||||
def __init__(self, host, cid, addr, afam):
|
||||
self.host = host
|
||||
@@ -94,8 +95,10 @@ class Connection:
|
||||
now = time.time()
|
||||
if not Null:
|
||||
d["addr"] = self.addr
|
||||
if self.rtts[-1]:
|
||||
d["rtt"] = "%0.1f" % self.rtts[-1]
|
||||
if self.state == Connection.ABSENT:
|
||||
d["rtt"] = ""
|
||||
elif self.rtts[-1]:
|
||||
d["rtt"] = "%d" % round(self.rtts[-1])
|
||||
elif self.state == Connection.UNKNOWN:
|
||||
d["rtt"] = ""
|
||||
else:
|
||||
@@ -297,13 +300,26 @@ class Host:
|
||||
self.plugin_retention = 100 # Keep last N samples per plugin
|
||||
# Alert state tracking: {metric_path: AlertState}
|
||||
self.alert_states = {}
|
||||
# Stale-data timers: {plugin_name: asyncio.TimerHandle}
|
||||
self.plugin_timers = {}
|
||||
# Client-reported protocol presence: {afam: {"present": bool, "ts": float}}
|
||||
self.presence: dict = {}
|
||||
# User access control
|
||||
self.owner: str | None = None # username of owner
|
||||
self.managers: list = [] # usernames with manager role
|
||||
self.monitors: list = [] # usernames with monitor role
|
||||
|
||||
def __getstate__(self):
|
||||
"""Prepare Host for pickling by excluding non-serializable timer objects."""
|
||||
state = self.__dict__.copy()
|
||||
# asyncio TimerHandles (and their lambda callbacks) can't be pickled.
|
||||
# They're recreated when the next PLG arrives after unpickling.
|
||||
state['plugin_timers'] = {}
|
||||
return state
|
||||
|
||||
def statedict(self):
|
||||
d = {}
|
||||
d["raw_name"] = self.name
|
||||
d["name"] = self.name
|
||||
if self.dyn:
|
||||
d["name"] += "*"
|
||||
@@ -311,6 +327,7 @@ class Host:
|
||||
d["name"] = "<b>%s</b>" % d["name"]
|
||||
d["dyn"] = str(self.dyn)
|
||||
d["num"] = self.num
|
||||
d["status"] = self.status()
|
||||
|
||||
# Add alert counts (split by acknowledged status)
|
||||
warning_unacked = 0
|
||||
@@ -364,7 +381,7 @@ class Host:
|
||||
def stateinfo(self):
|
||||
ddict = {}
|
||||
for d in self.__dict__:
|
||||
if d in ["alert_states", "plugin_data"]:
|
||||
if d in ["alert_states", "plugin_data", "plugin_timers"]:
|
||||
continue
|
||||
if d == "connections":
|
||||
cl = []
|
||||
@@ -417,6 +434,10 @@ class Host:
|
||||
ddict["alert_critical_unacked"] = critical_unacked
|
||||
ddict["alert_critical_acked"] = critical_acked
|
||||
|
||||
# Flap detection state (module-level in flap.py, never pickled)
|
||||
from . import flap
|
||||
ddict["flapping"] = flap.flapping_services(self.name)
|
||||
|
||||
# User access
|
||||
ddict["owner"] = getattr(self, "owner", None)
|
||||
ddict["managers"] = list(getattr(self, "managers", []))
|
||||
@@ -429,6 +450,7 @@ class Host:
|
||||
_, os_data = latest_os
|
||||
hbc_version = os_data.get("hbc_version")
|
||||
ddict["hbc_version"] = hbc_version
|
||||
ddict["status"] = self.status()
|
||||
|
||||
return ddict
|
||||
|
||||
@@ -444,6 +466,47 @@ class Host:
|
||||
else:
|
||||
return addr.find(".") > 0
|
||||
|
||||
def set_presence(self, afam, present, now):
|
||||
"""Record the client's report of whether it has internet connectivity on afam."""
|
||||
self.presence[afam] = {"present": bool(present), "ts": now}
|
||||
|
||||
def flag_present(self, afam):
|
||||
"""Client-reported presence; True when the client never reported (legacy clients)."""
|
||||
entry = getattr(self, "presence", {}).get(afam)
|
||||
return True if entry is None else entry["present"]
|
||||
|
||||
def family_present(self, afam):
|
||||
"""True if afam should be monitored: the client says it is present, or heartbeats
|
||||
are demonstrably arriving on it (real traffic overrides a false "absent")."""
|
||||
conn = self.connections.get(afam)
|
||||
return self.flag_present(afam) or (conn is not None and conn.state == Connection.UP)
|
||||
|
||||
def status(self):
|
||||
"""Host-level status: down, overdue, simplex, up or unknown.
|
||||
|
||||
overdue: every present family is overdue. simplex: both present, one up, one overdue
|
||||
(same severity as up). Families the client reports absent are not considered,
|
||||
unless none is present: then every connection not parked as ABSENT is considered.
|
||||
"""
|
||||
conns = self.connections
|
||||
if any(c.state == Connection.DOWN for c in conns.values()):
|
||||
return "down"
|
||||
present = [c for afam, c in conns.items() if self.family_present(afam)]
|
||||
if not present:
|
||||
# The client reports no protocol present (e.g. its internet probe fails on every
|
||||
# family) yet may still be heard: monitor whatever is not parked as absent, so a
|
||||
# host that then dies is "overdue", not silently "unknown".
|
||||
present = [c for c in conns.values() if c.state != Connection.ABSENT]
|
||||
up = sum(c.state == Connection.UP for c in present)
|
||||
overdue = sum(c.state == Connection.OVERDUE for c in present)
|
||||
if up and overdue:
|
||||
return "simplex"
|
||||
if up:
|
||||
return "up"
|
||||
if overdue:
|
||||
return "overdue"
|
||||
return "unknown"
|
||||
|
||||
def conndata(self, cid, addr, rtt, now):
|
||||
if addr[0:7] == "::ffff:":
|
||||
addr = addr[7:]
|
||||
@@ -482,6 +545,10 @@ class Host:
|
||||
self.managers = []
|
||||
if not hasattr(self, "monitors"):
|
||||
self.monitors = []
|
||||
if not hasattr(self, "plugin_timers"):
|
||||
self.plugin_timers = {}
|
||||
if not hasattr(self, "presence"):
|
||||
self.presence = {}
|
||||
|
||||
pass
|
||||
|
||||
@@ -541,6 +608,34 @@ class Host:
|
||||
"""
|
||||
return self.plugin_data
|
||||
|
||||
def reset_plugin_timer(self, plugin_name, timeout_seconds, callback):
|
||||
"""Reset the stale-data timer for a plugin.
|
||||
|
||||
If no new PLG data arrives within timeout_seconds, callback(host, plugin_name)
|
||||
is called so the caller can clear history and alerts.
|
||||
"""
|
||||
import asyncio
|
||||
existing = self.plugin_timers.get(plugin_name)
|
||||
if existing and not existing.cancelled():
|
||||
existing.cancel()
|
||||
|
||||
async def _fire():
|
||||
await callback(self, plugin_name)
|
||||
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
self.plugin_timers[plugin_name] = loop.call_later(
|
||||
timeout_seconds, lambda: asyncio.create_task(_fire())
|
||||
)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
def cancel_plugin_timer(self, plugin_name):
|
||||
"""Cancel the stale timer for a plugin, if any."""
|
||||
handle = self.plugin_timers.pop(plugin_name, None)
|
||||
if handle and not handle.cancelled():
|
||||
handle.cancel()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# User-role helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
+1036
-26
File diff suppressed because it is too large
Load Diff
+133
-1
@@ -11,7 +11,7 @@ import os
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
from typing import Dict, Any, Optional, Callable, Iterable, List, Tuple, Union
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -153,6 +153,31 @@ class MessageJournal:
|
||||
except Exception as e:
|
||||
logger.error(f"Error writing to journal: {e}")
|
||||
|
||||
async def log_event(self, event: Dict[str, Any]):
|
||||
"""Write a caller-provided dict verbatim as one JSONL line (with rotation)."""
|
||||
if not self.enabled or not self._initialized:
|
||||
return
|
||||
async with self._lock:
|
||||
try:
|
||||
line = json.dumps(event, separators=(',', ':')) + '\n'
|
||||
nbytes = len(line.encode('utf-8'))
|
||||
if self._current_size + nbytes > self.max_size:
|
||||
await self._rotate()
|
||||
if self._file_handle:
|
||||
self._file_handle.write(line)
|
||||
self._file_handle.flush()
|
||||
self._current_size += nbytes
|
||||
except Exception as e:
|
||||
logger.error(f"Error writing event to journal: {e}")
|
||||
|
||||
async def backfill(self, events: Iterable[Dict[str, Any]]):
|
||||
"""One-time seed: write *events* only if the journal file is currently empty."""
|
||||
if not self.enabled or not self._initialized or self._current_size > 0:
|
||||
return
|
||||
for ev in events:
|
||||
if isinstance(ev, dict):
|
||||
await self.log_event(ev)
|
||||
|
||||
async def _rotate(self):
|
||||
"""
|
||||
Rotate the journal file.
|
||||
@@ -309,6 +334,88 @@ class MessageJournal:
|
||||
}
|
||||
|
||||
|
||||
def _iter_journal_events(journal_dir: Union[str, Path], journal_file: str) -> Iterable[Dict[str, Any]]:
|
||||
"""Yield event dicts from the journal files, newest event first.
|
||||
|
||||
Reads the current file, then rotated backups newest-to-oldest (backup
|
||||
names embed rotation timestamps, so filename sort is chronological).
|
||||
"""
|
||||
dirp = Path(journal_dir)
|
||||
files = [dirp / journal_file]
|
||||
files.extend(sorted(dirp.glob(journal_file + '.*'), reverse=True))
|
||||
for f in files:
|
||||
if not f.is_file():
|
||||
continue
|
||||
try:
|
||||
lines = f.read_text(encoding='utf-8', errors='replace').splitlines()
|
||||
except OSError as e:
|
||||
logger.warning(f"Cannot read journal file {f}: {e}")
|
||||
continue
|
||||
for line in reversed(lines):
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(ev, dict):
|
||||
yield ev
|
||||
|
||||
|
||||
def filter_events(
|
||||
events: Iterable[Dict[str, Any]],
|
||||
limit: int = 100,
|
||||
before: Optional[float] = None,
|
||||
host: Optional[str] = None,
|
||||
level: Optional[str] = None,
|
||||
q: Optional[str] = None,
|
||||
predicate: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
||||
) -> Tuple[List[Dict[str, Any]], bool]:
|
||||
"""Filter an iterable of event dicts already ordered newest-first.
|
||||
|
||||
Returns (events, more): up to *limit* matching events, and whether at
|
||||
least one further matching event exists beyond the limit.
|
||||
"""
|
||||
host_l = host.lower() if host else None
|
||||
level_l = level.lower() if level else None
|
||||
q_l = q.lower() if q else None
|
||||
out: List[Dict[str, Any]] = []
|
||||
for ev in events:
|
||||
if not isinstance(ev, dict):
|
||||
continue
|
||||
ts = ev.get('ts')
|
||||
if before is not None and (not isinstance(ts, (int, float)) or ts >= before):
|
||||
continue
|
||||
if host_l and host_l not in str(ev.get('host') or '').lower():
|
||||
continue
|
||||
if level_l and str(ev.get('level') or '').lower() != level_l:
|
||||
continue
|
||||
if q_l and q_l not in str(ev.get('message') or '').lower():
|
||||
continue
|
||||
if predicate is not None and not predicate(ev):
|
||||
continue
|
||||
if len(out) >= limit:
|
||||
return out, True
|
||||
out.append(ev)
|
||||
return out, False
|
||||
|
||||
|
||||
def read_events(
|
||||
journal_dir: Union[str, Path],
|
||||
journal_file: str = 'events.journal',
|
||||
*,
|
||||
limit: int = 100,
|
||||
before: Optional[float] = None,
|
||||
host: Optional[str] = None,
|
||||
level: Optional[str] = None,
|
||||
q: Optional[str] = None,
|
||||
predicate: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
||||
) -> Tuple[List[Dict[str, Any]], bool]:
|
||||
"""Read filtered events newest-first from the events journal files."""
|
||||
return filter_events(
|
||||
_iter_journal_events(journal_dir, journal_file),
|
||||
limit=limit, before=before, host=host, level=level, q=q, predicate=predicate,
|
||||
)
|
||||
|
||||
|
||||
# Global journal instance
|
||||
_journal_instance: Optional[MessageJournal] = None
|
||||
|
||||
@@ -340,3 +447,28 @@ async def log_message(msg: Dict[str, Any], addr: tuple, timestamp: Optional[floa
|
||||
"""
|
||||
journal = get_journal()
|
||||
await journal.log_message(msg, addr, timestamp)
|
||||
|
||||
|
||||
# Global events journal instance (human-readable event log, written by notify.eventlog)
|
||||
_events_journal_instance: Optional[MessageJournal] = None
|
||||
|
||||
|
||||
def get_events_journal(config: Optional[Dict[str, Any]] = None) -> MessageJournal:
|
||||
"""Get or create the global events journal instance.
|
||||
|
||||
Uses the events_journal_* config keys; shares journal_dir and
|
||||
journal_enabled with the raw-datagram journal.
|
||||
"""
|
||||
global _events_journal_instance
|
||||
if _events_journal_instance is None:
|
||||
cfg = config or {}
|
||||
_events_journal_instance = MessageJournal(
|
||||
{
|
||||
'journal_dir': cfg.get('journal_dir', '/var/log/heartbeat'),
|
||||
'journal_file': cfg.get('events_journal_file', 'events.journal'),
|
||||
'journal_max_size': cfg.get('events_journal_max_size', 10 * 1024 * 1024),
|
||||
'journal_max_backups': cfg.get('events_journal_max_backups', 10),
|
||||
'journal_enabled': cfg.get('journal_enabled', True),
|
||||
}
|
||||
)
|
||||
return _events_journal_instance
|
||||
|
||||
+43
-14
@@ -78,9 +78,7 @@ async def reload_configuration(config_obj, config_path, components):
|
||||
True if reload succeeded, False otherwise
|
||||
"""
|
||||
try:
|
||||
logger.info("=" * 60)
|
||||
logger.info("Starting configuration reload...")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# Reload config file
|
||||
new_config = await config_obj.reload(config_path)
|
||||
@@ -101,9 +99,10 @@ async def reload_configuration(config_obj, config_path, components):
|
||||
access = config_mod.get_host_access(new_config, hostname)
|
||||
host.apply_access(access["owner"], access["managers"], access["monitors"])
|
||||
|
||||
# Reload threshold checker
|
||||
# Reload threshold checker and prune alerts orphaned by the new config
|
||||
if 'threshold_checker' in components:
|
||||
components['threshold_checker'].reload(new_config)
|
||||
components['threshold_checker'].purge_stale_alerts(hbdclass)
|
||||
|
||||
# Note: Changes to the following require restart:
|
||||
# - hb_port, hbd_port, ws_port (already bound)
|
||||
@@ -114,13 +113,11 @@ async def reload_configuration(config_obj, config_path, components):
|
||||
# These are reloadable and effective immediately:
|
||||
# - notification_channels
|
||||
# - threshold_configs
|
||||
# - hosts (watchhosts, dyndnshosts, notification_channels)
|
||||
# - hosts (watchhosts, dyndns, notification_channels)
|
||||
# - grace period (used on next heartbeat)
|
||||
# - debug/verbose flags (used on next message)
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("Configuration reload completed successfully")
|
||||
logger.info("=" * 60)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
@@ -168,6 +165,15 @@ async def _run_async(config, config_path=None):
|
||||
msg_journal = journal_mod.get_journal(config)
|
||||
await msg_journal.initialize()
|
||||
|
||||
# Initialize events journal (human-readable event log for the /log page)
|
||||
events_journal = journal_mod.get_events_journal(config)
|
||||
await events_journal.initialize()
|
||||
if data.msgs:
|
||||
# One-time seed on upgrade: only writes when the journal file is empty
|
||||
await events_journal.backfill(data.msgs)
|
||||
|
||||
notify_mod.eventlog(None, "INFO", f"hbd version {__version__} starting up")
|
||||
|
||||
# Initialize threshold checker
|
||||
threshold_checker = threshold_mod.ThresholdChecker(
|
||||
config=config,
|
||||
@@ -210,7 +216,6 @@ async def _run_async(config, config_path=None):
|
||||
ctx = dict(
|
||||
config=config,
|
||||
hbdclass=hbdclass,
|
||||
log=eventlog,
|
||||
msg_to_websockets=msg_to_websockets,
|
||||
msg_journal=msg_journal,
|
||||
threshold_checker=threshold_checker,
|
||||
@@ -237,12 +242,18 @@ async def _run_async(config, config_path=None):
|
||||
restore_ctx = dict(
|
||||
config=config,
|
||||
hbdclass=hbdclass,
|
||||
log=eventlog,
|
||||
msg_to_websockets=msg_to_websockets,
|
||||
threshold_checker=threshold_checker,
|
||||
)
|
||||
udp.restore_connection_timers(hbdclass, restore_ctx)
|
||||
|
||||
# Drop alert states that no longer have a matching threshold (stale after
|
||||
# upgrade or config change between runs).
|
||||
threshold_checker.purge_stale_alerts(hbdclass)
|
||||
|
||||
async def _http_reload_callback():
|
||||
await reload_configuration(config, config_path, components)
|
||||
|
||||
# HTTP server (asyncio-based via aiohttp)
|
||||
try:
|
||||
http_task = asyncio.create_task(
|
||||
@@ -252,9 +263,11 @@ async def _run_async(config, config_path=None):
|
||||
config=config,
|
||||
hbdclass=hbdclass,
|
||||
tcss=None,
|
||||
threshold_checker=threshold_checker,
|
||||
verbose=config.get("verbose", False),
|
||||
get_now=lambda: time.time(),
|
||||
VER="",
|
||||
reload_callback=_http_reload_callback,
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
@@ -375,6 +388,26 @@ async def _run_async(config, config_path=None):
|
||||
except Exception as e:
|
||||
logger.warning("Error closing message journal: %s", e)
|
||||
|
||||
# Journal the shutdown event now; run()'s shutdown eventlog fires after this journal closes
|
||||
try:
|
||||
await events_journal.log_event(
|
||||
{
|
||||
"ts": time.time(),
|
||||
"host": None,
|
||||
"level": "INFO",
|
||||
"service": None,
|
||||
"message": f"hbd version {__version__} shutdown",
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Could not journal shutdown event: %s", e)
|
||||
|
||||
# Close events journal
|
||||
try:
|
||||
await events_journal.close()
|
||||
except Exception as e:
|
||||
logger.warning("Error closing events journal: %s", e)
|
||||
|
||||
# Signal DNS worker to exit and await it
|
||||
try:
|
||||
if "dns_task" in locals() and dns_task:
|
||||
@@ -418,7 +451,6 @@ def load_pickled_hosts(config, hbdclass):
|
||||
pickfile = config.get("pickfile", "hbd.pickle")
|
||||
dyndnshosts = config_mod.get_dyndnshosts(config)
|
||||
watchhosts = config_mod.get_watchhosts(config)
|
||||
drophosts = config.get("drophosts", [])
|
||||
if 1 and os.path.exists(pickfile):
|
||||
if config.get("verbose", False):
|
||||
logger.info("opening pickls %s", pickfile)
|
||||
@@ -444,9 +476,6 @@ def load_pickled_hosts(config, hbdclass):
|
||||
hbdclass.Host.hosts[h].apply_access(
|
||||
access["owner"], access["managers"], access["monitors"]
|
||||
)
|
||||
for h in drophosts:
|
||||
if h in hbdclass.Host.hosts:
|
||||
del hbdclass.Host.hosts[h]
|
||||
if config.get("verbose", False):
|
||||
logger.info("%s pickled hosts loaded", len(hbdclass.Host.hosts))
|
||||
else:
|
||||
@@ -471,6 +500,8 @@ def run(config, config_path=None):
|
||||
if config.get("debug", 0) > 0:
|
||||
log_level = logging.DEBUG
|
||||
logging.basicConfig(level=log_level)
|
||||
if not config.get("debug", 0):
|
||||
logging.getLogger("aiohttp.access").propagate = False
|
||||
load_pickled_hosts(config, hbdclass)
|
||||
|
||||
notify_mod.initlog(logfile=config.get("logfile", "messages.log"))
|
||||
@@ -485,8 +516,6 @@ def run(config, config_path=None):
|
||||
except Exception as e:
|
||||
logger.warning("Failed to write pidfile %s: %s", pidfile, e)
|
||||
|
||||
eventlog(None, "INFO", f"hbd version {__version__} starting up")
|
||||
|
||||
if config_path:
|
||||
logger.info(f"Config file: {config_path} (reload with SIGHUP)")
|
||||
else:
|
||||
|
||||
+83
-59
@@ -15,7 +15,6 @@ their own ``notification_channels`` list. When no users are configured the
|
||||
server runs silently (no notifications sent).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import asyncio
|
||||
import logging
|
||||
import smtplib
|
||||
@@ -26,12 +25,11 @@ from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from . import data
|
||||
from . import flap as flap_mod
|
||||
from . import ws as ws_mod
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
msg_to_websockets = ws_mod.broadcast
|
||||
|
||||
# Module-level state set via setup()
|
||||
@@ -66,6 +64,7 @@ class Notification:
|
||||
body: str # detail message
|
||||
level: str # RECOVER | WARNING | CRITICAL | INFO
|
||||
url: str = "" # link to plugin metrics page
|
||||
service: str = "" # flap-detection key within the host ("" = the host itself)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -73,9 +72,10 @@ class Notification:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def setup(cfg: dict, loop: Optional[asyncio.AbstractEventLoop] = None):
|
||||
"""Initialize notifier from configuration dict and event loop."""
|
||||
"""Initialize notifier from configuration dict."""
|
||||
global _config, _loop
|
||||
_config = dict(cfg)
|
||||
flap_mod.setup(_config)
|
||||
if loop is not None:
|
||||
_loop = loop
|
||||
|
||||
@@ -84,6 +84,7 @@ def reload_config(cfg: dict):
|
||||
"""Reload notification configuration on SIGHUP."""
|
||||
global _config
|
||||
_config = dict(cfg)
|
||||
flap_mod.setup(_config)
|
||||
logger.info("Notification configuration reloaded")
|
||||
|
||||
|
||||
@@ -112,11 +113,23 @@ def closelog():
|
||||
|
||||
def eventlog(host, lvl, m, service=None):
|
||||
ts = time.time()
|
||||
msg = {
|
||||
"ts": ts,
|
||||
"host": host or None,
|
||||
"level": lvl,
|
||||
"service": service,
|
||||
"message": m,
|
||||
}
|
||||
data.msgs.append(msg)
|
||||
# Cap the in-memory buffer so it doesn't grow without bound; this list is
|
||||
# replayed to every dashboard client on connect and persisted in the pickle.
|
||||
cap = _config.get("msg_buffer_size", 500)
|
||||
if cap and len(data.msgs) > cap:
|
||||
del data.msgs[:-cap]
|
||||
s = f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))} {lvl} "
|
||||
if host:
|
||||
s += f"{host} "
|
||||
s += m
|
||||
data.msgs.append(s)
|
||||
logger.info(s)
|
||||
if logf:
|
||||
try:
|
||||
@@ -124,7 +137,22 @@ def eventlog(host, lvl, m, service=None):
|
||||
logf.flush()
|
||||
except Exception as e:
|
||||
logger.warning("failed to write to logfile: %s", e)
|
||||
msg_to_websockets("message", s)
|
||||
msg_to_websockets("message", msg)
|
||||
_journal_event(msg)
|
||||
|
||||
|
||||
def _journal_event(msg: dict):
|
||||
"""Schedule an async write of *msg* to the events journal (no-op without a loop)."""
|
||||
if _loop is None or not _loop.is_running():
|
||||
return
|
||||
from . import journal as journal_mod
|
||||
ej = journal_mod._events_journal_instance
|
||||
if ej is None or not ej.enabled:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(ej.log_event(msg), _loop)
|
||||
except Exception as e:
|
||||
logger.warning("failed to schedule events journal write: %s", e)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -139,10 +167,14 @@ def _send_pushover(channel_cfg: dict, notif: Notification) -> bool:
|
||||
if not token or not user:
|
||||
logger.warning("pushover: missing token or user")
|
||||
return False
|
||||
params: dict = {"token": token, "user": user, "title": notif.title, "message": notif.body}
|
||||
body = "%s: %s" % (notif.title, notif.body)
|
||||
title = ""
|
||||
params: dict = {"token": token, "user": user, "title": title, "message": body}
|
||||
if channel_cfg.get("sound"):
|
||||
params["sound"] = channel_cfg["sound"]
|
||||
if notif.url:
|
||||
params["url"] = notif.url
|
||||
params["url_title"] = "Plugin metrics"
|
||||
params["url_title"] = "Heartbeat"
|
||||
conn = http.client.HTTPSConnection("api.pushover.net:443")
|
||||
try:
|
||||
conn.request(
|
||||
@@ -215,7 +247,7 @@ def _send_mattermost(channel_cfg: dict, notif: Notification) -> bool:
|
||||
return False
|
||||
text = f"**{notif.title}**\n{notif.body}"
|
||||
if notif.url:
|
||||
text += f"\n[Plugin metrics]({notif.url})"
|
||||
text += f"\n[Plugin metrics] {notif.url}"
|
||||
ses = {"url": host, "scheme": "http", "basepath": "/api/v4", "port": 8065}
|
||||
mm = Driver(ses)
|
||||
payload: dict = {"text": text, "channel": channel, "username": channel_cfg.get("username", "hbd")}
|
||||
@@ -299,17 +331,6 @@ async def _send_sms_voipms_async(channel_cfg: dict, notif: Notification) -> bool
|
||||
return False
|
||||
|
||||
|
||||
def _send_sms_voipms(channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Dispatch voip.ms SMS send onto the shared event loop."""
|
||||
if _loop is None:
|
||||
logger.warning("sms_voipms: event loop not available")
|
||||
return False
|
||||
future = asyncio.run_coroutine_threadsafe(_send_sms_voipms_async(channel_cfg, notif), _loop)
|
||||
try:
|
||||
return future.result(timeout=15)
|
||||
except Exception as e:
|
||||
logger.error("sms_voipms send timed out or failed: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
|
||||
@@ -357,40 +378,26 @@ async def _send_matrix_async(channel_cfg: dict, notif: Notification) -> bool:
|
||||
await client.close()
|
||||
|
||||
|
||||
def _send_matrix(channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Dispatch matrix send onto the shared event loop."""
|
||||
if _loop is None:
|
||||
logger.warning("matrix: event loop not available")
|
||||
return False
|
||||
future = asyncio.run_coroutine_threadsafe(_send_matrix_async(channel_cfg, notif), _loop)
|
||||
try:
|
||||
return future.result(timeout=15)
|
||||
except Exception as e:
|
||||
logger.error("matrix send timed out or failed: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Channel dispatcher
|
||||
# Channel dispatcher (all async — sync drivers run in a thread executor)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Sync drivers kept for `hbd notify` CLI usage (asyncio.run wraps them there).
|
||||
_DRIVERS = {
|
||||
"pushover": _send_pushover,
|
||||
"email": _send_email,
|
||||
"mattermost": _send_mattermost,
|
||||
"signal": _send_signal,
|
||||
"sms_voipms": _send_sms_voipms,
|
||||
"matrix": _send_matrix,
|
||||
}
|
||||
|
||||
_TIMEOUT = 15 # seconds per channel send
|
||||
|
||||
def _dispatch_to_channel(channel_name: str, channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Send *notif* to a single named channel, honouring min_level.
|
||||
|
||||
RECOVER always bypasses min_level — a recovery is always relevant if the
|
||||
channel was configured for any alerting (handles the restart-then-recover case
|
||||
where _alerted_channels is empty and we fall through to the normal loop).
|
||||
"""
|
||||
async def _dispatch_to_channel(channel_name: str, channel_cfg: dict, notif: Notification) -> bool:
|
||||
"""Send *notif* to a single named channel, honouring min_level."""
|
||||
# Strip ownership metadata — notifier drivers only need delivery credentials.
|
||||
channel_cfg = {k: v for k, v in channel_cfg.items() if k not in ("owner", "private")}
|
||||
|
||||
level = notif.level.upper()
|
||||
if level != "RECOVER":
|
||||
min_level = channel_cfg.get("min_level", "WARNING").upper()
|
||||
@@ -398,14 +405,24 @@ def _dispatch_to_channel(channel_name: str, channel_cfg: dict, notif: Notificati
|
||||
logger.debug(
|
||||
"channel '%s': skipping level %s (min_level=%s)", channel_name, level, min_level
|
||||
)
|
||||
return True # not an error — filtered intentionally
|
||||
return True # filtered intentionally
|
||||
|
||||
ch_type = channel_cfg.get("type", "")
|
||||
driver = _DRIVERS.get(ch_type)
|
||||
if driver is None:
|
||||
logger.warning("unknown channel type '%s' for channel '%s'", ch_type, channel_name)
|
||||
try:
|
||||
if ch_type == "matrix":
|
||||
return await asyncio.wait_for(_send_matrix_async(channel_cfg, notif), timeout=_TIMEOUT)
|
||||
if ch_type == "sms_voipms":
|
||||
return await asyncio.wait_for(_send_sms_voipms_async(channel_cfg, notif), timeout=_TIMEOUT)
|
||||
sync_driver = _DRIVERS.get(ch_type)
|
||||
if sync_driver is None:
|
||||
logger.warning("unknown channel type '%s' for channel '%s'", ch_type, channel_name)
|
||||
return False
|
||||
return await asyncio.wait_for(
|
||||
asyncio.to_thread(sync_driver, channel_cfg, notif), timeout=_TIMEOUT
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error("channel '%s' timed out after %ds", channel_name, _TIMEOUT)
|
||||
return False
|
||||
return driver(channel_cfg, notif)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -416,21 +433,32 @@ def _build_url(host_name: str) -> str:
|
||||
base_url = _config.get("base_url", "").rstrip("/")
|
||||
if not base_url:
|
||||
return ""
|
||||
return f"{base_url}/plugins#{host_name}"
|
||||
return f"{base_url}/alerts?filter={host_name}"
|
||||
|
||||
|
||||
def send_notification(host_name: str, notif: Notification) -> dict:
|
||||
async def send_notification(host_name: str, notif: Notification) -> dict:
|
||||
"""Dispatch *notif* to all managers/owner of *host_name*.
|
||||
|
||||
Looks up the host's owner + managers, resolves each user's
|
||||
notification_channels, and dispatches. Silently does nothing if
|
||||
no users are configured.
|
||||
|
||||
Flap detection runs first: once *host_name*/*notif.service* is flapping the
|
||||
notification is dropped, and the one that trips the state carries
|
||||
``flap.FLAP_MARKER``.
|
||||
|
||||
Returns a dict of {channel_name: bool} results.
|
||||
"""
|
||||
from . import users as users_mod
|
||||
from . import hbdclass
|
||||
|
||||
action = flap_mod.observe(host_name, notif.service, notif.level)
|
||||
if action == flap_mod.SUPPRESS:
|
||||
logger.debug("flapping: suppressed %s notification for %s", notif.level, host_name)
|
||||
return {}
|
||||
if action == flap_mod.TRIP:
|
||||
notif.body = f"{notif.body} {flap_mod.FLAP_MARKER}"
|
||||
|
||||
if not users_mod.users_enabled():
|
||||
return {}
|
||||
|
||||
@@ -469,16 +497,12 @@ def send_notification(host_name: str, notif: Notification) -> dict:
|
||||
if not channel_cfg:
|
||||
continue
|
||||
try:
|
||||
ch_type = channel_cfg.get("type", "")
|
||||
driver = _DRIVERS.get(ch_type)
|
||||
if driver:
|
||||
ok = driver(channel_cfg, notif)
|
||||
results[channel_name] = ok
|
||||
if ok:
|
||||
logger.info("recover sent to channel '%s': %s", channel_name, notif.title)
|
||||
ok = await _dispatch_to_channel(channel_name, channel_cfg, notif)
|
||||
results[channel_name] = ok
|
||||
if ok:
|
||||
logger.info("recover sent to channel '%s': %s", channel_name, notif.title)
|
||||
except Exception as e:
|
||||
logger.error("error sending recover to channel '%s': %s", channel_name, e)
|
||||
# Clear the alerted set once recovery is delivered
|
||||
del _alerted_channels[host_name]
|
||||
return results
|
||||
|
||||
@@ -489,14 +513,14 @@ def send_notification(host_name: str, notif: Notification) -> dict:
|
||||
continue
|
||||
for channel_name in user.notification_channels:
|
||||
if channel_name in results:
|
||||
continue # already dispatched to this channel this notification
|
||||
continue
|
||||
channel_cfg = global_channels.get(channel_name)
|
||||
if not channel_cfg:
|
||||
logger.warning("channel '%s' not defined in notification_channels", channel_name)
|
||||
results[channel_name] = False
|
||||
continue
|
||||
try:
|
||||
ok = _dispatch_to_channel(channel_name, channel_cfg, notif)
|
||||
ok = await _dispatch_to_channel(channel_name, channel_cfg, notif)
|
||||
results[channel_name] = ok
|
||||
if ok:
|
||||
logger.info("notification sent to channel '%s': %s", channel_name, notif.title)
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
"""OAuth2 provider support.
|
||||
|
||||
Config shape (in ~/.hb.yaml):
|
||||
|
||||
oauth:
|
||||
my-gitea: # route slug → /login/oauth/my-gitea
|
||||
type: gitea # "gitea" | "github" | "nextcloud"
|
||||
# omit type to default to "gitea"
|
||||
url: https://git.example.com # required for gitea and nextcloud
|
||||
client_id: <client-id>
|
||||
client_secret: <client-secret>
|
||||
label: "Work Gitea" # optional display name on login button
|
||||
logo: https://example.com/logo.png # optional logo URL
|
||||
|
||||
github:
|
||||
type: github
|
||||
client_id: <client-id>
|
||||
client_secret: <client-secret>
|
||||
|
||||
nextcloud:
|
||||
type: nextcloud
|
||||
url: https://cloud.example.com
|
||||
client_id: <client-id>
|
||||
client_secret: <client-secret>
|
||||
|
||||
Register the OAuth app with each provider and set the redirect URI to:
|
||||
https://<hbd-host>/login/oauth/<name>/callback
|
||||
"""
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
import time
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass
|
||||
|
||||
import aiohttp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
STATE_TTL = 600 # 10 minutes
|
||||
|
||||
# state_token -> expiry timestamp
|
||||
_states: dict[str, float] = {}
|
||||
|
||||
|
||||
def make_state() -> str:
|
||||
"""Generate a CSRF state token, store it with TTL, and return it."""
|
||||
_purge_states()
|
||||
token = secrets.token_hex(32)
|
||||
_states[token] = time.time() + STATE_TTL
|
||||
return token
|
||||
|
||||
|
||||
def validate_state(state: str) -> bool:
|
||||
"""Return True if *state* is known and unexpired; always removes it."""
|
||||
expiry = _states.pop(state, None)
|
||||
if expiry is None:
|
||||
return False
|
||||
return time.time() < expiry
|
||||
|
||||
|
||||
def _purge_states() -> None:
|
||||
"""Remove all expired CSRF state tokens from the in-memory store."""
|
||||
now = time.time()
|
||||
expired = [k for k, exp in list(_states.items()) if exp < now]
|
||||
for k in expired:
|
||||
del _states[k]
|
||||
|
||||
|
||||
class OAuthError(Exception):
|
||||
"""Raised when the OAuth2 flow fails for any reason."""
|
||||
|
||||
|
||||
PROVIDER_DEFS: dict = {
|
||||
"gitea": {
|
||||
"authorize_url_tmpl": "{url}/login/oauth/authorize",
|
||||
"token_url_tmpl": "{url}/login/oauth/access_token",
|
||||
"profile_url_tmpl": "{url}/api/v1/user",
|
||||
"scope": "user:email",
|
||||
"field_map": {"username": "login", "full_name": "full_name", "avatar": "avatar_url"},
|
||||
"profile_data_path": [],
|
||||
"requires_url": True,
|
||||
"default_label": "Gitea",
|
||||
},
|
||||
"github": {
|
||||
"authorize_url_tmpl": "https://github.com/login/oauth/authorize",
|
||||
"token_url_tmpl": "https://github.com/login/oauth/access_token",
|
||||
"profile_url_tmpl": "https://api.github.com/user",
|
||||
"scope": "read:user",
|
||||
"field_map": {"username": "login", "full_name": "name", "avatar": "avatar_url"},
|
||||
"profile_data_path": [],
|
||||
"requires_url": False,
|
||||
"default_label": "GitHub",
|
||||
},
|
||||
"nextcloud": {
|
||||
"authorize_url_tmpl": "{url}/apps/oauth2/authorize",
|
||||
"token_url_tmpl": "{url}/apps/oauth2/api/v1/token",
|
||||
"profile_url_tmpl": "{url}/ocs/v2.php/cloud/user?format=json",
|
||||
"scope": "",
|
||||
"field_map": {"username": "id", "full_name": "display-name", "avatar": None},
|
||||
"profile_data_path": ["ocs", "data"],
|
||||
"requires_url": True,
|
||||
"default_label": "Nextcloud",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolvedProvider:
|
||||
"""A fully resolved OAuth2 provider instance, ready to use."""
|
||||
name: str
|
||||
type: str
|
||||
label: str
|
||||
logo: str
|
||||
authorize_url: str
|
||||
token_url: str
|
||||
profile_url: str
|
||||
scope: str
|
||||
client_id: str
|
||||
client_secret: str
|
||||
field_map: dict
|
||||
profile_data_path: list
|
||||
|
||||
|
||||
def get_providers(config: dict) -> list[ResolvedProvider]:
|
||||
"""Return a ResolvedProvider for every valid entry in config['oauth'].
|
||||
|
||||
Entries with missing required fields or unknown types are skipped with
|
||||
a warning log. Order follows config declaration order.
|
||||
"""
|
||||
result = []
|
||||
oauth_cfg = config.get("oauth", {})
|
||||
if not isinstance(oauth_cfg, dict):
|
||||
return result
|
||||
for name, entry in oauth_cfg.items():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
provider_type = entry.get("type", "gitea")
|
||||
defn = PROVIDER_DEFS.get(provider_type)
|
||||
if defn is None:
|
||||
logger.warning("OAuth: unknown provider type %r for %r, skipping", provider_type, name)
|
||||
continue
|
||||
client_id = entry.get("client_id", "")
|
||||
client_secret = entry.get("client_secret", "")
|
||||
if not client_id or not client_secret:
|
||||
logger.warning("OAuth: %r missing client_id or client_secret, skipping", name)
|
||||
continue
|
||||
url = entry.get("url", "").rstrip("/")
|
||||
if defn["requires_url"] and not url:
|
||||
logger.warning("OAuth: %r requires url but none configured, skipping", name)
|
||||
continue
|
||||
label = entry.get("label") or defn["default_label"]
|
||||
logo = entry.get("logo", "")
|
||||
result.append(ResolvedProvider(
|
||||
name=name,
|
||||
type=provider_type,
|
||||
label=label,
|
||||
logo=logo,
|
||||
authorize_url=defn["authorize_url_tmpl"].format(url=url),
|
||||
token_url=defn["token_url_tmpl"].format(url=url),
|
||||
profile_url=defn["profile_url_tmpl"].format(url=url),
|
||||
scope=defn["scope"],
|
||||
client_id=client_id,
|
||||
client_secret=client_secret,
|
||||
field_map=dict(defn["field_map"]),
|
||||
profile_data_path=list(defn["profile_data_path"]),
|
||||
))
|
||||
return result
|
||||
|
||||
|
||||
def is_enabled(config: dict) -> bool:
|
||||
"""Return True when at least one OAuth provider is fully configured."""
|
||||
return bool(get_providers(config))
|
||||
|
||||
|
||||
def build_auth_url(provider: ResolvedProvider, state: str, redirect_uri: str) -> str:
|
||||
"""Return the provider's OAuth2 authorization URL to redirect the browser to."""
|
||||
params: dict = {
|
||||
"client_id": provider.client_id,
|
||||
"redirect_uri": redirect_uri,
|
||||
"response_type": "code",
|
||||
"state": state,
|
||||
}
|
||||
if provider.scope:
|
||||
params["scope"] = provider.scope
|
||||
return f"{provider.authorize_url}?{urllib.parse.urlencode(params)}"
|
||||
|
||||
|
||||
async def exchange_code(provider: ResolvedProvider, code: str, redirect_uri: str) -> str:
|
||||
"""Exchange an authorization *code* for an access token.
|
||||
|
||||
Returns the access token string. Raises OAuthError on any failure.
|
||||
"""
|
||||
payload = {
|
||||
"client_id": provider.client_id,
|
||||
"client_secret": provider.client_secret,
|
||||
"code": code,
|
||||
"grant_type": "authorization_code",
|
||||
"redirect_uri": redirect_uri,
|
||||
}
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.post(
|
||||
provider.token_url,
|
||||
json=payload,
|
||||
headers={"Accept": "application/json"},
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
raise OAuthError(f"Token exchange failed ({resp.status}): {text}")
|
||||
data = await resp.json()
|
||||
token = data.get("access_token")
|
||||
if not token:
|
||||
raise OAuthError(f"No access_token in response: {data}")
|
||||
except aiohttp.ClientError as exc:
|
||||
raise OAuthError(f"Token exchange network error: {exc}") from exc
|
||||
return token
|
||||
|
||||
|
||||
async def fetch_user(provider: ResolvedProvider, token: str) -> dict:
|
||||
"""Fetch the authenticated user's profile from the provider.
|
||||
|
||||
Returns a dict with keys: login, full_name, avatar_url.
|
||||
Raises OAuthError on any failure.
|
||||
"""
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(
|
||||
provider.profile_url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
) as resp:
|
||||
if resp.status != 200:
|
||||
text = await resp.text()
|
||||
raise OAuthError(f"User fetch failed ({resp.status}): {text}")
|
||||
data = await resp.json()
|
||||
except aiohttp.ClientError as exc:
|
||||
raise OAuthError(f"User fetch network error: {exc}") from exc
|
||||
|
||||
try:
|
||||
for key in provider.profile_data_path:
|
||||
data = data.get(key, {})
|
||||
avatar_field = provider.field_map.get("avatar")
|
||||
return {
|
||||
"login": data.get(provider.field_map["username"], ""),
|
||||
"full_name": data.get(provider.field_map["full_name"], ""),
|
||||
"avatar_url": data.get(avatar_field, "") if avatar_field else "",
|
||||
}
|
||||
except AttributeError:
|
||||
raise OAuthError(f"Unexpected profile response structure from {provider.type}")
|
||||
+253
-38
@@ -21,19 +21,73 @@ editable bool Reserved for future use — currently always False
|
||||
sensitive bool True when the raw value must never be shown
|
||||
"""
|
||||
|
||||
from . import config_access
|
||||
|
||||
# Credential field names that should always be masked.
|
||||
_SECRET_KEYS = frozenset({
|
||||
"password", "token", "user_key", "api_key", "secret",
|
||||
"smtp_password", "smtp_user",
|
||||
"smtp_password", "smtp_user", "api_password", "access_token",
|
||||
})
|
||||
|
||||
_CHANNEL_TYPE_LABELS = {
|
||||
"pushover": "Pushover",
|
||||
"email": "E-mail",
|
||||
"signal": "Signal",
|
||||
"mattermost": "Mattermost",
|
||||
CHANNEL_TYPE_SCHEMAS = {
|
||||
"pushover": {
|
||||
"label": "Pushover",
|
||||
"fields": [
|
||||
{"key": "token", "label": "App token", "type": "secret", "required": True},
|
||||
{"key": "user", "label": "User key", "type": "secret", "required": True},
|
||||
{"key": "sound", "label": "Sound", "type": "text", "required": False},
|
||||
],
|
||||
},
|
||||
"email": {
|
||||
"label": "E-mail",
|
||||
"fields": [
|
||||
{"key": "recipients", "label": "Recipients (comma-separated)", "type": "list", "required": True},
|
||||
{"key": "sender", "label": "From address", "type": "text", "required": True},
|
||||
{"key": "smtp_server", "label": "SMTP server", "type": "text", "required": True},
|
||||
{"key": "smtp_port", "label": "SMTP port", "type": "port", "required": False},
|
||||
{"key": "smtp_user", "label": "SMTP username", "type": "text", "required": False},
|
||||
{"key": "smtp_password", "label": "SMTP password", "type": "secret", "required": False},
|
||||
],
|
||||
},
|
||||
"signal": {
|
||||
"label": "Signal",
|
||||
"fields": [
|
||||
{"key": "user", "label": "Sender number", "type": "text", "required": True},
|
||||
{"key": "recipient", "label": "Recipient number", "type": "text", "required": True},
|
||||
{"key": "cli_path", "label": "signal-cli path", "type": "text", "required": False},
|
||||
],
|
||||
},
|
||||
"matrix": {
|
||||
"label": "Matrix",
|
||||
"fields": [
|
||||
{"key": "homeserver", "label": "Homeserver URL", "type": "text", "required": True},
|
||||
{"key": "access_token", "label": "Access token", "type": "secret", "required": True},
|
||||
{"key": "room_id", "label": "Room ID", "type": "text", "required": True},
|
||||
],
|
||||
},
|
||||
"sms_voipms": {
|
||||
"label": "SMS (voip.ms)",
|
||||
"fields": [
|
||||
{"key": "api_user", "label": "API username", "type": "text", "required": True},
|
||||
{"key": "api_password", "label": "API password", "type": "secret", "required": True},
|
||||
{"key": "did", "label": "DID (from)", "type": "text", "required": True},
|
||||
{"key": "dst", "label": "Destination", "type": "text", "required": True},
|
||||
],
|
||||
},
|
||||
"mattermost": {
|
||||
"label": "Mattermost",
|
||||
"fields": [
|
||||
{"key": "host", "label": "Host", "type": "text", "required": True},
|
||||
{"key": "token", "label": "Webhook token", "type": "secret", "required": True},
|
||||
{"key": "channel", "label": "Channel", "type": "text", "required": True},
|
||||
{"key": "username", "label": "Bot username", "type": "text", "required": False},
|
||||
{"key": "icon", "label": "Icon URL", "type": "text", "required": False},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
_CHANNEL_TYPE_LABELS = {k: v["label"] for k, v in CHANNEL_TYPE_SCHEMAS.items()}
|
||||
|
||||
|
||||
def _mask(value):
|
||||
"""Return a masked placeholder for sensitive values."""
|
||||
@@ -88,9 +142,14 @@ def _sanitize_channel(name, cfg):
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_settings_sections(config: dict) -> list:
|
||||
def get_settings_sections(config: dict, threshold_checker=None, user=None) -> list:
|
||||
"""Return ordered list of setting sections for the settings page.
|
||||
|
||||
*user* is an object with ``username``/``admin`` attributes, or None for
|
||||
the unauthenticated (admin-equivalent) view. Non-admins get only the
|
||||
channels/hosts/thresholds sections, filtered to global items plus what
|
||||
they own or manage.
|
||||
|
||||
Each section:
|
||||
{
|
||||
"title": str,
|
||||
@@ -110,6 +169,9 @@ def get_settings_sections(config: dict) -> list:
|
||||
"sensitive": bool,
|
||||
}
|
||||
"""
|
||||
is_admin = user is None or getattr(user, "admin", False)
|
||||
username: str = getattr(user, "username", "") or ""
|
||||
|
||||
def field(key, label, ftype, description="", editable=False, sensitive=False):
|
||||
raw = config.get(key)
|
||||
if sensitive:
|
||||
@@ -143,14 +205,17 @@ def get_settings_sections(config: dict) -> list:
|
||||
}
|
||||
|
||||
# ---- Notification channels (complex, built separately) ----------------
|
||||
_METADATA_KEYS = {"type", "owner", "private", "min_level"}
|
||||
notif_channels = []
|
||||
for ch_name, ch_cfg in (config.get("notification_channels") or {}).items():
|
||||
for ch_name, ch_cfg in sorted((config.get("notification_channels") or {}).items()):
|
||||
if not isinstance(ch_cfg, dict):
|
||||
continue
|
||||
if not is_admin and not config_access.user_can_use(ch_cfg, username):
|
||||
continue
|
||||
ch_type = ch_cfg.get("type", "")
|
||||
fields = []
|
||||
for k, v in ch_cfg.items():
|
||||
if k == "type":
|
||||
if k in _METADATA_KEYS:
|
||||
continue
|
||||
sensitive = k in _SECRET_KEYS
|
||||
fields.append({
|
||||
@@ -165,60 +230,140 @@ def get_settings_sections(config: dict) -> list:
|
||||
"name": ch_name,
|
||||
"type": ch_type,
|
||||
"type_label": _CHANNEL_TYPE_LABELS.get(ch_type, ch_type.title()),
|
||||
"owner": ch_cfg.get("owner"),
|
||||
"editable": is_admin or ch_cfg.get("owner") == username,
|
||||
"min_level": ch_cfg.get("min_level", "WARNING"),
|
||||
"fields": fields,
|
||||
})
|
||||
|
||||
# ---- Users (show metadata only, never password hashes) ----------------
|
||||
users_list = []
|
||||
for username, attrs in (config.get("users") or {}).items():
|
||||
for uname, attrs in (config.get("users") or {}).items():
|
||||
if not isinstance(attrs, dict):
|
||||
continue
|
||||
users_list.append({
|
||||
"username": username,
|
||||
"username": uname,
|
||||
"full_name": attrs.get("full_name", ""),
|
||||
"admin": bool(attrs.get("admin", False)),
|
||||
"avatar": attrs.get("avatar", ""),
|
||||
"notification_channels": attrs.get("notification_channels", []),
|
||||
})
|
||||
|
||||
# ---- Threshold configurations -----------------------------------------
|
||||
def _tc_to_row(tc):
|
||||
return {
|
||||
"metric": tc.metric_path,
|
||||
"operator": tc.operator.value,
|
||||
"warning": tc.warning,
|
||||
"critical": tc.critical,
|
||||
"hysteresis": tc.hysteresis,
|
||||
"count": tc.count,
|
||||
"enabled": tc.enabled,
|
||||
"display": tc.display or "",
|
||||
"grace": tc.grace,
|
||||
}
|
||||
|
||||
threshold_config_list = []
|
||||
raw_threshold_cfgs = config.get("threshold_configs") or {}
|
||||
if threshold_checker is not None:
|
||||
if threshold_checker.threshold_configs:
|
||||
for cfg_name, cfg_metrics in sorted(threshold_checker.threshold_configs.items()):
|
||||
raw_cfg = raw_threshold_cfgs.get(cfg_name)
|
||||
tc_owner = raw_cfg.get("owner") if isinstance(raw_cfg, dict) else None
|
||||
if not is_admin and tc_owner and tc_owner != username:
|
||||
continue
|
||||
# For the default config use the merged effective set;
|
||||
# for named overrides use only the explicitly defined metrics
|
||||
# (threshold_raw_configs) so inherited defaults are not repeated.
|
||||
if cfg_name == "default":
|
||||
display_metrics = cfg_metrics
|
||||
else:
|
||||
display_metrics = threshold_checker.threshold_raw_configs.get(cfg_name, cfg_metrics)
|
||||
metrics = sorted(
|
||||
[_tc_to_row(tc) for tc in display_metrics.values()],
|
||||
key=lambda m: m["metric"],
|
||||
)
|
||||
threshold_config_list.append({
|
||||
"name": cfg_name,
|
||||
"metrics": metrics,
|
||||
"owner": tc_owner,
|
||||
"editable": is_admin or (tc_owner is not None and tc_owner == username),
|
||||
})
|
||||
elif threshold_checker.thresholds:
|
||||
metrics = sorted(
|
||||
[_tc_to_row(tc) for tc in threshold_checker.thresholds.values()],
|
||||
key=lambda m: m["metric"],
|
||||
)
|
||||
threshold_config_list.append({"name": "default", "metrics": metrics,
|
||||
"owner": None, "editable": is_admin})
|
||||
|
||||
# ---- Hosts summary ----------------------------------------------------
|
||||
hosts_list = []
|
||||
for hname, hcfg in (config.get("hosts") or {}).items():
|
||||
for hname, hcfg in sorted((config.get("hosts") or {}).items()):
|
||||
if not isinstance(hcfg, dict):
|
||||
continue
|
||||
managers = hcfg.get("managers", [])
|
||||
if isinstance(managers, str):
|
||||
managers = [managers]
|
||||
if not is_admin and hcfg.get("owner") != username and username not in managers:
|
||||
continue
|
||||
hosts_list.append({
|
||||
"name": hname,
|
||||
"watch": bool(hcfg.get("watch", False)),
|
||||
"watch": bool(hcfg.get("watch", True)),
|
||||
"dyndns": bool(hcfg.get("dyndns", False)),
|
||||
"owner": hcfg.get("owner", ""),
|
||||
"managers": hcfg.get("managers", []),
|
||||
"is_owner": is_admin or hcfg.get("owner") == username,
|
||||
"managers": managers,
|
||||
"monitors": hcfg.get("monitors", []),
|
||||
"threshold_config": hcfg.get("threshold_config", ""),
|
||||
"threshold_configs": (
|
||||
list(v) if isinstance(v := hcfg.get("threshold_config"), list)
|
||||
else ([v] if v else [])
|
||||
),
|
||||
"notification_channels": hcfg.get("notification_channels", []),
|
||||
})
|
||||
|
||||
return [
|
||||
# ---- OAuth providers -------------------------------------------------------
|
||||
oauth_providers = []
|
||||
for pname, pattrs in (config.get("oauth") or {}).items():
|
||||
if not isinstance(pattrs, dict):
|
||||
continue
|
||||
cs = pattrs.get("client_secret", "")
|
||||
oauth_providers.append({
|
||||
"name": pname,
|
||||
"type": pattrs.get("type", "gitea"),
|
||||
"url": pattrs.get("url", ""),
|
||||
"client_id": pattrs.get("client_id", ""),
|
||||
"client_secret": "•••" if cs else "",
|
||||
"label": pattrs.get("label", ""),
|
||||
"logo": pattrs.get("logo", ""),
|
||||
})
|
||||
|
||||
sections: list = [
|
||||
{
|
||||
"id": "network",
|
||||
"title": "Network",
|
||||
"description": "Ports and bind addresses for all server sockets.",
|
||||
"section_mode": "form",
|
||||
"api_section": "server",
|
||||
"fields": [
|
||||
field("hb_port", "Heartbeat UDP port", "port",
|
||||
"UDP port the server listens on for heartbeat datagrams."),
|
||||
"UDP port the server listens on for heartbeat datagrams.", editable=True),
|
||||
field("hbd_host", "HTTP bind address", "text",
|
||||
"Interface to bind the HTTP server to. Empty = all interfaces."),
|
||||
"Interface to bind the HTTP server to. Empty = all interfaces.", editable=True),
|
||||
field("hbd_port", "HTTP API port", "port",
|
||||
"TCP port for the HTTP API and web UI."),
|
||||
"TCP port for the HTTP API and web UI.", editable=True),
|
||||
field("ws_port", "WebSocket port", "port",
|
||||
"TCP port for the plain WebSocket server."),
|
||||
"TCP port for the plain WebSocket server.", editable=True),
|
||||
field("wss_port", "Secure WebSocket port", "port",
|
||||
"TCP port for WSS (TLS WebSocket). Leave empty to disable."),
|
||||
"TCP port for WSS (TLS WebSocket). Leave empty to disable.", editable=True),
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "tls",
|
||||
"title": "TLS / WebSocket Security",
|
||||
"description": "Certificate paths used when wss_port is set.",
|
||||
"section_mode": "form",
|
||||
"api_section": None,
|
||||
"fields": [
|
||||
field("cert_path", "Certificate directory", "path",
|
||||
"Directory containing the TLS certificate and key files."),
|
||||
@@ -232,73 +377,103 @@ def get_settings_sections(config: dict) -> list:
|
||||
"id": "monitoring",
|
||||
"title": "Monitoring",
|
||||
"description": "Heartbeat timing and alert re-notification behaviour.",
|
||||
"section_mode": "form",
|
||||
"api_section": "server",
|
||||
"fields": [
|
||||
field("interval", "Heartbeat interval", "duration",
|
||||
"Expected time between heartbeat messages from each client."),
|
||||
field("grace", "Grace multiplier", "number",
|
||||
"A host is marked overdue after interval × grace seconds of silence."),
|
||||
"Expected time between heartbeat messages from each client.", editable=True),
|
||||
field("grace", "Grace period", "number",
|
||||
"Extra seconds to wait after a missed heartbeat before sending notifications.", editable=True),
|
||||
field("threshold_renotify_interval", "Re-notify interval", "duration",
|
||||
"How often to re-send notifications for ongoing threshold alerts."),
|
||||
"How often to re-send notifications for ongoing threshold alerts.", editable=True),
|
||||
field("flap_count", "Flap count", "number",
|
||||
"Warning/critical notifications within the flap interval that mark a "
|
||||
"service as flapping and silence it. 0 disables flap detection.", editable=True),
|
||||
field("flap_interval", "Flap interval", "number",
|
||||
"Minutes: the window flap count is measured over, and how long a service "
|
||||
"must stay OK before flapping ends.", editable=True),
|
||||
field("autosave_interval", "Autosave interval", "duration",
|
||||
"How often the server saves its state to disk."),
|
||||
field("base_url", "Base URL", "text",
|
||||
"Base URL for notification links.", editable=True),
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "persistence",
|
||||
"title": "Persistence & Logging",
|
||||
"description": "State file and event log settings.",
|
||||
"section_mode": "form",
|
||||
"api_section": "server",
|
||||
"fields": [
|
||||
field("pickfile", "State file", "path",
|
||||
"Path to the pickle file used to persist host state across restarts."),
|
||||
"Path to the pickle file used to persist host state across restarts.", editable=True),
|
||||
field("logfile", "Event log", "path",
|
||||
"Path to the event log file."),
|
||||
"Path to the event log file.", editable=True),
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "journal",
|
||||
"title": "Message Journal",
|
||||
"description": "All received heartbeat and plugin messages are journalled here.",
|
||||
"section_mode": "form",
|
||||
"api_section": "server",
|
||||
"fields": [
|
||||
field("journal_enabled", "Enabled", "boolean",
|
||||
"Turn journalling on or off."),
|
||||
"Turn journalling on or off.", editable=True),
|
||||
field("journal_dir", "Journal directory","path",
|
||||
"Directory where journal files are written."),
|
||||
"Directory where journal files are written.", editable=True),
|
||||
field("journal_file", "Journal filename", "text",
|
||||
"Base filename for the journal (rotated copies get a numeric suffix)."),
|
||||
field("journal_max_size", "Max file size", "size",
|
||||
"Rotate the journal when it exceeds this size."),
|
||||
"Rotate the journal when it exceeds this size.", editable=True),
|
||||
field("journal_max_backups", "Backup count", "number",
|
||||
"Number of rotated journal files to keep."),
|
||||
"Number of rotated journal files to keep.", editable=True),
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "dns",
|
||||
"title": "Dynamic DNS",
|
||||
"description": "nsupdate-based DNS registration for dynamic hosts.",
|
||||
"description": "nsupdate-based DNS registration via nsupdate(8).",
|
||||
"section_mode": "form",
|
||||
"api_section": "dns",
|
||||
"fields": [
|
||||
field("nsupdate_bin", "nsupdate binary", "path",
|
||||
"Full path to the nsupdate executable."),
|
||||
field("dyndomains", "Dynamic domains", "list",
|
||||
"DNS zones managed by nsupdate for dynamic hosts."),
|
||||
field("drophosts", "Drop hosts", "list",
|
||||
"Hostnames to silently ignore — no state, no alerts."),
|
||||
"Path to the nsupdate binary.", editable=True),
|
||||
field("rndc_key", "RNDC key file", "path",
|
||||
"Path to the rndc key file used to authenticate DNS updates.", editable=True),
|
||||
field("dyndomains", "Dynamic domains", "list",
|
||||
"Domains updated via nsupdate when a host with dyndns: true reports in.",
|
||||
editable=True),
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "users",
|
||||
"title": "Users",
|
||||
"description": "Accounts defined in the config file. Password hashes are never shown.",
|
||||
"section_mode": "form",
|
||||
"api_section": "users",
|
||||
"users": users_list,
|
||||
"fields": [
|
||||
field("default_owner", "Default owner", "text",
|
||||
"Username that owns hosts with no explicit owner. "
|
||||
"Falls back to the first admin user."),
|
||||
"Falls back to the first admin user.", editable=True),
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "oauth",
|
||||
"title": "OAuth Providers",
|
||||
"description": "OAuth2 login providers. Client secrets are masked.",
|
||||
"section_mode": "form",
|
||||
"api_section": "oauth",
|
||||
"providers": oauth_providers,
|
||||
"fields": [],
|
||||
},
|
||||
{
|
||||
"id": "channels",
|
||||
"title": "Notification Channels",
|
||||
"description": "Named notification providers. Credentials are masked.",
|
||||
"section_mode": "channels",
|
||||
"api_section": "notification_channels",
|
||||
"channels": notif_channels,
|
||||
"fields": [
|
||||
field("default_notification_channels", "Default channels", "list",
|
||||
@@ -309,13 +484,29 @@ def get_settings_sections(config: dict) -> list:
|
||||
"id": "hosts",
|
||||
"title": "Hosts",
|
||||
"description": "Host definitions loaded from the config file.",
|
||||
"section_mode": "hosts",
|
||||
"api_section": "hosts",
|
||||
"hosts": hosts_list,
|
||||
"fields": [],
|
||||
},
|
||||
{
|
||||
"id": "thresholds",
|
||||
"title": "Threshold Configurations",
|
||||
"description": "Named alert threshold sets. Each defines warning/critical levels per metric.",
|
||||
"section_mode": "thresholds",
|
||||
"api_section": "thresholds",
|
||||
"threshold_configs": threshold_config_list,
|
||||
"fields": [
|
||||
field("default_threshold_config", "Default config", "text",
|
||||
"Threshold config used for hosts with no explicit mapping.", editable=True),
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "runtime",
|
||||
"title": "Runtime",
|
||||
"description": "Flags set at startup (require restart to change).",
|
||||
"section_mode": "form",
|
||||
"api_section": None,
|
||||
"fields": [
|
||||
field("foreground", "Foreground mode", "boolean",
|
||||
"Run in the foreground instead of daemonising."),
|
||||
@@ -326,3 +517,27 @@ def get_settings_sections(config: dict) -> list:
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
if not is_admin:
|
||||
sections = [s for s in sections if s["id"] in ("channels", "hosts", "thresholds")]
|
||||
for s in sections:
|
||||
s["fields"] = []
|
||||
return sections
|
||||
|
||||
|
||||
def get_settings_data(config: dict, threshold_checker=None, user=None) -> dict:
|
||||
"""Return sections list + auxiliary data for the settings template."""
|
||||
sections = get_settings_sections(config, threshold_checker=threshold_checker, user=user)
|
||||
is_admin = user is None or getattr(user, "admin", False)
|
||||
username: str = getattr(user, "username", "") or ""
|
||||
channels = config.get("notification_channels") or {}
|
||||
threshold_cfgs = config.get("threshold_configs") or {}
|
||||
if not is_admin:
|
||||
channels = config_access.user_channels(channels, username)
|
||||
threshold_cfgs = config_access.user_threshold_configs(threshold_cfgs, username)
|
||||
return {
|
||||
"sections": sections,
|
||||
"all_channel_names": sorted(channels.keys()),
|
||||
"all_usernames": sorted((config.get("users") or {}).keys()),
|
||||
"all_threshold_configs": sorted(threshold_cfgs.keys()),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
/* hbd-ui.css — shared design system for hbd pages.
|
||||
*
|
||||
* Green-biased neutrals, teal accent, mono identifiers. Pages link this
|
||||
* after static/style.css and build on the tokens + components below.
|
||||
* Introduced with the settings-page redesign; see settings.html for the
|
||||
* fullest use of the row/editor idiom.
|
||||
*/
|
||||
|
||||
:root {
|
||||
--st-paper: #f4f6f4;
|
||||
--st-surface: #ffffff;
|
||||
--st-surface-2: #eef1ee;
|
||||
--st-ink: #1d2420;
|
||||
--st-muted: #5f6a63;
|
||||
--st-faint: #8a948d;
|
||||
--st-line: #dde3de;
|
||||
--st-line-soft: #e8ece8;
|
||||
--st-accent: #0e7280;
|
||||
--st-accent-ink: #ffffff;
|
||||
--st-accent-soft: #e0eef0;
|
||||
--st-ok: #3f9c5a;
|
||||
--st-ok-soft: #e2f2e7;
|
||||
--st-warn: #d9982a;
|
||||
--st-warn-soft: #f9efdc;
|
||||
--st-crit: #d64545;
|
||||
--st-crit-soft: #f9e3e3;
|
||||
--st-badge-owner: #ede7f6; --st-badge-owner-ink: #5e35b1;
|
||||
--st-badge-mgr: #e3f2fd; --st-badge-mgr-ink: #1565c0;
|
||||
--st-mono: ui-monospace, "SF Mono", "Cascadia Mono", Menlo, Consolas, monospace;
|
||||
}
|
||||
html[data-theme="dark"] {
|
||||
--st-paper: #131816;
|
||||
--st-surface: #1b211e;
|
||||
--st-surface-2: #222925;
|
||||
--st-ink: #e4eae6;
|
||||
--st-muted: #9aa69e;
|
||||
--st-faint: #6e7a72;
|
||||
--st-line: #2c342f;
|
||||
--st-line-soft: #252c28;
|
||||
--st-accent: #4cc4d4;
|
||||
--st-accent-ink: #0d2326;
|
||||
--st-accent-soft: #143b40;
|
||||
--st-ok: #5cba77;
|
||||
--st-ok-soft: #1c2f22;
|
||||
--st-warn: #e0ad4e;
|
||||
--st-warn-soft: #33290f;
|
||||
--st-crit: #e06c6c;
|
||||
--st-crit-soft: #3a1a1a;
|
||||
--st-badge-owner: #33294a; --st-badge-owner-ink: #b39ddb;
|
||||
--st-badge-mgr: #1b3350; --st-badge-mgr-ink: #90caf9;
|
||||
}
|
||||
|
||||
/* ── sticky toolbar under the site nav ── */
|
||||
.st-toolbar {
|
||||
position: sticky; top: var(--nav-h, 48px); z-index: 90;
|
||||
display: flex; align-items: center; gap: 12px; flex-wrap: wrap;
|
||||
padding: 9px 20px;
|
||||
background: var(--st-surface); border-bottom: 1px solid var(--st-line);
|
||||
}
|
||||
.st-toolbar .brand { font-family: var(--st-mono); font-size: 15px; font-weight: 700; letter-spacing: -.02em; color: var(--st-ink); }
|
||||
.st-toolbar .brand .tld { color: var(--st-accent); }
|
||||
.st-toolbar .hintline { font-size: 12px; color: var(--st-faint); }
|
||||
.st-toolbar .spacer { flex: 1; }
|
||||
|
||||
/* ── ECG pulse divider ── */
|
||||
.pulse { display: block; width: 100%; height: 14px; }
|
||||
.pulse polyline { fill: none; stroke: var(--st-accent); stroke-width: 1.5; opacity: .55; }
|
||||
|
||||
/* ── buttons ── */
|
||||
.btn {
|
||||
font: 600 13px/1 inherit; border: 1px solid var(--st-line); border-radius: 6px;
|
||||
background: var(--st-surface); color: var(--st-ink); padding: 7px 14px; cursor: pointer;
|
||||
}
|
||||
.btn:focus-visible, a:focus-visible { outline: 2px solid var(--st-accent); outline-offset: 2px; }
|
||||
.btn.primary { background: var(--st-accent); border-color: var(--st-accent); color: var(--st-accent-ink); }
|
||||
.btn.quiet { border-color: transparent; background: transparent; color: var(--st-muted); }
|
||||
.btn.small { padding: 4px 10px; font-size: 12px; }
|
||||
.btn.danger { color: var(--st-crit); }
|
||||
.btn:disabled { opacity: .6; cursor: default; }
|
||||
|
||||
/* ── section headers as config keys ── */
|
||||
section.st { margin-top: 26px; scroll-margin-top: calc(var(--bars-h, 94px) + 12px); }
|
||||
.sec-head { display: flex; align-items: baseline; gap: 10px; margin-bottom: 8px; flex-wrap: wrap; }
|
||||
.sec-head h2 { margin: 0; font-family: var(--st-mono); font-size: 15px; font-weight: 700; color: var(--st-ink); }
|
||||
.sec-head h2 .colon { color: var(--st-accent); }
|
||||
.sec-head .count { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-faint); }
|
||||
.sec-head .sub { font-size: 12px; color: var(--st-muted); }
|
||||
.sec-head .add { margin-left: auto; }
|
||||
|
||||
/* ── record list + row ── */
|
||||
.list { background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 8px; overflow: hidden; }
|
||||
.row {
|
||||
display: grid; grid-template-columns: 190px minmax(0,1fr) auto;
|
||||
gap: 4px 14px; align-items: center;
|
||||
padding: 8px 14px; border-bottom: 1px solid var(--st-line-soft);
|
||||
transition: background .12s ease;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) { .row { transition: none; } }
|
||||
.row:last-child { border-bottom: none; }
|
||||
.row:hover { background: var(--st-surface-2); }
|
||||
.row .id { font-family: var(--st-mono); font-size: 13px; font-weight: 600; color: var(--st-ink); display: flex; align-items: center; gap: 8px; min-width: 0; }
|
||||
.row .id .name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.row .act { display: flex; gap: 4px; justify-content: flex-end; }
|
||||
|
||||
.dot { width: 8px; height: 8px; border-radius: 50%; flex: none; }
|
||||
.dot.ok { background: var(--st-ok); }
|
||||
.dot.warn { background: var(--st-warn); }
|
||||
.dot.crit { background: var(--st-crit); }
|
||||
.dot.off { background: transparent; border: 1.5px solid var(--st-faint); }
|
||||
|
||||
.facts { display: flex; flex-wrap: wrap; gap: 4px 6px; align-items: center; min-width: 0; }
|
||||
.chip {
|
||||
display: inline-flex; align-items: center; gap: 4px;
|
||||
font-family: var(--st-mono); font-size: 11px; line-height: 1.3;
|
||||
padding: 3px 7px; border-radius: 4px;
|
||||
background: var(--st-surface-2); color: var(--st-muted);
|
||||
}
|
||||
.chip.owner { background: var(--st-badge-owner); color: var(--st-badge-owner-ink); }
|
||||
.chip.mgr { background: var(--st-badge-mgr); color: var(--st-badge-mgr-ink); }
|
||||
.chip.level { background: transparent; border: 1px solid var(--st-line); }
|
||||
.chip.private { background: transparent; border: 1px dashed var(--st-badge-owner-ink); color: var(--st-badge-owner-ink); }
|
||||
.chip.ok { background: var(--st-ok-soft); color: var(--st-ok); }
|
||||
.chip.warn { background: var(--st-warn-soft); color: var(--st-warn); }
|
||||
.chip.crit { background: var(--st-crit-soft); color: var(--st-crit); }
|
||||
.chip .k { color: var(--st-faint); }
|
||||
.chip .w { color: var(--st-warn); font-weight: 700; }
|
||||
.chip .c { color: var(--st-crit); font-weight: 700; }
|
||||
|
||||
/* ── key/value rows (server groups, about page) ── */
|
||||
.kv .row { grid-template-columns: 230px minmax(0,1fr); padding: 6px 14px; }
|
||||
.group {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
padding: 7px 14px 5px; font-family: var(--st-mono); font-size: 11px; color: var(--st-faint);
|
||||
background: var(--st-surface); border-bottom: 1px solid var(--st-line-soft);
|
||||
}
|
||||
.group .gbtn { margin-left: auto; }
|
||||
.kv .val { font-family: var(--st-mono); font-size: 12.5px; color: var(--st-muted); min-width: 0; }
|
||||
.kv .val a { color: var(--st-accent); text-decoration: none; }
|
||||
.kv .val a:hover { text-decoration: underline; }
|
||||
.kv .desc { display: block; font-size: 11px; color: var(--st-faint); margin-top: 2px; }
|
||||
.val-masked { color: var(--st-faint); letter-spacing: 2px; }
|
||||
|
||||
.note { margin: 8px 2px 0; font-size: 12px; color: var(--st-faint); }
|
||||
|
||||
/* ── handheld defaults for rows ── */
|
||||
@media (max-width: 760px) {
|
||||
.st-toolbar { padding: 8px 10px; }
|
||||
.st-toolbar .hintline { display: none; }
|
||||
.row { grid-template-columns: minmax(0,1fr) auto; padding: 9px 12px; }
|
||||
.row .id { grid-row: 1; grid-column: 1; }
|
||||
.row .act { grid-row: 1; grid-column: 2; }
|
||||
.row .facts { grid-column: 1 / -1; }
|
||||
.kv .row { grid-template-columns: 1fr; }
|
||||
section.st { scroll-margin-top: calc(var(--bars-h, 94px) + 50px); }
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
/* hbd-ui.js — shared helpers for pages using hbd-ui.css.
|
||||
*
|
||||
* Keeps sticky toolbars stuck just below the fixed site nav, whose height
|
||||
* varies with viewport width and wrapping.
|
||||
*/
|
||||
|
||||
function _setStickyOffsets() {
|
||||
const nav = document.querySelector('.nav');
|
||||
const tb = document.querySelector('.st-toolbar');
|
||||
const navH = nav ? nav.offsetHeight : 48;
|
||||
const barsH = navH + (tb ? tb.offsetHeight : 46);
|
||||
document.documentElement.style.setProperty('--nav-h', navH + 'px');
|
||||
document.documentElement.style.setProperty('--bars-h', barsH + 'px');
|
||||
}
|
||||
window.addEventListener('resize', _setStickyOffsets);
|
||||
document.addEventListener('DOMContentLoaded', _setStickyOffsets);
|
||||
@@ -185,7 +185,7 @@
|
||||
/* Slightly larger tap targets in tables */
|
||||
#ntable td, #ntable th {
|
||||
padding: 4px 6px !important;
|
||||
font-size: 0.82em !important;
|
||||
font-size: 1.00em !important;
|
||||
}
|
||||
|
||||
/* Cards on plugin/alerts pages */
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
{% include 'head.html' %}
|
||||
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||||
<script src="/static/hbd-ui.js"></script>
|
||||
|
||||
<style>
|
||||
html, body { height: auto; overflow: visible; }
|
||||
body { background: var(--st-paper); padding: 60px 0 0; }
|
||||
|
||||
.about-main { max-width: 700px; margin: 0 auto; padding: 0 20px 60px; }
|
||||
|
||||
.hero {
|
||||
display: flex; align-items: baseline; gap: 14px; flex-wrap: wrap;
|
||||
padding: 26px 2px 0;
|
||||
}
|
||||
.hero .hb-logo {
|
||||
font-family: var(--st-mono); font-size: 30px; font-weight: 700;
|
||||
letter-spacing: -.03em; color: var(--st-ink);
|
||||
}
|
||||
.hero .hb-logo .tld { color: var(--st-accent); }
|
||||
.hero .version { font-family: var(--st-mono); font-size: 13px; color: var(--st-accent);
|
||||
background: var(--st-accent-soft); border-radius: 999px; padding: 3px 12px; }
|
||||
.hero .tagline { flex-basis: 100%; font-size: 13px; color: var(--st-muted); }
|
||||
</style>
|
||||
|
||||
<body>
|
||||
{% include 'nav.html' %}
|
||||
|
||||
<div class="st-toolbar">
|
||||
<span class="brand">hbd<span class="tld">·about</span></span>
|
||||
<span class="hintline">heartbeat monitoring system</span>
|
||||
</div>
|
||||
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||||
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||||
</svg>
|
||||
|
||||
<div class="about-main">
|
||||
<div class="hero">
|
||||
<span class="hb-logo">heart<span class="tld">beat</span></span>
|
||||
<span class="version">v{{ hbd_version }}</span>
|
||||
<span class="tagline">Lightweight host monitoring over UDP</span>
|
||||
</div>
|
||||
|
||||
<section class="st">
|
||||
<div class="sec-head">
|
||||
<h2>version<span class="colon">:</span></h2>
|
||||
</div>
|
||||
<div class="list kv">
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">server</span></span>
|
||||
<span class="val">{{ hbd_version }}</span>
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">python</span></span>
|
||||
<span class="val">{{ python_version }}</span>
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">license</span></span>
|
||||
<span class="val">MIT</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="st">
|
||||
<div class="sec-head">
|
||||
<h2>runtime<span class="colon">:</span></h2>
|
||||
</div>
|
||||
<div class="list kv">
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">host</span></span>
|
||||
<span class="val">{{ server_hostname }}</span>
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">started</span></span>
|
||||
<span class="val">{{ start_time_str }}</span>
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">uptime</span></span>
|
||||
<span class="val" id="uptime-value">{{ uptime_str }}</span>
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">hosts_monitored</span></span>
|
||||
<span class="val">{{ host_count }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="st">
|
||||
<div class="sec-head">
|
||||
<h2>contact<span class="colon">:</span></h2>
|
||||
</div>
|
||||
<div class="list kv">
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">author</span></span>
|
||||
<span class="val">Andreas Wrede</span>
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">email</span></span>
|
||||
<span class="val"><a href="mailto:aew.hbd@wrede.ca">aew.hbd@wrede.ca</a></span>
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="id"><span class="name">repository</span></span>
|
||||
<span class="val"><a href="https://git.wrede.ca/andreas/heartbeat" target="_blank" rel="noopener">git.wrede.ca/andreas/heartbeat</a></span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{% include 'foot.html' %}
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
var startEpoch = {{ start_epoch }};
|
||||
var el = document.getElementById('uptime-value');
|
||||
if (!el) return;
|
||||
function fmt(s) {
|
||||
var d = Math.floor(s / 86400);
|
||||
var h = Math.floor((s % 86400) / 3600);
|
||||
var m = Math.floor((s % 3600) / 60);
|
||||
var sec = s % 60;
|
||||
if (d > 0) return d + 'd ' + h + 'h ' + m + 'm';
|
||||
if (h > 0) return h + 'h ' + m + 'm ' + sec + 's';
|
||||
return m + 'm ' + sec + 's';
|
||||
}
|
||||
function tick() {
|
||||
var up = Math.floor(Date.now() / 1000 - startEpoch);
|
||||
el.textContent = fmt(up);
|
||||
}
|
||||
tick();
|
||||
setInterval(tick, 1000);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
+164
-427
@@ -1,523 +1,236 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
{% include 'head.html' %}
|
||||
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||||
<script src="/static/hbd-ui.js"></script>
|
||||
|
||||
<style>
|
||||
html, body { height: auto; overflow: visible; }
|
||||
body { background: var(--st-paper); padding: 60px 0 0; }
|
||||
|
||||
.container {
|
||||
max-width: 1400px;
|
||||
margin: 0 auto;
|
||||
.alerts-main { max-width: 1100px; margin: 0 auto; padding: 0 20px 60px; }
|
||||
|
||||
/* summary tiles */
|
||||
.stats { display: flex; gap: 10px; flex-wrap: wrap; padding-top: 18px; }
|
||||
.stat {
|
||||
flex: 1 1 140px; display: flex; align-items: baseline; gap: 10px;
|
||||
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 8px;
|
||||
padding: 10px 14px;
|
||||
}
|
||||
.stat .num { font-family: var(--st-mono); font-size: 24px; font-weight: 700; font-variant-numeric: tabular-nums; }
|
||||
.stat .lbl { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-faint); }
|
||||
.stat.crit .num { color: var(--st-crit); }
|
||||
.stat.warn .num { color: var(--st-warn); }
|
||||
.stat.ok .num { color: var(--st-ok); }
|
||||
|
||||
h1 { color: #333; margin-bottom: 10px; font-size: 1.5em; }
|
||||
|
||||
.subtitle {
|
||||
color: #666;
|
||||
margin-bottom: 30px;
|
||||
/* filter bar */
|
||||
.filterbar { display: flex; gap: 6px; align-items: center; flex-wrap: wrap; margin: 14px 0 0; }
|
||||
.filterbar .fbtn {
|
||||
font-family: var(--st-mono); font-size: 12px; cursor: pointer;
|
||||
color: var(--st-muted); background: var(--st-surface); border: 1px solid var(--st-line);
|
||||
border-radius: 999px; padding: 5px 12px;
|
||||
}
|
||||
|
||||
.summary-cards {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
margin-bottom: 16px;
|
||||
.filterbar .fbtn.active { background: var(--st-accent-soft); border-color: var(--st-accent); color: var(--st-accent); font-weight: 600; }
|
||||
.filterbar input {
|
||||
font: 12px var(--st-mono); color: var(--st-ink);
|
||||
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||||
padding: 5px 12px; min-width: 170px;
|
||||
}
|
||||
.filterbar input.invalid { border-color: var(--st-crit); }
|
||||
.filterbar .upd { margin-left: auto; font-size: 11.5px; color: var(--st-faint); }
|
||||
|
||||
.summary-card {
|
||||
background: white;
|
||||
border-radius: 6px;
|
||||
padding: 6px 14px;
|
||||
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
border-left: 4px solid #ddd;
|
||||
/* alert row specifics */
|
||||
.row.alert-crit { border-left: 3px solid var(--st-crit); padding-left: 11px; }
|
||||
.row.alert-warn { border-left: 3px solid var(--st-warn); padding-left: 11px; }
|
||||
.row.acked { opacity: .55; border-left-style: dashed; }
|
||||
.row .id a { color: inherit; text-decoration: none; }
|
||||
.row .id a:hover { color: var(--st-accent); }
|
||||
.acked-chip { color: var(--st-ok); font-size: 12px; white-space: nowrap; }
|
||||
|
||||
.empty {
|
||||
padding: 36px 14px; text-align: center; color: var(--st-faint); font-size: 13px;
|
||||
}
|
||||
|
||||
.summary-card.critical { border-left-color: #ea1e0f; }
|
||||
.summary-card.warning { border-left-color: #ff9800; }
|
||||
.summary-card.ok { border-left-color: #4caf50; }
|
||||
|
||||
.summary-number {
|
||||
font-size: 1.4em;
|
||||
font-weight: bold;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.summary-number.critical { color: #ea1e0f; }
|
||||
.summary-number.warning { color: #ff9800; }
|
||||
.summary-number.ok { color: #4caf50; }
|
||||
|
||||
.summary-label {
|
||||
color: #666;
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
.filters {
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
padding: 15px;
|
||||
margin-bottom: 20px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
display: flex;
|
||||
gap: 15px;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.filter-label {
|
||||
font-weight: bold;
|
||||
color: #555;
|
||||
}
|
||||
|
||||
.filter-button {
|
||||
padding: 8px 16px;
|
||||
border: 2px solid #ddd;
|
||||
background: white;
|
||||
border-radius: 20px;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.filter-button:hover {
|
||||
border-color: #2196f3;
|
||||
}
|
||||
|
||||
.filter-button.active {
|
||||
background: #2196f3;
|
||||
color: white;
|
||||
border-color: #2196f3;
|
||||
}
|
||||
|
||||
.alerts-container {
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
}
|
||||
|
||||
.alert-item {
|
||||
border-left: 5px solid #ddd;
|
||||
padding: 15px;
|
||||
margin-bottom: 15px;
|
||||
background: #fafafa;
|
||||
border-radius: 4px;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.alert-item.acknowledged {
|
||||
opacity: 0.8;
|
||||
background: #f0f0f0;
|
||||
}
|
||||
|
||||
.alert-item:hover {
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
transform: translateX(5px);
|
||||
}
|
||||
|
||||
.alert-item.critical {
|
||||
border-left-color: #f44336;
|
||||
background: #ffebee;
|
||||
}
|
||||
|
||||
.alert-item.warning {
|
||||
border-left-color: #ff9800;
|
||||
background: #fff3e0;
|
||||
}
|
||||
|
||||
.alert-item.unknown {
|
||||
border-left-color: #9e9e9e;
|
||||
background: #f5f5f5;
|
||||
}
|
||||
|
||||
.alert-main {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.alert-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 15px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.alert-level {
|
||||
padding: 4px 12px;
|
||||
border-radius: 12px;
|
||||
font-size: 0.75em;
|
||||
font-weight: bold;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.alert-level.critical {
|
||||
background: #f44336;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.alert-level.warning {
|
||||
background: #ff9800;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.alert-level.unknown {
|
||||
background: #9e9e9e;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.alert-hostname {
|
||||
font-weight: bold;
|
||||
color: #333;
|
||||
font-size: 1.1em;
|
||||
}
|
||||
|
||||
.alert-metric {
|
||||
color: #666;
|
||||
font-family: 'Courier New', monospace;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.alert-details {
|
||||
display: flex;
|
||||
gap: 20px;
|
||||
color: #666;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.alert-value {
|
||||
font-weight: bold;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.alert-duration {
|
||||
color: #999;
|
||||
font-size: 0.85em;
|
||||
}
|
||||
|
||||
.alert-actions {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
margin-left: 15px;
|
||||
}
|
||||
|
||||
.acknowledge-btn {
|
||||
padding: 8px 16px;
|
||||
background: #2196f3;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
font-size: 0.85em;
|
||||
transition: all 0.2s;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.acknowledge-btn:hover {
|
||||
background: #1976d2;
|
||||
transform: scale(1.05);
|
||||
}
|
||||
|
||||
.acknowledge-btn:disabled {
|
||||
background: #ccc;
|
||||
cursor: not-allowed;
|
||||
transform: none;
|
||||
}
|
||||
|
||||
.acknowledged-badge {
|
||||
padding: 4px 8px;
|
||||
background: #4caf50;
|
||||
color: white;
|
||||
border-radius: 4px;
|
||||
font-size: 0.75em;
|
||||
text-align: center;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.no-alerts {
|
||||
text-align: center;
|
||||
padding: 60px 20px;
|
||||
color: #999;
|
||||
}
|
||||
|
||||
.no-alerts-icon {
|
||||
font-size: 4em;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.loading {
|
||||
text-align: center;
|
||||
padding: 40px;
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.error {
|
||||
background: #ffebee;
|
||||
border-left: 4px solid #f44336;
|
||||
padding: 20px;
|
||||
margin: 20px 0;
|
||||
border-radius: 4px;
|
||||
color: #c62828;
|
||||
}
|
||||
|
||||
.refresh-info {
|
||||
text-align: center;
|
||||
color: #999;
|
||||
font-size: 0.85em;
|
||||
margin-top: 20px;
|
||||
padding-top: 20px;
|
||||
border-top: 1px solid #e0e0e0;
|
||||
}
|
||||
|
||||
.last-update {
|
||||
color: #666;
|
||||
font-size: 0.9em;
|
||||
text-align: right;
|
||||
margin-bottom: 15px;
|
||||
.empty .big { font-size: 26px; color: var(--st-ok); display: block; margin-bottom: 6px; }
|
||||
.errorbox {
|
||||
background: var(--st-crit-soft); color: var(--st-crit);
|
||||
border-radius: 8px; padding: 14px 16px; font-size: 13px;
|
||||
}
|
||||
</style>
|
||||
|
||||
<body>
|
||||
{% include 'nav.html' %}
|
||||
|
||||
<div class="container">
|
||||
<h1>{{ header }}</h1>
|
||||
<p class="subtitle">Real-time monitoring alerts and threshold violations</p>
|
||||
<div class="st-toolbar">
|
||||
<span class="brand">hbd<span class="tld">·alerts</span></span>
|
||||
<span class="hintline">threshold violations and reachability — refreshes every 15 s</span>
|
||||
</div>
|
||||
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||||
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||||
</svg>
|
||||
|
||||
<div class="summary-cards" id="summary-cards">
|
||||
<div class="summary-card critical">
|
||||
<div class="summary-label">Critical</div>
|
||||
<div class="summary-number critical" id="critical-count">-</div>
|
||||
</div>
|
||||
<div class="summary-card warning">
|
||||
<div class="summary-label">Warning</div>
|
||||
<div class="summary-number warning" id="warning-count">-</div>
|
||||
</div>
|
||||
<div class="summary-card ok">
|
||||
<div class="summary-label">Total Hosts</div>
|
||||
<div class="summary-number ok" id="host-count">-</div>
|
||||
</div>
|
||||
<div class="alerts-main">
|
||||
<div class="stats">
|
||||
<div class="stat crit"><span class="num" id="critical-count">–</span><span class="lbl">critical</span></div>
|
||||
<div class="stat warn"><span class="num" id="warning-count">–</span><span class="lbl">warning</span></div>
|
||||
<div class="stat ok"><span class="num" id="host-count">–</span><span class="lbl">hosts</span></div>
|
||||
</div>
|
||||
|
||||
<div class="filters">
|
||||
<span class="filter-label">Show:</span>
|
||||
<button class="filter-button active" onclick="filterAlerts('all')">All</button>
|
||||
<button class="filter-button" onclick="filterAlerts('critical')">Critical Only</button>
|
||||
<button class="filter-button" onclick="filterAlerts('warning')">Warning Only</button>
|
||||
<div class="filterbar">
|
||||
<button class="fbtn active" onclick="filterAlerts('all', this)">all</button>
|
||||
<button class="fbtn" onclick="filterAlerts('critical', this)">critical</button>
|
||||
<button class="fbtn" onclick="filterAlerts('warning', this)">warning</button>
|
||||
<input id="host-filter" type="text" placeholder="host filter (regex)" oninput="onHostFilterInput(this)">
|
||||
<span class="upd">updated <span id="last-update-time">never</span></span>
|
||||
</div>
|
||||
|
||||
<div class="alerts-container">
|
||||
<div class="last-update">Last updated: <span id="last-update-time">Never</span></div>
|
||||
<div id="alerts-list">
|
||||
<div class="loading">Loading alerts...</div>
|
||||
<section class="st" id="alerts">
|
||||
<div class="sec-head">
|
||||
<h2>alerts<span class="colon">:</span></h2>
|
||||
<span class="count" id="alert-count"></span>
|
||||
</div>
|
||||
<div class="refresh-info">
|
||||
Auto-refreshing every 15 seconds
|
||||
<div class="list" id="alerts-list">
|
||||
<div class="empty">Loading alerts…</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
let currentFilter = 'all';
|
||||
let allAlerts = [];
|
||||
let hostFilterRe = null;
|
||||
|
||||
function escHtml(s) {
|
||||
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');
|
||||
}
|
||||
|
||||
async function loadAlerts() {
|
||||
try {
|
||||
const response = await fetch('/api/0/alerts');
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
const data = await response.json();
|
||||
allAlerts = data.alerts;
|
||||
|
||||
// Update summary cards
|
||||
document.getElementById('critical-count').textContent = data.summary.critical || 0;
|
||||
document.getElementById('warning-count').textContent = data.summary.warning || 0;
|
||||
document.getElementById('host-count').textContent = data.host_count || 0;
|
||||
|
||||
// Update last update time
|
||||
document.getElementById('last-update-time').textContent = new Date().toLocaleTimeString();
|
||||
|
||||
// Render alerts
|
||||
renderAlerts(allAlerts);
|
||||
|
||||
} catch (error) {
|
||||
document.getElementById('alerts-list').innerHTML =
|
||||
`<div class="error">Failed to load alerts: ${error.message}</div>`;
|
||||
`<div class="errorbox">Failed to load alerts: ${escHtml(error.message)}. Retrying automatically.</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
function renderAlerts(alerts) {
|
||||
const container = document.getElementById('alerts-list');
|
||||
|
||||
// Filter alerts based on current filter
|
||||
let filteredAlerts = alerts;
|
||||
let filtered = alerts;
|
||||
if (currentFilter !== 'all') {
|
||||
filteredAlerts = alerts.filter(alert =>
|
||||
alert.level.toLowerCase() === currentFilter
|
||||
);
|
||||
filtered = filtered.filter(a => a.level.toLowerCase() === currentFilter);
|
||||
}
|
||||
if (hostFilterRe) {
|
||||
filtered = filtered.filter(a => hostFilterRe.test(a.hostname));
|
||||
}
|
||||
|
||||
if (filteredAlerts.length === 0) {
|
||||
if (currentFilter === 'all' && alerts.length === 0) {
|
||||
container.innerHTML = `
|
||||
<div class="no-alerts">
|
||||
<div class="no-alerts-icon">✓</div>
|
||||
<h2>All Systems Normal</h2>
|
||||
<p>No active alerts at this time</p>
|
||||
</div>
|
||||
`;
|
||||
} else {
|
||||
container.innerHTML = `
|
||||
<div class="no-alerts">
|
||||
<p>No ${currentFilter} alerts</p>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
document.getElementById('alert-count').textContent =
|
||||
filtered.length === alerts.length ? alerts.length : `${filtered.length} of ${alerts.length}`;
|
||||
|
||||
if (filtered.length === 0) {
|
||||
container.innerHTML = (currentFilter === 'all' && !hostFilterRe && alerts.length === 0)
|
||||
? `<div class="empty"><span class="big">✓</span>All systems normal — no active alerts.</div>`
|
||||
: `<div class="empty">No matching alerts.</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
let html = '';
|
||||
for (const alert of filteredAlerts) {
|
||||
html += renderAlert(alert);
|
||||
}
|
||||
container.innerHTML = html;
|
||||
container.innerHTML = filtered.map(renderAlert).join('');
|
||||
}
|
||||
|
||||
function renderAlert(alert) {
|
||||
const level = alert.level.toLowerCase();
|
||||
const cls = level === 'critical' ? 'crit' : (level === 'warning' ? 'warn' : 'level');
|
||||
const duration = getDuration(alert.since);
|
||||
const acknowledged = alert.acknowledged || false;
|
||||
const acked = alert.acknowledged || false;
|
||||
|
||||
// Use formatted message if available, otherwise build from individual fields
|
||||
let valueText = `Value: <span class="alert-value">${formatValue(alert.last_value)}</span>`;
|
||||
const metric = (alert.metric_path.includes('.')
|
||||
? alert.metric_path.slice(alert.metric_path.indexOf('.') + 1)
|
||||
: alert.metric_path).replace(/_status_code$/, '');
|
||||
|
||||
let chips = `<span class="chip ${cls}">${escHtml(alert.level)}</span>`;
|
||||
chips += `<span class="chip"><span class="k">metric</span> ${escHtml(metric)}</span>`;
|
||||
if (alert.formatted_message) {
|
||||
valueText += ` <span class="threshold-info">${alert.formatted_message}</span>`;
|
||||
} else if (alert.threshold_value !== undefined && alert.threshold_value !== null && alert.operator) {
|
||||
valueText += ` <span class="threshold-info">(threshold: ${alert.operator} ${formatValue(alert.threshold_value)})</span>`;
|
||||
}
|
||||
|
||||
// Build actions section
|
||||
let actionsHtml = '';
|
||||
if (acknowledged) {
|
||||
actionsHtml = `
|
||||
<div class="alert-actions">
|
||||
<div class="acknowledged-badge">✓ Acknowledged</div>
|
||||
</div>
|
||||
`;
|
||||
chips += `<span class="chip">${escHtml(alert.formatted_message)}</span>`;
|
||||
} else {
|
||||
actionsHtml = `
|
||||
<div class="alert-actions">
|
||||
<button class="acknowledge-btn" onclick="acknowledgeAlert('${alert.hostname}', '${alert.metric_path}', event)">
|
||||
Acknowledge
|
||||
</button>
|
||||
</div>
|
||||
`;
|
||||
chips += `<span class="chip"><span class="k">value</span> ${escHtml(formatValue(alert.last_value))}</span>`;
|
||||
if (alert.threshold_value !== undefined && alert.threshold_value !== null && alert.operator) {
|
||||
chips += `<span class="chip level">${escHtml(alert.operator)} ${escHtml(formatValue(alert.threshold_value))}</span>`;
|
||||
}
|
||||
}
|
||||
if (alert.recovery_threshold !== undefined && alert.recovery_threshold !== null) {
|
||||
const recOp = (alert.operator === '>' || alert.operator === '>=') ? '<' : '>';
|
||||
chips += `<span class="chip level">recovers ${recOp} ${escHtml(formatValue(alert.recovery_threshold))}</span>`;
|
||||
}
|
||||
chips += `<span class="chip"><span class="k">for</span> ${duration}</span>`;
|
||||
|
||||
const act = acked
|
||||
? `<span class="acked-chip">✓ acknowledged</span>`
|
||||
: `<button class="btn small" onclick="acknowledgeAlert('${escHtml(alert.hostname)}', '${escHtml(alert.metric_path)}', event)">Acknowledge</button>`;
|
||||
|
||||
return `
|
||||
<div class="alert-item ${level} ${acknowledged ? 'acknowledged' : ''}">
|
||||
<div class="alert-main">
|
||||
<div class="alert-header">
|
||||
<span class="alert-level ${level}">${alert.level}</span>
|
||||
<span class="alert-hostname">${alert.hostname}</span>
|
||||
</div>
|
||||
<div class="alert-metric">${alert.metric_path}</div>
|
||||
<div class="alert-details">
|
||||
<span>${valueText}</span>
|
||||
<span class="alert-duration">Active for ${duration}</span>
|
||||
</div>
|
||||
</div>
|
||||
${actionsHtml}
|
||||
</div>
|
||||
`;
|
||||
<div class="row alert-${cls === 'level' ? 'warn' : cls}${acked ? ' acked' : ''}">
|
||||
<span class="id">
|
||||
<span class="dot ${cls === 'crit' ? 'crit' : 'warn'}"></span>
|
||||
<a class="name" href="/plugins#${encodeURIComponent(alert.hostname)}">${escHtml(alert.hostname)}</a>
|
||||
</span>
|
||||
<span class="facts">${chips}</span>
|
||||
<span class="act">${act}</span>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function formatValue(value) {
|
||||
if (typeof value === 'number') {
|
||||
if (value > 1000) {
|
||||
return value.toLocaleString();
|
||||
}
|
||||
if (value > 1000) return value.toLocaleString();
|
||||
return value.toFixed(2);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function getDuration(timestamp) {
|
||||
const now = Date.now() / 1000;
|
||||
const seconds = Math.floor(now - timestamp);
|
||||
|
||||
if (seconds < 60) {
|
||||
return `${seconds}s`;
|
||||
} else if (seconds < 3600) {
|
||||
return `${Math.floor(seconds / 60)}m`;
|
||||
} else if (seconds < 86400) {
|
||||
const hours = Math.floor(seconds / 3600);
|
||||
const minutes = Math.floor((seconds % 3600) / 60);
|
||||
return `${hours}h ${minutes}m`;
|
||||
} else {
|
||||
const days = Math.floor(seconds / 86400);
|
||||
const hours = Math.floor((seconds % 86400) / 3600);
|
||||
return `${days}d ${hours}h`;
|
||||
const seconds = Math.floor(Date.now() / 1000 - timestamp);
|
||||
if (seconds < 60) return `${seconds}s`;
|
||||
if (seconds < 3600) return `${Math.floor(seconds / 60)}m`;
|
||||
if (seconds < 86400) {
|
||||
return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
|
||||
}
|
||||
return `${Math.floor(seconds / 86400)}d ${Math.floor((seconds % 86400) / 3600)}h`;
|
||||
}
|
||||
|
||||
function filterAlerts(filter) {
|
||||
function filterAlerts(filter, btn) {
|
||||
currentFilter = filter;
|
||||
|
||||
// Update active button
|
||||
document.querySelectorAll('.filter-button').forEach(btn => {
|
||||
btn.classList.remove('active');
|
||||
});
|
||||
event.target.classList.add('active');
|
||||
|
||||
// Re-render with new filter
|
||||
document.querySelectorAll('.filterbar .fbtn').forEach(b => b.classList.remove('active'));
|
||||
btn.classList.add('active');
|
||||
renderAlerts(allAlerts);
|
||||
}
|
||||
|
||||
async function acknowledgeAlert(hostname, metricPath, event) {
|
||||
// Prevent event bubbling
|
||||
if (event) {
|
||||
event.stopPropagation();
|
||||
}
|
||||
|
||||
// Disable the button
|
||||
if (event) event.stopPropagation();
|
||||
const button = event.target;
|
||||
button.disabled = true;
|
||||
button.textContent = 'Acknowledging...';
|
||||
|
||||
button.textContent = 'Acknowledging…';
|
||||
try {
|
||||
const response = await fetch('/api/0/alerts/acknowledge', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
hostname: hostname,
|
||||
metric_path: metricPath,
|
||||
}),
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({hostname: hostname, metric_path: metricPath}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`);
|
||||
}
|
||||
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||
const result = await response.json();
|
||||
|
||||
// Update the alert in our local data
|
||||
const alert = allAlerts.find(a => a.hostname === hostname && a.metric_path === metricPath);
|
||||
if (alert) {
|
||||
alert.acknowledged = true;
|
||||
alert.acknowledged_at = result.acknowledged_at;
|
||||
const a = allAlerts.find(x => x.hostname === hostname && x.metric_path === metricPath);
|
||||
if (a) {
|
||||
a.acknowledged = true;
|
||||
a.acknowledged_at = result.acknowledged_at;
|
||||
}
|
||||
|
||||
// Re-render alerts
|
||||
renderAlerts(allAlerts);
|
||||
|
||||
} catch (error) {
|
||||
alert(`Failed to acknowledge alert: ${error.message}`);
|
||||
button.disabled = false;
|
||||
@@ -525,10 +238,34 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-refresh every 15 seconds
|
||||
function onHostFilterInput(input) {
|
||||
const val = input.value.trim();
|
||||
if (!val) {
|
||||
hostFilterRe = null;
|
||||
input.classList.remove('invalid');
|
||||
} else {
|
||||
try {
|
||||
hostFilterRe = new RegExp(val, 'i');
|
||||
input.classList.remove('invalid');
|
||||
} catch (_) {
|
||||
hostFilterRe = null;
|
||||
input.classList.add('invalid');
|
||||
}
|
||||
}
|
||||
renderAlerts(allAlerts);
|
||||
}
|
||||
|
||||
setInterval(loadAlerts, 15000);
|
||||
|
||||
// Initial load
|
||||
(function () {
|
||||
const param = new URLSearchParams(window.location.search).get('filter');
|
||||
if (param) {
|
||||
const input = document.getElementById('host-filter');
|
||||
input.value = param;
|
||||
onHostFilterInput(input);
|
||||
}
|
||||
})();
|
||||
|
||||
loadAlerts();
|
||||
</script>
|
||||
</body>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<footer>
|
||||
<div id="copyright">
|
||||
©2002-2026 <A HREF="mailto:andreas@wrede.ca">Andreas Wrede</A> All Rights Reserved.</p>
|
||||
©2002-2026 <A HREF="mailto:aew.hbd@wrede.ca">Andreas Wrede</A> All Rights Reserved.</p>
|
||||
</div>
|
||||
</footer>
|
||||
+125
-16
@@ -5,7 +5,68 @@
|
||||
<link rel="icon" href="/static/images/favicon.ico" sizes="32x32" />
|
||||
<title>{{ title }}</title>
|
||||
{% if extra_scripts %}<script src="{{ extra_scripts }}"></script>{% endif %}
|
||||
<script>
|
||||
/* Apply saved theme before first paint to avoid flash */
|
||||
(function() {
|
||||
try {
|
||||
var p = localStorage.getItem('hbd_theme') || 'auto';
|
||||
var dark = p === 'dark' || (p === 'auto' && window.matchMedia('(prefers-color-scheme: dark)').matches);
|
||||
if (dark) document.documentElement.setAttribute('data-theme', 'dark');
|
||||
} catch(e) {}
|
||||
})();
|
||||
</script>
|
||||
<style>
|
||||
/* ── Theme variables ── */
|
||||
:root {
|
||||
--bg: #f5f5f5;
|
||||
--surface: #ffffff;
|
||||
--surface-2: #f8f8f8;
|
||||
--surface-3: #f5f5f5;
|
||||
--text: #222222;
|
||||
--text-2: #333333;
|
||||
--text-3: #555555;
|
||||
--text-sec: #666666;
|
||||
--text-muted: #888888;
|
||||
--text-dim: #aaaaaa;
|
||||
--text-ghost: #cccccc;
|
||||
--border: #e0e0e0;
|
||||
--border-2: #eeeeee;
|
||||
--border-3: #f0f0f0;
|
||||
--border-4: #f5f5f5;
|
||||
--link: #0066cc;
|
||||
--nav-bg: #ffffff;
|
||||
--input-bg: #ffffff;
|
||||
--input-border: #cccccc;
|
||||
--shadow-sm: rgba(0,0,0,.08);
|
||||
--shadow: rgba(0,0,0,.10);
|
||||
--shadow-nav: rgba(0,0,0,.10);
|
||||
}
|
||||
html[data-theme="dark"] {
|
||||
color-scheme: dark;
|
||||
--bg: #111827;
|
||||
--surface: #1f2937;
|
||||
--surface-2: #283447;
|
||||
--surface-3: #374151;
|
||||
--text: #e5e7eb;
|
||||
--text-2: #d1d5db;
|
||||
--text-3: #9ca3af;
|
||||
--text-sec: #9ca3af;
|
||||
--text-muted: #6b7280;
|
||||
--text-dim: #4b5563;
|
||||
--text-ghost: #374151;
|
||||
--border: #374151;
|
||||
--border-2: #2d3748;
|
||||
--border-3: #253040;
|
||||
--border-4: #1e2a38;
|
||||
--link: #60a5fa;
|
||||
--nav-bg: #1f2937;
|
||||
--input-bg: #283447;
|
||||
--input-border: #4b5563;
|
||||
--shadow-sm: rgba(0,0,0,.30);
|
||||
--shadow: rgba(0,0,0,.40);
|
||||
--shadow-nav: rgba(0,0,0,.40);
|
||||
}
|
||||
|
||||
/* ── Reset / shared baseline ── */
|
||||
*, *::before, *::after { box-sizing: border-box; }
|
||||
html {
|
||||
@@ -15,19 +76,24 @@
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 10px;
|
||||
background: #f5f5f5;
|
||||
padding-top: 60px;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
}
|
||||
h1 { font-size: 1.5em; color: #333; margin: 0 0 5px; }
|
||||
h2 { font-size: 1.1em; color: #333; margin: 0 0 8px; }
|
||||
h1 { font-size: 1.5em; color: var(--text-2); margin: 0 0 5px; }
|
||||
h2 { font-size: 1.1em; color: var(--text-2); margin: 0 0 8px; }
|
||||
p { margin: 0; }
|
||||
|
||||
/* Navigation bar — shared across all pages */
|
||||
.nav {
|
||||
background: #fff;
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
z-index: 200;
|
||||
background: var(--nav-bg);
|
||||
padding: 6px 12px;
|
||||
margin-bottom: 10px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,.1);
|
||||
border-radius: 4px;
|
||||
box-shadow: 0 2px 4px var(--shadow-nav);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
@@ -38,25 +104,25 @@
|
||||
.nav a {
|
||||
margin-right: 20px;
|
||||
text-decoration: none;
|
||||
color: #0066cc;
|
||||
color: var(--link);
|
||||
font-weight: 500;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
.nav a:hover { text-decoration: underline; }
|
||||
.nav a.active { color: #333; font-weight: bold; }
|
||||
.nav a.active { color: var(--text-2); font-weight: bold; }
|
||||
.nav-user {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
text-decoration: none;
|
||||
color: #333;
|
||||
color: var(--text-2);
|
||||
font-size: 0.9em;
|
||||
font-weight: 500;
|
||||
padding: 4px 8px;
|
||||
border-radius: 20px;
|
||||
transition: background 0.15s;
|
||||
}
|
||||
.nav-user:hover { background: #f0f4ff; text-decoration: none; }
|
||||
.nav-user:hover { background: var(--surface-2); text-decoration: none; }
|
||||
.nav-username {
|
||||
max-width: 0;
|
||||
overflow: hidden;
|
||||
@@ -77,7 +143,7 @@
|
||||
.nav-initials {
|
||||
width: 28px; height: 28px;
|
||||
border-radius: 50%;
|
||||
background: #0066cc;
|
||||
background: var(--link);
|
||||
color: #fff;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -102,7 +168,7 @@
|
||||
.nav-hamburger span {
|
||||
display: block;
|
||||
height: 3px;
|
||||
background: #555;
|
||||
background: var(--text-muted);
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
@@ -114,19 +180,51 @@
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
padding-top: 8px;
|
||||
border-top: 1px solid #eee;
|
||||
border-top: 1px solid var(--border-2);
|
||||
order: 3;
|
||||
}
|
||||
.nav-links.nav-open { display: flex; }
|
||||
.nav-links a { margin-right: 0; padding: 6px 0; font-size: 1em; }
|
||||
}
|
||||
|
||||
/* ── Global dark-mode: inputs ── */
|
||||
html[data-theme="dark"] input:not([type=checkbox]):not([type=radio]),
|
||||
html[data-theme="dark"] select,
|
||||
html[data-theme="dark"] textarea {
|
||||
background-color: var(--input-bg);
|
||||
border-color: var(--input-border);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
/* Pending config publish button */
|
||||
.nav-publish-btn {
|
||||
background: #e65100;
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
padding: 4px 10px;
|
||||
font-size: 0.82em;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
flex-shrink: 0;
|
||||
white-space: nowrap;
|
||||
margin-left: auto;
|
||||
}
|
||||
.nav-publish-btn:hover { background: #bf360c; }
|
||||
.nav-publish-btn:disabled { opacity: 0.7; cursor: default; }
|
||||
|
||||
/* Swiss railway clock — nav */
|
||||
.nav-clock {
|
||||
.nav-pie {
|
||||
flex-shrink: 0;
|
||||
line-height: 0;
|
||||
margin-left: auto;
|
||||
padding: 4px 4px 4px 0;
|
||||
}
|
||||
#alert-pie { display: block; cursor: default; }
|
||||
.nav-clock {
|
||||
flex-shrink: 0;
|
||||
line-height: 0;
|
||||
padding: 4px 4px 4px 0;
|
||||
cursor: pointer;
|
||||
}
|
||||
#swiss-clock { display: block; }
|
||||
@@ -204,7 +302,7 @@
|
||||
ctx.restore();
|
||||
}
|
||||
|
||||
hand((m + s / 60) / 60 * Math.PI * 2 - Math.PI / 2,
|
||||
hand((sFrac >= 58.5 ? m + 1 : m) / 60 * Math.PI * 2 - Math.PI / 2,
|
||||
R * 0.88, -R * 0.12, SIZE * 0.027, '#222'); /* minute */
|
||||
hand((h + m / 60) / 12 * Math.PI * 2 - Math.PI / 2,
|
||||
R * 0.58, -R * 0.12, SIZE * 0.039, '#222'); /* hour */
|
||||
@@ -252,6 +350,17 @@
|
||||
setTimeout(clockTick, delay);
|
||||
}
|
||||
|
||||
/* Keep auto-theme in sync with system setting changes */
|
||||
try {
|
||||
window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', function(e) {
|
||||
var pref = localStorage.getItem('hbd_theme') || 'auto';
|
||||
if (pref === 'auto') {
|
||||
if (e.matches) { document.documentElement.setAttribute('data-theme', 'dark'); }
|
||||
else { document.documentElement.removeAttribute('data-theme'); }
|
||||
}
|
||||
});
|
||||
} catch(e) {}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
/* Start the shared tick loop */
|
||||
clockTick();
|
||||
|
||||
+383
-537
@@ -1,565 +1,411 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
{% include 'head.html' %}
|
||||
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||||
<script src="/static/hbd-ui.js"></script>
|
||||
|
||||
<style>
|
||||
body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100vh;
|
||||
overflow: hidden;
|
||||
}
|
||||
html, body { height: auto; overflow: visible; }
|
||||
body { background: var(--st-paper); padding: 60px 0 0; }
|
||||
|
||||
@media (max-width: 640px) {
|
||||
body {
|
||||
height: auto;
|
||||
min-height: 100vh;
|
||||
overflow: auto;
|
||||
flex-direction: column;
|
||||
}
|
||||
.container {
|
||||
max-height: none;
|
||||
overflow: visible;
|
||||
}
|
||||
.table-section {
|
||||
max-height: 55vh;
|
||||
}
|
||||
.log-section {
|
||||
flex: none;
|
||||
max-height: 40vh;
|
||||
}
|
||||
}
|
||||
.live-main { max-width: 1180px; margin: 0 auto; padding: 0 20px 70px; }
|
||||
|
||||
.container {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
max-width: 1600px;
|
||||
width: 100%;
|
||||
margin: 0 auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 15px;
|
||||
overflow: hidden;
|
||||
.st-toolbar input {
|
||||
font: 12px var(--st-mono); color: var(--st-ink);
|
||||
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||||
padding: 5px 12px; min-width: 160px;
|
||||
}
|
||||
.live-dot { display: inline-flex; align-items: center; gap: 6px; font-size: 12px; font-weight: 600; color: var(--st-ok); }
|
||||
.live-dot::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: var(--st-ok); }
|
||||
.live-dot.reconnecting { color: var(--st-crit); }
|
||||
.live-dot.reconnecting::before { background: var(--st-crit); }
|
||||
|
||||
h1 {
|
||||
color: #333;
|
||||
margin-bottom: 5px;
|
||||
font-size: 1.5em;
|
||||
/* ── host rows: 6 columns ── */
|
||||
.hrow, .hhead {
|
||||
display: grid;
|
||||
grid-template-columns: 175px 105px 70px 125px 85px minmax(0,1fr);
|
||||
gap: 4px 14px; align-items: center;
|
||||
padding: 6px 14px; border-bottom: 1px solid var(--st-line-soft);
|
||||
}
|
||||
|
||||
h2 {
|
||||
color: #333;
|
||||
margin-bottom: 10px;
|
||||
font-size: 1.2em;
|
||||
padding: 10px 15px;
|
||||
background: white;
|
||||
border-radius: 6px;
|
||||
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
|
||||
.hrow:last-child { border-bottom: none; }
|
||||
.hrow:hover { background: var(--st-surface-2); }
|
||||
.hhead {
|
||||
font-family: var(--st-mono); font-size: 10.5px; text-transform: uppercase; letter-spacing: .05em;
|
||||
color: var(--st-faint); background: var(--st-surface-2); border-radius: 8px 8px 0 0;
|
||||
}
|
||||
.hhead .num, .hrow .num { text-align: right; }
|
||||
|
||||
.subtitle {
|
||||
color: #666;
|
||||
margin-bottom: 15px;
|
||||
font-size: 0.9em;
|
||||
.hrow.sev-warn { border-left: 3px solid var(--st-warn); padding-left: 11px; }
|
||||
.hrow.sev-crit { border-left: 3px solid var(--st-crit); padding-left: 11px; }
|
||||
|
||||
.host { font-family: var(--st-mono); font-size: 13px; font-weight: 400; display: flex; align-items: center; gap: 7px; min-width: 0; }
|
||||
.host.watched { font-weight: 700; }
|
||||
.host a { color: inherit; text-decoration: none; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.host a:hover { color: var(--st-accent); }
|
||||
.dot.down { background: var(--st-crit); box-shadow: 0 0 0 3px var(--st-crit-soft); }
|
||||
.dot.overdue { background: transparent; border: 2px solid var(--st-warn); }
|
||||
.stale { color: var(--st-faint); font-size: 10px; border: 1px solid var(--st-line); border-radius: 4px; padding: 1px 4px; font-family: var(--st-mono); white-space: nowrap; }
|
||||
.flap { color: var(--st-warn); font-size: 10px; border: 1px solid var(--st-warn); border-radius: 4px; padding: 1px 4px; font-family: var(--st-mono); white-space: nowrap; }
|
||||
.chip.outline { background: transparent; border: 1px solid var(--st-line); }
|
||||
|
||||
.lat { font-family: var(--st-mono); font-size: 12px; color: var(--st-muted); font-variant-numeric: tabular-nums; }
|
||||
.since { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-faint); font-variant-numeric: tabular-nums; }
|
||||
|
||||
.lastalert { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.lastalert .lvl { font-weight: 700; }
|
||||
.lastalert .lvl.crit { color: var(--st-crit); }
|
||||
.lastalert .lvl.warn { color: var(--st-warn); }
|
||||
.lastalert .ago { color: var(--st-faint); }
|
||||
.lastalert.quiet { color: var(--st-faint); }
|
||||
|
||||
/* ── hover card on the status chip ── */
|
||||
.tip { position: relative; display: inline-flex; }
|
||||
.tipbox {
|
||||
display: none; position: absolute; z-index: 40; top: calc(100% + 6px); left: 0;
|
||||
width: 340px; background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 8px;
|
||||
box-shadow: 0 8px 28px var(--shadow); padding: 10px 12px;
|
||||
}
|
||||
.tip:hover .tipbox, .tip:focus-within .tipbox { display: block; }
|
||||
.tipbox table { border-collapse: collapse; width: 100%; font-family: var(--st-mono); font-size: 11.5px; }
|
||||
.tipbox th { text-align: left; font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: var(--st-faint); padding: 2px 6px 4px; border-bottom: 1px solid var(--st-line-soft); }
|
||||
.tipbox td { padding: 3px 6px; color: var(--st-muted); }
|
||||
.tipbox td:first-child { color: var(--st-faint); }
|
||||
.tipbox .up { color: var(--st-ok); font-weight: 700; }
|
||||
.tipbox .dn { color: var(--st-crit); font-weight: 700; }
|
||||
.tipbox .ov { color: var(--st-warn); font-weight: 700; }
|
||||
.tipbox .ab { color: var(--st-faint); font-weight: 700; }
|
||||
.tipbox .meta { margin-top: 6px; padding-top: 6px; border-top: 1px solid var(--st-line-soft); font-size: 11px; color: var(--st-faint); }
|
||||
|
||||
.content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 15px;
|
||||
}
|
||||
|
||||
.table-section {
|
||||
background: white;
|
||||
border-radius: 6px;
|
||||
padding: 15px;
|
||||
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
|
||||
overflow-x: auto;
|
||||
overflow-y: auto;
|
||||
max-height: 60vh;
|
||||
}
|
||||
|
||||
.log-section {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
background: white;
|
||||
border-radius: 6px;
|
||||
padding: 15px;
|
||||
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
#ntable {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
#ntable td,
|
||||
#ntable th {
|
||||
border: 1px solid #e0e0e0;
|
||||
text-align: left;
|
||||
padding: 2px 4px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
#ntable tr:nth-child(even) {
|
||||
background-color: #fafafa;
|
||||
}
|
||||
|
||||
#ntable tr:hover {
|
||||
background-color: #e3f2fd;
|
||||
}
|
||||
|
||||
#ntable tbody tr.row-warning {
|
||||
background-color: #fff8c5;
|
||||
}
|
||||
|
||||
#ntable tbody tr.row-critical {
|
||||
background-color: #fde8e8;
|
||||
}
|
||||
|
||||
#ntable tbody tr.row-warning:hover {
|
||||
background-color: #fff0a0;
|
||||
}
|
||||
|
||||
#ntable tbody tr.row-critical:hover {
|
||||
background-color: #f9c8c8;
|
||||
}
|
||||
|
||||
#ntable th {
|
||||
padding: 6px 8px;
|
||||
background-color: #2196f3;
|
||||
color: white;
|
||||
font-weight: 600;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
}
|
||||
|
||||
#ntable
|
||||
th:not(.sorttable_sorted):not(.sorttable_sorted_reverse):not(.sorttable_nosort):after {
|
||||
content: " ⇅";
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
/* Alert count column styling */
|
||||
#ntable td.alert-warning {
|
||||
color: #ff9800;
|
||||
font-weight: bold;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
#ntable td.alert-critical {
|
||||
color: #f44336;
|
||||
font-weight: bold;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
/* Scrollbar styling */
|
||||
.log-section::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
}
|
||||
|
||||
.log-section::-webkit-scrollbar-track {
|
||||
background: #f1f1f1;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.log-section::-webkit-scrollbar-thumb {
|
||||
background: #888;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.log-section::-webkit-scrollbar-thumb:hover {
|
||||
background: #555;
|
||||
}
|
||||
|
||||
/* Message styling */
|
||||
#messages {
|
||||
font-size: 0.85em;
|
||||
line-height: 1.0;
|
||||
}
|
||||
|
||||
#messages div {
|
||||
padding: 5px 0;
|
||||
border-bottom: 1px solid #f0f0f0;
|
||||
}
|
||||
|
||||
/* Modal for connection status messages */
|
||||
.connection-modal {
|
||||
display: none;
|
||||
position: fixed;
|
||||
z-index: 1000;
|
||||
left: 0;
|
||||
top: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
|
||||
.connection-modal.show {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.connection-modal-content {
|
||||
background-color: white;
|
||||
padding: 30px 40px;
|
||||
border-radius: 8px;
|
||||
text-align: center;
|
||||
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
|
||||
min-width: 300px;
|
||||
}
|
||||
|
||||
.connection-modal-content p {
|
||||
margin: 0;
|
||||
font-size: 16px;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
/* State indicators */
|
||||
.state-up {
|
||||
color: #4caf50;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.state-down {
|
||||
color: #f44336;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.state-overdue {
|
||||
color: #ff9800;
|
||||
font-weight: 700;
|
||||
@media (max-width: 760px) {
|
||||
.hhead { display: none; }
|
||||
.hrow { grid-template-columns: minmax(0,1fr) auto auto; padding: 8px 12px; }
|
||||
.hrow .num { text-align: left; }
|
||||
.hrow .since { display: none; }
|
||||
.lastalert { grid-column: 1 / -1; }
|
||||
.tipbox { left: auto; right: 0; }
|
||||
}
|
||||
</style>
|
||||
<script type="text/javascript">
|
||||
var cnt = 0;
|
||||
var nTable = document;
|
||||
var name_idx = {};
|
||||
var c = 0;
|
||||
var HBD_VERSION = "{{ hbd_version }}";
|
||||
|
||||
function hostNameHtml(data) {
|
||||
var nameHtml = data.name;
|
||||
if (!data.hbc_version || data.hbc_version !== HBD_VERSION) {
|
||||
nameHtml += ' 🥀';
|
||||
}
|
||||
return data.dyn ? '<b>' + nameHtml + '</b>' : nameHtml;
|
||||
}
|
||||
|
||||
function setup() {
|
||||
name_idx = {};
|
||||
nTable = document.getElementById("ntable");
|
||||
for (var i = 0, row; (row = nTable.rows[i]); i++) {
|
||||
if (i == 0) continue;
|
||||
var cell = nTable.rows[i].cells[0];
|
||||
var name = cell.dataset.name || cell.innerText.replace(/\s*🥀\s*$/, '').trim();
|
||||
name_idx[name] = nTable.rows[i];
|
||||
}
|
||||
}
|
||||
|
||||
function updateRowAlert(row, data) {
|
||||
var criticalUnacked = data.alert_critical_unacked || 0;
|
||||
var criticalAcked = data.alert_critical_acked || 0;
|
||||
var warningUnacked = data.alert_warning_unacked || 0;
|
||||
var warningAcked = data.alert_warning_acked || 0;
|
||||
row.classList.remove('row-warning', 'row-critical');
|
||||
if (criticalUnacked > 0 || criticalAcked > 0) {
|
||||
row.classList.add('row-critical');
|
||||
} else if (warningUnacked > 0 || warningAcked > 0) {
|
||||
row.classList.add('row-warning');
|
||||
}
|
||||
}
|
||||
|
||||
function createRow(data) {
|
||||
var row = document.createElement("tr");
|
||||
var c_name = document.createElement("td");
|
||||
var c_warning = document.createElement("td");
|
||||
c_warning.style.textAlign = "center";
|
||||
c_warning.style.color = "#ff9800";
|
||||
c_warning.style.fontWeight = "bold";
|
||||
var c_critical = document.createElement("td");
|
||||
c_critical.style.textAlign = "center";
|
||||
c_critical.style.color = "#f44336";
|
||||
c_critical.style.fontWeight = "bold";
|
||||
var c_ipv4addr = document.createElement("td");
|
||||
var c_ipv4state = document.createElement("td");
|
||||
var c_ipv4latency = document.createElement("td");
|
||||
c_ipv4latency.style.textAlign = "right";
|
||||
var c_ipv4statets = document.createElement("td");
|
||||
c_ipv4statets.style.textAlign = "right";
|
||||
var c_ipv6addr = document.createElement("td");
|
||||
var c_ipv6state = document.createElement("td");
|
||||
var c_ipv6latency = document.createElement("td");
|
||||
c_ipv6latency.style.textAlign = "right";
|
||||
var c_ipv6statets = document.createElement("td");
|
||||
c_ipv6statets.style.textAlign = "right";
|
||||
row.appendChild(c_name);
|
||||
row.appendChild(c_warning);
|
||||
row.appendChild(c_critical);
|
||||
row.appendChild(c_ipv4addr);
|
||||
row.appendChild(c_ipv4state);
|
||||
row.appendChild(c_ipv4latency);
|
||||
row.appendChild(c_ipv4statets);
|
||||
row.appendChild(c_ipv6addr);
|
||||
row.appendChild(c_ipv6state);
|
||||
row.appendChild(c_ipv6latency);
|
||||
row.appendChild(c_ipv6statets);
|
||||
c_name.dataset.name = data.name;
|
||||
c_name.innerHTML = hostNameHtml(data);
|
||||
|
||||
// Set alert counts in "x/y" format (unacked/acked)
|
||||
var warningUnacked = data.alert_warning_unacked || 0;
|
||||
var warningAcked = data.alert_warning_acked || 0;
|
||||
var criticalUnacked = data.alert_critical_unacked || 0;
|
||||
var criticalAcked = data.alert_critical_acked || 0;
|
||||
|
||||
if (warningUnacked > 0 || warningAcked > 0) {
|
||||
c_warning.innerHTML = warningAcked > 0 ? warningUnacked + "/" + warningAcked : warningUnacked;
|
||||
} else {
|
||||
c_warning.innerHTML = "";
|
||||
}
|
||||
|
||||
if (criticalUnacked > 0 || criticalAcked > 0) {
|
||||
c_critical.innerHTML = criticalAcked > 0 ? criticalUnacked + "/" + criticalAcked : criticalUnacked;
|
||||
} else {
|
||||
c_critical.innerHTML = "";
|
||||
}
|
||||
|
||||
c_ipv4addr.innerHTML = data.connections[0].addr;
|
||||
c_ipv4state.innerHTML = data.connections[0].state;
|
||||
if (data.connections.length > 1) {
|
||||
c_ipv6addr.innerHTML = data.connections[1].addr;
|
||||
c_ipv6state.innerHTML = data.connections[1].state;
|
||||
}
|
||||
var table = document.getElementById("ntablebody"); // find table to append to
|
||||
table.appendChild(row); // append row to table
|
||||
name_idx[c_name] = row;
|
||||
updateRowAlert(row, data);
|
||||
}
|
||||
|
||||
function formatTS(ts) {
|
||||
const now = new Date();
|
||||
const d = new Date(ts * 1000);
|
||||
|
||||
const pad = n => String(n).padStart(2, '0');
|
||||
const timeStr = `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
|
||||
|
||||
// Same calendar day → show time only
|
||||
if (d.toDateString() === now.toDateString()) {
|
||||
return timeStr;
|
||||
}
|
||||
|
||||
// Within 8 days → show "-X d hh:mm:ss"
|
||||
const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
const dStart = new Date(d.getFullYear(), d.getMonth(), d.getDate());
|
||||
const diffDays = Math.round((todayStart - dStart) / 86400000);
|
||||
if (diffDays < 8) {
|
||||
return `-${diffDays}d ${timeStr}`;
|
||||
}
|
||||
|
||||
// Older → date only
|
||||
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
|
||||
}
|
||||
|
||||
function update_table(data) {
|
||||
if (!(data.name in name_idx)) {
|
||||
createRow(data);
|
||||
setup();
|
||||
}
|
||||
|
||||
// Update name cell (version indicator)
|
||||
var nameCell = name_idx[data.name].cells[0];
|
||||
nameCell.dataset.name = data.name;
|
||||
nameCell.innerHTML = hostNameHtml(data);
|
||||
|
||||
// Update warning and critical counts in "x/y" format (unacked/acked)
|
||||
var warningUnacked = data.alert_warning_unacked || 0;
|
||||
var warningAcked = data.alert_warning_acked || 0;
|
||||
var criticalUnacked = data.alert_critical_unacked || 0;
|
||||
var criticalAcked = data.alert_critical_acked || 0;
|
||||
|
||||
if (warningUnacked > 0 || warningAcked > 0) {
|
||||
name_idx[data.name].cells[1].innerHTML = warningAcked > 0 ? warningUnacked + "/" + warningAcked : warningUnacked;
|
||||
} else {
|
||||
name_idx[data.name].cells[1].innerHTML = "";
|
||||
}
|
||||
|
||||
if (criticalUnacked > 0 || criticalAcked > 0) {
|
||||
name_idx[data.name].cells[2].innerHTML = criticalAcked > 0 ? criticalUnacked + "/" + criticalAcked : criticalUnacked;
|
||||
} else {
|
||||
name_idx[data.name].cells[2].innerHTML = "";
|
||||
}
|
||||
|
||||
for (var i = 0; i < data.connections.length; i++) {
|
||||
// Offset by 2 for the warning/critical count columns
|
||||
name_idx[data.name].cells[3 + i * 4].innerHTML = data.connections[i].addr;
|
||||
name_idx[data.name].cells[6 + i * 4].innerHTML = formatTS(
|
||||
data.connections[i].statetime
|
||||
);
|
||||
if (data.connections[i].state == "up") {
|
||||
state = '<span class="state-up">up</span>';
|
||||
latency = Number.parseFloat(data.connections[i].rtts[0]).toFixed(2);
|
||||
} else {
|
||||
if (data.connections[i].state == "unknown") {
|
||||
state = "";
|
||||
latency = "";
|
||||
name_idx[data.name].cells[3 + i * 4].innerHTML = "";
|
||||
name_idx[data.name].cells[6 + i * 4].innerHTML = "";
|
||||
} else if (data.connections[i].state == "down") {
|
||||
state = '<span class="state-down">down</span>';
|
||||
latency = "-";
|
||||
} else if (data.connections[i].state == "overdue") {
|
||||
state = '<span class="state-overdue">overdue</span>';
|
||||
latency = "-";
|
||||
} else {
|
||||
state = "<b>" + data.connections[i].state + "</b>";
|
||||
latency = "-";
|
||||
}
|
||||
}
|
||||
name_idx[data.name].cells[4 + i * 4].innerHTML = state;
|
||||
name_idx[data.name].cells[5 + i * 4].innerHTML = latency;
|
||||
}
|
||||
updateRowAlert(name_idx[data.name], data);
|
||||
}
|
||||
|
||||
function WS_Connect() {
|
||||
if ("WebSocket" in window) {
|
||||
//N.B: subprotocol field causes chrome to error 1006
|
||||
var ws_hbd = new WebSocket("{{heartbeat_ws_url}}", /* "hdb" */ );
|
||||
|
||||
ws_hbd.onopen = function () {
|
||||
// Web Socket is connected, send data using send()
|
||||
console.log("ws connect {{heartbeat_ws_url}}");
|
||||
// Hide modal window if visible
|
||||
var modal = document.getElementById("connectionModal");
|
||||
if (modal) {
|
||||
modal.classList.remove("show");
|
||||
}
|
||||
ws_hbd.send("heartbeat_web");
|
||||
};
|
||||
|
||||
ws_hbd.onerror = function (event) {
|
||||
console.log(event);
|
||||
};
|
||||
|
||||
ws_hbd.onmessage = function (event) {
|
||||
/* console.log(event.data); */
|
||||
var state = JSON.parse(event.data);
|
||||
/* console.log("State: " + state.type); */
|
||||
if (state.type == "host") {
|
||||
update_table(state.data);
|
||||
} else if (state.type == "message") {
|
||||
var msgs = document.getElementById("messages");
|
||||
msgs.insertAdjacentHTML("afterbegin", "<div>" + state.data + "</div>");
|
||||
}
|
||||
cnt++;
|
||||
};
|
||||
|
||||
ws_hbd.onclose = function (event) {
|
||||
/* console.log(event); */
|
||||
console.log("Connection is closed, reopening");
|
||||
// Show modal window
|
||||
var modal = document.getElementById("connectionModal");
|
||||
if (modal) {
|
||||
modal.classList.add("show");
|
||||
}
|
||||
setTimeout(function () {
|
||||
WS_Connect();
|
||||
}, 3000);
|
||||
};
|
||||
} else {
|
||||
// The browser doesn't support WebSocket
|
||||
console.log("WebSocket NOT supported by your Browser!");
|
||||
}
|
||||
}
|
||||
WS_Connect();
|
||||
</script>
|
||||
<body>
|
||||
{% include 'nav.html' %}
|
||||
|
||||
{% include 'menu.html' %}
|
||||
|
||||
<div class="container">
|
||||
<div>
|
||||
<h1>{{ header }}</h1>
|
||||
<p class="subtitle">Real-time host monitoring and event log</p>
|
||||
</div>
|
||||
|
||||
<div class="table-section">
|
||||
<table id="ntable" class="sortable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th style="text-align: center" title="Warning Alerts">⚠️</th>
|
||||
<th style="text-align: center" title="Critical Alerts">🔴</th>
|
||||
<th>IPv4 Addr</th>
|
||||
<th>State</th>
|
||||
<th style="text-align: right">Latency</th>
|
||||
<th style="text-align: right">Last State</th>
|
||||
<th>IPv6 Addr</th>
|
||||
<th>State</th>
|
||||
<th style="text-align: right">Latency</th>
|
||||
<th style="text-align: right">Last State</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="ntablebody">
|
||||
{% for host in hosts %}
|
||||
<tr class="{% if host.alert_critical_unacked > 0 or host.alert_critical_acked > 0 %}row-critical{% elif host.alert_warning_unacked > 0 or host.alert_warning_acked > 0 %}row-warning{% endif %}">
|
||||
<td data-name="{{ host.name }}">{{ host.name }}{% if not host.hbc_version or host.hbc_version != hbd_version %} 🥀{% endif %}</td>
|
||||
<td style="text-align: center; color: #ff9800; font-weight: bold;">
|
||||
{%- set warning_unacked = host.alert_warning_unacked -%}
|
||||
{%- set warning_acked = host.alert_warning_acked -%}
|
||||
{%- if warning_unacked > 0 or warning_acked > 0 -%}
|
||||
{{ warning_unacked }}{% if warning_acked > 0 %}/{{ warning_acked }}{% endif %}
|
||||
{%- endif -%}
|
||||
</td>
|
||||
<td style="text-align: center; color: #f44336; font-weight: bold;">
|
||||
{%- set critical_unacked = host.alert_critical_unacked -%}
|
||||
{%- set critical_acked = host.alert_critical_acked -%}
|
||||
{%- if critical_unacked > 0 or critical_acked > 0 -%}
|
||||
{{ critical_unacked }}{% if critical_acked > 0 %}/{{ critical_acked }}{% endif %}
|
||||
{%- endif -%}
|
||||
</td>
|
||||
{% for conn in host.connections %}
|
||||
<td>{{ conn.addr if conn.addr else '' }}</td>
|
||||
<td>{{ conn.state if conn.state else '' }}</td>
|
||||
<td style="text-align: right">{{ conn.latency if conn.latency else '' }}</td>
|
||||
<td style="text-align: right">{{ conn.last_state_ts if conn.last_state_ts else '' }}</td>
|
||||
{% endfor %}
|
||||
{% if host.connections|length == 0 %}
|
||||
<td></td><td></td><td></td><td></td>
|
||||
<td></td><td></td><td></td><td></td>
|
||||
{% elif host.connections|length == 1 %}
|
||||
<td></td><td></td><td></td><td></td>
|
||||
{% endif %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="log-section">
|
||||
<h2>Log of Events</h2>
|
||||
<div id="messages"></div>
|
||||
</div>
|
||||
<div class="st-toolbar">
|
||||
<span class="brand">hbd<span class="tld">·live</span></span>
|
||||
<span class="hintline">every host, one line — details on hover, history in the overview</span>
|
||||
<span class="spacer"></span>
|
||||
<input type="text" id="host-filter" placeholder="host filter (regex)">
|
||||
<span class="live-dot reconnecting" id="ws-status">connecting…</span>
|
||||
</div>
|
||||
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||||
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||||
</svg>
|
||||
|
||||
{% include 'foot.html' %}
|
||||
|
||||
<!-- Connection status modal -->
|
||||
<div id="connectionModal" class="connection-modal">
|
||||
<div class="connection-modal-content">
|
||||
<p>⚠️ Connection is closed, reopening...</p>
|
||||
</div>
|
||||
<div class="live-main">
|
||||
<section class="st">
|
||||
<div class="sec-head">
|
||||
<h2>hosts<span class="colon">:</span></h2>
|
||||
<span class="count" id="host-counts"></span>
|
||||
<span class="sub">sorted by severity, then name</span>
|
||||
</div>
|
||||
<div class="list">
|
||||
<div class="hhead">
|
||||
<span>host</span><span>status</span><span class="num">latency</span>
|
||||
<span>alerts</span><span>since</span><span>last alert</span>
|
||||
</div>
|
||||
<div id="host-rows"></div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
setup();
|
||||
var HBD_VERSION = "{{ hbd_version }}";
|
||||
var INITIAL_HOSTS = {{ hosts | tojson }};
|
||||
|
||||
var hostData = {}; // name -> latest stateinfo payload
|
||||
var lastAlert = {}; // name -> {level, message, ts}
|
||||
var hostFilterRe = null;
|
||||
|
||||
function escHtml(s) {
|
||||
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');
|
||||
}
|
||||
|
||||
// ---- derived per-host facts ----------------------------------------
|
||||
var STATE_RANK = { down: 3, overdue: 2, up: 1 };
|
||||
|
||||
function worstConnection(data) {
|
||||
var worst = null;
|
||||
(data.connections || []).forEach(function (c) {
|
||||
if (!c.state || c.state === 'unknown' || c.state === 'absent') return;
|
||||
if (!worst || (STATE_RANK[c.state] || 0) > (STATE_RANK[worst.state] || 0)) worst = c;
|
||||
});
|
||||
return worst;
|
||||
}
|
||||
|
||||
function severity(data) {
|
||||
var st = data.status;
|
||||
if ((data.alert_critical_unacked || 0) > 0 || (data.alert_critical_acked || 0) > 0) return 'crit';
|
||||
if (st === 'down') return 'crit';
|
||||
if ((data.alert_warning_unacked || 0) > 0 || (data.alert_warning_acked || 0) > 0) return 'warn';
|
||||
if (st === 'overdue') return 'warn';
|
||||
return 'ok';
|
||||
}
|
||||
var SEV_ORDER = { crit: 0, warn: 1, ok: 2 };
|
||||
|
||||
function relTime(ts) {
|
||||
if (!ts) return '—';
|
||||
var s = Math.max(0, Math.floor(Date.now() / 1000 - ts));
|
||||
if (s < 60) return s + 's';
|
||||
if (s < 3600) return Math.floor(s / 60) + 'm';
|
||||
if (s < 86400) return Math.floor(s / 3600) + 'h ' + Math.floor((s % 3600) / 60) + 'm';
|
||||
if (s < 86400 * 30) return Math.floor(s / 86400) + 'd ' + Math.floor((s % 86400) / 3600) + 'h';
|
||||
return Math.floor(s / 86400) + 'd';
|
||||
}
|
||||
|
||||
function fmtClock(ts) {
|
||||
var d = new Date(ts * 1000);
|
||||
function p(n) { return n < 10 ? '0' + n : '' + n; }
|
||||
var t = p(d.getHours()) + ':' + p(d.getMinutes()) + ':' + p(d.getSeconds());
|
||||
return (d.toDateString() === new Date().toDateString())
|
||||
? t
|
||||
: d.getFullYear() + '-' + p(d.getMonth() + 1) + '-' + p(d.getDate()) + ' ' + t;
|
||||
}
|
||||
|
||||
// ---- host row rendering ---------------------------------------------
|
||||
function statusChip(data) {
|
||||
var st = data.status;
|
||||
if (!st || st === 'unknown') return '<span class="chip outline">—</span>';
|
||||
var cls = (st === 'up' || st === 'simplex') ? 'ok' : (st === 'down' ? 'crit' : 'warn');
|
||||
return '<span class="chip ' + cls + '">' + escHtml(st) + '</span>';
|
||||
}
|
||||
|
||||
function tipCard(data) {
|
||||
var rows = '';
|
||||
(data.connections || []).forEach(function (c) {
|
||||
if (!c.state || c.state === 'unknown') return;
|
||||
var cls = c.state === 'up' ? 'up' : (c.state === 'down' ? 'dn' : (c.state === 'absent' ? 'ab' : 'ov'));
|
||||
var fam = (c.afam || '').replace('IPv', 'v');
|
||||
var rtt = (c.state === 'up' && c.rtts && c.rtts[0]) ? Math.round(c.rtts[0]) + ' ms' : '—';
|
||||
rows += '<tr><td>' + escHtml(fam) + '</td><td>' + escHtml(c.addr || '—') + '</td>'
|
||||
+ '<td class="' + cls + '">' + escHtml(c.state) + '</td>'
|
||||
+ '<td>' + rtt + '</td><td>' + (c.statetime ? fmtClock(c.statetime) : '—') + '</td></tr>';
|
||||
});
|
||||
if (!rows) rows = '<tr><td colspan="5">no connections yet</td></tr>';
|
||||
var meta = [];
|
||||
meta.push('agent ' + (data.hbc_version ? 'v' + escHtml(String(data.hbc_version)) : 'unknown'));
|
||||
if (data.dyn) meta.push('dyndns');
|
||||
if (data.watched) meta.push('watched');
|
||||
if (data.interval) meta.push('interval ' + escHtml(String(data.interval)) + ' s');
|
||||
return '<span class="tipbox" role="tooltip">'
|
||||
+ '<table><thead><tr><th></th><th>address</th><th>state</th><th>rtt</th><th>since</th></tr></thead>'
|
||||
+ '<tbody>' + rows + '</tbody></table>'
|
||||
+ '<div class="meta">' + meta.join(' · ') + '</div></span>';
|
||||
}
|
||||
|
||||
function alertChips(data) {
|
||||
var cu = data.alert_critical_unacked || 0, ca = data.alert_critical_acked || 0;
|
||||
var wu = data.alert_warning_unacked || 0, wa = data.alert_warning_acked || 0;
|
||||
var out = '';
|
||||
if (cu) out += '<span class="chip crit"><b>' + cu + '</b> crit</span> ';
|
||||
if (wu) out += '<span class="chip warn"><b>' + wu + '</b> warn</span> ';
|
||||
var acked = ca + wa;
|
||||
if (acked) out += '<span class="chip outline">' + acked + ' ack’d</span>';
|
||||
return out;
|
||||
}
|
||||
|
||||
function lastAlertHtml(name) {
|
||||
var a = lastAlert[name];
|
||||
if (!a) return '<span class="lastalert quiet">—</span>';
|
||||
var lvl = (a.level || '').toUpperCase();
|
||||
var quiet = (lvl !== 'WARNING' && lvl !== 'CRITICAL');
|
||||
var lvlCls = lvl === 'CRITICAL' ? 'crit' : (lvl === 'WARNING' ? 'warn' : '');
|
||||
var short = lvl === 'CRITICAL' ? 'CRIT' : (lvl === 'WARNING' ? 'WARN' : escHtml(lvl));
|
||||
return '<span class="lastalert' + (quiet ? ' quiet' : '') + '" title="' + escHtml(a.message) + '">'
|
||||
+ '<span class="lvl ' + lvlCls + '">' + short + '</span> '
|
||||
+ escHtml(a.message)
|
||||
+ (a.ts ? ' <span class="ago">· ' + relTime(a.ts) + ' ago</span>' : '')
|
||||
+ '</span>';
|
||||
}
|
||||
|
||||
function renderRow(data) {
|
||||
var name = data.name;
|
||||
var sev = severity(data);
|
||||
var row = document.querySelector('.hrow[data-host="' + CSS.escape(name) + '"]');
|
||||
var isNew = false;
|
||||
if (!row) {
|
||||
row = document.createElement('div');
|
||||
row.dataset.host = name;
|
||||
isNew = true;
|
||||
}
|
||||
row.className = 'hrow' + (sev !== 'ok' ? ' sev-' + sev : '');
|
||||
row.dataset.sev = sev;
|
||||
|
||||
var w = worstConnection(data);
|
||||
var up = (data.connections || []).find(function (c) { return c.state === 'up' && c.rtts && c.rtts.length; });
|
||||
var lat = up ? Math.round(up.rtts[0]) + ' ms' : '—';
|
||||
var since = w ? relTime(w.statetime) : '—';
|
||||
var staleTag = '';
|
||||
if (!data.hbc_version) staleTag = '<span class="stale" title="agent version unknown">agent?</span>';
|
||||
else if (data.hbc_version !== HBD_VERSION) {
|
||||
staleTag = '<span class="stale" title="agent v' + escHtml(String(data.hbc_version)) + ' ≠ server v' + HBD_VERSION + '">v' + escHtml(String(data.hbc_version)) + '</span>';
|
||||
}
|
||||
var st = data.status;
|
||||
var dotCls = (!st || st === 'unknown') ? 'off'
|
||||
: ((st === 'up' || st === 'simplex') ? sev : (st === 'down' ? 'down' : 'overdue'));
|
||||
var flapTag = '';
|
||||
var flapping = data.flapping || [];
|
||||
if (flapping.length) {
|
||||
var svcs = flapping.map(function (s) { return s || 'host'; }).join(', ');
|
||||
flapTag = '<span class="flap" title="notifications suppressed: ' + escHtml(svcs) + '">flapping</span>';
|
||||
}
|
||||
|
||||
row.innerHTML =
|
||||
'<span class="host' + (data.watched ? ' watched' : '') + '">'
|
||||
+ '<span class="dot ' + dotCls + '"></span>'
|
||||
+ '<a href="/plugins#' + encodeURIComponent(name) + '">' + escHtml(name) + '</a>' + staleTag + flapTag + '</span>'
|
||||
+ '<span class="tip" tabindex="0">' + statusChip(data) + tipCard(data) + '</span>'
|
||||
+ '<span class="lat num">' + lat + '</span>'
|
||||
+ '<span>' + alertChips(data) + '</span>'
|
||||
+ '<span class="since">' + since + '</span>'
|
||||
+ lastAlertHtml(name);
|
||||
|
||||
if (isNew) {
|
||||
document.getElementById('host-rows').appendChild(row);
|
||||
sortRows();
|
||||
}
|
||||
applyHostFilter(row);
|
||||
}
|
||||
|
||||
function sortRows() {
|
||||
var container = document.getElementById('host-rows');
|
||||
var rows = [...container.children];
|
||||
rows.sort(function (a, b) {
|
||||
var d = (SEV_ORDER[a.dataset.sev] ?? 2) - (SEV_ORDER[b.dataset.sev] ?? 2);
|
||||
return d !== 0 ? d : a.dataset.host.localeCompare(b.dataset.host);
|
||||
});
|
||||
rows.forEach(function (r) { container.appendChild(r); });
|
||||
}
|
||||
|
||||
function updateCounts() {
|
||||
var total = 0, up = 0, overdue = 0, down = 0;
|
||||
Object.values(hostData).forEach(function (d) {
|
||||
total++;
|
||||
var st = d.status;
|
||||
if (!st || st === 'unknown') return;
|
||||
if (st === 'up' || st === 'simplex') up++;
|
||||
else if (st === 'overdue') overdue++;
|
||||
else if (st === 'down') down++;
|
||||
});
|
||||
var parts = [total + ' hosts', up + ' up'];
|
||||
if (overdue) parts.push(overdue + ' overdue');
|
||||
if (down) parts.push(down + ' down');
|
||||
document.getElementById('host-counts').textContent = parts.join(' · ');
|
||||
}
|
||||
|
||||
function updateHost(data) {
|
||||
var prev = hostData[data.name];
|
||||
var prevSev = prev ? severity(prev) : null;
|
||||
hostData[data.name] = data;
|
||||
renderRow(data);
|
||||
if (prevSev !== severity(data)) sortRows();
|
||||
updateCounts();
|
||||
}
|
||||
|
||||
// ---- host filter -----------------------------------------------------
|
||||
function applyHostFilter(row) {
|
||||
var rows = row ? [row] : [...document.querySelectorAll('.hrow')];
|
||||
rows.forEach(function (r) {
|
||||
r.style.display = (!hostFilterRe || hostFilterRe.test(r.dataset.host)) ? '' : 'none';
|
||||
});
|
||||
}
|
||||
document.getElementById('host-filter').addEventListener('input', function () {
|
||||
var v = this.value.trim();
|
||||
try {
|
||||
hostFilterRe = v ? new RegExp(v, 'i') : null;
|
||||
this.style.borderColor = '';
|
||||
} catch (_) {
|
||||
hostFilterRe = null;
|
||||
this.style.borderColor = 'var(--st-crit)';
|
||||
}
|
||||
applyHostFilter();
|
||||
});
|
||||
|
||||
// ---- last alert tracking ---------------------------------------------
|
||||
var ALERT_LEVELS = { WARNING: 1, CRITICAL: 1, RECOVER: 1 };
|
||||
|
||||
function noteAlert(host, level, message, ts) {
|
||||
if (!host || !ALERT_LEVELS[(level || '').toUpperCase()]) return;
|
||||
var cur = lastAlert[host];
|
||||
if (cur && cur.ts && ts && ts < cur.ts) return; // keep the newest
|
||||
lastAlert[host] = { level: level, message: message, ts: ts };
|
||||
if (hostData[host]) renderRow(hostData[host]);
|
||||
}
|
||||
|
||||
async function seedActiveAlerts() {
|
||||
try {
|
||||
var r = await fetch('/api/0/alerts');
|
||||
if (!r.ok) return;
|
||||
var data = await r.json();
|
||||
(data.alerts || []).forEach(function (a) {
|
||||
var metric = (a.metric_path.includes('.')
|
||||
? a.metric_path.slice(a.metric_path.indexOf('.') + 1)
|
||||
: a.metric_path).replace(/_status_code$/, '');
|
||||
var msg = metric + ' = ' + a.last_value
|
||||
+ (a.formatted_message ? ' ' + a.formatted_message
|
||||
: (a.operator && a.threshold_value != null ? ' (' + a.operator + ' ' + a.threshold_value + ')' : ''));
|
||||
noteAlert(a.hostname, a.level, msg, a.since);
|
||||
});
|
||||
} catch (_) { /* dashboard still works without the seed */ }
|
||||
}
|
||||
|
||||
// ---- websocket -----------------------------------------------------------
|
||||
function setWsStatus(ok) {
|
||||
var el = document.getElementById('ws-status');
|
||||
el.classList.toggle('reconnecting', !ok);
|
||||
el.textContent = ok ? 'live' : 'reconnecting…';
|
||||
}
|
||||
|
||||
function WS_Connect() {
|
||||
if (!("WebSocket" in window)) {
|
||||
console.log("WebSocket NOT supported by your Browser!");
|
||||
return;
|
||||
}
|
||||
var ws_hbd = new WebSocket("{{ heartbeat_ws_url }}");
|
||||
|
||||
ws_hbd.onopen = function () {
|
||||
setWsStatus(true);
|
||||
ws_hbd.send("heartbeat_web");
|
||||
};
|
||||
|
||||
ws_hbd.onerror = function (event) { console.log(event); };
|
||||
|
||||
ws_hbd.onmessage = function (event) {
|
||||
var state = JSON.parse(event.data);
|
||||
if (state.type === "host") {
|
||||
updateHost(state.data);
|
||||
} else if (state.type === "message") {
|
||||
var msg = state.data;
|
||||
noteAlert(msg.host, msg.level, msg.message, msg.ts);
|
||||
}
|
||||
};
|
||||
|
||||
ws_hbd.onclose = function () {
|
||||
setWsStatus(false);
|
||||
setTimeout(WS_Connect, 3000);
|
||||
};
|
||||
}
|
||||
|
||||
// ---- boot -----------------------------------------------------------------
|
||||
INITIAL_HOSTS.forEach(function (h) { hostData[h.name] = h; });
|
||||
Object.values(hostData).forEach(renderRow);
|
||||
sortRows();
|
||||
updateCounts();
|
||||
seedActiveAlerts();
|
||||
WS_Connect();
|
||||
|
||||
// refresh relative times once a minute
|
||||
setInterval(function () {
|
||||
Object.values(hostData).forEach(renderRow);
|
||||
}, 60000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
{% include 'head.html' %}
|
||||
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||||
<script src="/static/hbd-ui.js"></script>
|
||||
|
||||
<style>
|
||||
html, body { height: auto; overflow: visible; }
|
||||
body { background: var(--st-paper); padding: 60px 0 0; }
|
||||
|
||||
.log-main { max-width: 1100px; margin: 0 auto; padding: 0 20px 60px; }
|
||||
|
||||
.st-toolbar input, .st-toolbar select {
|
||||
font: 12px var(--st-mono); color: var(--st-ink);
|
||||
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||||
padding: 5px 12px;
|
||||
}
|
||||
.live-dot { display: inline-flex; align-items: center; gap: 6px; font-size: 12px; font-weight: 600; color: var(--st-ok); }
|
||||
.live-dot::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: var(--st-ok); }
|
||||
.live-dot.reconnecting { color: var(--st-crit); }
|
||||
.live-dot.reconnecting::before { background: var(--st-crit); }
|
||||
|
||||
.logrow {
|
||||
display: grid; grid-template-columns: 130px 74px 150px minmax(0,1fr);
|
||||
gap: 4px 14px; align-items: baseline;
|
||||
padding: 4px 14px; border-bottom: 1px solid var(--st-line-soft);
|
||||
font-family: var(--st-mono); font-size: 11.5px;
|
||||
}
|
||||
.logrow:last-child { border-bottom: none; }
|
||||
.logrow .t { color: var(--st-faint); font-variant-numeric: tabular-nums; white-space: nowrap; }
|
||||
.logrow .h { color: var(--st-ink); font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.logrow .m { color: var(--st-muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
|
||||
.empty { padding: 24px 14px; text-align: center; color: var(--st-faint); font-size: 12.5px; }
|
||||
.loadmore {
|
||||
display: block; margin: 14px auto 0; cursor: pointer;
|
||||
font-family: var(--st-mono); font-size: 12px; color: var(--st-muted);
|
||||
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||||
padding: 6px 16px;
|
||||
}
|
||||
.loadmore:hover { color: var(--st-accent); border-color: var(--st-accent); }
|
||||
.loadmore:disabled { opacity: .5; cursor: default; }
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.logrow { grid-template-columns: 66px minmax(0,1fr); }
|
||||
.logrow .lvlc, .logrow .h { display: none; }
|
||||
}
|
||||
</style>
|
||||
|
||||
<body>
|
||||
{% include 'nav.html' %}
|
||||
|
||||
<div class="st-toolbar">
|
||||
<span class="brand">hbd<span class="tld">·log</span></span>
|
||||
<span class="hintline">connectivity and alert events — full journal history</span>
|
||||
<span class="spacer"></span>
|
||||
<input type="text" id="filter-host" placeholder="host">
|
||||
<select id="filter-level">
|
||||
<option value="">level</option>
|
||||
<option value="info">INFO</option>
|
||||
<option value="warning">WARNING</option>
|
||||
<option value="critical">CRITICAL</option>
|
||||
<option value="recover">RECOVER</option>
|
||||
<option value="unknown">UNKNOWN</option>
|
||||
</select>
|
||||
<input type="text" id="filter-msg" placeholder="message">
|
||||
<span class="live-dot reconnecting" id="ws-status">connecting…</span>
|
||||
</div>
|
||||
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||||
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||||
</svg>
|
||||
|
||||
<div class="log-main">
|
||||
<section class="st">
|
||||
<div class="sec-head">
|
||||
<h2>log<span class="colon">:</span></h2>
|
||||
<span class="count" id="log-count"></span>
|
||||
</div>
|
||||
<div class="list" id="messages">
|
||||
<div class="empty">Loading events…</div>
|
||||
</div>
|
||||
<button class="loadmore" id="load-older" style="display:none">load older</button>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var PAGE_SIZE = 200;
|
||||
var oldestTs = null; // ts of the oldest loaded row (pagination cursor)
|
||||
var seq = 0; // guards against out-of-order fetch responses
|
||||
var newestTs = null; // ts of the newest loaded row (dedup guard for the WS tail)
|
||||
|
||||
function escHtml(s) {
|
||||
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');
|
||||
}
|
||||
|
||||
function fmtClock(ts) {
|
||||
var d = new Date(ts * 1000);
|
||||
function p(n) { return n < 10 ? '0' + n : '' + n; }
|
||||
var t = p(d.getHours()) + ':' + p(d.getMinutes()) + ':' + p(d.getSeconds());
|
||||
return (d.toDateString() === new Date().toDateString())
|
||||
? t
|
||||
: d.getFullYear() + '-' + p(d.getMonth() + 1) + '-' + p(d.getDate()) + ' ' + t;
|
||||
}
|
||||
|
||||
function logRowHtml(msg) {
|
||||
var lvl = (msg.level || 'INFO');
|
||||
var lc = lvl.toLowerCase();
|
||||
var chipCls = lc === 'critical' ? 'crit' : (lc === 'warning' ? 'warn' : (lc === 'recover' ? 'ok' : ''));
|
||||
var short = lc === 'critical' ? 'CRIT' : (lc === 'warning' ? 'WARN' : escHtml(lvl));
|
||||
return '<div class="logrow">'
|
||||
+ '<span class="t">' + fmtClock(msg.ts) + '</span>'
|
||||
+ '<span class="lvlc"><span class="chip ' + chipCls + '">' + short + '</span></span>'
|
||||
+ '<span class="h">' + escHtml(msg.host || '') + (msg.service ? ' · ' + escHtml(msg.service) : '') + '</span>'
|
||||
+ '<span class="m">' + escHtml(msg.message || '') + '</span>'
|
||||
+ '</div>';
|
||||
}
|
||||
|
||||
function currentFilters() {
|
||||
return {
|
||||
host: document.getElementById('filter-host').value.trim(),
|
||||
level: document.getElementById('filter-level').value,
|
||||
q: document.getElementById('filter-msg').value.trim()
|
||||
};
|
||||
}
|
||||
|
||||
function matchesFilters(msg) {
|
||||
var f = currentFilters();
|
||||
if (f.host && !(msg.host || '').toLowerCase().includes(f.host.toLowerCase())) return false;
|
||||
if (f.level && (msg.level || '').toLowerCase() !== f.level) return false;
|
||||
if (f.q && !(msg.message || '').toLowerCase().includes(f.q.toLowerCase())) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function updateCount() {
|
||||
var n = document.querySelectorAll('#messages .logrow').length;
|
||||
document.getElementById('log-count').textContent = n ? n + ' events loaded' : '';
|
||||
}
|
||||
|
||||
async function fetchLog(reset) {
|
||||
var mySeq = ++seq;
|
||||
var box = document.getElementById('messages');
|
||||
var btn = document.getElementById('load-older');
|
||||
btn.disabled = true;
|
||||
var f = currentFilters();
|
||||
var params = new URLSearchParams();
|
||||
params.set('limit', String(PAGE_SIZE));
|
||||
if (f.host) params.set('host', f.host);
|
||||
if (f.level) params.set('level', f.level);
|
||||
if (f.q) params.set('q', f.q);
|
||||
if (!reset && oldestTs != null) params.set('before', String(oldestTs));
|
||||
var data;
|
||||
try {
|
||||
var r = await fetch('/api/0/log?' + params.toString());
|
||||
if (!r.ok) throw new Error('HTTP ' + r.status);
|
||||
data = await r.json();
|
||||
} catch (e) {
|
||||
if (mySeq === seq && reset) {
|
||||
box.innerHTML = '<div class="empty">Failed to load events: ' + escHtml(e.message) + '</div>';
|
||||
}
|
||||
btn.disabled = false;
|
||||
return;
|
||||
}
|
||||
if (mySeq !== seq) return; // a newer request superseded this one
|
||||
if (reset) { box.innerHTML = ''; oldestTs = null; }
|
||||
var html = '';
|
||||
data.events.forEach(function (ev) {
|
||||
html += logRowHtml(ev);
|
||||
if (ev.ts != null && (oldestTs == null || ev.ts < oldestTs)) oldestTs = ev.ts;
|
||||
if (ev.ts != null && (newestTs == null || ev.ts > newestTs)) newestTs = ev.ts;
|
||||
});
|
||||
box.insertAdjacentHTML('beforeend', html);
|
||||
if (!box.children.length) box.innerHTML = '<div class="empty">No events found.</div>';
|
||||
btn.style.display = data.more ? '' : 'none';
|
||||
btn.disabled = false;
|
||||
updateCount();
|
||||
}
|
||||
|
||||
function onFilterChange() { fetchLog(true); }
|
||||
|
||||
function debounce(fn, ms) {
|
||||
var t;
|
||||
return function () { clearTimeout(t); t = setTimeout(fn, ms); };
|
||||
}
|
||||
document.getElementById('filter-host').addEventListener('input', debounce(onFilterChange, 300));
|
||||
document.getElementById('filter-msg').addEventListener('input', debounce(onFilterChange, 300));
|
||||
document.getElementById('filter-level').addEventListener('change', onFilterChange);
|
||||
document.getElementById('load-older').addEventListener('click', function () { fetchLog(false); });
|
||||
|
||||
// ---- live tail over the shared websocket --------------------------------
|
||||
function setWsStatus(ok) {
|
||||
var el = document.getElementById('ws-status');
|
||||
el.classList.toggle('reconnecting', !ok);
|
||||
el.textContent = ok ? 'live' : 'reconnecting…';
|
||||
}
|
||||
|
||||
function WS_Connect() {
|
||||
if (!("WebSocket" in window)) return;
|
||||
var ws_hbd = new WebSocket("{{ heartbeat_ws_url }}");
|
||||
ws_hbd.onopen = function () {
|
||||
setWsStatus(true);
|
||||
ws_hbd.send("heartbeat_web");
|
||||
};
|
||||
ws_hbd.onmessage = function (event) {
|
||||
var state = JSON.parse(event.data);
|
||||
// history replays are covered by the API seed; only tail new events
|
||||
if (state.type !== "message" || state.history) return;
|
||||
var msg = state.data;
|
||||
if (!matchesFilters(msg)) return;
|
||||
if (msg.ts != null && newestTs != null && msg.ts <= newestTs) return; // already loaded via the API
|
||||
if (msg.ts != null && (newestTs == null || msg.ts > newestTs)) newestTs = msg.ts;
|
||||
var box = document.getElementById('messages');
|
||||
var placeholder = box.querySelector('.empty');
|
||||
if (placeholder) placeholder.remove();
|
||||
box.insertAdjacentHTML('afterbegin', logRowHtml(msg));
|
||||
updateCount();
|
||||
};
|
||||
ws_hbd.onclose = function () {
|
||||
setWsStatus(false);
|
||||
setTimeout(WS_Connect, 3000);
|
||||
};
|
||||
}
|
||||
|
||||
// ---- boot ----------------------------------------------------------------
|
||||
fetchLog(true);
|
||||
WS_Connect();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,2 +0,0 @@
|
||||
<!-- <label for="drawer-toggle" id="drawer-toggle-label"></label>
|
||||
s<header>{{ header }}</header> -->
|
||||
@@ -4,11 +4,19 @@
|
||||
</button>
|
||||
<div class="nav-links" id="nav-links">
|
||||
<a href="/live"{% if active_page == "live" %} class="active"{% endif %}>Live Dashboard</a>
|
||||
<a href="/plugins"{% if active_page == "plugins" %} class="active"{% endif %}>Plugin Metrics</a>
|
||||
<a href="/plugins"{% if active_page == "plugins" %} class="active"{% endif %}>Host Overview</a>
|
||||
<a href="/alerts"{% if active_page == "alerts" %} class="active"{% endif %}>Alerts</a>
|
||||
{% if current_user and current_user.admin %}
|
||||
<a href="/log"{% if active_page == "log" %} class="active"{% endif %}>Log</a>
|
||||
{% if current_user %}
|
||||
<a href="/settings"{% if active_page == "settings" %} class="active"{% endif %}>Settings</a>
|
||||
{% endif %}
|
||||
<a href="/about"{% if active_page == "about" %} class="active"{% endif %}>About</a>
|
||||
</div>
|
||||
{% if current_user %}
|
||||
<button id="nav-publish-btn" class="nav-publish-btn" onclick="navPublishConfig()" style="display:none" title="Publish pending config changes to .hb.yaml">⚠ Publish Config</button>
|
||||
{% endif %}
|
||||
<div class="nav-pie" title="Host alert status">
|
||||
<canvas id="alert-pie" width="44" height="44"></canvas>
|
||||
</div>
|
||||
<div class="nav-clock" title="Click for full-screen clock">
|
||||
<canvas id="swiss-clock" width="44" height="44"></canvas>
|
||||
@@ -41,4 +49,87 @@
|
||||
});
|
||||
}
|
||||
})();
|
||||
|
||||
function drawAlertPie(critical, warning, ok) {
|
||||
var canvas = document.getElementById('alert-pie');
|
||||
if (!canvas) return;
|
||||
var ctx = canvas.getContext('2d');
|
||||
var SIZE = canvas.width;
|
||||
var R = SIZE / 2;
|
||||
ctx.clearRect(0, 0, SIZE, SIZE);
|
||||
var total = critical + warning + ok;
|
||||
if (total === 0) {
|
||||
ctx.beginPath();
|
||||
ctx.arc(R, R, R - 1, 0, Math.PI * 2);
|
||||
ctx.fillStyle = '#ccc';
|
||||
ctx.fill();
|
||||
return;
|
||||
}
|
||||
var slices = [
|
||||
{ value: critical, color: '#e53935' },
|
||||
{ value: warning, color: '#ffb300' },
|
||||
{ value: ok, color: '#43a047' }
|
||||
];
|
||||
var start = -Math.PI / 2;
|
||||
slices.forEach(function(s) {
|
||||
if (s.value === 0) return;
|
||||
var sweep = (s.value / total) * Math.PI * 2;
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(R, R);
|
||||
ctx.arc(R, R, R - 1, start, start + sweep);
|
||||
ctx.closePath();
|
||||
ctx.fillStyle = s.color;
|
||||
ctx.fill();
|
||||
start += sweep;
|
||||
});
|
||||
}
|
||||
|
||||
function updateAlertPie() {
|
||||
fetch('/api/0/alert_summary').then(function(r) {
|
||||
if (!r.ok) return;
|
||||
return r.json();
|
||||
}).then(function(d) {
|
||||
if (d) drawAlertPie(d.critical || 0, d.warning || 0, d.ok || 0);
|
||||
}).catch(function() {});
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
updateAlertPie();
|
||||
setInterval(updateAlertPie, 30000);
|
||||
navCheckPendingConfig();
|
||||
window.addEventListener('storage', navCheckPendingConfig);
|
||||
});
|
||||
|
||||
function navCheckPendingConfig() {
|
||||
var btn = document.getElementById('nav-publish-btn');
|
||||
if (!btn) return;
|
||||
btn.style.display = localStorage.getItem('hbd_pending_config') ? '' : 'none';
|
||||
}
|
||||
|
||||
async function navPublishConfig() {
|
||||
var btn = document.getElementById('nav-publish-btn');
|
||||
var pending = localStorage.getItem('hbd_pending_config');
|
||||
if (!pending) return;
|
||||
var staged;
|
||||
try { staged = JSON.parse(pending); } catch(e) { return; }
|
||||
if (btn) { btn.disabled = true; btn.textContent = 'Saving…'; }
|
||||
try {
|
||||
var resp = await fetch('/api/0/config', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: pending
|
||||
});
|
||||
if (resp.ok) {
|
||||
localStorage.removeItem('hbd_pending_config');
|
||||
window.location.reload();
|
||||
} else {
|
||||
var err = await resp.json().catch(function() { return {}; });
|
||||
alert('Error: ' + (err.error || resp.statusText));
|
||||
if (btn) { btn.disabled = false; btn.textContent = '⚠ Publish Config'; }
|
||||
}
|
||||
} catch(e) {
|
||||
alert('Network error: ' + e.message);
|
||||
if (btn) { btn.disabled = false; btn.textContent = '⚠ Publish Config'; }
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
+1354
-909
File diff suppressed because it is too large
Load Diff
@@ -96,7 +96,7 @@
|
||||
border-radius: 4px;
|
||||
background: #f44336;
|
||||
color: #fff;
|
||||
font-size: 0.85em;
|
||||
font-size: 1.00em;
|
||||
font-weight: 500;
|
||||
text-decoration: none;
|
||||
transition: background 0.15s;
|
||||
@@ -157,7 +157,7 @@
|
||||
gap: 6px;
|
||||
padding: 4px 12px;
|
||||
border-radius: 16px;
|
||||
font-size: 0.85em;
|
||||
font-size: 1.00em;
|
||||
font-weight: 500;
|
||||
text-decoration: none;
|
||||
}
|
||||
@@ -204,6 +204,119 @@
|
||||
}
|
||||
|
||||
.channel-name { color: #333; }
|
||||
|
||||
.edit-section { margin-top: 20px; }
|
||||
.edit-section h4 { font-size: .88em; font-weight: 600; color: #333; margin: 0 0 10px; text-transform: uppercase; letter-spacing: .04em; border-bottom: 1px solid #eee; padding-bottom: 6px; }
|
||||
.edit-field { margin-bottom: 10px; }
|
||||
.edit-field label { display: block; font-size: .82em; color: #666; margin-bottom: 3px; }
|
||||
.edit-input { width: 100%; border: 1px solid #ccc; border-radius: 4px; padding: 5px 8px; font-size: .88em; box-sizing: border-box; }
|
||||
.edit-input:focus { border-color: #0066cc; outline: none; }
|
||||
.status-msg { font-size: .82em; margin-left: 8px; }
|
||||
.save-row { display: flex; align-items: center; margin-top: 8px; }
|
||||
.btn-save { background: #0066cc; color: #fff; border: none; border-radius: 4px; padding: 5px 14px; font-size: .85em; cursor: pointer; }
|
||||
.btn-save:hover { background: #0055aa; }
|
||||
/* ---- Channel chip picker ---- */
|
||||
.ch-picker { }
|
||||
.ch-picker-label { font-size: .8em; font-weight: 600; color: #888; text-transform: uppercase; letter-spacing: .04em; margin-bottom: 6px; }
|
||||
.ch-chips { display: flex; flex-wrap: wrap; gap: 6px; min-height: 32px; margin-bottom: 10px; }
|
||||
.ch-chip {
|
||||
display: inline-flex; align-items: center; gap: 5px;
|
||||
padding: 4px 10px; border-radius: 14px; font-size: .85em; font-weight: 500; cursor: pointer;
|
||||
border: none; font-family: inherit;
|
||||
}
|
||||
.ch-chip.selected { background: #e3f2fd; color: #1565c0; }
|
||||
.ch-chip.selected:hover { background: #bbdefb; }
|
||||
.ch-chip.available { background: #f1f3f4; color: #555; }
|
||||
.ch-chip.available:hover { background: #e8eaf6; color: #283593; }
|
||||
.ch-chip-x { font-size: .9em; line-height: 1; color: inherit; opacity: .7; }
|
||||
|
||||
/* ---- My Channels card list ---- */
|
||||
.my-ch-card {
|
||||
border: 1px solid #e8eaf6; border-radius: 6px; margin-bottom: 8px; overflow: hidden;
|
||||
}
|
||||
.my-ch-header {
|
||||
display: flex; align-items: center; gap: 8px; padding: 8px 12px;
|
||||
background: #f8f9ff; border-bottom: 1px solid #e8eaf6;
|
||||
}
|
||||
.my-ch-name { font-weight: 600; font-size: .9em; color: #222; }
|
||||
.my-ch-type { padding: 2px 7px; border-radius: 8px; font-size: .72em; font-weight: 600; background: #e8eaf6; color: #3949ab; }
|
||||
.my-ch-actions { margin-left: auto; display: flex; gap: 5px; }
|
||||
.btn-sm-edit { background: #888; color: #fff; border: none; border-radius: 4px; padding: 2px 8px; font-size: .78em; cursor: pointer; }
|
||||
.btn-sm-edit:hover { background: #666; }
|
||||
.btn-sm-del { background: transparent; color: #c62828; border: 1px solid #e0e0e0; border-radius: 4px; padding: 2px 7px; font-size: .78em; cursor: pointer; }
|
||||
.btn-sm-del:hover { background: #fce4ec; }
|
||||
|
||||
/* ---- Theme picker ---- */
|
||||
.theme-btns { display: flex; gap: 6px; }
|
||||
.theme-btn {
|
||||
padding: 5px 14px;
|
||||
border: 1px solid var(--border, #e0e0e0);
|
||||
border-radius: 4px;
|
||||
background: var(--surface-3, #f5f5f5);
|
||||
color: var(--text-sec, #666);
|
||||
cursor: pointer;
|
||||
font-size: .88em;
|
||||
font-family: inherit;
|
||||
}
|
||||
.theme-btn:hover { border-color: var(--link, #0066cc); color: var(--link, #0066cc); }
|
||||
.theme-btn.active { background: var(--link, #0066cc); color: #fff; border-color: var(--link, #0066cc); }
|
||||
|
||||
/* ── Dark mode ── */
|
||||
html[data-theme="dark"] h1 { color: var(--text); }
|
||||
html[data-theme="dark"] .subtitle { color: var(--text-sec); }
|
||||
html[data-theme="dark"] .profile-card { background: var(--surface); box-shadow: 0 1px 6px var(--shadow); }
|
||||
html[data-theme="dark"] .profile-name { color: var(--text); }
|
||||
html[data-theme="dark"] .profile-username { color: var(--text-sec); }
|
||||
html[data-theme="dark"] .badge-admin { background: #1a3255; color: #7aa8f0; }
|
||||
html[data-theme="dark"] .badge-user { background: var(--surface-3); color: var(--text-sec); }
|
||||
html[data-theme="dark"] .section { background: var(--surface); box-shadow: 0 1px 6px var(--shadow); }
|
||||
html[data-theme="dark"] .section h2 { color: var(--text); border-bottom-color: var(--border); }
|
||||
html[data-theme="dark"] .settings-row { border-bottom-color: var(--border-4); }
|
||||
html[data-theme="dark"] .settings-label { color: var(--text-sec); }
|
||||
html[data-theme="dark"] .settings-value { color: var(--text); }
|
||||
html[data-theme="dark"] .settings-empty { color: var(--text-dim); }
|
||||
html[data-theme="dark"] .edit-section h4 { color: var(--text); border-bottom-color: var(--border); }
|
||||
html[data-theme="dark"] .edit-field label { color: var(--text-sec); }
|
||||
html[data-theme="dark"] .edit-input { background: var(--input-bg); border-color: var(--input-border); color: var(--text); }
|
||||
html[data-theme="dark"] .channel-row { border-bottom-color: var(--border-4); }
|
||||
html[data-theme="dark"] .channel-name { color: var(--text); }
|
||||
html[data-theme="dark"] .ch-picker-label { color: var(--text-sec); }
|
||||
html[data-theme="dark"] .ch-chip.selected { background: #1a3255; color: #60a5fa; }
|
||||
html[data-theme="dark"] .ch-chip.available { background: var(--surface-3); color: var(--text-sec); }
|
||||
html[data-theme="dark"] .ch-chip.available:hover { background: var(--border); color: var(--link); }
|
||||
html[data-theme="dark"] .my-ch-card { border-color: var(--border); }
|
||||
html[data-theme="dark"] .my-ch-header { background: var(--surface-2); border-bottom-color: var(--border); }
|
||||
html[data-theme="dark"] .my-ch-name { color: var(--text); }
|
||||
html[data-theme="dark"] .host-chip.owner { background: #0d2e17; color: #66bb6a; }
|
||||
html[data-theme="dark"] .host-chip.manager { background: #0d1f40; color: #64b5f6; }
|
||||
html[data-theme="dark"] .host-chip.monitor { background: #1e0d30; color: #ba68c8; }
|
||||
html[data-theme="dark"] .no-hosts { color: var(--text-dim); }
|
||||
html[data-theme="dark"] .ch-modal-box { background: var(--surface); color: var(--text); }
|
||||
html[data-theme="dark"] .ch-modal-box h3 { color: var(--text); }
|
||||
html[data-theme="dark"] .ch-form-row label { color: var(--text-sec); }
|
||||
html[data-theme="dark"] .ch-form-divider { color: var(--text-muted); border-top-color: var(--border); }
|
||||
|
||||
/* ---- Channel modal (for My Channels CRUD) ---- */
|
||||
.ch-modal-overlay {
|
||||
position: fixed; inset: 0; background: rgba(0,0,0,.4);
|
||||
display: flex; align-items: center; justify-content: center; z-index: 1001;
|
||||
}
|
||||
.ch-modal-box {
|
||||
background: #fff; border-radius: 8px; padding: 24px;
|
||||
min-width: 360px; max-width: 520px; width: 95%;
|
||||
box-shadow: 0 8px 32px rgba(0,0,0,.2);
|
||||
}
|
||||
.ch-modal-box h3 { margin: 0 0 16px; font-size: 1em; }
|
||||
.ch-form-row { margin-bottom: 12px; }
|
||||
.ch-form-row label { display: block; font-size: .83em; font-weight: 600; color: #555; margin-bottom: 3px; }
|
||||
.ch-form-row input[type=text], .ch-form-row input[type=password], .ch-form-row select {
|
||||
width: 100%; border: 1px solid #ccc; border-radius: 4px; padding: 5px 8px;
|
||||
font-size: .88em; box-sizing: border-box; font-family: inherit;
|
||||
}
|
||||
.ch-form-row input:focus, .ch-form-row select:focus { border-color: #0066cc; outline: none; }
|
||||
.ch-form-divider { font-size: .78em; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; color: #888; margin: 14px 0 8px; border-top: 1px solid #eee; padding-top: 10px; }
|
||||
.ch-modal-footer { display: flex; justify-content: flex-end; gap: 8px; margin-top: 18px; }
|
||||
.ch-modal-status { font-size: .83em; margin-top: 8px; }
|
||||
</style>
|
||||
|
||||
<body>
|
||||
@@ -266,19 +379,158 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Notification channels -->
|
||||
{% if current_user %}
|
||||
<!-- ---- Editable identity ---- -->
|
||||
<div class="section edit-section">
|
||||
<h4>Identity</h4>
|
||||
<div class="edit-field">
|
||||
<label for="profile-fullname">Display name</label>
|
||||
<input id="profile-fullname" class="edit-input" type="text" value="{{ current_user.full_name | e }}" placeholder="Full name">
|
||||
</div>
|
||||
<div class="edit-field">
|
||||
<label for="profile-avatar">Avatar URL or path</label>
|
||||
<input id="profile-avatar" class="edit-input" type="text" value="{{ current_user.avatar | e }}" placeholder="/path/to/avatar.png or https://…">
|
||||
</div>
|
||||
<div class="save-row">
|
||||
<button class="btn-save" onclick="saveIdentity()">Save</button>
|
||||
<span id="identity-status" class="status-msg"></span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ---- Change password ---- -->
|
||||
<div class="section edit-section">
|
||||
<h4>Change password</h4>
|
||||
<div class="edit-field">
|
||||
<label for="profile-current-pw">Current password</label>
|
||||
<input id="profile-current-pw" class="edit-input" type="password" autocomplete="current-password">
|
||||
</div>
|
||||
<div class="edit-field">
|
||||
<label for="profile-new-pw">New password</label>
|
||||
<input id="profile-new-pw" class="edit-input" type="password" autocomplete="new-password">
|
||||
</div>
|
||||
<div class="save-row">
|
||||
<button class="btn-save" onclick="changePassword()">Change password</button>
|
||||
<span id="password-status" class="status-msg"></span>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Notification channels — chip picker -->
|
||||
<div class="section">
|
||||
<h2>Notification Channels</h2>
|
||||
{% if notification_channels %}
|
||||
{% for ch in notification_channels %}
|
||||
<div class="channel-row">
|
||||
<span class="channel-type">{{ ch.type }}</span>
|
||||
<span class="channel-name">{{ ch.name }}</span>
|
||||
{% if current_user %}
|
||||
<p style="font-size:.82em;color:#888;margin:0 0 12px">Click a channel to add or remove it from your alert list.</p>
|
||||
{% if all_channels %}
|
||||
<div class="ch-picker">
|
||||
<div class="ch-picker-label">Selected</div>
|
||||
<div id="selected-chips" class="ch-chips">
|
||||
{% for ch in all_channels %}
|
||||
{% if ch.name in (current_user.notification_channels or []) %}
|
||||
<button class="ch-chip selected" data-ch="{{ ch.name | e }}" onclick="toggleChip(this)">
|
||||
{{ ch.name | e }} <span class="ch-chip-x">×</span>
|
||||
</button>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% set selected_set = current_user.notification_channels or [] %}
|
||||
{% set has_selected = selected_set | length > 0 %}
|
||||
{% if not has_selected %}
|
||||
<span style="font-size:.83em;color:#bbb;font-style:italic;align-self:center">None selected</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="ch-picker-label">Available</div>
|
||||
<div id="available-chips" class="ch-chips">
|
||||
{% for ch in all_channels %}
|
||||
{% if ch.name not in (current_user.notification_channels or []) %}
|
||||
<button class="ch-chip available" data-ch="{{ ch.name | e }}" onclick="toggleChip(this)">
|
||||
+ {{ ch.name | e }}
|
||||
</button>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
<span class="no-hosts">No personal notification channels configured.</span>
|
||||
<p style="font-size:.83em;color:#bbb;font-style:italic">No notification channels available. You can create your own below.</p>
|
||||
{% endif %}
|
||||
<div class="save-row">
|
||||
<button class="btn-save" onclick="saveChannels()">Save channels</button>
|
||||
<span id="channels-status" class="status-msg"></span>
|
||||
</div>
|
||||
{% else %}
|
||||
<span class="no-hosts">Log in to manage notification channels.</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- My Channels — create/edit/delete own channels -->
|
||||
{% if current_user %}
|
||||
<div class="section">
|
||||
<h2>My Channels</h2>
|
||||
<p style="font-size:.82em;color:#888;margin:0 0 12px">Channels you own are private to you. Global channels are managed by administrators.</p>
|
||||
<div id="my-channels-list">
|
||||
{% set my_channels = all_channels | selectattr('owner', 'equalto', current_user.username) | list %}
|
||||
{% for ch in my_channels %}
|
||||
<div class="my-ch-card" id="mychcard-{{ ch.name | e }}">
|
||||
<div class="my-ch-header">
|
||||
<span class="my-ch-name">{{ ch.name | e }}</span>
|
||||
<span class="my-ch-type">{{ ch.type | e }}</span>
|
||||
<span class="my-ch-actions">
|
||||
<button class="btn-sm-edit" onclick="openMyChModal('{{ ch.name | e }}')">Edit</button>
|
||||
<button class="btn-sm-del" onclick="deleteMyChannel('{{ ch.name | e }}')">✕</button>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% if not my_channels %}
|
||||
<p id="my-channels-empty" style="font-size:.83em;color:#bbb;font-style:italic">No channels yet.</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="save-row" style="margin-top:8px">
|
||||
<button class="btn-save" onclick="openMyChModal()">+ New channel</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- My Channels modal -->
|
||||
<div id="my-ch-modal" class="ch-modal-overlay" style="display:none" onclick="if(event.target===this)closeMyChModal()">
|
||||
<div class="ch-modal-box">
|
||||
<h3 id="my-ch-modal-title">New Channel</h3>
|
||||
<div class="ch-form-row">
|
||||
<label>Channel name</label>
|
||||
<input type="text" id="my-ch-name" placeholder="e.g. my_pushover" autocomplete="off">
|
||||
</div>
|
||||
<div class="ch-form-row">
|
||||
<label>Type</label>
|
||||
<select id="my-ch-type" onchange="onMyChTypeChange()">
|
||||
<option value="">— select —</option>
|
||||
</select>
|
||||
</div>
|
||||
<div id="my-ch-type-fields"></div>
|
||||
<div class="ch-form-divider">Options</div>
|
||||
<div class="ch-form-row">
|
||||
<label>Minimum alert level</label>
|
||||
<select id="my-ch-min-level">
|
||||
<option value="WARNING">WARNING (and above)</option>
|
||||
<option value="CRITICAL">CRITICAL only</option>
|
||||
</select>
|
||||
</div>
|
||||
<div id="my-ch-modal-status" class="ch-modal-status"></div>
|
||||
<div class="ch-modal-footer">
|
||||
<button class="btn-save" style="background:#888" onclick="closeMyChModal()">Cancel</button>
|
||||
<button class="btn-save" onclick="saveMyChannel()">Save</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- Appearance -->
|
||||
<div class="section">
|
||||
<h2>Appearance</h2>
|
||||
<div class="settings-row">
|
||||
<span class="settings-label">Theme</span>
|
||||
<div class="theme-btns">
|
||||
<button class="theme-btn" data-theme-val="auto" onclick="setTheme('auto')">Auto</button>
|
||||
<button class="theme-btn" data-theme-val="light" onclick="setTheme('light')">Light</button>
|
||||
<button class="theme-btn" data-theme-val="dark" onclick="setTheme('dark')">Dark</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Host access -->
|
||||
@@ -326,5 +578,255 @@
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<script>
|
||||
// ---- Theme ----
|
||||
function applyTheme(pref) {
|
||||
var dark = pref === 'dark' ||
|
||||
(pref === 'auto' && window.matchMedia('(prefers-color-scheme: dark)').matches);
|
||||
if (dark) { document.documentElement.setAttribute('data-theme', 'dark'); }
|
||||
else { document.documentElement.removeAttribute('data-theme'); }
|
||||
}
|
||||
function setTheme(pref) {
|
||||
try { localStorage.setItem('hbd_theme', pref); } catch(e) {}
|
||||
applyTheme(pref);
|
||||
document.querySelectorAll('.theme-btn').forEach(function(b) {
|
||||
b.classList.toggle('active', b.dataset.themeVal === pref);
|
||||
});
|
||||
}
|
||||
(function() {
|
||||
var pref = 'auto';
|
||||
try { pref = localStorage.getItem('hbd_theme') || 'auto'; } catch(e) {}
|
||||
document.querySelectorAll('.theme-btn').forEach(function(b) {
|
||||
b.classList.toggle('active', b.dataset.themeVal === pref);
|
||||
});
|
||||
})();
|
||||
|
||||
// ---- Identity ----
|
||||
async function saveIdentity() {
|
||||
const full_name = document.getElementById('profile-fullname').value;
|
||||
const avatar = document.getElementById('profile-avatar').value;
|
||||
const resp = await fetch('/api/0/users/me', {
|
||||
method: 'PUT',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({full_name, avatar}),
|
||||
});
|
||||
if (resp.ok) {
|
||||
showStatus('identity-status', 'Saved', '#2e7d32');
|
||||
} else {
|
||||
const err = await resp.json().catch(() => ({}));
|
||||
showStatus('identity-status', err.error || 'Error saving', '#c62828');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Password ----
|
||||
async function changePassword() {
|
||||
const current = document.getElementById('profile-current-pw').value;
|
||||
const newpw = document.getElementById('profile-new-pw').value;
|
||||
if (!current || !newpw) {
|
||||
showStatus('password-status', 'Both fields are required', '#c62828');
|
||||
return;
|
||||
}
|
||||
const resp = await fetch('/api/0/users/me', {
|
||||
method: 'PUT',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({password: {current, new: newpw}}),
|
||||
});
|
||||
if (resp.ok) {
|
||||
document.getElementById('profile-current-pw').value = '';
|
||||
document.getElementById('profile-new-pw').value = '';
|
||||
showStatus('password-status', 'Password changed', '#2e7d32');
|
||||
} else {
|
||||
const err = await resp.json().catch(() => ({}));
|
||||
showStatus('password-status', err.error || 'Error', '#c62828');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Channel chip picker ----
|
||||
function toggleChip(btn) {
|
||||
const name = btn.dataset.ch;
|
||||
const isSelected = btn.classList.contains('selected');
|
||||
if (isSelected) {
|
||||
// Move to available
|
||||
btn.classList.remove('selected');
|
||||
btn.classList.add('available');
|
||||
btn.innerHTML = '+ ' + escHtml(name);
|
||||
btn.onclick = function() { toggleChip(this); };
|
||||
document.getElementById('available-chips').appendChild(btn);
|
||||
// Remove "None selected" placeholder if it exists
|
||||
} else {
|
||||
// Move to selected
|
||||
btn.classList.remove('available');
|
||||
btn.classList.add('selected');
|
||||
btn.innerHTML = escHtml(name) + ' <span class="ch-chip-x">×</span>';
|
||||
btn.onclick = function() { toggleChip(this); };
|
||||
document.getElementById('selected-chips').appendChild(btn);
|
||||
}
|
||||
// Update placeholder visibility
|
||||
const sel = document.getElementById('selected-chips');
|
||||
const placeholder = sel.querySelector('span[style]');
|
||||
const hasChips = sel.querySelectorAll('.ch-chip.selected').length > 0;
|
||||
if (placeholder) placeholder.style.display = hasChips ? 'none' : '';
|
||||
}
|
||||
|
||||
async function saveChannels() {
|
||||
const notification_channels = [
|
||||
...document.querySelectorAll('#selected-chips .ch-chip.selected')
|
||||
].map(b => b.dataset.ch);
|
||||
const resp = await fetch('/api/0/users/me', {
|
||||
method: 'PUT',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({notification_channels}),
|
||||
});
|
||||
if (resp.ok) {
|
||||
showStatus('channels-status', 'Saved', '#2e7d32');
|
||||
} else {
|
||||
const err = await resp.json().catch(() => ({}));
|
||||
showStatus('channels-status', err.error || 'Error saving', '#c62828');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- My Channels CRUD ----
|
||||
let _myChSchemas = {};
|
||||
let _myChEditName = null;
|
||||
|
||||
async function _loadMyChSchemas() {
|
||||
try {
|
||||
const r = await fetch('/api/0/notification_channel_types');
|
||||
_myChSchemas = await r.json();
|
||||
const sel = document.getElementById('my-ch-type');
|
||||
if (!sel) return;
|
||||
Object.entries(_myChSchemas).forEach(([k, v]) => {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = k; opt.textContent = v.label;
|
||||
sel.appendChild(opt);
|
||||
});
|
||||
} catch(e) { console.warn('Could not load channel schemas', e); }
|
||||
}
|
||||
|
||||
function onMyChTypeChange() {
|
||||
const type = document.getElementById('my-ch-type').value;
|
||||
const container = document.getElementById('my-ch-type-fields');
|
||||
container.innerHTML = '';
|
||||
if (!type || !_myChSchemas[type]) return;
|
||||
const divider = document.createElement('div');
|
||||
divider.className = 'ch-form-divider';
|
||||
divider.textContent = _myChSchemas[type].label + ' settings';
|
||||
container.appendChild(divider);
|
||||
(_myChSchemas[type].fields || []).forEach(sf => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'ch-form-row';
|
||||
const lbl = document.createElement('label');
|
||||
lbl.textContent = sf.label + (sf.required ? ' *' : '');
|
||||
const inp = document.createElement('input');
|
||||
inp.type = sf.type === 'secret' ? 'password' : 'text';
|
||||
inp.id = 'mychf-' + sf.key;
|
||||
inp.placeholder = sf.required ? '(required)' : '(optional)';
|
||||
inp.autocomplete = 'off';
|
||||
row.appendChild(lbl);
|
||||
row.appendChild(inp);
|
||||
container.appendChild(row);
|
||||
});
|
||||
}
|
||||
|
||||
async function openMyChModal(name) {
|
||||
_myChEditName = name || null;
|
||||
document.getElementById('my-ch-modal-status').textContent = '';
|
||||
document.getElementById('my-ch-modal-title').textContent = name ? 'Edit Channel' : 'New Channel';
|
||||
document.getElementById('my-ch-name').value = name || '';
|
||||
document.getElementById('my-ch-name').disabled = !!name;
|
||||
document.getElementById('my-ch-type').value = '';
|
||||
document.getElementById('my-ch-type-fields').innerHTML = '';
|
||||
document.getElementById('my-ch-min-level').value = 'WARNING';
|
||||
|
||||
if (name) {
|
||||
try {
|
||||
const r = await fetch('/api/0/notification_channels');
|
||||
const channels = await r.json();
|
||||
const ch = channels.find(c => c.name === name);
|
||||
if (ch) {
|
||||
document.getElementById('my-ch-type').value = ch.type;
|
||||
onMyChTypeChange();
|
||||
document.getElementById('my-ch-min-level').value = ch.min_level || 'WARNING';
|
||||
(ch.fields || []).forEach(f => {
|
||||
const inp = document.getElementById('mychf-' + f.key);
|
||||
if (inp) inp.value = f.value || '';
|
||||
});
|
||||
}
|
||||
} catch(e) { console.warn('Failed to load channel', e); }
|
||||
}
|
||||
document.getElementById('my-ch-modal').style.display = 'flex';
|
||||
}
|
||||
|
||||
function closeMyChModal() {
|
||||
document.getElementById('my-ch-modal').style.display = 'none';
|
||||
}
|
||||
|
||||
async function saveMyChannel() {
|
||||
const name = document.getElementById('my-ch-name').value.trim();
|
||||
const type = document.getElementById('my-ch-type').value;
|
||||
const minLevel = document.getElementById('my-ch-min-level').value;
|
||||
const statusEl = document.getElementById('my-ch-modal-status');
|
||||
statusEl.textContent = '';
|
||||
|
||||
if (!name) { statusEl.textContent = 'Name is required.'; statusEl.style.color = '#c62828'; return; }
|
||||
if (!type) { statusEl.textContent = 'Please select a type.'; statusEl.style.color = '#c62828'; return; }
|
||||
|
||||
const body = { name, type, min_level: minLevel };
|
||||
if (_myChSchemas[type]) {
|
||||
(_myChSchemas[type].fields || []).forEach(sf => {
|
||||
const inp = document.getElementById('mychf-' + sf.key);
|
||||
if (inp) body[sf.key] = inp.value;
|
||||
});
|
||||
}
|
||||
|
||||
const isEdit = !!_myChEditName;
|
||||
const url = isEdit
|
||||
? '/api/0/notification_channels/' + encodeURIComponent(_myChEditName)
|
||||
: '/api/0/notification_channels';
|
||||
const method = isEdit ? 'PUT' : 'POST';
|
||||
try {
|
||||
const r = await fetch(url, { method, headers: {'Content-Type': 'application/json'}, body: JSON.stringify(body) });
|
||||
if (r.ok) {
|
||||
closeMyChModal();
|
||||
window.location.reload();
|
||||
} else {
|
||||
const err = await r.json().catch(() => ({}));
|
||||
statusEl.textContent = err.error || 'Error saving.';
|
||||
statusEl.style.color = '#c62828';
|
||||
}
|
||||
} catch(e) {
|
||||
statusEl.textContent = 'Network error: ' + e.message;
|
||||
statusEl.style.color = '#c62828';
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteMyChannel(name) {
|
||||
if (!confirm('Delete channel "' + name + '"?')) return;
|
||||
try {
|
||||
const r = await fetch('/api/0/notification_channels/' + encodeURIComponent(name), { method: 'DELETE' });
|
||||
if (r.ok) {
|
||||
window.location.reload();
|
||||
} else {
|
||||
const err = await r.json().catch(() => ({}));
|
||||
alert('Error: ' + (err.error || 'Could not delete.'));
|
||||
}
|
||||
} catch(e) { alert('Network error: ' + e.message); }
|
||||
}
|
||||
|
||||
// ---- Utilities ----
|
||||
function showStatus(id, msg, color) {
|
||||
const el = document.getElementById(id);
|
||||
if (!el) return;
|
||||
el.textContent = msg;
|
||||
el.style.color = color;
|
||||
setTimeout(() => { el.textContent = ''; }, 3000);
|
||||
}
|
||||
|
||||
function escHtml(s) {
|
||||
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', _loadMyChSchemas);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
+1271
-439
File diff suppressed because it is too large
Load Diff
+495
-130
@@ -9,10 +9,11 @@ This module provides a flexible threshold checking system that:
|
||||
- Supports multiple comparison operators
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from enum import Enum
|
||||
from typing import Dict, Any, Optional, Tuple, Callable
|
||||
from typing import Dict, List, Any, Optional, Tuple, Callable
|
||||
from . import notify as notify_mod
|
||||
from .config import THRESHOLD_DEFAULTS
|
||||
|
||||
@@ -29,12 +30,13 @@ class AlertLevel(Enum):
|
||||
|
||||
class ComparisonOperator(Enum):
|
||||
"""Supported comparison operators for threshold checks."""
|
||||
GT = ">" # Greater than
|
||||
GTE = ">=" # Greater than or equal
|
||||
LT = "<" # Less than
|
||||
LTE = "<=" # Less than or equal
|
||||
EQ = "==" # Equal to
|
||||
NEQ = "!=" # Not equal to
|
||||
GT = ">" # Greater than
|
||||
GTE = ">=" # Greater than or equal
|
||||
LT = "<" # Less than
|
||||
LTE = "<=" # Less than or equal
|
||||
EQ = "==" # Equal to
|
||||
NEQ = "!=" # Not equal to
|
||||
NAGIOS = "nagios" # Nagios exit-code semantics: 0=OK 1=WARNING 2=CRITICAL 3=UNKNOWN
|
||||
|
||||
|
||||
class AlertState:
|
||||
@@ -56,6 +58,7 @@ class AlertState:
|
||||
self.last_notification = None
|
||||
self.threshold_value = None # The threshold value that triggered alert
|
||||
self.operator = None # The comparison operator (>, <, >=, etc.)
|
||||
self.hysteresis: Optional[float] = None # Hysteresis fraction used for recovery
|
||||
self.formatted_message = None # Formatted display message for UI
|
||||
self.acknowledged = False # Whether alert has been acknowledged
|
||||
self.acknowledged_at = None # Timestamp when acknowledged
|
||||
@@ -151,6 +154,15 @@ class AlertState:
|
||||
if self.formatted_message is not None:
|
||||
result["formatted_message"] = self.formatted_message
|
||||
|
||||
# Compute and expose the recovery threshold so the UI can display it
|
||||
if (self.hysteresis and self.threshold_value is not None
|
||||
and self.operator is not None):
|
||||
ha = abs(self.threshold_value * self.hysteresis)
|
||||
if self.operator in ('>', '>='):
|
||||
result["recovery_threshold"] = round(self.threshold_value - ha, 4)
|
||||
elif self.operator in ('<', '<='):
|
||||
result["recovery_threshold"] = round(self.threshold_value + ha, 4)
|
||||
|
||||
return result
|
||||
|
||||
def __setstate__(self, state):
|
||||
@@ -158,6 +170,8 @@ class AlertState:
|
||||
self.__dict__.update(state)
|
||||
if not hasattr(self, 'consecutive_count'):
|
||||
self.consecutive_count = 0
|
||||
if not hasattr(self, 'hysteresis'):
|
||||
self.hysteresis = None
|
||||
|
||||
def acknowledge(self):
|
||||
"""Acknowledge this alert to stop reminder notifications."""
|
||||
@@ -181,6 +195,7 @@ class ThresholdConfig:
|
||||
hysteresis: float = 0.0,
|
||||
enabled: bool = True,
|
||||
count: int = 1,
|
||||
grace: Optional[float] = None,
|
||||
):
|
||||
"""
|
||||
Initialize threshold configuration.
|
||||
@@ -193,6 +208,7 @@ class ThresholdConfig:
|
||||
hysteresis: Hysteresis percentage to prevent flapping (0.0-1.0)
|
||||
enabled: Whether this threshold is enabled
|
||||
count: Number of consecutive exceedances required before alerting (default 1)
|
||||
grace: Per-metric grace period in seconds; overrides global grace when set
|
||||
"""
|
||||
self.metric_path = metric_path
|
||||
self.warning = warning
|
||||
@@ -201,6 +217,7 @@ class ThresholdConfig:
|
||||
self.hysteresis = hysteresis
|
||||
self.display = display
|
||||
self.count = max(1, int(count))
|
||||
self.grace = float(grace) if grace is not None else None
|
||||
|
||||
# Parse operator
|
||||
try:
|
||||
@@ -226,6 +243,16 @@ class ThresholdConfig:
|
||||
if not self.enabled:
|
||||
return AlertLevel.OK
|
||||
|
||||
# Nagios exit-code semantics: value IS the severity
|
||||
if self.operator == ComparisonOperator.NAGIOS:
|
||||
try:
|
||||
code = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return AlertLevel.UNKNOWN
|
||||
return {0: AlertLevel.OK, 1: AlertLevel.WARNING, 2: AlertLevel.CRITICAL}.get(
|
||||
code, AlertLevel.UNKNOWN
|
||||
)
|
||||
|
||||
try:
|
||||
# Convert value to float for comparison
|
||||
value = float(value)
|
||||
@@ -262,6 +289,10 @@ class ThresholdConfig:
|
||||
"""
|
||||
new_level = self.evaluate(value)
|
||||
|
||||
# Nagios exit codes are discrete integers — hysteresis doesn't apply
|
||||
if self.operator == ComparisonOperator.NAGIOS:
|
||||
return new_level
|
||||
|
||||
# If no hysteresis, return new level
|
||||
if self.hysteresis == 0.0:
|
||||
return new_level
|
||||
@@ -328,14 +359,17 @@ class ThresholdChecker:
|
||||
renotify_interval: Seconds between repeat notifications (default: 1 hour)
|
||||
journal: Optional MessageJournal instance for logging threshold events
|
||||
"""
|
||||
# Named threshold configurations: {config_name: {metric_path: ThresholdConfig}}
|
||||
# Named threshold configurations (pre-merged: defaults + overrides): {config_name: {metric_path: ThresholdConfig}}
|
||||
self.threshold_configs = {}
|
||||
|
||||
# Raw overrides only for each named config (no defaults baked in): {config_name: {metric_path: ThresholdConfig}}
|
||||
self.threshold_raw_configs: Dict[str, Dict[str, ThresholdConfig]] = {}
|
||||
|
||||
# Single threshold set for backward compatibility: {metric_path: ThresholdConfig}
|
||||
self.thresholds = {}
|
||||
|
||||
# Host to config name mapping: {host_name: config_name}
|
||||
self.host_config_mapping = {}
|
||||
# Host to ordered list of config names: {host_name: [config_name, ...]}
|
||||
self.host_config_mapping: Dict[str, List[str]] = {}
|
||||
|
||||
# Default config name to use when no mapping exists
|
||||
self.default_config = "default"
|
||||
@@ -372,6 +406,7 @@ class ThresholdChecker:
|
||||
|
||||
# Clear old configuration
|
||||
self.threshold_configs.clear()
|
||||
self.threshold_raw_configs.clear()
|
||||
self.thresholds.clear()
|
||||
self.host_config_mapping.clear()
|
||||
self.grace_seconds = float(config.get("grace", 2))
|
||||
@@ -391,10 +426,24 @@ class ThresholdChecker:
|
||||
Supports two formats:
|
||||
1. Legacy format with direct 'thresholds' section
|
||||
2. New format with 'threshold_configs' and 'host_threshold_mapping'
|
||||
|
||||
In all cases, THRESHOLD_DEFAULTS are seeded into threshold_configs["default"]
|
||||
so the Settings page always shows the built-in defaults.
|
||||
_parse_multi_config() overwrites this with the fully-merged effective defaults.
|
||||
"""
|
||||
# Always expose built-in defaults through threshold_configs["default"] so
|
||||
# the Settings page has something to display even in legacy/no-config mode.
|
||||
seed: Dict[str, ThresholdConfig] = {}
|
||||
for plugin_name, plugin_thresholds in THRESHOLD_DEFAULTS.get("thresholds", {}).items():
|
||||
if isinstance(plugin_thresholds, dict):
|
||||
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=seed)
|
||||
if seed:
|
||||
self.threshold_configs["default"] = seed
|
||||
self.threshold_raw_configs["default"] = {}
|
||||
|
||||
# Check for new multi-config format
|
||||
if "threshold_configs" in config:
|
||||
self._parse_multi_config(config)
|
||||
self._parse_multi_config(config) # overwrites threshold_configs["default"]
|
||||
elif "thresholds" in config:
|
||||
# Legacy single threshold configuration
|
||||
self._parse_legacy_config(config)
|
||||
@@ -424,9 +473,10 @@ class ThresholdChecker:
|
||||
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=effective_defaults)
|
||||
|
||||
self.threshold_configs["default"] = dict(effective_defaults)
|
||||
self.threshold_raw_configs["default"] = {}
|
||||
logger.info("Registered 'default' threshold config with %d metrics", len(effective_defaults))
|
||||
|
||||
# Parse each named configuration, seeding it with effective_defaults first
|
||||
# Parse each named configuration
|
||||
for config_name, config_data in threshold_configs.items():
|
||||
if config_name == "default":
|
||||
continue # already handled above
|
||||
@@ -440,33 +490,61 @@ class ThresholdChecker:
|
||||
continue
|
||||
|
||||
logger.info("Parsing threshold configuration: %s", config_name)
|
||||
self.threshold_configs[config_name] = dict(effective_defaults)
|
||||
|
||||
# Raw overrides only (used for multi-config layering)
|
||||
raw_overrides: Dict[str, ThresholdConfig] = {}
|
||||
thresholds_config = config_data["thresholds"]
|
||||
for plugin_name, plugin_thresholds in thresholds_config.items():
|
||||
if not isinstance(plugin_thresholds, dict):
|
||||
continue
|
||||
plugin_enabled = plugin_thresholds.get('enabled', plugin_thresholds.get('enable', True))
|
||||
if not plugin_enabled:
|
||||
# raw_overrides is empty at this point so there's nothing to delete.
|
||||
# Instead, inject disabled stubs for every matching effective_default so
|
||||
# the merge step overwrites the inherited defaults.
|
||||
for key, tc in effective_defaults.items():
|
||||
if key.startswith(f"{plugin_name}."):
|
||||
raw_overrides[key] = ThresholdConfig(
|
||||
metric_path=key,
|
||||
warning=tc.warning,
|
||||
critical=tc.critical,
|
||||
operator=tc.operator.value,
|
||||
enabled=False,
|
||||
)
|
||||
logger.info(
|
||||
"Plugin-level disable in config '%s': disabled all thresholds for %s",
|
||||
config_name, plugin_name,
|
||||
)
|
||||
else:
|
||||
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=raw_overrides)
|
||||
self.threshold_raw_configs[config_name] = raw_overrides
|
||||
|
||||
self._parse_plugin_thresholds(
|
||||
plugin_name,
|
||||
plugin_thresholds,
|
||||
target_dict=self.threshold_configs[config_name]
|
||||
)
|
||||
# Pre-merged version (defaults + overrides) for single-config fast path
|
||||
self.threshold_configs[config_name] = dict(effective_defaults)
|
||||
self.threshold_configs[config_name].update(raw_overrides)
|
||||
|
||||
# Parse host to config mapping from two possible sources
|
||||
# 1. New format: hosts section with threshold_config attribute
|
||||
# Parse host → config list mapping from two possible sources
|
||||
|
||||
def _normalise(value) -> List[str]:
|
||||
"""Accept a string or list; always return a list."""
|
||||
if isinstance(value, list):
|
||||
return [str(v) for v in value]
|
||||
return [str(value)]
|
||||
|
||||
# 1. hosts section with threshold_config attribute (string or list)
|
||||
if "hosts" in config:
|
||||
hosts_config = config["hosts"]
|
||||
if isinstance(hosts_config, dict):
|
||||
for host_name, host_attrs in hosts_config.items():
|
||||
if isinstance(host_attrs, dict) and "threshold_config" in host_attrs:
|
||||
self.host_config_mapping[host_name] = host_attrs["threshold_config"]
|
||||
self.host_config_mapping[host_name] = _normalise(host_attrs["threshold_config"])
|
||||
|
||||
# 2. Legacy format: host_threshold_mapping section (for backward compatibility)
|
||||
# 2. Legacy host_threshold_mapping section (string values only)
|
||||
if "host_threshold_mapping" in config:
|
||||
legacy_mapping = config.get("host_threshold_mapping", {})
|
||||
if isinstance(legacy_mapping, dict):
|
||||
self.host_config_mapping.update(legacy_mapping)
|
||||
for host_name, value in legacy_mapping.items():
|
||||
self.host_config_mapping[host_name] = _normalise(value)
|
||||
|
||||
# Set default config (first one alphabetically or explicitly set)
|
||||
self.default_config = config.get("default_threshold_config", "default")
|
||||
@@ -516,14 +594,26 @@ class ThresholdChecker:
|
||||
self._parse_rtt_thresholds(thresholds, target_dict)
|
||||
return
|
||||
|
||||
# Plugin-level enabled: false (also accept 'enable' as a common typo) removes all
|
||||
# thresholds for this plugin — e.g. memory_monitor: {enabled: false}.
|
||||
plugin_enabled = thresholds.get('enabled', thresholds.get('enable', True))
|
||||
if not plugin_enabled:
|
||||
for key in [k for k in target_dict if k.startswith(f"{plugin_name}.")]:
|
||||
del target_dict[key]
|
||||
logger.info("Plugin-level disable: removed all thresholds for %s", plugin_name)
|
||||
return
|
||||
|
||||
for metric_name, threshold_config in thresholds.items():
|
||||
if not isinstance(threshold_config, dict):
|
||||
continue
|
||||
|
||||
# Handle nested metrics (e.g., partitions./.percent)
|
||||
# Handle nested metrics (e.g., partitions./.percent or pools.*.status)
|
||||
if metric_name == "partitions":
|
||||
self._parse_partition_thresholds(plugin_name, threshold_config, target_dict)
|
||||
continue
|
||||
if metric_name == "pools":
|
||||
self._parse_pool_thresholds(plugin_name, threshold_config, target_dict)
|
||||
continue
|
||||
|
||||
metric_path = f"{plugin_name}.{metric_name}"
|
||||
|
||||
@@ -531,11 +621,15 @@ class ThresholdChecker:
|
||||
warning = threshold_config.get("warning")
|
||||
critical = threshold_config.get("critical")
|
||||
operator = threshold_config.get("operator", ">")
|
||||
display = threshold_config.get("display", "(threshold: {op_symbol} {threshold_value})")
|
||||
hysteresis = threshold_config.get("hysteresis", 0.1) # 10% default
|
||||
# Nagios operator maps exit codes directly; no numeric thresholds needed
|
||||
is_nagios_op = (operator == "nagios")
|
||||
default_display = "{check_name}: {output}" if is_nagios_op else "(threshold: {op_symbol} {threshold_value})"
|
||||
display = threshold_config.get("display", default_display)
|
||||
hysteresis = threshold_config.get("hysteresis", 0.0 if is_nagios_op else 0.02)
|
||||
enabled = threshold_config.get("enabled", True)
|
||||
grace = threshold_config.get("grace", None)
|
||||
|
||||
if warning is None and critical is None:
|
||||
if warning is None and critical is None and not is_nagios_op:
|
||||
logger.warning("No thresholds defined for %s, skipping", metric_path)
|
||||
continue
|
||||
|
||||
@@ -546,7 +640,8 @@ class ThresholdChecker:
|
||||
operator=operator,
|
||||
hysteresis=hysteresis,
|
||||
enabled=enabled,
|
||||
display=display
|
||||
display=display,
|
||||
grace=grace,
|
||||
)
|
||||
|
||||
target_dict[metric_path] = threshold
|
||||
@@ -591,6 +686,7 @@ class ThresholdChecker:
|
||||
hysteresis = threshold_config.get("hysteresis", 0.1)
|
||||
enabled = threshold_config.get("enabled", True)
|
||||
display = threshold_config.get("display")
|
||||
grace = threshold_config.get("grace", None)
|
||||
if warning is None and critical is None:
|
||||
continue
|
||||
|
||||
@@ -601,11 +697,64 @@ class ThresholdChecker:
|
||||
operator=operator,
|
||||
hysteresis=hysteresis,
|
||||
enabled=enabled,
|
||||
display=display
|
||||
display=display,
|
||||
grace=grace,
|
||||
)
|
||||
|
||||
target_dict[metric_path] = threshold
|
||||
|
||||
def _parse_pool_thresholds(
|
||||
self,
|
||||
plugin_name: str,
|
||||
pools: Dict[str, Any],
|
||||
target_dict: Optional[Dict[str, ThresholdConfig]] = None,
|
||||
):
|
||||
"""Parse ZFS pool thresholds. Pool names may be literal or '*' (all pools).
|
||||
|
||||
Config shape::
|
||||
|
||||
zfs_monitor:
|
||||
pools:
|
||||
'*':
|
||||
status:
|
||||
warning: 1
|
||||
critical: 2
|
||||
operator: '>'
|
||||
tank:
|
||||
capacity:
|
||||
warning: 80
|
||||
critical: 90
|
||||
"""
|
||||
if target_dict is None:
|
||||
target_dict = self.thresholds
|
||||
|
||||
for pool_name, metrics in pools.items():
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
for metric_name, threshold_config in metrics.items():
|
||||
if not isinstance(threshold_config, dict):
|
||||
continue
|
||||
metric_path = f"{plugin_name}.{pool_name}.{metric_name}"
|
||||
warning = threshold_config.get("warning")
|
||||
critical = threshold_config.get("critical")
|
||||
operator = threshold_config.get("operator", ">")
|
||||
hysteresis = threshold_config.get("hysteresis", 0.02)
|
||||
enabled = threshold_config.get("enabled", True)
|
||||
display = threshold_config.get("display")
|
||||
grace = threshold_config.get("grace", None)
|
||||
if warning is None and critical is None:
|
||||
continue
|
||||
target_dict[metric_path] = ThresholdConfig(
|
||||
metric_path=metric_path,
|
||||
warning=warning,
|
||||
critical=critical,
|
||||
operator=operator,
|
||||
hysteresis=hysteresis,
|
||||
enabled=enabled,
|
||||
display=display,
|
||||
grace=grace,
|
||||
)
|
||||
|
||||
def _parse_rtt_thresholds(
|
||||
self,
|
||||
rtt_thresholds: Dict[str, Any],
|
||||
@@ -635,10 +784,11 @@ class ThresholdChecker:
|
||||
warning = rtt_thresholds.get("warning")
|
||||
critical = rtt_thresholds.get("critical")
|
||||
operator = rtt_thresholds.get("operator", ">")
|
||||
hysteresis = rtt_thresholds.get("hysteresis", 0.1) # 10% default
|
||||
hysteresis = rtt_thresholds.get("hysteresis", 0.02) # 2% default
|
||||
enabled = rtt_thresholds.get("enabled", True)
|
||||
display = rtt_thresholds.get("display")
|
||||
count = rtt_thresholds.get("count", 1)
|
||||
grace = rtt_thresholds.get("grace", None)
|
||||
|
||||
if warning is None and critical is None:
|
||||
logger.warning("No RTT thresholds defined, skipping")
|
||||
@@ -653,6 +803,7 @@ class ThresholdChecker:
|
||||
enabled=enabled,
|
||||
display=display,
|
||||
count=count,
|
||||
grace=grace,
|
||||
)
|
||||
|
||||
target_dict[metric_path] = threshold
|
||||
@@ -664,7 +815,10 @@ class ThresholdChecker:
|
||||
)
|
||||
|
||||
def get_thresholds_for_host(self, host_name: str) -> Dict[str, ThresholdConfig]:
|
||||
"""Get the appropriate threshold configuration for a host.
|
||||
"""Get the effective threshold configuration for a host.
|
||||
|
||||
When threshold_config is a list, configs are applied left-to-right on top
|
||||
of the default thresholds so earlier entries can be overridden by later ones.
|
||||
|
||||
Args:
|
||||
host_name: Name of the host
|
||||
@@ -676,23 +830,40 @@ class ThresholdChecker:
|
||||
if self.thresholds and not self.threshold_configs:
|
||||
return self.thresholds
|
||||
|
||||
# Multi-config mode: look up host-specific configuration
|
||||
if self.threshold_configs:
|
||||
config_name = self.host_config_mapping.get(host_name, self.default_config)
|
||||
if not self.threshold_configs:
|
||||
return {}
|
||||
|
||||
if config_name in self.threshold_configs:
|
||||
return self.threshold_configs[config_name]
|
||||
else:
|
||||
config_names = self.host_config_mapping.get(host_name)
|
||||
|
||||
# No host-specific mapping → return pre-merged default
|
||||
if not config_names:
|
||||
return self.threshold_configs.get(self.default_config, {})
|
||||
|
||||
# Single config → fast path using pre-merged copy
|
||||
if len(config_names) == 1:
|
||||
name = config_names[0]
|
||||
if name in self.threshold_configs:
|
||||
return self.threshold_configs[name]
|
||||
logger.warning(
|
||||
"Threshold config '%s' not found for host '%s', using default '%s'",
|
||||
name, host_name, self.default_config,
|
||||
)
|
||||
return self.threshold_configs.get(self.default_config, {})
|
||||
|
||||
# Multiple configs → start from defaults, layer raw overrides in order
|
||||
result = dict(self.threshold_configs.get(self.default_config, {}))
|
||||
for name in config_names:
|
||||
if name == self.default_config:
|
||||
continue # defaults already the base
|
||||
raw = self.threshold_raw_configs.get(name)
|
||||
if raw is None:
|
||||
logger.warning(
|
||||
"Threshold config '%s' not found for host '%s', using default '%s'",
|
||||
config_name,
|
||||
host_name,
|
||||
self.default_config
|
||||
"Threshold config '%s' not found for host '%s', skipping",
|
||||
name, host_name,
|
||||
)
|
||||
return self.threshold_configs.get(self.default_config, {})
|
||||
|
||||
# No thresholds configured
|
||||
return {}
|
||||
else:
|
||||
result.update(raw)
|
||||
return result
|
||||
|
||||
def check_value(
|
||||
self,
|
||||
@@ -760,6 +931,12 @@ class ThresholdChecker:
|
||||
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||
threshold_value = threshold.warning
|
||||
|
||||
# Keep hysteresis on the state so the UI can show the recovery threshold
|
||||
if new_level != AlertLevel.OK:
|
||||
alert_state.hysteresis = threshold.hysteresis
|
||||
else:
|
||||
alert_state.hysteresis = None
|
||||
|
||||
# Update state and check for changes
|
||||
old_level = alert_state.level
|
||||
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
||||
@@ -769,6 +946,36 @@ class ThresholdChecker:
|
||||
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, None)
|
||||
|
||||
return None
|
||||
def _find_threshold(
|
||||
self, thresholds: Dict[str, "ThresholdConfig"], metric_path: str
|
||||
) -> Tuple[Optional["ThresholdConfig"], Optional[str]]:
|
||||
"""Return (threshold, check_name) for *metric_path*, falling back to suffix matches.
|
||||
|
||||
Allows generic thresholds like ``nagios_runner.status_code`` to match
|
||||
fully-qualified paths like ``nagios_runner.check_disk_root_status_code``.
|
||||
The exact match is always tried first; then successive leading
|
||||
underscore-delimited segments are stripped from the field name until
|
||||
a match is found or no segments remain.
|
||||
|
||||
Returns:
|
||||
(ThresholdConfig, None) for an exact match.
|
||||
(ThresholdConfig, "check_disk_root") for a suffix match — the second
|
||||
element is the stripped prefix, available as ``{check_name}`` in
|
||||
display format templates.
|
||||
(None, None) when no threshold is found.
|
||||
"""
|
||||
if metric_path in thresholds:
|
||||
return thresholds[metric_path], None
|
||||
plugin, sep, field = metric_path.partition(".")
|
||||
if not sep:
|
||||
return None, None
|
||||
parts = field.split("_")
|
||||
for i in range(1, len(parts)):
|
||||
candidate = plugin + "." + "_".join(parts[i:])
|
||||
if candidate in thresholds:
|
||||
return thresholds[candidate], "_".join(parts[:i])
|
||||
return None, None
|
||||
|
||||
def check_plugin_data(
|
||||
self,
|
||||
host_name: str,
|
||||
@@ -797,11 +1004,10 @@ class ThresholdChecker:
|
||||
for metric_name, value in data.items():
|
||||
metric_path = f"{plugin_name}.{metric_name}"
|
||||
|
||||
if metric_path not in thresholds:
|
||||
threshold, check_name = self._find_threshold(thresholds, metric_path)
|
||||
if threshold is None:
|
||||
continue
|
||||
|
||||
threshold = thresholds[metric_path]
|
||||
|
||||
# Get or create alert state
|
||||
if metric_path not in alert_states:
|
||||
alert_states[metric_path] = AlertState(metric_path)
|
||||
@@ -821,13 +1027,15 @@ class ThresholdChecker:
|
||||
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||
threshold_value = threshold.warning
|
||||
|
||||
alert_state.hysteresis = threshold.hysteresis if new_level != AlertLevel.OK else None
|
||||
|
||||
# Update state and check for changes
|
||||
old_level = alert_state.level
|
||||
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
||||
state_changes.append((metric_path, old_level, new_level, value))
|
||||
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, data)
|
||||
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, data, check_name=check_name, metric_name=metric_name)
|
||||
elif new_level != AlertLevel.OK:
|
||||
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, data)
|
||||
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, data, check_name=check_name, metric_name=metric_name)
|
||||
|
||||
# Check nested metrics (e.g., partition data in disk_monitor)
|
||||
self._check_nested_metrics(
|
||||
@@ -852,6 +1060,44 @@ class ThresholdChecker:
|
||||
# Get host-specific thresholds
|
||||
thresholds = self.get_thresholds_for_host(host_name)
|
||||
|
||||
# ZFS pool health checks
|
||||
if plugin_name == "zfs_monitor" and "pools" in data:
|
||||
pools = data["pools"]
|
||||
if isinstance(pools, dict):
|
||||
for pool_name, pool_metrics in pools.items():
|
||||
if not isinstance(pool_metrics, dict):
|
||||
continue
|
||||
# Synthesize status from health string for older clients
|
||||
# that predate the status field.
|
||||
pool_metrics_effective = dict(pool_metrics)
|
||||
if "health" in pool_metrics and "status" not in pool_metrics:
|
||||
pool_metrics_effective["status"] = 0 if pool_metrics["health"] == "ONLINE" else 1
|
||||
for metric_name, value in pool_metrics_effective.items():
|
||||
# Try specific pool name first, then wildcard '*'
|
||||
metric_path = f"{plugin_name}.{pool_name}.{metric_name}"
|
||||
wildcard_path = f"{plugin_name}.*.{metric_name}"
|
||||
threshold = thresholds.get(metric_path) or thresholds.get(wildcard_path)
|
||||
if threshold is None:
|
||||
continue
|
||||
if metric_path not in alert_states:
|
||||
alert_states[metric_path] = AlertState(metric_path)
|
||||
alert_state = alert_states[metric_path]
|
||||
new_level = threshold.evaluate_with_hysteresis(value, alert_state.level)
|
||||
threshold_value = None
|
||||
if new_level == AlertLevel.CRITICAL and threshold.critical is not None:
|
||||
threshold_value = threshold.critical
|
||||
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||
threshold_value = threshold.warning
|
||||
alert_state.hysteresis = threshold.hysteresis if new_level != AlertLevel.OK else None
|
||||
pool_context = dict(pool_metrics_effective)
|
||||
pool_context["pool_name"] = pool_name
|
||||
old_level = alert_state.level
|
||||
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
||||
state_changes.append((metric_path, old_level, new_level, value))
|
||||
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, pool_context, metric_name=pool_name)
|
||||
elif new_level != AlertLevel.OK:
|
||||
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, pool_context, metric_name=pool_name)
|
||||
|
||||
# Look for partition data in disk_monitor
|
||||
if plugin_name == "disk_monitor" and "partitions" in data:
|
||||
partitions = data["partitions"]
|
||||
@@ -887,6 +1133,8 @@ class ThresholdChecker:
|
||||
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||
threshold_value = threshold.warning
|
||||
|
||||
alert_state.hysteresis = threshold.hysteresis if new_level != AlertLevel.OK else None
|
||||
|
||||
old_level = alert_state.level
|
||||
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
||||
state_changes.append((metric_path, old_level, new_level, value))
|
||||
@@ -903,6 +1151,8 @@ class ThresholdChecker:
|
||||
value: Any,
|
||||
threshold: ThresholdConfig,
|
||||
plugin_data: Optional[Dict[str, Any]] = None,
|
||||
check_name: Optional[str] = None,
|
||||
metric_name: Optional[str] = None,
|
||||
):
|
||||
"""Trigger a notification for an alert state change.
|
||||
|
||||
@@ -925,55 +1175,53 @@ class ThresholdChecker:
|
||||
# Format operator symbol
|
||||
op_symbol = threshold.operator.value
|
||||
|
||||
# Short metric label: strip the plugin-name prefix and _status_code suffix
|
||||
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
|
||||
|
||||
# Use a display-friendly value (inf is the sentinel for "overdue")
|
||||
import math
|
||||
display_value = "overdue" if isinstance(value, float) and math.isinf(value) else value
|
||||
|
||||
# Format message
|
||||
if new_level == AlertLevel.OK:
|
||||
lvl = "RECOVER"
|
||||
message = f"{metric_path} = {display_value} ({old_level.name} -> OK)"
|
||||
elif new_level == AlertLevel.WARNING:
|
||||
lvl = "WARNING"
|
||||
if threshold_value is not None:
|
||||
threshold_info = self._format_display(
|
||||
threshold.display,
|
||||
value=display_value,
|
||||
threshold_value=threshold_value,
|
||||
op_symbol=op_symbol,
|
||||
plugin_data=plugin_data
|
||||
)
|
||||
message = f"{metric_path} = {display_value} {threshold_info}"
|
||||
else:
|
||||
message = f"{metric_path} = {display_value}"
|
||||
elif new_level == AlertLevel.CRITICAL:
|
||||
lvl = "CRITICAL"
|
||||
if threshold_value is not None:
|
||||
threshold_info = self._format_display(
|
||||
threshold.display,
|
||||
value=display_value,
|
||||
threshold_value=threshold_value,
|
||||
op_symbol=op_symbol,
|
||||
plugin_data=plugin_data
|
||||
)
|
||||
message = f"{metric_path} = {display_value} {threshold_info}"
|
||||
else:
|
||||
message = f"{metric_path} = {display_value}"
|
||||
else:
|
||||
lvl = "UNKNOWN"
|
||||
message = f"{metric_path} = {display_value}"
|
||||
# Format message — for the nagios operator there is no numeric threshold_value;
|
||||
# render the display template whenever one is available.
|
||||
has_display = threshold_value is not None or threshold.operator == ComparisonOperator.NAGIOS
|
||||
|
||||
# Return the formatted threshold info for storing in AlertState
|
||||
formatted_threshold_msg = None
|
||||
if threshold_value is not None and new_level != AlertLevel.OK:
|
||||
formatted_threshold_msg = self._format_display(
|
||||
def _fmt():
|
||||
return self._format_display(
|
||||
threshold.display,
|
||||
value=display_value,
|
||||
threshold_value=threshold_value,
|
||||
op_symbol=op_symbol,
|
||||
plugin_data=plugin_data
|
||||
plugin_data=plugin_data,
|
||||
check_name=check_name,
|
||||
metric_name=metric_name,
|
||||
)
|
||||
|
||||
if new_level == AlertLevel.OK:
|
||||
lvl = "RECOVER"
|
||||
message = f"{short_path} = {display_value} ({old_level.name} -> OK)"
|
||||
elif new_level == AlertLevel.WARNING:
|
||||
lvl = "WARNING"
|
||||
if has_display:
|
||||
message = f"{short_path} = {display_value} {_fmt()}"
|
||||
else:
|
||||
message = f"{short_path} = {display_value}"
|
||||
elif new_level == AlertLevel.CRITICAL:
|
||||
lvl = "CRITICAL"
|
||||
if has_display:
|
||||
message = f"{short_path} = {display_value} {_fmt()}"
|
||||
else:
|
||||
message = f"{short_path} = {display_value}"
|
||||
else:
|
||||
lvl = "UNKNOWN"
|
||||
if has_display:
|
||||
message = f"{short_path} = {display_value} {_fmt()}"
|
||||
else:
|
||||
message = f"{short_path} = {display_value}"
|
||||
|
||||
# Formatted threshold info stored on AlertState for the UI
|
||||
formatted_threshold_msg = _fmt() if has_display and new_level != AlertLevel.OK else None
|
||||
|
||||
return lvl, message, formatted_threshold_msg
|
||||
|
||||
def _send_notification(
|
||||
@@ -987,23 +1235,29 @@ class ThresholdChecker:
|
||||
value: Any,
|
||||
):
|
||||
"""Send notification and log to journal/eventlog."""
|
||||
try:
|
||||
notify_mod.send_notification(
|
||||
host_name,
|
||||
notify_mod.Notification(
|
||||
title=f"[{lvl}] {host_name}",
|
||||
body=message,
|
||||
level=lvl,
|
||||
),
|
||||
)
|
||||
logger.info("Notification sent: %s", message)
|
||||
except Exception as e:
|
||||
logger.error("Failed to send notification: %s", e)
|
||||
from . import hbdclass
|
||||
host = hbdclass.Host.hosts.get(host_name)
|
||||
if host is not None and not host.watched:
|
||||
eventlog(host_name, lvl, message, service="threshold")
|
||||
return
|
||||
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
|
||||
title = f"[{lvl}] {host_name} {short_path}"
|
||||
# Strip the "metric = " prefix from message so body is just the value/detail
|
||||
prefix = short_path + " = "
|
||||
body = message[len(prefix):] if message.startswith(prefix) else message
|
||||
asyncio.get_event_loop().create_task(notify_mod.send_notification(
|
||||
host_name,
|
||||
notify_mod.Notification(
|
||||
title=title,
|
||||
body=body,
|
||||
level=lvl,
|
||||
service=short_path,
|
||||
),
|
||||
))
|
||||
|
||||
# Log to journal
|
||||
if self.journal is not None:
|
||||
try:
|
||||
import asyncio
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.create_task(self.journal.log_threshold_event(
|
||||
host_name=host_name,
|
||||
@@ -1021,33 +1275,62 @@ class ThresholdChecker:
|
||||
self,
|
||||
display_format: str,
|
||||
value: Any,
|
||||
threshold_value: float,
|
||||
threshold_value: Optional[float],
|
||||
op_symbol: str,
|
||||
plugin_data: Optional[Dict[str, Any]] = None,
|
||||
check_name: Optional[str] = None,
|
||||
metric_name: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Format the display string using available data.
|
||||
|
||||
Args:
|
||||
display_format: Format string from threshold config
|
||||
value: Current metric value
|
||||
threshold_value: Threshold value that was exceeded
|
||||
op_symbol: Comparison operator symbol
|
||||
plugin_data: Optional dictionary of plugin data fields
|
||||
Available template variables:
|
||||
{value} - current metric value
|
||||
{threshold_value} - threshold that was exceeded
|
||||
{op_symbol} - comparison operator (>, <, >=, <=, ==, !=)
|
||||
{check_name} - prefix stripped for generic threshold match
|
||||
(e.g. "check_disk_root" when metric
|
||||
"check_disk_root_status_code" matched generic
|
||||
threshold "status_code")
|
||||
{metric_name} - field name within the plugin data dict
|
||||
Any key from plugin_data is also available.
|
||||
|
||||
Returns:
|
||||
Formatted display string
|
||||
"""
|
||||
if not display_format:
|
||||
display_format = "(threshold: {op_symbol} {threshold_value})" if threshold_value is not None else ""
|
||||
|
||||
# Build format context with standard variables
|
||||
format_context = {
|
||||
'value': value,
|
||||
'threshold_value': threshold_value,
|
||||
'op_symbol': op_symbol,
|
||||
}
|
||||
if threshold_value is not None:
|
||||
format_context['threshold_value'] = threshold_value
|
||||
|
||||
# Add generic-match context variables when available
|
||||
if check_name is not None:
|
||||
format_context['check_name'] = check_name
|
||||
if metric_name is not None:
|
||||
format_context['metric_name'] = metric_name
|
||||
|
||||
# Add all plugin data fields if available
|
||||
if plugin_data:
|
||||
format_context.update(plugin_data)
|
||||
|
||||
# For nagios_runner generic matches, expose the matched check's output
|
||||
# and status as short aliases {output} and {status} so display templates
|
||||
# don't need to use the full {check_disk_root_output} form.
|
||||
if check_name and plugin_data:
|
||||
if 'output' not in format_context:
|
||||
output = plugin_data.get(f"{check_name}_output")
|
||||
if output is not None:
|
||||
format_context['output'] = output
|
||||
if 'status' not in format_context:
|
||||
status = plugin_data.get(f"{check_name}_status")
|
||||
if status is not None:
|
||||
format_context['status'] = status
|
||||
|
||||
try:
|
||||
# Format the display string
|
||||
return display_format.format(**format_context)
|
||||
@@ -1077,17 +1360,26 @@ class ThresholdChecker:
|
||||
value: Any,
|
||||
threshold: ThresholdConfig,
|
||||
plugin_data: Optional[Dict[str, Any]],
|
||||
check_name: Optional[str] = None,
|
||||
metric_name: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Handle a state-change transition with grace-period logic.
|
||||
|
||||
Transitioning INTO alert: defers the notification for grace_seconds.
|
||||
Transitioning INTO alert (worsening): defers the notification for the effective
|
||||
grace period (threshold.grace if set, else self.grace_seconds). Grace of 0 fires
|
||||
the notification immediately with no deferral.
|
||||
De-escalation within alert states (e.g. CRITICAL→WARNING): no new notification;
|
||||
the metric is still alerting so no RECOVER was sent.
|
||||
Transitioning TO OK:
|
||||
- Still in grace window (pending_since set): suppresses both the alert
|
||||
and the recovery — the spike never warranted a page.
|
||||
- Past grace: fires the RECOVER notification normally.
|
||||
"""
|
||||
effective_grace = threshold.grace if threshold.grace is not None else self.grace_seconds
|
||||
|
||||
lvl, message, formatted_msg = self._trigger_notification(
|
||||
host_name, metric_path, old_level, new_level, value, threshold, plugin_data
|
||||
host_name, metric_path, old_level, new_level, value, threshold, plugin_data,
|
||||
check_name=check_name, metric_name=metric_name,
|
||||
)
|
||||
alert_state.formatted_message = formatted_msg
|
||||
|
||||
@@ -1095,16 +1387,31 @@ class ThresholdChecker:
|
||||
if alert_state.pending_since is not None:
|
||||
logger.info(
|
||||
"Alert suppressed (recovered within %.0fs grace): %s on %s",
|
||||
self.grace_seconds, metric_path, host_name,
|
||||
effective_grace, metric_path, host_name,
|
||||
)
|
||||
alert_state.pending_since = None
|
||||
else:
|
||||
self._send_notification(host_name, lvl, message, metric_path, old_level, new_level, value)
|
||||
elif new_level.value > old_level.value:
|
||||
# Worsening (OK→WARNING, OK→CRITICAL, WARNING→CRITICAL).
|
||||
if effective_grace <= 0:
|
||||
# No grace period — fire immediately.
|
||||
self._send_notification(host_name, lvl, message, metric_path, old_level, new_level, value)
|
||||
now = time.time()
|
||||
alert_state.last_notification = now
|
||||
alert_state.notification_count = 1
|
||||
else:
|
||||
alert_state.pending_since = time.time()
|
||||
logger.debug(
|
||||
"Alert deferred (%.0fs grace): %s on %s = %s",
|
||||
effective_grace, metric_path, host_name, value,
|
||||
)
|
||||
else:
|
||||
alert_state.pending_since = time.time()
|
||||
# De-escalation within alert states (e.g. CRITICAL→WARNING): metric is still
|
||||
# alerting but did not recover, so no new notification.
|
||||
logger.debug(
|
||||
"Alert deferred (%.0fs grace): %s on %s = %s",
|
||||
self.grace_seconds, metric_path, host_name, value,
|
||||
"De-escalation %s→%s for %s on %s, no notification",
|
||||
old_level.name, new_level.name, metric_path, host_name,
|
||||
)
|
||||
|
||||
def _check_pending_or_renotify(
|
||||
@@ -1115,25 +1422,43 @@ class ThresholdChecker:
|
||||
value: Any,
|
||||
threshold: ThresholdConfig,
|
||||
plugin_data: Optional[Dict[str, Any]],
|
||||
check_name: Optional[str] = None,
|
||||
metric_name: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Called when alert level is unchanged and non-OK.
|
||||
|
||||
If a deferred notification is pending and grace_seconds have elapsed,
|
||||
fires it now. Otherwise falls through to normal reminder logic.
|
||||
"""
|
||||
effective_grace = threshold.grace if threshold.grace is not None else self.grace_seconds
|
||||
if alert_state.pending_since is not None:
|
||||
if time.time() - alert_state.pending_since >= self.grace_seconds:
|
||||
if time.time() - alert_state.pending_since >= effective_grace:
|
||||
lvl, message, formatted_msg = self._trigger_notification(
|
||||
host_name, metric_path, AlertLevel.OK, alert_state.level, value, threshold, plugin_data
|
||||
host_name, metric_path, AlertLevel.OK, alert_state.level, value, threshold, plugin_data,
|
||||
check_name=check_name, metric_name=metric_name,
|
||||
)
|
||||
alert_state.formatted_message = formatted_msg
|
||||
self._send_notification(
|
||||
host_name, lvl, message, metric_path, AlertLevel.OK, alert_state.level, value
|
||||
)
|
||||
alert_state.pending_since = None
|
||||
now = time.time()
|
||||
alert_state.last_notification = now
|
||||
alert_state.notification_count = 1
|
||||
# else: still within grace window, do nothing
|
||||
else:
|
||||
self._check_renotify(host_name, alert_state, metric_path, value, threshold, plugin_data)
|
||||
self._check_renotify(host_name, alert_state, metric_path, value, threshold, plugin_data, check_name=check_name, metric_name=metric_name)
|
||||
|
||||
@staticmethod
|
||||
def _human_duration(seconds: float) -> str:
|
||||
s = int(seconds)
|
||||
if s < 120:
|
||||
return f"{s}s"
|
||||
if s < 3600:
|
||||
return f"{s // 60}m {s % 60}s"
|
||||
h, rem = divmod(s, 3600)
|
||||
m = rem // 60
|
||||
return f"{h}h {m}m" if m else f"{h}h"
|
||||
|
||||
def _check_renotify(
|
||||
self,
|
||||
@@ -1143,6 +1468,8 @@ class ThresholdChecker:
|
||||
value: Any,
|
||||
threshold: ThresholdConfig,
|
||||
plugin_data: Optional[Dict[str, Any]] = None,
|
||||
check_name: Optional[str] = None,
|
||||
metric_name: Optional[str] = None,
|
||||
):
|
||||
"""Check if we should send a repeat notification.
|
||||
|
||||
@@ -1180,6 +1507,7 @@ class ThresholdChecker:
|
||||
|
||||
# Format operator symbol
|
||||
op_symbol = threshold.operator.value
|
||||
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
|
||||
|
||||
# Time to re-notify
|
||||
if threshold_value is not None:
|
||||
@@ -1189,26 +1517,63 @@ class ThresholdChecker:
|
||||
value=value,
|
||||
threshold_value=threshold_value,
|
||||
op_symbol=op_symbol,
|
||||
plugin_data=plugin_data
|
||||
plugin_data=plugin_data,
|
||||
check_name=check_name,
|
||||
metric_name=metric_name,
|
||||
)
|
||||
message = f"REMINDER ({alert_state.level.name}): {host_name} - {metric_path} = {value} {threshold_info}, ongoing for {int(now - alert_state.since)}s"
|
||||
body = f"{value} {threshold_info}, ongoing for {self._human_duration(now - alert_state.since)}"
|
||||
else:
|
||||
message = f"REMINDER ({alert_state.level.name}): {host_name} - {metric_path} = {value} (ongoing for {int(now - alert_state.since)}s)"
|
||||
body = f"{value} (ongoing for {self._human_duration(now - alert_state.since)})"
|
||||
message = f"REMINDER ({alert_state.level.name}): {host_name} - {short_path} = {body}"
|
||||
|
||||
try:
|
||||
notify_mod.send_notification(
|
||||
from . import hbdclass
|
||||
host = hbdclass.Host.hosts.get(host_name)
|
||||
if host is None or host.watched:
|
||||
asyncio.get_event_loop().create_task(notify_mod.send_notification(
|
||||
host_name,
|
||||
notify_mod.Notification(
|
||||
title=f"[REMINDER/{alert_state.level.name}] {host_name}",
|
||||
body=message,
|
||||
title=f"[REMINDER/{alert_state.level.name}] {host_name} {short_path}",
|
||||
body=body,
|
||||
level=alert_state.level.name,
|
||||
service=short_path,
|
||||
),
|
||||
)
|
||||
alert_state.last_notification = now
|
||||
alert_state.notification_count += 1
|
||||
))
|
||||
logger.info("Re-notification sent: %s", message)
|
||||
except Exception as e:
|
||||
logger.error("Failed to send re-notification: %s", e)
|
||||
alert_state.last_notification = now
|
||||
alert_state.notification_count += 1
|
||||
|
||||
def purge_stale_alerts(self, hbdclass) -> None:
|
||||
"""Remove alert states that have no matching threshold configuration.
|
||||
|
||||
Called after startup (pickle restore) and after each config reload so
|
||||
that alerts orphaned by configuration changes do not linger forever.
|
||||
Alerts whose metric_path is not present in the current threshold config
|
||||
for that host are silently dropped.
|
||||
"""
|
||||
for hostname, host in hbdclass.Host.hosts.items():
|
||||
if not host.alert_states:
|
||||
continue
|
||||
configured = self.get_thresholds_for_host(hostname)
|
||||
stale = []
|
||||
for mp in host.alert_states:
|
||||
# connectivity.* and rtt are managed by the connection state
|
||||
# machine, not by threshold config — never purge them.
|
||||
if mp == "rtt" or mp.startswith("connectivity"):
|
||||
continue
|
||||
if self._find_threshold(configured, mp)[0] is not None:
|
||||
continue
|
||||
# Also match wildcard pool/partition thresholds (e.g. "zfs_monitor.*.status"
|
||||
# covers alert state "zfs_monitor.tank.status").
|
||||
parts = mp.split(".")
|
||||
if len(parts) == 3 and f"{parts[0]}.*.{parts[2]}" in configured:
|
||||
continue
|
||||
stale.append(mp)
|
||||
for mp in stale:
|
||||
logger.info(
|
||||
"Purging stale alert state for %s / %s (no threshold configured)",
|
||||
hostname, mp,
|
||||
)
|
||||
del host.alert_states[mp]
|
||||
|
||||
def get_active_alerts(self, alert_states: Dict[str, AlertState]) -> list:
|
||||
"""
|
||||
|
||||
+237
-45
@@ -16,6 +16,11 @@ from . import notify as notify_mod
|
||||
logger = logging.getLogger(__name__)
|
||||
eventlog = notify_mod.eventlog
|
||||
|
||||
# Plugin data is declared stale after this many collection intervals without a
|
||||
# fresh sample. >= 2 so a single missed sample never purges live data (the user
|
||||
# directive: wait at least two full intervals after recovery before going stale).
|
||||
_STALE_INTERVAL_MULTIPLIER = 3
|
||||
|
||||
# SO_TIMESTAMP: kernel attaches a struct timeval to each received datagram.
|
||||
# Supported on Linux, FreeBSD, and macOS. The constant is not exposed by
|
||||
# Python's socket module on all platforms
|
||||
@@ -189,6 +194,40 @@ def _set_connectivity_alert(host, afam, level_name):
|
||||
state.update(level, level_name)
|
||||
|
||||
|
||||
_PRESENCE_KEYS = {"p4": "IPv4", "p6": "IPv6"}
|
||||
|
||||
|
||||
def _apply_presence(host, msg, now, uname, ctx, skip_conn):
|
||||
"""Record client-reported protocol presence and settle the affected connections.
|
||||
|
||||
A family reported absent whose connection is already overdue/unknown is parked as
|
||||
ABSENT (timer cancelled, connectivity alert cleared). One still UP is left alone; its
|
||||
timer expiry parks it. A family reported present again while ABSENT restarts a grace
|
||||
timer. The connection that carried this very heartbeat (skip_conn) is never touched.
|
||||
"""
|
||||
cfg = ctx.get("config", {})
|
||||
for key, afam in _PRESENCE_KEYS.items():
|
||||
if key not in msg:
|
||||
continue
|
||||
present = bool(msg[key])
|
||||
host.set_presence(afam, present, now)
|
||||
conn = host.connections.get(afam)
|
||||
if conn is None or conn is skip_conn:
|
||||
continue
|
||||
cls = conn.__class__
|
||||
if not present:
|
||||
if conn.getstate() in (cls.OVERDUE, cls.UNKNOWN):
|
||||
conn.cancel_overdue_timer()
|
||||
conn.newstate(cls.ABSENT, now)
|
||||
_set_connectivity_alert(host, afam, "OK")
|
||||
elif conn.getstate() == cls.ABSENT:
|
||||
conn.newstate(cls.UNKNOWN, now)
|
||||
interval = int(msg.get("interval", 0) or 0) or host.interval
|
||||
if interval > 0:
|
||||
on_overdue, _ = _make_timer_callbacks(uname, host, ctx)
|
||||
conn.reset_overdue_timer(interval + cfg.get("grace", 2), on_overdue)
|
||||
|
||||
|
||||
def _make_timer_callbacks(uname, host, ctx):
|
||||
"""Return (on_overdue, on_unknown) async callbacks for connection timer logic.
|
||||
|
||||
@@ -205,25 +244,52 @@ def _make_timer_callbacks(uname, host, ctx):
|
||||
msg_to_websockets("host", host.stateinfo())
|
||||
|
||||
async def on_overdue(connection):
|
||||
if connection.getstate() != connection.__class__.UP:
|
||||
cls = connection.__class__
|
||||
if connection.getstate() not in (cls.UP, cls.UNKNOWN):
|
||||
return
|
||||
now = time.time()
|
||||
connection.newstate(connection.__class__.OVERDUE, now, cfg.get("grace", 2))
|
||||
msg = f"{connection.afam} overdue"
|
||||
eventlog(uname, "CRITICAL", msg)
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[CRITICAL] {uname}", body=msg, level="CRITICAL"),
|
||||
)
|
||||
# Track in alert_states so the Alerts Dashboard shows this
|
||||
_set_connectivity_alert(host, connection.afam, "CRITICAL")
|
||||
if threshold_checker:
|
||||
threshold_checker.check_value(
|
||||
host_name=uname,
|
||||
metric_path="rtt",
|
||||
value=float("inf"),
|
||||
alert_states=host.alert_states,
|
||||
)
|
||||
afam = connection.afam
|
||||
# Park only while some other family is still present: the last monitored family
|
||||
# must never be parked silently, or a host whose client reports no internet on any
|
||||
# protocol would die without an alert.
|
||||
if not host.flag_present(afam) and any(
|
||||
af != afam and host.family_present(af) for af in host.connections):
|
||||
# The client says this protocol has no connectivity: not an outage.
|
||||
connection.newstate(cls.ABSENT, now)
|
||||
connection.cancel_overdue_timer()
|
||||
_set_connectivity_alert(host, afam, "OK")
|
||||
if msg_to_websockets:
|
||||
msg_to_websockets("host", host.stateinfo())
|
||||
return
|
||||
|
||||
# If any connection is DOWN (clean shutdown reported) Host.status() is "down", so a
|
||||
# later overdue of the other family raises no separate event/alert; the shutdown did.
|
||||
prev = host.status()
|
||||
connection.newstate(cls.OVERDUE, now, cfg.get("grace", 2))
|
||||
new = host.status()
|
||||
if new == "overdue" and prev != "overdue":
|
||||
names = ", ".join(sorted(af for af, c in host.connections.items()
|
||||
if c.getstate() == cls.OVERDUE))
|
||||
msg = f"{names} overdue"
|
||||
eventlog(uname, "CRITICAL", msg)
|
||||
if host.watched:
|
||||
asyncio.create_task(notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[CRITICAL] {uname}", body=msg, level="CRITICAL"),
|
||||
))
|
||||
# Track in alert_states so the Alerts Dashboard shows this
|
||||
for af, c in host.connections.items():
|
||||
if c.getstate() == cls.OVERDUE:
|
||||
_set_connectivity_alert(host, af, "CRITICAL")
|
||||
if threshold_checker:
|
||||
threshold_checker.check_value(
|
||||
host_name=uname,
|
||||
metric_path="rtt",
|
||||
value=float("inf"),
|
||||
alert_states=host.alert_states,
|
||||
)
|
||||
elif new == "simplex":
|
||||
eventlog(uname, "INFO", f"{afam} overdue (simplex)")
|
||||
if msg_to_websockets:
|
||||
msg_to_websockets("host", host.stateinfo())
|
||||
connection.reset_overdue_timer(DROPOVERDUE, on_unknown)
|
||||
@@ -231,6 +297,23 @@ def _make_timer_callbacks(uname, host, ctx):
|
||||
return on_overdue, on_unknown
|
||||
|
||||
|
||||
def _make_plugin_stale_callback(uname, ctx):
|
||||
"""Return an async callback that clears stale plugin data and its alerts."""
|
||||
msg_to_websockets = ctx.get("msg_to_websockets")
|
||||
|
||||
async def on_plugin_stale(host, plugin_name):
|
||||
host.plugin_data.pop(plugin_name, None)
|
||||
stale_keys = [k for k in host.alert_states if k.startswith(f"{plugin_name}.")]
|
||||
for k in stale_keys:
|
||||
del host.alert_states[k]
|
||||
eventlog(uname, "INFO", f"plugin data stale: {plugin_name}")
|
||||
if msg_to_websockets:
|
||||
msg_to_websockets("plugin_stale", {"host": uname, "plugin": plugin_name})
|
||||
msg_to_websockets("host", host.stateinfo())
|
||||
|
||||
return on_plugin_stale
|
||||
|
||||
|
||||
def restore_connection_timers(hbdclass, ctx):
|
||||
"""Restore overdue timers for all loaded connections after a pickle restore.
|
||||
|
||||
@@ -247,11 +330,18 @@ def restore_connection_timers(hbdclass, ctx):
|
||||
interval = host.interval
|
||||
for afam, conn in list(host.connections.items()):
|
||||
state = conn.getstate()
|
||||
if state == hbdclass.Connection.ABSENT:
|
||||
continue
|
||||
if state == hbdclass.Connection.DOWN:
|
||||
_set_connectivity_alert(host, afam, "CRITICAL")
|
||||
continue
|
||||
|
||||
on_overdue, on_unknown = _make_timer_callbacks(uname, host, ctx)
|
||||
|
||||
if state == hbdclass.Connection.UNKNOWN:
|
||||
_set_connectivity_alert(host, afam, "CRITICAL")
|
||||
continue
|
||||
|
||||
if state == hbdclass.Connection.UP and interval > 0:
|
||||
elapsed = now - conn.lastbeat
|
||||
# Give hosts one full (interval + grace) of extra time on startup
|
||||
@@ -267,6 +357,7 @@ def restore_connection_timers(hbdclass, ctx):
|
||||
restored += 1
|
||||
|
||||
elif state == hbdclass.Connection.OVERDUE:
|
||||
host_overdue = host.status() == "overdue"
|
||||
elapsed_overdue = now - conn.statetime
|
||||
remaining = DROPOVERDUE - elapsed_overdue
|
||||
if remaining <= 1:
|
||||
@@ -276,17 +367,35 @@ def restore_connection_timers(hbdclass, ctx):
|
||||
"Marking %s/%s UNKNOWN (overdue %.1f days)",
|
||||
uname, afam, elapsed_overdue / 86400,
|
||||
)
|
||||
# A connection dropped to UNKNOWN keeps its alert, like the
|
||||
# UNKNOWN branch above and on_unknown do.
|
||||
host_overdue = True
|
||||
else:
|
||||
conn.reset_overdue_timer(remaining, on_unknown)
|
||||
logger.debug(
|
||||
"Restored OVERDUE timer %s/%s: %.0fs remaining",
|
||||
uname, afam, remaining,
|
||||
)
|
||||
# Ensure the connectivity alert is set — it may be missing if
|
||||
# hbd was shut down before the on_overdue callback had a chance
|
||||
# to record it.
|
||||
if host_overdue:
|
||||
_set_connectivity_alert(host, afam, "CRITICAL")
|
||||
restored += 1
|
||||
|
||||
logger.info("Restored timers for %d connection(s)", restored)
|
||||
|
||||
|
||||
def _is_rtt_key(plugin_name: str) -> bool:
|
||||
"""True for synthetic RTT-history keys (rtt_ipv4/rtt_ipv6), not real plugin data."""
|
||||
return plugin_name.startswith("rtt_")
|
||||
|
||||
|
||||
def _has_real_plugin_data(plugin_data: dict) -> bool:
|
||||
"""True if plugin_data holds any real (non-RTT) client-collected plugin data."""
|
||||
return any(not _is_rtt_key(k) for k in plugin_data)
|
||||
|
||||
|
||||
def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
"""Handle a parsed datagram message.
|
||||
|
||||
@@ -315,7 +424,6 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
|
||||
cfg = ctx.get("config", {})
|
||||
hbdcls = ctx.get("hbdclass")
|
||||
log = ctx.get("log")
|
||||
msg_to_websockets = ctx.get("msg_to_websockets")
|
||||
DEBUG = ctx.get("DEBUG", 0)
|
||||
verbose = ctx.get("verbose", False)
|
||||
@@ -333,11 +441,12 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
# Use new config function to check dyndns
|
||||
dyndnshosts = config_mod.get_dyndnshosts(cfg)
|
||||
host.dyn = uname in dyndnshosts
|
||||
watchhosts = config_mod.get_watchhosts(cfg)
|
||||
host.watched = uname in watchhosts
|
||||
# Apply user-access settings from config
|
||||
access = config_mod.get_host_access(cfg, uname)
|
||||
host.apply_access(access["owner"], access["managers"], access["monitors"])
|
||||
if verbose:
|
||||
print(("XX: New host, num now %s" % (len(hbdcls.Host.hosts))))
|
||||
logger.info("New host signed on: %s (dyn=%s, access=%s)", uname, host.dyn, access)
|
||||
newh = True
|
||||
else:
|
||||
host = hbdcls.Host.hosts[uname]
|
||||
@@ -351,8 +460,10 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
|
||||
if msg.get("ID") == "HTB":
|
||||
host.doesack = msg.get("acks", -1)
|
||||
# send ACK back
|
||||
# send ACK back; ask client to resend plugin info when we have none yet
|
||||
rmsg = {"time": time.time()}
|
||||
if not _has_real_plugin_data(host.plugin_data):
|
||||
rmsg["request_update"] = 1
|
||||
opkt = dicttos("ACK", rmsg)
|
||||
try:
|
||||
transport.sendto(opkt, addr)
|
||||
@@ -365,10 +476,52 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
plugin_name = msg.get("plugin")
|
||||
if plugin_name:
|
||||
# Extract plugin fields, dropping protocol metadata fields
|
||||
# (_interval is the client-declared collection interval, not a metric).
|
||||
plugin_data = {k: v for k, v in msg.items()
|
||||
if k not in ("ID", "plugin", "id", "name")}
|
||||
if k not in ("ID", "plugin", "id", "name", "_interval")}
|
||||
# Store plugin data with timestamp
|
||||
host.add_plugin_data(plugin_name, plugin_data, timestamp=now)
|
||||
|
||||
# Set the stale timer. Prefer the interval the client declares: it is
|
||||
# the true collection cadence and is correct from the very first
|
||||
# post-recovery sample, so we never purge data too early after an
|
||||
# outage. interval == 0 means a collect-once InfoPlugin that never
|
||||
# goes stale. Legacy clients omit _interval, so fall back to inferring
|
||||
# the cadence from the gap between the last two received samples.
|
||||
history = host.plugin_data.get(plugin_name, [])
|
||||
declared = msg.get("_interval")
|
||||
if declared is not None:
|
||||
if declared > 0:
|
||||
host.reset_plugin_timer(plugin_name, declared * _STALE_INTERVAL_MULTIPLIER,
|
||||
_make_plugin_stale_callback(uname, ctx))
|
||||
else:
|
||||
host.cancel_plugin_timer(plugin_name)
|
||||
elif len(history) >= 2:
|
||||
plugin_interval = max(history[-1][0] - history[-2][0], 1)
|
||||
host.reset_plugin_timer(plugin_name, plugin_interval * _STALE_INTERVAL_MULTIPLIER,
|
||||
_make_plugin_stale_callback(uname, ctx))
|
||||
else:
|
||||
host.cancel_plugin_timer(plugin_name)
|
||||
|
||||
# Remove alert states for metrics present in the previous sample but
|
||||
# absent now (e.g. a nagios check removed from configuration).
|
||||
if len(history) >= 2:
|
||||
prev_keys = set(history[-2][1].keys())
|
||||
curr_keys = set(plugin_data.keys())
|
||||
for metric_name in prev_keys - curr_keys:
|
||||
metric_path = f"{plugin_name}.{metric_name}"
|
||||
if host.alert_states.pop(metric_path, None) is not None:
|
||||
eventlog(uname, "INFO", f"stale check removed: {metric_path}")
|
||||
if (prev_keys - curr_keys) and msg_to_websockets:
|
||||
msg_to_websockets("host", host.stateinfo())
|
||||
|
||||
# If os_info reports an owner and none is configured server-side, apply it
|
||||
if plugin_name == "os_info":
|
||||
config_owner = config_mod.get_host_access(cfg, uname).get("owner")
|
||||
default_owner = config_mod.get_default_owner(cfg)
|
||||
inferred_owner = config_owner or plugin_data.get("owner") or default_owner
|
||||
host.owner = inferred_owner
|
||||
logger.info(f"owner for {uname} is {host.owner}")
|
||||
if DEBUG > 1:
|
||||
print(f"Stored plugin data for {uname}: {plugin_name}")
|
||||
|
||||
@@ -408,10 +561,11 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
|
||||
if res:
|
||||
eventlog(uname, "WARNING", res)
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[WARNING] {uname}", body=res, level="WARNING"),
|
||||
)
|
||||
if host.watched:
|
||||
asyncio.create_task(notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[WARNING] {uname}", body=res, level="WARNING"),
|
||||
))
|
||||
|
||||
interval = int(msg.get("interval", 0) or 0)
|
||||
shutdown = msg.get("shutdown", 0)
|
||||
@@ -420,31 +574,66 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
boot = msg.get("boot", 0)
|
||||
|
||||
if boot:
|
||||
# hbc was stared with a -b flag
|
||||
eventlog(uname, "INFO", "booted")
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[INFO] {uname}", body=f"{host.name} booted", level="INFO"),
|
||||
)
|
||||
if host.watched:
|
||||
asyncio.create_task(notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[INFO] {uname}", body=f"{host.name} booted", level="INFO"),
|
||||
))
|
||||
if message:
|
||||
eventlog(uname, "INFO", "msg: %s" % message, service=service)
|
||||
eventlog(uname, "INFO", message, service=service)
|
||||
|
||||
_apply_presence(host, msg, now, uname, ctx, conn)
|
||||
prev_status = host.status()
|
||||
|
||||
if conn.getstate() != hbdcls.Connection.UP:
|
||||
# Transition to UP and log/notify if appropriate
|
||||
lasts = conn.state
|
||||
d = conn.newstate(hbdcls.Connection.UP, now)
|
||||
# Clear connectivity alert now that the host is back up
|
||||
if boot:
|
||||
# On reboot, pre-boot plugin data and derived alerts are stale.
|
||||
# Cancel all plugin timers and wipe plugin state so timers restart
|
||||
# cleanly from the first two post-boot samples. An ordinary
|
||||
# reconnect (no boot flag) doesn't invalidate the client's
|
||||
# already-collected data, so it's left alone — this keeps chart
|
||||
# history intact across a transient network blip.
|
||||
for pname in list(host.plugin_timers):
|
||||
host.cancel_plugin_timer(pname)
|
||||
for pname in [k for k in host.plugin_data if not _is_rtt_key(k)]:
|
||||
del host.plugin_data[pname]
|
||||
stale_plugin_keys = [
|
||||
k for k in host.alert_states
|
||||
if k not in ("rtt",) and not k.startswith("connectivity.")
|
||||
]
|
||||
for k in stale_plugin_keys:
|
||||
del host.alert_states[k]
|
||||
# Clear connectivity alert now that the host is back up; once the host is no longer
|
||||
# overdue no family may keep a connectivity alert (e.g. the simplex case).
|
||||
_set_connectivity_alert(host, conn.afam, "OK")
|
||||
if host.status() in ("up", "simplex"):
|
||||
for af in host.connections:
|
||||
_set_connectivity_alert(host, af, "OK")
|
||||
# Don't log/notify RECOVER for a brand-new host seen for the first time —
|
||||
# it was never down, it just hasn't been seen before.
|
||||
if not newh:
|
||||
if d == 0 or lasts == "unknown":
|
||||
m = "%s is up" % (conn.afam)
|
||||
elif d < 4:
|
||||
# Transient blip (likely client restart) — skip log and notification
|
||||
m = None
|
||||
else:
|
||||
m = "%s back after being %s for %s" % (conn.afam, lasts, dur(d))
|
||||
eventlog(uname, "RECOVER", m)
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[RECOVER] {uname}", body=m, level="RECOVER"),
|
||||
)
|
||||
if m:
|
||||
# RECOVER (and a notification) only if the host itself was overdue/down;
|
||||
# a single family coming back while another stayed up is informational.
|
||||
level = "RECOVER" if prev_status in ("overdue", "down", "unknown") else "INFO"
|
||||
eventlog(uname, level, m)
|
||||
if host.watched and level == "RECOVER":
|
||||
asyncio.create_task(notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[RECOVER] {uname}", body=m, level="RECOVER"),
|
||||
))
|
||||
|
||||
if boot or newh:
|
||||
host.upcount = host.doesack
|
||||
@@ -454,16 +643,21 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
if shutdown:
|
||||
m = "%s shutdown" % conn.afam
|
||||
eventlog(uname, "INFO", m)
|
||||
notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[INFO] {uname}", body=m, level="INFO"),
|
||||
)
|
||||
if host.watched:
|
||||
asyncio.create_task(notify_mod.send_notification(
|
||||
uname,
|
||||
notify_mod.Notification(title=f"[INFO] {uname}", body=m, level="INFO"),
|
||||
))
|
||||
conn.newstate(hbdcls.Connection.DOWN, now)
|
||||
_set_connectivity_alert(host, conn.afam, "CRITICAL")
|
||||
|
||||
if interval > 0:
|
||||
host.interval = interval
|
||||
|
||||
# Record RTT history for charting
|
||||
if rtt is not None:
|
||||
host.add_plugin_data(f"rtt_{conn.afam.lower()}", {"rtt": rtt}, timestamp=now)
|
||||
|
||||
# Timer-based reachability monitoring
|
||||
# Reset overdue timer on every heartbeat
|
||||
if interval > 0 and conn.getstate() != hbdcls.Connection.DOWN:
|
||||
@@ -491,12 +685,10 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
||||
op, rmsg = host.cmds[0]
|
||||
if op == "CMD":
|
||||
del host.cmds[0]
|
||||
if log:
|
||||
log(uname, "command sent")
|
||||
eventlog(uname, "INFO", "command sent")
|
||||
elif op == "UPD":
|
||||
del host.cmds[0]
|
||||
if log:
|
||||
log(uname, "update initiated")
|
||||
eventlog(uname, "INFO", "update initiated")
|
||||
opkt = dicttos(op, rmsg)
|
||||
try:
|
||||
transport.sendto(opkt, addr)
|
||||
|
||||
@@ -146,9 +146,14 @@ def load_users(config: dict) -> dict:
|
||||
Returns the new ``users`` dict.
|
||||
"""
|
||||
global users
|
||||
old_users = dict(users) # snapshot before rebuild
|
||||
users_cfg = config.get("users", {})
|
||||
if not isinstance(users_cfg, dict):
|
||||
users = {}
|
||||
# Preserve OAuth-provisioned users (password_hash == "") that aren't in config.
|
||||
for username, existing_user in old_users.items():
|
||||
if not existing_user.password_hash and username not in users:
|
||||
users[username] = existing_user
|
||||
return users
|
||||
|
||||
result: dict = {}
|
||||
@@ -166,6 +171,10 @@ def load_users(config: dict) -> dict:
|
||||
)
|
||||
|
||||
users = result
|
||||
# Preserve OAuth-provisioned users (password_hash == "") that aren't in config.
|
||||
for username, existing_user in old_users.items():
|
||||
if not existing_user.password_hash and username not in users:
|
||||
users[username] = existing_user
|
||||
logger.info("Loaded %d user(s) from config", len(users))
|
||||
return users
|
||||
|
||||
@@ -187,6 +196,26 @@ def authenticate(username: str, password: str) -> "User | None":
|
||||
return None
|
||||
|
||||
|
||||
def provision_oauth_user(username: str, full_name: str, avatar: str) -> "User":
|
||||
"""Create or update a user sourced from an OAuth2 provider.
|
||||
|
||||
New users are inserted with no password_hash — they can only authenticate
|
||||
via OAuth. Existing users (e.g. defined in config with a password) have
|
||||
their display name and avatar refreshed; all other attributes are preserved.
|
||||
"""
|
||||
user = users.get(username)
|
||||
if user is None:
|
||||
user = User(username=username, full_name=full_name, avatar=avatar)
|
||||
users[username] = user
|
||||
logger.info("Provisioned OAuth user %r", username)
|
||||
else:
|
||||
if full_name:
|
||||
user.full_name = full_name
|
||||
if avatar:
|
||||
user.avatar = avatar
|
||||
return user
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session management
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+62
-13
@@ -13,7 +13,8 @@ from . import data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_connections: set = set()
|
||||
# Map of WebSocket → User object (or None when auth is disabled)
|
||||
_connections: dict = {}
|
||||
_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
_get_hosts: Optional[Callable[[], Iterable]] = None
|
||||
_verbose: bool = False
|
||||
@@ -34,31 +35,65 @@ def setup(
|
||||
_verbose = verbose
|
||||
|
||||
|
||||
def _user_can_see_host(user, host_name: str) -> bool:
|
||||
"""Return True if *user* may see updates for *host_name* (manager or higher)."""
|
||||
from . import hbdclass, users as users_mod
|
||||
if user is None or not users_mod.users_enabled():
|
||||
return True
|
||||
if user.admin:
|
||||
return True
|
||||
host = hbdclass.Host.hosts.get(host_name)
|
||||
if host is None:
|
||||
return False
|
||||
return host.is_manager(user.username)
|
||||
|
||||
|
||||
def _get_token(request) -> str:
|
||||
"""Extract session token from request (mirrors logic in http.py)."""
|
||||
auth = request.headers.get("Authorization", "")
|
||||
if auth.startswith("Bearer "):
|
||||
return auth[7:].strip()
|
||||
token = request.headers.get("X-Auth-Token", "")
|
||||
if token:
|
||||
return token
|
||||
return request.cookies.get("hbd_session", "")
|
||||
|
||||
|
||||
async def handler(request):
|
||||
"""aiohttp WebSocket upgrade handler — register as GET /ws."""
|
||||
from aiohttp import web
|
||||
from . import users as users_mod
|
||||
|
||||
ws = web.WebSocketResponse()
|
||||
await ws.prepare(request)
|
||||
|
||||
_connections.add(ws)
|
||||
token = _get_token(request)
|
||||
user = users_mod.get_session_user(token) if token else None
|
||||
|
||||
_connections[ws] = user
|
||||
remote = request.remote
|
||||
logger.info("WebSocket connected from %s", remote)
|
||||
|
||||
try:
|
||||
# Send current host state to the new client
|
||||
# Send current host state, filtered to hosts this user may see
|
||||
if _get_hosts:
|
||||
try:
|
||||
for h in list(_get_hosts()):
|
||||
await ws.send_str(json.dumps({"type": "host", "data": h}))
|
||||
host_name = h.get("raw_name") or h.get("name", "")
|
||||
if _user_can_see_host(user, host_name):
|
||||
await ws.send_str(json.dumps({"type": "host", "data": h}))
|
||||
except Exception as e:
|
||||
logger.error("Error sending initial hosts: %s", e)
|
||||
|
||||
# Send recent messages
|
||||
# Send recent messages newest-first so the client can append them in
|
||||
# display order without reordering on arrival (tagged history=True so
|
||||
# the client knows to append rather than prepend).
|
||||
if data.msgs:
|
||||
try:
|
||||
for m in data.msgs:
|
||||
await ws.send_str(json.dumps({"type": "message", "data": m}))
|
||||
for m in reversed(data.msgs[-30:]):
|
||||
host_name = m.get("host") if isinstance(m, dict) else None
|
||||
if not host_name or _user_can_see_host(user, host_name):
|
||||
await ws.send_str(json.dumps({"type": "message", "data": m, "history": True}))
|
||||
except Exception as e:
|
||||
logger.error("Error sending initial messages: %s", e)
|
||||
|
||||
@@ -74,7 +109,7 @@ async def handler(request):
|
||||
except Exception as e:
|
||||
logger.exception("WebSocket handler error from %s: %s", remote, e)
|
||||
finally:
|
||||
_connections.discard(ws)
|
||||
_connections.pop(ws, None)
|
||||
logger.info("WebSocket disconnected from %s", remote)
|
||||
|
||||
return ws
|
||||
@@ -83,25 +118,39 @@ async def handler(request):
|
||||
def broadcast(typ: str, payload) -> bool:
|
||||
"""Thread-safe broadcast to all connected WebSocket clients.
|
||||
|
||||
For host and plugin updates, only sends to clients whose user has
|
||||
manager-or-higher access to that host. Other message types are
|
||||
broadcast to all clients.
|
||||
|
||||
Can be called from any thread; schedules sends on the event loop.
|
||||
Returns False if the loop is not running yet.
|
||||
"""
|
||||
if not _loop:
|
||||
return False
|
||||
|
||||
# Determine the host name for access-filtered message types
|
||||
host_name: Optional[str] = None
|
||||
if typ in ("host", "plugin"):
|
||||
host_name = payload.get("raw_name") or payload.get("host") or payload.get("name")
|
||||
elif typ == "message" and isinstance(payload, dict):
|
||||
host_name = payload.get("host")
|
||||
|
||||
jmsg = json.dumps({"type": typ, "data": payload})
|
||||
|
||||
async def _send_all():
|
||||
dead = set()
|
||||
for ws in list(_connections):
|
||||
for ws, user in list(_connections.items()):
|
||||
try:
|
||||
if not ws.closed:
|
||||
await ws.send_str(jmsg)
|
||||
else:
|
||||
if ws.closed:
|
||||
dead.add(ws)
|
||||
continue
|
||||
if host_name is not None and not _user_can_see_host(user, host_name):
|
||||
continue
|
||||
await ws.send_str(jmsg)
|
||||
except Exception:
|
||||
dead.add(ws)
|
||||
for ws in dead:
|
||||
_connections.discard(ws)
|
||||
_connections.pop(ws, None)
|
||||
|
||||
asyncio.run_coroutine_threadsafe(_send_all(), _loop)
|
||||
return True
|
||||
|
||||
+27
-8
@@ -4,20 +4,32 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "hbd"
|
||||
version = "5.1.2"
|
||||
version = "5.4.5"
|
||||
description = "Heartbeat monitoring system — client (hbc) and server (hbd)"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
license = "MIT"
|
||||
keywords = ["heartbeat", "monitoring", "dns", "websocket", "system-monitoring"]
|
||||
authors = [
|
||||
{ name = "heartbeat contributors" }
|
||||
]
|
||||
|
||||
# Core dependencies (required for both client and server)
|
||||
dependencies = [
|
||||
"PyYAML>=6.0",
|
||||
]
|
||||
license = "MIT"
|
||||
license-files = ["LICENSE.md"]
|
||||
keywords = ["heartbeat", "monitoring", "dns", "websocket", "system-monitoring"]
|
||||
authors = [
|
||||
{ name = "Andreas Wrede" }
|
||||
]
|
||||
classifiers = [
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Operating System :: POSIX :: Linux",
|
||||
"Operating System :: POSIX :: BSD",
|
||||
"Topic :: System :: Monitoring",
|
||||
"Topic :: System :: Networking :: Monitoring",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Repository = "https://git.wrede.ca/andreas/heartbeat"
|
||||
|
||||
[project.optional-dependencies]
|
||||
# Client-only dependencies (hbc - system monitoring client)
|
||||
@@ -32,8 +44,12 @@ server = [
|
||||
"aiohttp>=3.11",
|
||||
"Jinja2>=3.1.6",
|
||||
"matrix-nio>=0.24",
|
||||
"ruamel.yaml>=0.18",
|
||||
]
|
||||
|
||||
# Minimal client — hbc_mini only, no external dependencies
|
||||
mini = []
|
||||
|
||||
# Install both client and server
|
||||
all = [
|
||||
"hbd[client,server]",
|
||||
@@ -54,6 +70,9 @@ dev = [
|
||||
hbd = "hbd.server.cli:main"
|
||||
hbc = "hbd.client.main:main"
|
||||
|
||||
[tool.setuptools]
|
||||
script-files = ["scripts/hb_install.sh", "scripts/hbc_mini.py"]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["hbd*"]
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
key "rndc-key" {
|
||||
algorithm hmac-md5;
|
||||
secret "qlGa+AYKtyOgWNuozqECMw==";
|
||||
};
|
||||
+20
-1
@@ -4,12 +4,31 @@ set -e
|
||||
uv version --bump patch
|
||||
VER=$(uv version --short)
|
||||
sed -i".bak" "s/__version__ = \"[0-9.]*\"\(.*\)$/__version__ = \"$VER\"\1/" hbd/__init__.py
|
||||
sed -i".bak" "s/__version__ = \"[0-9.]*\"\(.*\)$/__version__ = \"$VER\"\1/" scripts/hbc_mini.py
|
||||
sed -i".bak" "s/\*\*Package:\*\* \`hbd\` v[0-9.]*/\*\*Package:\*\* \`hbd\` v$VER/" README.md
|
||||
|
||||
# Update CHANGELOG.md with commits since last tag
|
||||
LASTTAG=$(git describe --tags --abbrev=0 2>/dev/null || true)
|
||||
# Matches both unscoped ("feat: ...") and scoped ("feat(server): ...") conventional-commit
|
||||
# subjects; the scope is dropped from the changelog line.
|
||||
ADDED=$(git log "${LASTTAG:+$LASTTAG..}HEAD" --pretty="%s" | grep -E "^feat(\([^)]*\))?:" | sed -E 's/^feat(\([^)]*\))?: /- /')
|
||||
FIXED=$(git log "${LASTTAG:+$LASTTAG..}HEAD" --pretty="%s" | grep -E "^fix(\([^)]*\))?:" | sed -E 's/^fix(\([^)]*\))?: /- /')
|
||||
{
|
||||
printf "## [%s]\n" "$VER"
|
||||
[ -n "$ADDED" ] && printf "\n### Added\n%s\n" "$ADDED"
|
||||
[ -n "$FIXED" ] && printf "\n### Fixed\n%s\n" "$FIXED"
|
||||
printf "\n---\n\n"
|
||||
} > /tmp/changelog_entry.txt
|
||||
sed -i".bak" "4r /tmp/changelog_entry.txt" CHANGELOG.md
|
||||
rm /tmp/changelog_entry.txt CHANGELOG.md.bak
|
||||
|
||||
# commit pyproject.toml
|
||||
git commit -m "version $VER" pyproject.toml hbd/__init__.py
|
||||
git commit -m "version $VER" pyproject.toml hbd/__init__.py scripts/hbc_mini.py README.md CHANGELOG.md
|
||||
git push
|
||||
# tag version
|
||||
git tag -a v$VER -m "Version $VER"
|
||||
git push --tags
|
||||
|
||||
rm hbd/__init__.py.bak
|
||||
rm scripts/hbc_mini.py.bak
|
||||
rm README.md.bak
|
||||
@@ -0,0 +1,3 @@
|
||||
hbc_mini
|
||||
hbc_mini_dbg
|
||||
test_probe
|
||||
@@ -0,0 +1,27 @@
|
||||
CC ?= cc
|
||||
CFLAGS = -O2 -Wall -Wextra -std=c11
|
||||
LDFLAGS = -lz -lpthread -lm
|
||||
TARGET = hbc_mini
|
||||
SRC = hbc_mini.c
|
||||
|
||||
# FreeBSD/NetBSD keep zlib in base; no extra flags needed.
|
||||
# On some NetBSD installs pthreads may need -lpthread from pkgsrc.
|
||||
|
||||
.PHONY: all clean debug test
|
||||
|
||||
all: $(TARGET)
|
||||
|
||||
$(TARGET): $(SRC)
|
||||
$(CC) $(CFLAGS) -o $@ $< $(LDFLAGS)
|
||||
|
||||
debug: $(SRC)
|
||||
$(CC) -g -fsanitize=address,undefined -o $(TARGET)_dbg $< $(LDFLAGS)
|
||||
|
||||
test: test_probe
|
||||
./test_probe
|
||||
|
||||
test_probe: test_probe.c $(SRC)
|
||||
$(CC) $(CFLAGS) -o $@ test_probe.c $(LDFLAGS)
|
||||
|
||||
clean:
|
||||
rm -f $(TARGET) $(TARGET)_dbg test_probe
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,243 @@
|
||||
/* Unit tests for the connectivity-probe helpers in hbc_mini.c.
|
||||
* Build and run: make test (live check of this host: ./test_probe --live)
|
||||
* The source file is included directly so its static functions can be exercised; its main()
|
||||
* is renamed out of the way.
|
||||
*/
|
||||
#define main hbc_mini_main
|
||||
#include "hbc_mini.c"
|
||||
#undef main
|
||||
|
||||
static int g_run = 0, g_fail = 0;
|
||||
#define CHECK(cond) do { g_run++; if (!(cond)) { g_fail++; \
|
||||
fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); } } while (0)
|
||||
#define CHECK_STR(a, b) do { g_run++; if (strcmp((a), (b)) != 0) { g_fail++; \
|
||||
fprintf(stderr, "FAIL %s:%d: \"%s\" != \"%s\"\n", __FILE__, __LINE__, (a), (b)); } } while (0)
|
||||
|
||||
static const char PROC_V4[] =
|
||||
"Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n"
|
||||
"wlan0\t00000000\t0100A8C0\t0003\t0\t0\t600\t00000000\t0\t0\t0\n"
|
||||
"eth0\t00000000\t0101A8C0\t0003\t0\t0\t100\t00000000\t0\t0\t0\n"
|
||||
"wlan0\t0000A8C0\t00000000\t0001\t0\t0\t600\t00FFFFFF\t0\t0\t0\n";
|
||||
|
||||
static const char PROC_V6[] =
|
||||
"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "
|
||||
"fe800000000000000000000000000001 00000400 00000001 00000000 00000003 wlan0\n"
|
||||
"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "
|
||||
"00000000000000000000000000000000 ffffffff 00000001 00000000 00200200 lo\n"
|
||||
"fe800000000000000000000000000000 40 00000000000000000000000000000000 00 "
|
||||
"00000000000000000000000000000000 00000100 00000001 00000000 00000001 wlan0\n";
|
||||
|
||||
static const char ROUTE_GET[] =
|
||||
" route to: 0.0.0.0\ndestination: 0.0.0.0\n mask: 0.0.0.0\n"
|
||||
" gateway: 192.0.2.1\n fib: 0\n interface: vtnet0\n";
|
||||
|
||||
static const char LINUX_PING[] =
|
||||
"PING 1.1.1.1 (1.1.1.1) 56(84) bytes of data.\n"
|
||||
"64 bytes from 1.1.1.1: icmp_seq=1 ttl=57 time=11.2 ms\n\n"
|
||||
"--- 1.1.1.1 ping statistics ---\n"
|
||||
"1 packets transmitted, 1 received, 0% packet loss, time 0ms\n"
|
||||
"rtt min/avg/max/mdev = 11.234/11.234/11.234/0.000 ms\n";
|
||||
|
||||
static const char FREEBSD_PING[] =
|
||||
"PING 1.1.1.1 (1.1.1.1): 56 data bytes\n"
|
||||
"64 bytes from 1.1.1.1: icmp_seq=0 ttl=59 time=4.006 ms\n\n"
|
||||
"--- 1.1.1.1 ping statistics ---\n"
|
||||
"1 packets transmitted, 1 packets received, 0.0% packet loss, 1 packets out of wait time\n"
|
||||
"round-trip min/avg/max/stddev = 4.006/4.006/4.006/0.000 ms\n";
|
||||
|
||||
static void test_gateway_parsers(void) {
|
||||
char gw[64];
|
||||
gw_parse_proc_v4(PROC_V4, gw, sizeof(gw));
|
||||
CHECK_STR(gw, "192.168.1.1"); /* eth0, metric 100 beats wlan0, 600 */
|
||||
gw_parse_proc_v4("Iface\tDestination\n", gw, sizeof(gw));
|
||||
CHECK_STR(gw, "");
|
||||
gw_parse_proc_v6(PROC_V6, gw, sizeof(gw));
|
||||
CHECK_STR(gw, "fe80::1"); /* reject and non-default lines skipped */
|
||||
gw_parse_route_get(ROUTE_GET, gw, sizeof(gw));
|
||||
CHECK_STR(gw, "192.0.2.1");
|
||||
gw_parse_route_get("route: writing to routing socket: not in table\n", gw, sizeof(gw));
|
||||
CHECK_STR(gw, "");
|
||||
}
|
||||
|
||||
static void test_ping_parser(void) {
|
||||
CHECK(ping_parse_rtt(LINUX_PING) > 11.23 && ping_parse_rtt(LINUX_PING) < 11.24);
|
||||
CHECK(ping_parse_rtt(FREEBSD_PING) > 4.0 && ping_parse_rtt(FREEBSD_PING) < 4.01);
|
||||
CHECK(ping_parse_rtt("1 packets transmitted, 0 received, 100% packet loss\n") == -1.0);
|
||||
CHECK(ping_parse_rtt("sh: 1: ping: not found\n") == -2.0);
|
||||
CHECK(ping_parse_rtt("ping: command not found\n") == -2.0);
|
||||
}
|
||||
|
||||
/* ---- scriptable network for probe_step ------------------------------------------------ */
|
||||
static struct {
|
||||
char addr[2][INET6_ADDRSTRLEN]; /* "" = no route */
|
||||
char gw[2][INET6_ADDRSTRLEN];
|
||||
double rtt[2]; /* -1 no reply, -2 ping unavailable */
|
||||
int unsupported[2];
|
||||
int npings[2];
|
||||
} net;
|
||||
|
||||
static int fake_local(int af, const char *t, char *out, size_t n) {
|
||||
int f = af == AF_INET6; (void)t;
|
||||
if (net.unsupported[f]) return -1;
|
||||
if (!net.addr[f][0]) return 0;
|
||||
snprintf(out, n, "%s", net.addr[f]);
|
||||
return 1;
|
||||
}
|
||||
static double fake_ping(int af, const char *t) { int f = af == AF_INET6; (void)t; net.npings[f]++; return net.rtt[f]; }
|
||||
static void fake_gw(int af, char *out, size_t n) { snprintf(out, n, "%s", net.gw[af == AF_INET6]); }
|
||||
static const probe_ops_t FAKE = { fake_local, fake_ping, fake_gw };
|
||||
|
||||
static void net_reset(void) {
|
||||
memset(&net, 0, sizeof(net));
|
||||
snprintf(net.addr[0], INET6_ADDRSTRLEN, "192.0.2.5");
|
||||
snprintf(net.addr[1], INET6_ADDRSTRLEN, "2001:db8::5");
|
||||
snprintf(net.gw[0], INET6_ADDRSTRLEN, "192.0.2.1");
|
||||
snprintf(net.gw[1], INET6_ADDRSTRLEN, "fe80::1");
|
||||
net.rtt[0] = 10.0; net.rtt[1] = 20.0;
|
||||
}
|
||||
|
||||
static void probe_reset(probe_t *p) {
|
||||
memset(p, 0, sizeof(*p));
|
||||
snprintf(p->target[0], sizeof(p->target[0]), "1.1.1.1");
|
||||
snprintf(p->target[1], sizeof(p->target[1]), "2606:4700:4700::1111");
|
||||
p->probe_interval = 60; p->retry_interval = 10; p->cheap_gateway = true;
|
||||
p->fam[0].present = p->fam[1].present = PROBE_UNKNOWN;
|
||||
p->fam[0].rtt = p->fam[1].rtt = -1;
|
||||
pthread_mutex_init(&p->mu, NULL);
|
||||
}
|
||||
|
||||
static void step_both(probe_t *p, double now) {
|
||||
probe_step(p, &p->fam[0], 0, &FAKE, now);
|
||||
probe_step(p, &p->fam[1], 1, &FAKE, now);
|
||||
}
|
||||
|
||||
static void test_present_with_details(void) {
|
||||
net_reset(); probe_t p; probe_reset(&p);
|
||||
step_both(&p, 1000);
|
||||
CHECK(p.fam[0].present == 1 && p.fam[1].present == 1);
|
||||
CHECK_STR(p.fam[0].addr, "192.0.2.5");
|
||||
CHECK_STR(p.fam[0].gateway, "192.0.2.1");
|
||||
CHECK(p.fam[0].rtt == 10.0);
|
||||
kvdict_t d; kv_clear(&d);
|
||||
probe_flags(&p, &d);
|
||||
CHECK_STR(kv_get(&d, "p4") ? kv_get(&d, "p4") : "(null)", "1");
|
||||
CHECK_STR(kv_get(&d, "p6") ? kv_get(&d, "p6") : "(null)", "1");
|
||||
char js[512];
|
||||
CHECK(probe_metric_json(&p, js, sizeof(js)) > 0);
|
||||
CHECK(strstr(js, "\"ipv6\":{\"present\":true,\"addr\":\"2001:db8::5\",\"gateway\":\"fe80::1\"") != NULL);
|
||||
}
|
||||
|
||||
static void test_no_route_is_absent_and_wastes_no_ping(void) {
|
||||
net_reset(); net.addr[1][0] = '\0'; probe_t p; probe_reset(&p);
|
||||
step_both(&p, 1000);
|
||||
CHECK(p.fam[1].present == 0 && net.npings[1] == 0 && net.npings[0] == 1);
|
||||
kvdict_t d; kv_clear(&d);
|
||||
probe_flags(&p, &d);
|
||||
CHECK_STR(kv_get(&d, "p6") ? kv_get(&d, "p6") : "(null)", "0");
|
||||
char js[512];
|
||||
probe_metric_json(&p, js, sizeof(js));
|
||||
CHECK(strstr(js, "\"ipv6\":{\"present\":false,\"addr\":null,\"gateway\":null,\"rtt\":null}") != NULL);
|
||||
}
|
||||
|
||||
static void test_debounce_and_rate_limit(void) {
|
||||
net_reset(); probe_t p; probe_reset(&p);
|
||||
step_both(&p, 1000);
|
||||
step_both(&p, 1000); /* cached: no second ping */
|
||||
CHECK(net.npings[0] == 1 && net.npings[1] == 1);
|
||||
net.rtt[1] = -1;
|
||||
step_both(&p, 1060); /* first failure: still present */
|
||||
CHECK(p.fam[1].present == 1);
|
||||
step_both(&p, 1070); /* retry interval elapsed: flips */
|
||||
CHECK(p.fam[1].present == 0);
|
||||
net.rtt[1] = 25;
|
||||
step_both(&p, 1080); /* first success recovers */
|
||||
CHECK(p.fam[1].present == 1);
|
||||
}
|
||||
|
||||
static void test_first_failure_is_unknown_and_omitted(void) {
|
||||
net_reset(); net.rtt[0] = -1; probe_t p; probe_reset(&p);
|
||||
step_both(&p, 1000);
|
||||
CHECK(p.fam[0].present == PROBE_UNKNOWN);
|
||||
kvdict_t d; kv_clear(&d);
|
||||
probe_flags(&p, &d);
|
||||
CHECK(kv_get(&d, "p4") == NULL && kv_get(&d, "p6") != NULL);
|
||||
char js[512];
|
||||
probe_metric_json(&p, js, sizeof(js));
|
||||
CHECK(strstr(js, "\"ipv4\":{\"present\":null") != NULL);
|
||||
}
|
||||
|
||||
static void test_address_or_gateway_change_repings(void) {
|
||||
net_reset(); probe_t p; probe_reset(&p);
|
||||
step_both(&p, 1000);
|
||||
snprintf(net.addr[0], INET6_ADDRSTRLEN, "198.51.100.7");
|
||||
step_both(&p, 1000);
|
||||
CHECK(net.npings[0] == 2);
|
||||
snprintf(net.gw[0], INET6_ADDRSTRLEN, "192.0.2.254");
|
||||
step_both(&p, 1000);
|
||||
CHECK(net.npings[0] == 3);
|
||||
}
|
||||
|
||||
static void test_route_loss_then_return_pings_immediately(void) {
|
||||
net_reset(); probe_t p; probe_reset(&p);
|
||||
step_both(&p, 1000);
|
||||
net.addr[1][0] = '\0';
|
||||
step_both(&p, 1000);
|
||||
CHECK(p.fam[1].present == 0);
|
||||
snprintf(net.addr[1], INET6_ADDRSTRLEN, "2001:db8::5");
|
||||
step_both(&p, 1000);
|
||||
CHECK(net.npings[1] == 2 && p.fam[1].present == 1);
|
||||
}
|
||||
|
||||
static void test_missing_ping_stays_unknown_even_after_route_loss(void) {
|
||||
net_reset(); net.rtt[0] = -2; probe_t p; probe_reset(&p);
|
||||
step_both(&p, 1000);
|
||||
step_both(&p, 1600);
|
||||
CHECK(p.fam[0].present == PROBE_UNKNOWN && net.npings[0] == 1);
|
||||
net.addr[0][0] = '\0';
|
||||
step_both(&p, 1700);
|
||||
CHECK(p.fam[0].present == 0); /* no route is definitive */
|
||||
snprintf(net.addr[0], INET6_ADDRSTRLEN, "192.0.2.5");
|
||||
step_both(&p, 1800);
|
||||
CHECK(p.fam[0].present == PROBE_UNKNOWN && net.npings[0] == 1);
|
||||
}
|
||||
|
||||
static void test_unsupported_family_is_unknown(void) {
|
||||
net_reset(); net.unsupported[1] = 1; probe_t p; probe_reset(&p);
|
||||
step_both(&p, 1000);
|
||||
CHECK(p.fam[1].present == PROBE_UNKNOWN && p.fam[0].present == 1);
|
||||
}
|
||||
|
||||
static int live_mode(void) {
|
||||
static const probe_ops_t sys_ops = { sys_local_address, sys_ping, sys_gateway };
|
||||
probe_t p; probe_reset(&p);
|
||||
p.cheap_gateway = false;
|
||||
snprintf(p.target[0], sizeof(p.target[0]), "1.1.1.1");
|
||||
for (int round = 0; round < 3; round++) {
|
||||
double now = mono_ts();
|
||||
for (int f = 0; f < 2; f++) probe_step(&p, &p.fam[f], f, &sys_ops, now);
|
||||
sleep(round < 2 ? 11 : 0);
|
||||
}
|
||||
char js[512];
|
||||
probe_metric_json(&p, js, sizeof(js));
|
||||
kvdict_t d; kv_clear(&d);
|
||||
probe_flags(&p, &d);
|
||||
printf("%s\np4=%s p6=%s\n", js, kv_get(&d, "p4") ? kv_get(&d, "p4") : "-",
|
||||
kv_get(&d, "p6") ? kv_get(&d, "p6") : "-");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
if (argc > 1 && !strcmp(argv[1], "--live")) return live_mode();
|
||||
test_gateway_parsers();
|
||||
test_ping_parser();
|
||||
test_present_with_details();
|
||||
test_no_route_is_absent_and_wastes_no_ping();
|
||||
test_debounce_and_rate_limit();
|
||||
test_first_failure_is_unknown_and_omitted();
|
||||
test_address_or_gateway_change_repings();
|
||||
test_route_loss_then_return_pings_immediately();
|
||||
test_missing_ping_stays_unknown_even_after_route_loss();
|
||||
test_unsupported_family_is_unknown();
|
||||
printf("%d checks, %d failed\n", g_run, g_fail);
|
||||
return g_fail ? 1 : 0;
|
||||
}
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Helper script to install the heartbeat tools. By default, it will only
|
||||
# install the heartbeat client, hbc. The server is installed when the arg 'server' is passed
|
||||
# to the script. The script will install the heartbeat tools in a python
|
||||
# virtual environment in ~/venvs/hbd. The hbd and hbc commands will be
|
||||
# installed from the wheel and symlinked to ~/bin/hbd and ~/bin/hbc,
|
||||
# respectively. If the virtual environment already exists, it will be
|
||||
# reused. The script will also remove any existing symlinks for hbd and hbc
|
||||
# in ~/bin before creating new ones.
|
||||
|
||||
set -e
|
||||
what=$1
|
||||
on_ha=0
|
||||
where=""
|
||||
venv=""
|
||||
[ "$2" = "HA" ] && on_ha=1
|
||||
[ -z "$what" ] && what="client"
|
||||
|
||||
if [ -d /homeassistant ]; then # if running from HA command line
|
||||
echo "HA, running \"docker exec homeassistant /config/bin/hb_install.sh $@\""
|
||||
docker exec homeassistant /config/bin/hb_install.sh $@ HA
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ]; then
|
||||
echo "Failed to install heartbeat in HA, please check the logs for more details"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ $on_ha -eq 1 ] || [ -r /.dockerenv ] && [ -d /config/bin ]; then
|
||||
# Installing under docker on Home Assistant OS, using /config/bin for executables and /config/venvs for virtual environments
|
||||
echo "Home Assistant OS detected, installing under docker"
|
||||
where="/config/bin"
|
||||
venv="/config/venvs"
|
||||
else
|
||||
if [ ! -d $HOME/.local/bin ] && [ ! -d $HOME/bin ]; then
|
||||
echo "No suitable bin directory found in PATH, please add either $HOME/.local/bin or $HOME/bin to your PATH"
|
||||
exit 1
|
||||
fi
|
||||
for where in $HOME/bin $HOME/.local/bin notset ; do
|
||||
if echo ":$PATH:" | grep -q ":$where:" ; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$where" = "notset" ]; then
|
||||
echo "No suitable bin directory found in PATH, please add either $HOME/.local/bin or $HOME/bin to your PATH"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$what" = "mini" ]; then
|
||||
venv=""
|
||||
else
|
||||
venv="$HOME/venvs"
|
||||
fi
|
||||
fi
|
||||
echo "Installing $what to $where"
|
||||
if [ ! -z "$venv" ]; then
|
||||
echo "Using virtual environment at $venv/hbd"
|
||||
fi
|
||||
|
||||
if [ "$venv" != "" ] && [ ! -d $venv/hbd ]; then
|
||||
arg=""
|
||||
have_pip=$(python3 -c "import pip" 2>/dev/null &> /dev/null && echo "Installed" || echo "Not Installed")
|
||||
if [ "$have_pip" = "Not Installed" ]; then
|
||||
# some systems do not have pip installed by default, so we need to fetch get-pip.py and install pip
|
||||
echo "pip is not installed, fetching get-pip.py and installing pip"
|
||||
arg="--without-pip"
|
||||
fi
|
||||
mkdir -p $venv
|
||||
have_venv=$(python3 -c "import venv" 2>/dev/null &> /dev/null && echo "Installed" || echo "Not Installed")
|
||||
if [ "$have_venv" = "Not Installed" ]; then
|
||||
if [ "$have_pip" = "Not Installed" ]; then
|
||||
echo "python has no venv, and no pip to install virtualenv, cannot continue"
|
||||
exit 1
|
||||
fi
|
||||
echo "python venv module not found, installing virtualenv"
|
||||
python3 -m pip install --user virtualenv
|
||||
python3 -m virtualenv $venv/hbd --system-site-packages $arg
|
||||
else
|
||||
python3 -m venv $venv/hbd --system-site-packages $arg
|
||||
fi
|
||||
. $venv/hbd/bin/activate
|
||||
if [ -n "$arg" ]; then
|
||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py
|
||||
fi
|
||||
deactivate
|
||||
fi
|
||||
|
||||
if [ ! -z "$venv" ]; then
|
||||
. $venv/hbd/bin/activate
|
||||
fi
|
||||
if [ "$what" = "mini" ]; then
|
||||
curl -s -o $where/hbc_mini https://git.wrede.ca/andreas/heartbeat/raw/branch/master/scripts/hbc_mini.py
|
||||
chmod +x $where/hbc_mini
|
||||
else
|
||||
python3 -mpip install --upgrade --index-url https://git.wrede.ca/api/packages/andreas/pypi/simple/ --extra-index-url https://pypi.org/simple hbd[$what]
|
||||
fi
|
||||
|
||||
if [ ! -z "$venv" ]; then
|
||||
echo "linking executables to $where"
|
||||
if [ "$what" = "server" ]; then
|
||||
rm -f $where/hbd
|
||||
ln -sf $(which hbd) $where/hbd
|
||||
elif [ "$what" = "client" ]; then
|
||||
rm -f $where/hbc
|
||||
ln -sf $(which hbc) $where/hbc
|
||||
fi
|
||||
rm -f $where/hb_install.sh
|
||||
ln -sf $(which hb_install.sh) $where/hb_install.sh
|
||||
fi
|
||||
echo "Installation complete. To upgrade, run the following:"
|
||||
echo " $where/hb_install.sh $what"
|
||||
echo "To install on another machine, run the following obtain the install script and run it:"
|
||||
echo "from https://git.wrede.ca/andreas/heartbeat/raw/branch/master/scripts/hb_install.sh"
|
||||
echo "and then run sh hb_install.sh [mini|client]"
|
||||
Executable
+1551
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,49 @@
|
||||
# PyInstaller spec for hbc_windows.exe
|
||||
# Build with: pyinstaller hbc_windows.spec
|
||||
#
|
||||
# Requirements (on Windows):
|
||||
# pip install pyinstaller
|
||||
|
||||
block_cipher = None
|
||||
|
||||
a = Analysis(
|
||||
['hbc_windows.py'],
|
||||
pathex=[],
|
||||
binaries=[],
|
||||
datas=[],
|
||||
hiddenimports=[],
|
||||
hookspath=[],
|
||||
hooksconfig={},
|
||||
runtime_hooks=[],
|
||||
excludes=['tkinter', 'unittest', 'email', 'html', 'http', 'urllib', 'xml'],
|
||||
win_no_prefer_redirects=False,
|
||||
win_private_assemblies=False,
|
||||
cipher=block_cipher,
|
||||
noarchive=False,
|
||||
)
|
||||
|
||||
pyz = PYZ(a.pure, a.zlib_archive, cipher=block_cipher)
|
||||
|
||||
exe = EXE(
|
||||
pyz,
|
||||
a.scripts,
|
||||
a.binaries,
|
||||
a.zipfiles,
|
||||
a.datas,
|
||||
[],
|
||||
name='hbc_windows',
|
||||
debug=False,
|
||||
bootloader_ignore_signals=False,
|
||||
strip=False,
|
||||
upx=False,
|
||||
upx_exclude=[],
|
||||
runtime_tmpdir=None,
|
||||
console=True,
|
||||
disable_windowed_traceback=False,
|
||||
argv_emulation=False,
|
||||
target_arch=None,
|
||||
codesign_identity=None,
|
||||
entitlements_file=None,
|
||||
icon=None,
|
||||
version=None,
|
||||
)
|
||||
@@ -1,88 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
# install the heartbeat client, hbc. The server is installed when the arg 'server' is passed
|
||||
# install the heartbeat client, hbc. The server is installed when the arg 'server' is passed
|
||||
# to the script. The script will install the heartbeat tools in a python
|
||||
# virtual environment in ~/venvs/hbd. The hbd and hbc commands will be
|
||||
# installed from the wheel and symlinked to ~/bin/hbd and ~/bin/hbc,
|
||||
# respectively. If the virtual environment already exists, it will be
|
||||
# reused. The script will also remove any existing symlinks for hbd and hbc
|
||||
# in ~/bin before creating new ones.
|
||||
|
||||
|
||||
# hbd/hbc from wheel and create symlinks for hbd and hbc in ~/bin
|
||||
|
||||
set -e
|
||||
what=$1
|
||||
on_ha=0
|
||||
[ -z "$what" ] && what="client"
|
||||
|
||||
if [ -d /homeassistant ]; then
|
||||
echo "cannot install in HA, run \"docker exec -it homeassistant $0 $@\""
|
||||
exit 1
|
||||
fi
|
||||
if [ -d /config ]; then
|
||||
echo "Installing on HA"
|
||||
where="/config/bin"
|
||||
venv="/config/venvs"
|
||||
on_ha=1
|
||||
else
|
||||
if [ ! -d $HOME/.local/bin ] && [ ! -d $HOME/bin ]; then
|
||||
echo "No suitable bin directory found in PATH, please add either $HOME/.local/bin or $HOME/bin to your PATH"
|
||||
exit 1
|
||||
fi
|
||||
for where in $HOME/bin $HOME/.local/bin notset ; do
|
||||
if echo ":$PATH:" | grep -q ":$where:" ; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$where" = "notset" ]; then
|
||||
echo "No suitable bin directory found in PATH, please add either $HOME/.local/bin or $HOME/bin to your PATH"
|
||||
exit 1
|
||||
fi
|
||||
venv="$HOME/venvs"
|
||||
fi
|
||||
|
||||
echo "Installing heartbeat $what"
|
||||
|
||||
if [ ! -d $venv/hbd ]; then
|
||||
python3 -m pip --version > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
# truenas does not have pip installed by default, so we need to fetch get-pip.py and install pip
|
||||
echo "pip is not installed, fetching get-pip.py and installing pip"
|
||||
arg="--without-pip"
|
||||
fi
|
||||
mkdir -p $venv
|
||||
have_venv=$(python3 -c "import venv" &> /dev/null && echo "Installed" || echo "Not Installed")
|
||||
if [ "$have_venv" = "Not Installed" ]; then
|
||||
echo "python venv module not found, installing virtualenv"
|
||||
python3 -m pip install --user virtualenv
|
||||
python3 -m virtualenv $venv/hbd --system-site-packages $arg
|
||||
else
|
||||
python3 -m venv $venv/hbd --system-site-packages $arg
|
||||
fi
|
||||
. $venv/hbd/bin/activate
|
||||
if [ -n "$arg" ]; then
|
||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py
|
||||
fi
|
||||
deactivate
|
||||
fi
|
||||
|
||||
. $venv/hbd/bin/activate
|
||||
python3 -mpip install --upgrade --index-url https://git.wrede.ca/api/packages/andreas/pypi/simple/ --extra-index-url https://pypi.org/simple hbd[$what]
|
||||
|
||||
if [ "$what" = "server" ]; then
|
||||
rm -f $where/hbd
|
||||
ln -sf $(which hbd) $where/hbd
|
||||
echo "hbd installed, you can run it with \"$where/hbd\" or \"hbd\" if $where is in your PATH"
|
||||
else
|
||||
rm -f $where/hbc
|
||||
ln -sf $(which hbc) $where/hbc
|
||||
if [ $on_ha -eq 1 ]; then
|
||||
echo "restarting hbc "
|
||||
job=$(grep run_hbc configuration.yaml | sed 's/run_hbc://')
|
||||
$job
|
||||
else
|
||||
echo "hbc installed, you can run it with \"$where/hbc\" or \"hbc\" if $where is in your PATH"
|
||||
fi
|
||||
fi
|
||||
@@ -0,0 +1,126 @@
|
||||
#Requires -RunAsAdministrator
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Install hbc_windows.exe as a Windows Service using NSSM.
|
||||
|
||||
.DESCRIPTION
|
||||
Installs the HeartBeat Client as a Windows Service that starts automatically.
|
||||
Requires NSSM (Non-Sucking Service Manager) in PATH or alongside this script.
|
||||
Requires hbc_windows.exe built via: pyinstaller hbc_windows.spec
|
||||
|
||||
.PARAMETER Server
|
||||
HBD server hostname or IP address (required).
|
||||
|
||||
.PARAMETER ExePath
|
||||
Path to hbc_windows.exe. Defaults to the directory containing this script.
|
||||
|
||||
.PARAMETER ServiceName
|
||||
Windows service name. Default: heartbeat-client
|
||||
|
||||
.PARAMETER ConfigFile
|
||||
Path to hbc.json config file. Optional.
|
||||
|
||||
.PARAMETER LogFile
|
||||
Path to log file. Default: C:\ProgramData\heartbeat\hbc.log
|
||||
|
||||
.PARAMETER Interval
|
||||
Heartbeat interval in seconds. Default: 10
|
||||
|
||||
.EXAMPLE
|
||||
.\install_hbc_windows.ps1 -Server hbd.example.com
|
||||
.\install_hbc_windows.ps1 -Server hbd.example.com -ConfigFile C:\ProgramData\heartbeat\hbc.json
|
||||
#>
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$Server,
|
||||
|
||||
[string]$ExePath = "",
|
||||
[string]$ServiceName = "heartbeat-client",
|
||||
[string]$ConfigFile = "",
|
||||
[string]$LogFile = "C:\ProgramData\heartbeat\hbc.log",
|
||||
[int]$Interval = 10
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# Locate hbc_windows.exe
|
||||
if ($ExePath -eq "") {
|
||||
$ExePath = Join-Path $PSScriptRoot "hbc_windows.exe"
|
||||
}
|
||||
if (-not (Test-Path $ExePath)) {
|
||||
Write-Error "hbc_windows.exe not found at: $ExePath`nBuild it first with: pyinstaller hbc_windows.spec"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Locate NSSM
|
||||
$nssm = Get-Command nssm -ErrorAction SilentlyContinue
|
||||
if (-not $nssm) {
|
||||
$nssmLocal = Join-Path $PSScriptRoot "nssm.exe"
|
||||
if (Test-Path $nssmLocal) {
|
||||
$nssm = $nssmLocal
|
||||
} else {
|
||||
Write-Error "nssm.exe not found in PATH or alongside this script.`nDownload from https://nssm.cc/download"
|
||||
exit 1
|
||||
}
|
||||
} else {
|
||||
$nssm = $nssm.Source
|
||||
}
|
||||
|
||||
# Build argument list
|
||||
$args_list = "--daemon $Server"
|
||||
if ($ConfigFile -ne "") {
|
||||
$args_list = "--daemon -c `"$ConfigFile`" $Server"
|
||||
}
|
||||
if ($LogFile -ne "") {
|
||||
$args_list = "$args_list --log-file `"$LogFile`""
|
||||
}
|
||||
|
||||
# Create data directory
|
||||
$dataDir = "C:\ProgramData\heartbeat"
|
||||
if (-not (Test-Path $dataDir)) {
|
||||
New-Item -ItemType Directory -Path $dataDir | Out-Null
|
||||
Write-Host "Created $dataDir"
|
||||
}
|
||||
|
||||
# Remove existing service if present
|
||||
$existing = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
|
||||
if ($existing) {
|
||||
Write-Host "Removing existing service '$ServiceName'..."
|
||||
& $nssm stop $ServiceName 2>$null
|
||||
& $nssm remove $ServiceName confirm
|
||||
}
|
||||
|
||||
# Install service
|
||||
Write-Host "Installing service '$ServiceName'..."
|
||||
& $nssm install $ServiceName $ExePath $args_list
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Error "nssm install failed (exit $LASTEXITCODE)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Configure service
|
||||
& $nssm set $ServiceName DisplayName "HeartBeat Client"
|
||||
& $nssm set $ServiceName Description "Sends heartbeat and plugin metrics to the HBD monitoring server."
|
||||
& $nssm set $ServiceName Start SERVICE_AUTO_START
|
||||
& $nssm set $ServiceName AppStdout (Join-Path $dataDir "nssm_stdout.log")
|
||||
& $nssm set $ServiceName AppStderr (Join-Path $dataDir "nssm_stderr.log")
|
||||
& $nssm set $ServiceName AppRotateFiles 1
|
||||
& $nssm set $ServiceName AppRotateBytes 5242880
|
||||
|
||||
# Start service
|
||||
Write-Host "Starting service '$ServiceName'..."
|
||||
& $nssm start $ServiceName
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Warning "Service installed but failed to start — check logs in $dataDir"
|
||||
} else {
|
||||
Write-Host "Service '$ServiceName' started successfully."
|
||||
Write-Host "Log file: $LogFile"
|
||||
Write-Host ""
|
||||
Write-Host "Useful commands:"
|
||||
Write-Host " nssm status $ServiceName"
|
||||
Write-Host " nssm stop $ServiceName"
|
||||
Write-Host " nssm restart $ServiceName"
|
||||
Write-Host " nssm remove $ServiceName confirm"
|
||||
}
|
||||
@@ -68,8 +68,7 @@ async def test_nagios_runner():
|
||||
print(f" ✓ Collected {len(data)} data points")
|
||||
|
||||
print(f"\n4. Results:")
|
||||
print(f" Overall Status: {data.get('overall_status')} (code: {data.get('overall_status_code')})")
|
||||
print(f" Plugins Executed: {data.get('plugin_count')}")
|
||||
print(f" Data points collected: {len(data)}")
|
||||
|
||||
# Show individual plugin results
|
||||
print(f"\n5. Individual Plugin Results:")
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Tests for the allow_remote_command gate on CMD packets in hbc."""
|
||||
import asyncio
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from hbd.client import main as client_main
|
||||
from hbd.client.config import CLIENT_DEFAULTS, load_config
|
||||
|
||||
|
||||
class FakeConn:
|
||||
"""Stand-in for AsyncConnection that records what the client sends back."""
|
||||
|
||||
def __init__(self):
|
||||
self.sent = []
|
||||
|
||||
async def sendto(self, msg, msg_id="HTB"):
|
||||
self.sent.append(msg)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def conn(monkeypatch):
|
||||
"""A fake connection with remote commands disabled and subprocess booby-trapped."""
|
||||
monkeypatch.setattr(client_main, "allow_remote_command", False)
|
||||
|
||||
def explode(*a, **kw):
|
||||
raise AssertionError("subprocess must not run when the command is refused")
|
||||
|
||||
monkeypatch.setattr(subprocess, "check_output", explode)
|
||||
return FakeConn()
|
||||
|
||||
|
||||
def test_default_is_false():
|
||||
assert CLIENT_DEFAULTS["allow_remote_command"] is False
|
||||
assert load_config("/nonexistent/hbc.yaml")["allow_remote_command"] is False
|
||||
assert client_main.allow_remote_command is False
|
||||
|
||||
|
||||
def test_refused_when_disabled(conn):
|
||||
asyncio.run(client_main.handle_command(conn, {"cmd": "id"}))
|
||||
assert conn.sent == [{
|
||||
"service": "command",
|
||||
"msg": "Refused: allow_remote_command is false",
|
||||
}]
|
||||
|
||||
|
||||
def test_refusal_reported_for_every_attempt(conn):
|
||||
for _ in range(3):
|
||||
asyncio.run(client_main.handle_command(conn, {"cmd": "rm -rf /"}))
|
||||
assert len(conn.sent) == 3
|
||||
|
||||
|
||||
def test_empty_command_is_ignored_either_way(conn, monkeypatch):
|
||||
asyncio.run(client_main.handle_command(conn, {}))
|
||||
monkeypatch.setattr(client_main, "allow_remote_command", True)
|
||||
asyncio.run(client_main.handle_command(conn, {"cmd": ""}))
|
||||
assert conn.sent == []
|
||||
|
||||
|
||||
def test_executed_when_enabled(conn, monkeypatch):
|
||||
monkeypatch.setattr(client_main, "allow_remote_command", True)
|
||||
monkeypatch.setattr(subprocess, "check_output", lambda *a, **kw: b"uid=0(root)")
|
||||
asyncio.run(client_main.handle_command(conn, {"cmd": "id"}))
|
||||
assert conn.sent == [{"service": "command", "msg": "OK uid=0(root)"}]
|
||||
|
||||
|
||||
def test_failure_reported_when_enabled(conn, monkeypatch):
|
||||
monkeypatch.setattr(client_main, "allow_remote_command", True)
|
||||
|
||||
def fail(*a, **kw):
|
||||
raise subprocess.CalledProcessError(1, "false")
|
||||
|
||||
monkeypatch.setattr(subprocess, "check_output", fail)
|
||||
asyncio.run(client_main.handle_command(conn, {"cmd": "false"}))
|
||||
assert len(conn.sent) == 1
|
||||
assert conn.sent[0]["msg"].startswith("CalledProcessError ")
|
||||
|
||||
|
||||
def test_config_file_can_opt_in(tmp_path):
|
||||
cfg_file = tmp_path / "hbc.yaml"
|
||||
cfg_file.write_text("allow_remote_command: true\n", encoding="utf-8")
|
||||
assert load_config(str(cfg_file))["allow_remote_command"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("configured, expected", [(True, True), (False, False), (None, False)])
|
||||
def test_async_main_sets_the_flag_from_config(monkeypatch, configured, expected):
|
||||
"""async_main must apply the config value before any CMD packet can arrive."""
|
||||
monkeypatch.setattr(client_main, "allow_remote_command", not expected)
|
||||
|
||||
class Bail(Exception):
|
||||
pass
|
||||
|
||||
# gethostname() is the first call after the flag is assigned — bail there so the
|
||||
# test never opens a socket.
|
||||
def bail():
|
||||
raise Bail
|
||||
|
||||
monkeypatch.setattr(client_main.socket, "gethostname", bail)
|
||||
|
||||
cfg = {} if configured is None else {"allow_remote_command": configured}
|
||||
with pytest.raises(Bail):
|
||||
asyncio.run(client_main.async_main(object(), cfg))
|
||||
assert client_main.allow_remote_command is expected
|
||||
@@ -0,0 +1,28 @@
|
||||
from hbd.client.main import build_heartbeat_msg
|
||||
|
||||
|
||||
class _Conn:
|
||||
ackcount = 3
|
||||
rtts = [0, 12.5]
|
||||
|
||||
|
||||
class _Conn2:
|
||||
def __init__(self, flags):
|
||||
self._flags = flags
|
||||
|
||||
def flags(self):
|
||||
return self._flags
|
||||
|
||||
|
||||
def test_heartbeat_without_connectivity_is_unchanged():
|
||||
assert build_heartbeat_msg(_Conn(), 60) == {"acks": 3, "rtt": 12.5, "interval": 60}
|
||||
|
||||
|
||||
def test_heartbeat_carries_presence_flags():
|
||||
msg = build_heartbeat_msg(_Conn(), 60, _Conn2({"p4": True, "p6": False}))
|
||||
assert msg == {"acks": 3, "rtt": 12.5, "interval": 60, "p4": True, "p6": False}
|
||||
|
||||
|
||||
def test_heartbeat_omits_unknown_flags():
|
||||
msg = build_heartbeat_msg(_Conn(), 60, _Conn2({}))
|
||||
assert "p4" not in msg and "p6" not in msg
|
||||
@@ -0,0 +1,247 @@
|
||||
"""Tests for ownership rules and scoped config merges (hbd.server.config_access)."""
|
||||
import pytest
|
||||
from hbd.server import config_access as ca
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Visibility helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_is_global_when_no_owner():
|
||||
assert ca.is_global({"type": "pushover"})
|
||||
assert ca.is_global({"type": "email", "owner": ""})
|
||||
assert ca.is_global(None) # non-dict is treated as global
|
||||
assert not ca.is_global({"type": "email", "owner": "alice"})
|
||||
|
||||
|
||||
def test_user_can_use_global_or_own():
|
||||
assert ca.user_can_use({"type": "pushover"}, "alice")
|
||||
assert ca.user_can_use({"owner": "alice"}, "alice")
|
||||
assert not ca.user_can_use({"owner": "bob"}, "alice")
|
||||
|
||||
|
||||
HOSTS = {
|
||||
"web1": {"owner": "alice", "watch": True},
|
||||
"web2": {"owner": "bob", "managers": ["alice"]},
|
||||
"web3": {"owner": "bob", "managers": "carol"}, # string manager form
|
||||
"web4": {"watch": True}, # unowned
|
||||
}
|
||||
|
||||
|
||||
def test_user_hosts_owner_and_manager():
|
||||
assert set(ca.user_hosts(HOSTS, "alice")) == {"web1", "web2"}
|
||||
assert set(ca.user_hosts(HOSTS, "carol")) == {"web3"}
|
||||
assert ca.user_hosts(HOSTS, "dave") == {}
|
||||
assert ca.user_hosts(None, "alice") == {}
|
||||
|
||||
|
||||
CHANNELS = {
|
||||
"global_ch": {"type": "pushover"},
|
||||
"alice_ch": {"type": "email", "owner": "alice"},
|
||||
"bob_ch": {"type": "signal", "owner": "bob"},
|
||||
}
|
||||
|
||||
|
||||
def test_user_channels_global_plus_own():
|
||||
assert set(ca.user_channels(CHANNELS, "alice")) == {"global_ch", "alice_ch"}
|
||||
assert set(ca.user_channels(CHANNELS, "dave")) == {"global_ch"}
|
||||
|
||||
|
||||
TCS = {
|
||||
"default": {"thresholds": {}},
|
||||
"alice_tc": {"owner": "alice", "thresholds": {}},
|
||||
}
|
||||
|
||||
|
||||
def test_user_threshold_configs_global_plus_own():
|
||||
assert set(ca.user_threshold_configs(TCS, "alice")) == {"default", "alice_tc"}
|
||||
assert set(ca.user_threshold_configs(TCS, "bob")) == {"default"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# merge_hosts_scoped
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MERGE_HOSTS = {
|
||||
"mine": {"owner": "alice", "watch": True, "notification_channels": ["global_ch"]},
|
||||
"managed": {"owner": "bob", "managers": ["alice"], "watch": True,
|
||||
"notification_channels": ["bob_ch"]},
|
||||
"foreign": {"owner": "bob", "watch": False},
|
||||
}
|
||||
MERGE_CHANNELS = {
|
||||
"global_ch": {"type": "pushover"},
|
||||
"alice_ch": {"type": "email", "owner": "alice"},
|
||||
"bob_ch": {"type": "signal", "owner": "bob"},
|
||||
}
|
||||
MERGE_TCS = {"alice_tc": {"owner": "alice"}, "bob_tc": {"owner": "bob"}}
|
||||
|
||||
|
||||
def _merge_hosts(payload, existing=None):
|
||||
return ca.merge_hosts_scoped(
|
||||
existing if existing is not None else dict(MERGE_HOSTS),
|
||||
payload, "alice", MERGE_CHANNELS, MERGE_TCS,
|
||||
)
|
||||
|
||||
|
||||
def _full_payload(**overrides):
|
||||
"""Payload covering alice's full visible subset, with per-host overrides."""
|
||||
p = {
|
||||
"mine": {"owner": "alice", "watch": True, "notification_channels": ["global_ch"]},
|
||||
"managed": {"watch": True, "notification_channels": ["bob_ch"]},
|
||||
}
|
||||
p.update(overrides)
|
||||
return p
|
||||
|
||||
|
||||
def test_merge_hosts_preserves_foreign_hosts():
|
||||
result = _merge_hosts(_full_payload())
|
||||
assert result["foreign"] == {"owner": "bob", "watch": False}
|
||||
|
||||
|
||||
def test_merge_hosts_manager_edits_settings():
|
||||
result = _merge_hosts(_full_payload(managed={"watch": False, "dyndns": True,
|
||||
"notification_channels": ["bob_ch"]}))
|
||||
assert result["managed"]["watch"] is False
|
||||
assert result["managed"]["dyndns"] is True
|
||||
# access fields carried over untouched
|
||||
assert result["managed"]["owner"] == "bob"
|
||||
assert result["managed"]["managers"] == ["alice"]
|
||||
|
||||
|
||||
def test_merge_hosts_manager_cannot_change_owner():
|
||||
with pytest.raises(ca.ScopedMergeError, match="managed"):
|
||||
_merge_hosts(_full_payload(managed={"watch": True, "owner": "alice"}))
|
||||
|
||||
|
||||
def test_merge_hosts_manager_cannot_change_managers():
|
||||
with pytest.raises(ca.ScopedMergeError, match="managers"):
|
||||
_merge_hosts(_full_payload(managed={"watch": True, "managers": ["alice", "carol"]}))
|
||||
|
||||
|
||||
def test_merge_hosts_manager_same_access_values_ok():
|
||||
# Submitting unchanged access fields is not a violation.
|
||||
result = _merge_hosts(_full_payload(managed={"watch": True, "owner": "bob",
|
||||
"managers": ["alice"],
|
||||
"notification_channels": ["bob_ch"]}))
|
||||
assert result["managed"]["owner"] == "bob"
|
||||
|
||||
|
||||
def test_merge_hosts_manager_cannot_delete():
|
||||
payload = {"mine": {"owner": "alice", "watch": True}} # 'managed' missing
|
||||
with pytest.raises(ca.ScopedMergeError, match="managed"):
|
||||
_merge_hosts(payload)
|
||||
|
||||
|
||||
def test_merge_hosts_owner_can_delete():
|
||||
payload = _full_payload()
|
||||
del payload["mine"]
|
||||
result = _merge_hosts(payload)
|
||||
assert "mine" not in result
|
||||
assert "managed" in result and "foreign" in result
|
||||
|
||||
|
||||
def test_merge_hosts_owner_can_transfer_ownership():
|
||||
result = _merge_hosts(_full_payload(mine={"owner": "bob", "watch": True}))
|
||||
assert result["mine"]["owner"] == "bob"
|
||||
|
||||
|
||||
def test_merge_hosts_new_host_owner_forced():
|
||||
result = _merge_hosts(_full_payload(newhost={"watch": True, "owner": "bob"}))
|
||||
assert result["newhost"]["owner"] == "alice"
|
||||
|
||||
|
||||
def test_merge_hosts_cannot_touch_foreign_host():
|
||||
with pytest.raises(ca.ScopedMergeError, match="foreign"):
|
||||
_merge_hosts(_full_payload(foreign={"watch": True}))
|
||||
|
||||
|
||||
def test_merge_hosts_cannot_add_foreign_private_channel():
|
||||
with pytest.raises(ca.ScopedMergeError, match="bob_ch"):
|
||||
_merge_hosts(_full_payload(mine={"owner": "alice", "watch": True,
|
||||
"notification_channels": ["bob_ch"]}))
|
||||
|
||||
|
||||
def test_merge_hosts_keeps_preexisting_foreign_assignment():
|
||||
# 'managed' already has bob_ch; keeping it is fine.
|
||||
result = _merge_hosts(_full_payload())
|
||||
assert result["managed"]["notification_channels"] == ["bob_ch"]
|
||||
|
||||
|
||||
def test_merge_hosts_can_add_global_and_own_channel():
|
||||
result = _merge_hosts(_full_payload(mine={"owner": "alice", "watch": True,
|
||||
"notification_channels": ["global_ch", "alice_ch"]}))
|
||||
assert result["mine"]["notification_channels"] == ["global_ch", "alice_ch"]
|
||||
|
||||
|
||||
def test_merge_hosts_threshold_assignment_validated():
|
||||
with pytest.raises(ca.ScopedMergeError, match="bob_tc"):
|
||||
_merge_hosts(_full_payload(mine={"owner": "alice", "watch": True,
|
||||
"threshold_config": ["bob_tc"]}))
|
||||
result = _merge_hosts(_full_payload(mine={"owner": "alice", "watch": True,
|
||||
"threshold_config": ["default", "alice_tc"]}))
|
||||
assert result["mine"]["threshold_config"] == ["default", "alice_tc"]
|
||||
|
||||
|
||||
def test_merge_hosts_rejects_unknown_fields():
|
||||
with pytest.raises(ca.ScopedMergeError, match="sneaky"):
|
||||
_merge_hosts(_full_payload(mine={"owner": "alice", "watch": True, "sneaky": 1}))
|
||||
|
||||
|
||||
def test_merge_hosts_owner_clearing_list_removes_key():
|
||||
result = _merge_hosts(_full_payload(mine={"owner": "alice", "watch": True}))
|
||||
assert "notification_channels" not in result["mine"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# merge_threshold_configs_scoped
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MERGE_EXISTING_TCS = {
|
||||
"default": {"thresholds": {"cpu": {"load": {"warning": 2}}}},
|
||||
"alice_tc": {"owner": "alice", "thresholds": {"cpu": {"load": {"warning": 3}}}},
|
||||
"bob_tc": {"owner": "bob", "thresholds": {}},
|
||||
"global_tc": {"thresholds": {}},
|
||||
}
|
||||
|
||||
|
||||
def _merge_tcs(payload):
|
||||
return ca.merge_threshold_configs_scoped(dict(MERGE_EXISTING_TCS), payload, "alice")
|
||||
|
||||
|
||||
def test_merge_tcs_new_config_owner_forced():
|
||||
result = _merge_tcs({"alice_tc": {"thresholds": {}},
|
||||
"new_tc": {"thresholds": {}, "owner": "bob"}})
|
||||
assert result["new_tc"]["owner"] == "alice"
|
||||
|
||||
|
||||
def test_merge_tcs_edit_own():
|
||||
result = _merge_tcs({"alice_tc": {"thresholds": {"mem": {"used": {"warning": 90}}}}})
|
||||
assert result["alice_tc"]["thresholds"] == {"mem": {"used": {"warning": 90}}}
|
||||
assert result["alice_tc"]["owner"] == "alice"
|
||||
|
||||
|
||||
def test_merge_tcs_delete_own_when_missing():
|
||||
result = _merge_tcs({})
|
||||
assert "alice_tc" not in result
|
||||
|
||||
|
||||
def test_merge_tcs_preserves_global_and_foreign():
|
||||
result = _merge_tcs({"alice_tc": {"thresholds": {}}})
|
||||
assert result["default"] == MERGE_EXISTING_TCS["default"]
|
||||
assert result["bob_tc"] == MERGE_EXISTING_TCS["bob_tc"]
|
||||
assert result["global_tc"] == MERGE_EXISTING_TCS["global_tc"]
|
||||
|
||||
|
||||
def test_merge_tcs_rejects_default():
|
||||
with pytest.raises(ca.ScopedMergeError, match="default"):
|
||||
_merge_tcs({"alice_tc": {"thresholds": {}}, "default": {"thresholds": {}}})
|
||||
|
||||
|
||||
def test_merge_tcs_rejects_global_name_collision():
|
||||
with pytest.raises(ca.ScopedMergeError, match="global_tc"):
|
||||
_merge_tcs({"alice_tc": {"thresholds": {}}, "global_tc": {"thresholds": {}}})
|
||||
|
||||
|
||||
def test_merge_tcs_rejects_foreign_owned():
|
||||
with pytest.raises(ca.ScopedMergeError, match="bob_tc"):
|
||||
_merge_tcs({"alice_tc": {"thresholds": {}}, "bob_tc": {"thresholds": {}}})
|
||||
@@ -0,0 +1,162 @@
|
||||
import glob
|
||||
import os
|
||||
import pytest
|
||||
from hbd.server import configio
|
||||
|
||||
SAMPLE_YAML = """\
|
||||
# Server configuration
|
||||
hbd_port: 50004 # HTTP API port
|
||||
interval: 20
|
||||
users:
|
||||
alice:
|
||||
full_name: Alice Smith
|
||||
admin: true
|
||||
notification_channels:
|
||||
pushover_ops:
|
||||
type: pushover
|
||||
token: abc123
|
||||
"""
|
||||
|
||||
|
||||
def test_read_roundtrip_loads_values(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
assert data["hbd_port"] == 50004
|
||||
assert data["interval"] == 20
|
||||
assert data["users"]["alice"]["full_name"] == "Alice Smith"
|
||||
|
||||
|
||||
def test_write_config_creates_backup(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
data["interval"] = 30
|
||||
configio.write_config(str(f), data)
|
||||
backups = configio.list_backups(str(f))
|
||||
assert len(backups) == 1
|
||||
assert ".bak." in backups[0]
|
||||
|
||||
|
||||
def test_write_config_preserves_comments(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
data["interval"] = 30
|
||||
configio.write_config(str(f), data)
|
||||
content = f.read_text()
|
||||
assert "# Server configuration" in content
|
||||
assert "# HTTP API port" in content
|
||||
|
||||
|
||||
def test_write_config_atomically_replaces_file(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
data["interval"] = 99
|
||||
configio.write_config(str(f), data)
|
||||
assert not (tmp_path / ".hb.yaml.tmp").exists()
|
||||
data2 = configio.read_roundtrip(str(f))
|
||||
assert data2["interval"] == 99
|
||||
|
||||
|
||||
def test_write_config_backup_rotation(tmp_path):
|
||||
cfg = tmp_path / ".hb.yaml"
|
||||
cfg.write_text(SAMPLE_YAML)
|
||||
# Pre-create 10 existing backups with old timestamps
|
||||
for i in range(10):
|
||||
(tmp_path / f".hb.yaml.bak.20260101-{i:06d}").write_text("old")
|
||||
data = configio.read_roundtrip(str(cfg))
|
||||
configio.write_config(str(cfg), data)
|
||||
backups = configio.list_backups(str(cfg))
|
||||
assert len(backups) == 10
|
||||
assert not (tmp_path / ".hb.yaml.bak.20260101-000000").exists()
|
||||
|
||||
|
||||
def test_list_backups_newest_first(tmp_path):
|
||||
cfg = tmp_path / ".hb.yaml"
|
||||
cfg.write_text(SAMPLE_YAML)
|
||||
for i in range(3):
|
||||
(tmp_path / f".hb.yaml.bak.20260101-{i:02d}0000").write_text("b")
|
||||
backups = configio.list_backups(str(cfg))
|
||||
assert len(backups) == 3
|
||||
assert backups == sorted(backups, reverse=True)
|
||||
|
||||
|
||||
def test_apply_structured_section_server_updates_keys(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
configio.apply_structured_section(data, "server", {"interval": 60, "hbd_port": 8080})
|
||||
assert data["interval"] == 60
|
||||
assert data["hbd_port"] == 8080
|
||||
|
||||
|
||||
def test_apply_structured_section_server_ignores_unknown_keys(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
configio.apply_structured_section(data, "server", {"interval": 60, "not_a_key": "x"})
|
||||
assert "not_a_key" not in data
|
||||
|
||||
|
||||
def test_apply_structured_section_users_replaces_dict(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
new_users = {"bob": {"full_name": "Bob Jones", "admin": False}}
|
||||
configio.apply_structured_section(data, "users", new_users)
|
||||
assert "alice" not in data["users"]
|
||||
assert data["users"]["bob"]["full_name"] == "Bob Jones"
|
||||
|
||||
|
||||
def test_apply_yaml_section_notification_channels(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
new_yaml = "email_ops:\n type: email\n recipients: [ops@example.com]\n"
|
||||
configio.apply_yaml_section(data, "notification_channels", new_yaml)
|
||||
assert "email_ops" in data["notification_channels"]
|
||||
assert "pushover_ops" not in data["notification_channels"]
|
||||
|
||||
|
||||
def test_apply_yaml_section_thresholds_maps_to_threshold_configs(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
configio.apply_yaml_section(data, "thresholds", "default:\n cpu: 80\n")
|
||||
assert "threshold_configs" in data
|
||||
assert data["threshold_configs"]["default"]["cpu"] == 80
|
||||
|
||||
|
||||
def test_apply_yaml_section_dns_replaces_each_key(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
configio.apply_yaml_section(
|
||||
data, "dns",
|
||||
"nsupdate_bin: /usr/bin/nsupdate\ndyndomains: [dyn.example.com]\n"
|
||||
)
|
||||
assert data["nsupdate_bin"] == "/usr/bin/nsupdate"
|
||||
assert data["dyndomains"] == ["dyn.example.com"]
|
||||
|
||||
|
||||
def test_apply_yaml_section_unknown_raises(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
with pytest.raises(ValueError, match="Unknown YAML section"):
|
||||
configio.apply_yaml_section(data, "nope", "x: 1\n")
|
||||
|
||||
|
||||
def test_apply_structured_section_unknown_raises(tmp_path):
|
||||
f = tmp_path / ".hb.yaml"
|
||||
f.write_text(SAMPLE_YAML)
|
||||
data = configio.read_roundtrip(str(f))
|
||||
with pytest.raises(ValueError, match="Unknown structured section"):
|
||||
configio.apply_structured_section(data, "nope", {"x": 1})
|
||||
|
||||
|
||||
def test_read_roundtrip_missing_file_raises(tmp_path):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
configio.read_roundtrip(str(tmp_path / "nonexistent.yaml"))
|
||||
@@ -0,0 +1,173 @@
|
||||
import asyncio
|
||||
|
||||
from hbd.client.connectivity import Connectivity, ProbeUnavailable
|
||||
from hbd.client.pingutil import PingUnavailable
|
||||
|
||||
|
||||
class Net:
|
||||
"""Scriptable network: addr None = no route, 'unavailable' = family unsupported."""
|
||||
|
||||
def __init__(self):
|
||||
self.addr = {"ipv4": "192.0.2.5", "ipv6": "2001:db8::5"}
|
||||
self.gw = {"ipv4": "192.0.2.1", "ipv6": "fe80::1"}
|
||||
self.rtt = {"ipv4": 10.0, "ipv6": 20.0} # None = ping fails, 'unavailable' = no ping binary
|
||||
self.pings = []
|
||||
self.now = 1000.0
|
||||
|
||||
def local(self, fam, target):
|
||||
a = self.addr[fam]
|
||||
if a == "unavailable":
|
||||
raise ProbeUnavailable("no such family")
|
||||
return a
|
||||
|
||||
async def ping(self, target, fam):
|
||||
self.pings.append(fam)
|
||||
r = self.rtt[fam]
|
||||
if r == "unavailable":
|
||||
raise PingUnavailable("no ping")
|
||||
return r
|
||||
|
||||
async def gateway(self, fam):
|
||||
return self.gw[fam]
|
||||
|
||||
def clock(self):
|
||||
return self.now
|
||||
|
||||
|
||||
def make(net, **kw):
|
||||
c = Connectivity(local_address_fn=net.local, ping_fn=net.ping, gateway_fn=net.gateway,
|
||||
clock=net.clock, **kw)
|
||||
c._cheap_gateway = True
|
||||
return c
|
||||
|
||||
|
||||
def refresh(c):
|
||||
asyncio.run(c.refresh())
|
||||
|
||||
|
||||
def test_present_with_addr_gateway_rtt():
|
||||
net = Net()
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
s = c.snapshot()["ipv4"]
|
||||
assert (s.present, s.addr, s.gateway, s.rtt) == (True, "192.0.2.5", "192.0.2.1", 10.0)
|
||||
assert c.flags() == {"p4": True, "p6": True}
|
||||
assert c.as_metric()["ipv6"]["gateway"] == "fe80::1"
|
||||
|
||||
|
||||
def test_no_route_is_immediately_absent():
|
||||
net = Net()
|
||||
net.addr["ipv6"] = None
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
assert c.flags() == {"p4": True, "p6": False}
|
||||
assert net.pings == ["ipv4"] # no ping wasted on a family without a route
|
||||
assert c.snapshot()["ipv6"].addr is None
|
||||
|
||||
|
||||
def test_ping_is_debounced_and_rate_limited():
|
||||
net = Net()
|
||||
c = make(net, probe_interval=60.0, retry_interval=10.0)
|
||||
refresh(c)
|
||||
refresh(c) # same instant: cached, no second ping
|
||||
assert net.pings == ["ipv4", "ipv6"]
|
||||
net.now += 60
|
||||
net.rtt["ipv6"] = None # first failure: still present (debounce)
|
||||
refresh(c)
|
||||
assert c.flags()["p6"] is True
|
||||
net.now += 10 # retry_interval elapsed: second failure flips it
|
||||
refresh(c)
|
||||
assert c.flags()["p6"] is False
|
||||
|
||||
|
||||
def test_first_probe_failing_once_is_unknown():
|
||||
net = Net()
|
||||
net.rtt["ipv4"] = None
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
assert "p4" not in c.flags() and c.flags()["p6"] is True
|
||||
|
||||
|
||||
def test_recovers_on_first_success():
|
||||
net = Net()
|
||||
net.rtt["ipv6"] = None
|
||||
c = make(net, retry_interval=10.0)
|
||||
refresh(c)
|
||||
net.now += 10
|
||||
refresh(c)
|
||||
assert c.flags()["p6"] is False
|
||||
net.rtt["ipv6"] = 25.0
|
||||
net.now += 10
|
||||
refresh(c)
|
||||
assert c.flags()["p6"] is True
|
||||
|
||||
|
||||
def test_address_change_invalidates_cache():
|
||||
net = Net()
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
net.addr["ipv4"] = "198.51.100.7" # moved networks, no time has passed
|
||||
refresh(c)
|
||||
assert net.pings.count("ipv4") == 2
|
||||
assert c.snapshot()["ipv4"].addr == "198.51.100.7"
|
||||
|
||||
|
||||
def test_gateway_change_invalidates_cache():
|
||||
net = Net()
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
net.gw["ipv4"] = "192.0.2.254"
|
||||
refresh(c)
|
||||
assert net.pings.count("ipv4") == 2
|
||||
|
||||
|
||||
def test_route_loss_then_return_pings_immediately():
|
||||
net = Net()
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
net.addr["ipv6"] = None
|
||||
refresh(c)
|
||||
net.addr["ipv6"] = "2001:db8::5"
|
||||
refresh(c)
|
||||
assert net.pings.count("ipv6") == 2 and c.flags()["p6"] is True
|
||||
|
||||
|
||||
def test_missing_ping_binary_leaves_presence_unknown_and_stops_retrying():
|
||||
net = Net()
|
||||
net.rtt["ipv4"] = "unavailable"
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
net.now += 600
|
||||
refresh(c)
|
||||
assert "p4" not in c.flags()
|
||||
assert net.pings.count("ipv4") == 1
|
||||
|
||||
|
||||
def test_unsupported_family_is_unknown():
|
||||
net = Net()
|
||||
net.addr["ipv6"] = "unavailable"
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
assert "p6" not in c.flags() and c.flags()["p4"] is True
|
||||
|
||||
|
||||
def test_from_config():
|
||||
c = Connectivity.from_config({"probe_v4": "9.9.9.9", "probe_interval": 30})
|
||||
assert c.probes["ipv4"] == "9.9.9.9"
|
||||
assert c.probes["ipv6"] == "2606:4700:4700::1111"
|
||||
assert c.probe_interval == 30.0
|
||||
|
||||
|
||||
def test_ping_less_family_stays_unknown_after_route_returns():
|
||||
net = Net()
|
||||
net.rtt["ipv4"] = "unavailable"
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
assert "p4" not in c.flags()
|
||||
net.addr["ipv4"] = None
|
||||
refresh(c)
|
||||
assert c.flags()["p4"] is False # no route is definitive
|
||||
net.addr["ipv4"] = "192.0.2.5"
|
||||
refresh(c)
|
||||
assert "p4" not in c.flags()
|
||||
assert net.pings.count("ipv4") == 1 # missing binary is not retried
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Tests for the dedicated events journal (write path and read path)."""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from hbd.server import journal
|
||||
|
||||
|
||||
def _make_journal(tmp_path, **overrides):
|
||||
cfg = {"journal_dir": str(tmp_path), "journal_file": "events.journal"}
|
||||
cfg.update(overrides)
|
||||
j = journal.MessageJournal(cfg)
|
||||
assert asyncio.run(j.initialize())
|
||||
return j
|
||||
|
||||
|
||||
def _read_lines(tmp_path, name="events.journal"):
|
||||
return (tmp_path / name).read_text(encoding="utf-8").splitlines()
|
||||
|
||||
|
||||
EV1 = {"ts": 1000.0, "host": "h1", "level": "INFO", "service": None, "message": "host up"}
|
||||
EV2 = {"ts": 2000.0, "host": "h2", "level": "CRITICAL", "service": "cpu", "message": "cpu high"}
|
||||
|
||||
|
||||
def test_log_event_writes_one_json_line(tmp_path):
|
||||
j = _make_journal(tmp_path)
|
||||
asyncio.run(j.log_event(EV1))
|
||||
asyncio.run(j.close())
|
||||
lines = _read_lines(tmp_path)
|
||||
assert len(lines) == 1
|
||||
assert json.loads(lines[0]) == EV1
|
||||
|
||||
|
||||
def test_log_event_appends_in_order(tmp_path):
|
||||
j = _make_journal(tmp_path)
|
||||
asyncio.run(j.log_event(EV1))
|
||||
asyncio.run(j.log_event(EV2))
|
||||
asyncio.run(j.close())
|
||||
lines = _read_lines(tmp_path)
|
||||
assert [json.loads(ln)["ts"] for ln in lines] == [1000.0, 2000.0]
|
||||
|
||||
|
||||
def test_log_event_rotates_at_max_size(tmp_path):
|
||||
# max_size fits one serialized event line (~77 bytes) but not two, so the
|
||||
# second write triggers exactly one rotation
|
||||
j = _make_journal(tmp_path, journal_max_size=120)
|
||||
asyncio.run(j.log_event(EV1))
|
||||
asyncio.run(j.log_event(EV2))
|
||||
asyncio.run(j.close())
|
||||
backups = list(tmp_path.glob("events.journal.*"))
|
||||
assert len(backups) == 1
|
||||
assert json.loads(backups[0].read_text().splitlines()[0]) == EV1
|
||||
assert json.loads(_read_lines(tmp_path)[0]) == EV2
|
||||
|
||||
|
||||
def test_log_event_noop_when_disabled(tmp_path):
|
||||
j = journal.MessageJournal(
|
||||
{"journal_dir": str(tmp_path), "journal_file": "events.journal", "journal_enabled": False}
|
||||
)
|
||||
asyncio.run(j.initialize())
|
||||
asyncio.run(j.log_event(EV1))
|
||||
assert not (tmp_path / "events.journal").exists()
|
||||
|
||||
|
||||
def test_backfill_seeds_empty_journal(tmp_path):
|
||||
j = _make_journal(tmp_path)
|
||||
asyncio.run(j.backfill([EV1, EV2]))
|
||||
asyncio.run(j.close())
|
||||
lines = _read_lines(tmp_path)
|
||||
assert len(lines) == 2
|
||||
assert json.loads(lines[0]) == EV1
|
||||
|
||||
|
||||
def test_backfill_skipped_when_journal_nonempty(tmp_path):
|
||||
(tmp_path / "events.journal").write_text(json.dumps(EV1) + "\n")
|
||||
j = _make_journal(tmp_path) # initialize() picks up the existing size
|
||||
asyncio.run(j.backfill([EV2]))
|
||||
asyncio.run(j.close())
|
||||
assert len(_read_lines(tmp_path)) == 1
|
||||
|
||||
|
||||
def test_get_events_journal_uses_events_config_keys(tmp_path):
|
||||
journal._events_journal_instance = None
|
||||
try:
|
||||
ej = journal.get_events_journal(
|
||||
{
|
||||
"journal_dir": str(tmp_path),
|
||||
"events_journal_file": "ev.jsonl",
|
||||
"events_journal_max_size": 12345,
|
||||
"events_journal_max_backups": 3,
|
||||
}
|
||||
)
|
||||
assert ej.journal_file == "ev.jsonl"
|
||||
assert ej.max_size == 12345
|
||||
assert ej.max_backups == 3
|
||||
assert ej.journal_dir == tmp_path
|
||||
# singleton: second call returns the same instance
|
||||
assert journal.get_events_journal() is ej
|
||||
finally:
|
||||
journal._events_journal_instance = None
|
||||
|
||||
|
||||
def test_get_events_journal_defaults(tmp_path):
|
||||
journal._events_journal_instance = None
|
||||
try:
|
||||
ej = journal.get_events_journal({"journal_dir": str(tmp_path)})
|
||||
assert ej.journal_file == "events.journal"
|
||||
assert ej.max_size == 10 * 1024 * 1024
|
||||
assert ej.max_backups == 10
|
||||
finally:
|
||||
journal._events_journal_instance = None
|
||||
|
||||
|
||||
# ---- read path -------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_journal(path, events):
|
||||
path.write_text("".join(json.dumps(e) + "\n" for e in events), encoding="utf-8")
|
||||
|
||||
|
||||
def _evts(*ts_list):
|
||||
return [
|
||||
{"ts": float(t), "host": f"host{i}", "level": "INFO", "service": None, "message": f"msg {t}"}
|
||||
for i, t in enumerate(ts_list)
|
||||
]
|
||||
|
||||
|
||||
def test_read_events_newest_first(tmp_path):
|
||||
_write_journal(tmp_path / "events.journal", _evts(1, 2, 3))
|
||||
events, more = journal.read_events(tmp_path)
|
||||
assert [e["ts"] for e in events] == [3.0, 2.0, 1.0]
|
||||
assert more is False
|
||||
|
||||
|
||||
def test_read_events_limit_and_more(tmp_path):
|
||||
_write_journal(tmp_path / "events.journal", _evts(1, 2, 3))
|
||||
events, more = journal.read_events(tmp_path, limit=2)
|
||||
assert [e["ts"] for e in events] == [3.0, 2.0]
|
||||
assert more is True
|
||||
|
||||
|
||||
def test_read_events_before_cursor(tmp_path):
|
||||
_write_journal(tmp_path / "events.journal", _evts(1, 2, 3))
|
||||
events, _ = journal.read_events(tmp_path, before=3.0)
|
||||
assert [e["ts"] for e in events] == [2.0, 1.0]
|
||||
|
||||
|
||||
def test_read_events_spans_rotated_files(tmp_path):
|
||||
# rotated backup holds the oldest events; current file the newest
|
||||
_write_journal(tmp_path / "events.journal.20260101-000000", _evts(1, 2))
|
||||
_write_journal(tmp_path / "events.journal.20260201-000000", _evts(3, 4))
|
||||
_write_journal(tmp_path / "events.journal", _evts(5, 6))
|
||||
events, more = journal.read_events(tmp_path, limit=10)
|
||||
assert [e["ts"] for e in events] == [6.0, 5.0, 4.0, 3.0, 2.0, 1.0]
|
||||
assert more is False
|
||||
|
||||
|
||||
def test_read_events_pagination_across_files(tmp_path):
|
||||
_write_journal(tmp_path / "events.journal.20260101-000000", _evts(1, 2))
|
||||
_write_journal(tmp_path / "events.journal", _evts(3, 4))
|
||||
page1, more1 = journal.read_events(tmp_path, limit=3)
|
||||
assert [e["ts"] for e in page1] == [4.0, 3.0, 2.0]
|
||||
assert more1 is True
|
||||
page2, more2 = journal.read_events(tmp_path, limit=3, before=page1[-1]["ts"])
|
||||
assert [e["ts"] for e in page2] == [1.0]
|
||||
assert more2 is False
|
||||
|
||||
|
||||
def test_read_events_host_filter_substring_case_insensitive(tmp_path):
|
||||
evs = [
|
||||
{"ts": 1.0, "host": "Wentworth", "level": "INFO", "service": None, "message": "a"},
|
||||
{"ts": 2.0, "host": "winter", "level": "INFO", "service": None, "message": "b"},
|
||||
]
|
||||
_write_journal(tmp_path / "events.journal", evs)
|
||||
events, _ = journal.read_events(tmp_path, host="went")
|
||||
assert [e["host"] for e in events] == ["Wentworth"]
|
||||
|
||||
|
||||
def test_read_events_level_filter_exact_case_insensitive(tmp_path):
|
||||
evs = [
|
||||
{"ts": 1.0, "host": "h", "level": "CRITICAL", "service": None, "message": "a"},
|
||||
{"ts": 2.0, "host": "h", "level": "INFO", "service": None, "message": "b"},
|
||||
]
|
||||
_write_journal(tmp_path / "events.journal", evs)
|
||||
events, _ = journal.read_events(tmp_path, level="critical")
|
||||
assert [e["level"] for e in events] == ["CRITICAL"]
|
||||
|
||||
|
||||
def test_read_events_message_filter(tmp_path):
|
||||
evs = [
|
||||
{"ts": 1.0, "host": "h", "level": "INFO", "service": None, "message": "disk almost full"},
|
||||
{"ts": 2.0, "host": "h", "level": "INFO", "service": None, "message": "all quiet"},
|
||||
]
|
||||
_write_journal(tmp_path / "events.journal", evs)
|
||||
events, _ = journal.read_events(tmp_path, q="Disk")
|
||||
assert [e["ts"] for e in events] == [1.0]
|
||||
|
||||
|
||||
def test_read_events_skips_malformed_lines(tmp_path):
|
||||
p = tmp_path / "events.journal"
|
||||
p.write_text('{"ts": 1.0, "host": "h", "level": "INFO", "message": "ok"}\nnot json\n[1,2]\n')
|
||||
events, _ = journal.read_events(tmp_path)
|
||||
assert [e["ts"] for e in events] == [1.0]
|
||||
|
||||
|
||||
def test_read_events_predicate(tmp_path):
|
||||
_write_journal(tmp_path / "events.journal", _evts(1, 2))
|
||||
events, _ = journal.read_events(tmp_path, predicate=lambda e: e["host"] == "host0")
|
||||
assert [e["host"] for e in events] == ["host0"]
|
||||
|
||||
|
||||
def test_read_events_missing_dir(tmp_path):
|
||||
events, more = journal.read_events(tmp_path / "nope")
|
||||
assert events == [] and more is False
|
||||
|
||||
|
||||
def test_filter_events_over_in_memory_ring():
|
||||
ring = _evts(1, 2, 3) # oldest-first, like data.msgs
|
||||
events, more = journal.filter_events(reversed(ring), limit=2)
|
||||
assert [e["ts"] for e in events] == [3.0, 2.0]
|
||||
assert more is True
|
||||
|
||||
|
||||
def test_filter_events_skips_non_dict_entries():
|
||||
ring = [
|
||||
{"ts": 1.0, "host": "h", "level": "INFO", "service": None, "message": "a"},
|
||||
"legacy string entry",
|
||||
None,
|
||||
{"ts": 2.0, "host": "h", "level": "INFO", "service": None, "message": "b"},
|
||||
]
|
||||
events, more = journal.filter_events(reversed(ring), limit=10)
|
||||
assert [e["ts"] for e in events] == [2.0, 1.0]
|
||||
assert more is False
|
||||
|
||||
|
||||
# ---- eventlog wiring --------------------------------------------------------
|
||||
|
||||
|
||||
def test_eventlog_writes_to_events_journal(tmp_path):
|
||||
from hbd.server import data, notify
|
||||
|
||||
journal._events_journal_instance = None
|
||||
saved_msgs = data.msgs
|
||||
data.msgs = []
|
||||
try:
|
||||
ej = journal.get_events_journal({"journal_dir": str(tmp_path)})
|
||||
|
||||
async def scenario():
|
||||
await ej.initialize()
|
||||
notify.setup({}, loop=asyncio.get_running_loop())
|
||||
notify.eventlog("h1", "INFO", "hello world")
|
||||
await asyncio.sleep(0.05) # let the scheduled journal write run
|
||||
await ej.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
lines = _read_lines(tmp_path)
|
||||
assert len(lines) == 1
|
||||
ev = json.loads(lines[0])
|
||||
assert ev["host"] == "h1"
|
||||
assert ev["level"] == "INFO"
|
||||
assert ev["message"] == "hello world"
|
||||
assert isinstance(ev["ts"], float)
|
||||
finally:
|
||||
journal._events_journal_instance = None
|
||||
data.msgs = saved_msgs
|
||||
notify._loop = None
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Tests for flap detection (hbd.server.flap) and its hook in notify.send_notification."""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from hbd.server import flap, hbdclass, notify, users as users_mod
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_flap_state():
|
||||
"""Every test starts with an empty state and a 3-in-10-minutes config."""
|
||||
flap._state.clear()
|
||||
flap.setup({"flap_count": 3, "flap_interval": 10})
|
||||
yield
|
||||
flap._state.clear()
|
||||
flap.setup({})
|
||||
|
||||
|
||||
def alerts(n, host="h1", service="cpu", level="CRITICAL"):
|
||||
return [flap.observe(host, service, level) for _ in range(n)]
|
||||
|
||||
|
||||
def advance(seconds, host="h1", service="cpu"):
|
||||
"""Backdate a key's recorded timestamps to simulate the clock moving on."""
|
||||
st = flap._state[(host, service)]
|
||||
st["events"] = [t - seconds for t in st["events"]]
|
||||
if st["ok_since"] is not None:
|
||||
st["ok_since"] -= seconds
|
||||
|
||||
|
||||
# --- tripping ---------------------------------------------------------------
|
||||
|
||||
def test_alerts_up_to_flap_count_pass():
|
||||
assert alerts(3) == [flap.PASS] * 3
|
||||
assert flap.flapping_services("h1") == []
|
||||
|
||||
|
||||
def test_exceeding_flap_count_trips_then_suppresses():
|
||||
assert alerts(5) == [flap.PASS, flap.PASS, flap.PASS, flap.TRIP, flap.SUPPRESS]
|
||||
assert flap.flapping_services("h1") == ["cpu"]
|
||||
|
||||
|
||||
def test_warning_and_critical_both_count():
|
||||
flap.observe("h1", "cpu", "WARNING")
|
||||
flap.observe("h1", "cpu", "CRITICAL")
|
||||
flap.observe("h1", "cpu", "WARNING")
|
||||
assert flap.observe("h1", "cpu", "CRITICAL") == flap.TRIP
|
||||
|
||||
|
||||
def test_alerts_outside_the_window_do_not_count():
|
||||
alerts(3)
|
||||
advance(11 * 60) # older than flap_interval
|
||||
assert alerts(3) == [flap.PASS] * 3
|
||||
assert flap.flapping_services("h1") == []
|
||||
|
||||
|
||||
def test_recover_alone_never_trips():
|
||||
assert [flap.observe("h1", "cpu", "RECOVER") for _ in range(5)] == [flap.PASS] * 5
|
||||
|
||||
|
||||
# --- suppression while flapping --------------------------------------------
|
||||
|
||||
def test_recover_and_info_are_suppressed_while_flapping():
|
||||
alerts(4)
|
||||
assert flap.observe("h1", "cpu", "RECOVER") == flap.SUPPRESS
|
||||
assert flap.observe("h1", "cpu", "INFO") == flap.SUPPRESS
|
||||
|
||||
|
||||
def test_info_passes_when_not_flapping():
|
||||
assert flap.observe("h1", "cpu", "INFO") == flap.PASS
|
||||
|
||||
|
||||
# --- clearing ---------------------------------------------------------------
|
||||
|
||||
def test_clears_silently_one_interval_after_recover():
|
||||
alerts(4)
|
||||
flap.observe("h1", "cpu", "RECOVER")
|
||||
advance(10 * 60)
|
||||
assert flap.flapping_services("h1") == []
|
||||
assert flap.observe("h1", "cpu", "CRITICAL") == flap.PASS
|
||||
|
||||
|
||||
def test_still_flapping_before_the_interval_elapses():
|
||||
alerts(4)
|
||||
flap.observe("h1", "cpu", "RECOVER")
|
||||
advance(9 * 60)
|
||||
assert flap.flapping_services("h1") == ["cpu"]
|
||||
assert flap.observe("h1", "cpu", "CRITICAL") == flap.SUPPRESS
|
||||
|
||||
|
||||
def test_alert_during_the_quiet_window_keeps_it_flapping():
|
||||
alerts(4)
|
||||
flap.observe("h1", "cpu", "RECOVER")
|
||||
advance(9 * 60)
|
||||
flap.observe("h1", "cpu", "CRITICAL") # resets the quiet window
|
||||
advance(2 * 60)
|
||||
assert flap.flapping_services("h1") == ["cpu"]
|
||||
|
||||
|
||||
# --- host removal ------------------------------------------------------------
|
||||
|
||||
def test_clear_host_drops_flapping_state_even_without_a_recover():
|
||||
# A host dropped mid-flap (no RECOVER ever received) must not stay
|
||||
# flapping forever: nothing will ever set ok_since for it again.
|
||||
alerts(4)
|
||||
assert flap.flapping_services("h1") == ["cpu"]
|
||||
flap.clear_host("h1")
|
||||
assert flap.flapping_services("h1") == []
|
||||
assert flap.observe("h1", "cpu", "CRITICAL") == flap.PASS
|
||||
|
||||
|
||||
def test_clear_host_only_affects_the_named_host():
|
||||
alerts(4, host="h1")
|
||||
alerts(4, host="h2")
|
||||
flap.clear_host("h1")
|
||||
assert flap.flapping_services("h1") == []
|
||||
assert flap.flapping_services("h2") == ["cpu"]
|
||||
|
||||
|
||||
# --- keying -----------------------------------------------------------------
|
||||
|
||||
def test_services_and_hosts_are_tracked_independently():
|
||||
alerts(4, service="cpu")
|
||||
assert flap.observe("h1", "disk", "CRITICAL") == flap.PASS
|
||||
assert flap.observe("h2", "cpu", "CRITICAL") == flap.PASS
|
||||
assert flap.flapping_services("h1") == ["cpu"]
|
||||
assert flap.flapping_services("h2") == []
|
||||
|
||||
|
||||
def test_host_level_events_use_the_empty_service_key():
|
||||
alerts(4, service="")
|
||||
assert flap.flapping_services("h1") == [""]
|
||||
|
||||
|
||||
# --- disabled ---------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("cfg", [
|
||||
{"flap_count": 0, "flap_interval": 10},
|
||||
{"flap_count": 3, "flap_interval": 0},
|
||||
{},
|
||||
])
|
||||
def test_disabled_never_suppresses(cfg):
|
||||
flap.setup(cfg)
|
||||
assert alerts(20) == [flap.PASS] * 20
|
||||
assert flap.flapping_services("h1") == []
|
||||
|
||||
|
||||
# --- integration through the real dispatch path -----------------------------
|
||||
|
||||
NOTIFY_CFG = {
|
||||
"flap_count": 3,
|
||||
"flap_interval": 10,
|
||||
"notification_channels": {"ch1": {"type": "pushover", "token": "t", "user": "u"}},
|
||||
"users": {"alice": {"notification_channels": ["ch1"]}},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def delivered(monkeypatch):
|
||||
"""Wire up a host, a user and a stub channel driver; collect what gets delivered."""
|
||||
sent = []
|
||||
monkeypatch.setattr(notify, "_config", dict(notify._config)) # restored on teardown
|
||||
notify.setup(NOTIFY_CFG)
|
||||
users_mod.load_users(NOTIFY_CFG)
|
||||
monkeypatch.setitem(notify._DRIVERS, "pushover", lambda cfg, n: sent.append(n.body) or True)
|
||||
host = hbdclass.Host("flaphost")
|
||||
host.watched = True
|
||||
host.owner = "alice"
|
||||
yield sent, host
|
||||
hbdclass.Host.hosts.pop("flaphost", None)
|
||||
users_mod.load_users({})
|
||||
|
||||
|
||||
def _notify(level, service, body):
|
||||
n = notify.Notification(title=f"[{level}] flaphost", body=body, level=level, service=service)
|
||||
asyncio.run(notify.send_notification("flaphost", n))
|
||||
|
||||
|
||||
def test_marker_on_the_tripping_notification_then_nothing(delivered):
|
||||
sent, host = delivered
|
||||
for i in range(5):
|
||||
_notify("CRITICAL", "cpu", f"cpu = 9{i}")
|
||||
_notify("RECOVER", "cpu", "cpu = 12")
|
||||
|
||||
assert sent == ["cpu = 90", "cpu = 91", "cpu = 92", f"cpu = 93 {flap.FLAP_MARKER}"]
|
||||
assert host.stateinfo()["flapping"] == ["cpu"]
|
||||
|
||||
|
||||
def test_a_flapping_service_does_not_silence_another(delivered):
|
||||
sent, host = delivered
|
||||
for i in range(5):
|
||||
_notify("CRITICAL", "cpu", "cpu = 99")
|
||||
sent.clear()
|
||||
_notify("CRITICAL", "disk", "disk = 91")
|
||||
assert sent == ["disk = 91"]
|
||||
|
||||
|
||||
def test_notifications_resume_after_the_state_clears(delivered):
|
||||
sent, host = delivered
|
||||
for i in range(5):
|
||||
_notify("CRITICAL", "cpu", "cpu = 99")
|
||||
_notify("RECOVER", "cpu", "cpu = 12")
|
||||
advance(10 * 60, host="flaphost")
|
||||
assert host.stateinfo()["flapping"] == []
|
||||
sent.clear()
|
||||
_notify("CRITICAL", "cpu", "cpu = 99")
|
||||
assert sent == ["cpu = 99"]
|
||||
|
||||
|
||||
def test_stateinfo_flapping_survives_json_encoding(delivered):
|
||||
sent, host = delivered
|
||||
for i in range(5):
|
||||
_notify("CRITICAL", "", "IPv4 overdue") # host-level, empty service key
|
||||
info = host.stateinfo()
|
||||
assert info["flapping"] == [""]
|
||||
assert json.loads(json.dumps(info))["flapping"] == [""]
|
||||
@@ -0,0 +1,75 @@
|
||||
import asyncio
|
||||
import struct
|
||||
|
||||
from hbd.client import gateway
|
||||
|
||||
PROC_V4 = (
|
||||
"Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n"
|
||||
"wlan0\t00000000\t0100A8C0\t0003\t0\t0\t600\t00000000\t0\t0\t0\n"
|
||||
"eth0\t00000000\t0101A8C0\t0003\t0\t0\t100\t00000000\t0\t0\t0\n"
|
||||
"wlan0\t0000A8C0\t00000000\t0001\t0\t0\t600\t00FFFFFF\t0\t0\t0\n"
|
||||
)
|
||||
|
||||
PROC_V6 = (
|
||||
"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "
|
||||
"fe800000000000000000000000000001 00000400 00000001 00000000 00000003 wlan0\n"
|
||||
"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "
|
||||
"00000000000000000000000000000000 ffffffff 00000001 00000000 00200200 lo\n"
|
||||
"fe800000000000000000000000000000 40 00000000000000000000000000000000 00 "
|
||||
"00000000000000000000000000000000 00000100 00000001 00000000 00000001 wlan0\n"
|
||||
)
|
||||
|
||||
MACOS = """ route to: default
|
||||
destination: default
|
||||
mask: default
|
||||
gateway: 192.168.1.1
|
||||
interface: en0
|
||||
"""
|
||||
|
||||
WIN_V4 = """IPv4 Route Table
|
||||
Active Routes:
|
||||
Network Destination Netmask Gateway Interface Metric
|
||||
0.0.0.0 0.0.0.0 192.168.1.1 192.168.1.50 25
|
||||
0.0.0.0 0.0.0.0 10.0.0.1 10.0.0.9 50
|
||||
127.0.0.0 255.0.0.0 On-link 127.0.0.1 331
|
||||
"""
|
||||
|
||||
WIN_V6 = """IPv6 Route Table
|
||||
Active Routes:
|
||||
If Metric Network Destination Gateway
|
||||
12 281 ::/0 fe80::1
|
||||
1 331 ::1/128 On-link
|
||||
"""
|
||||
|
||||
|
||||
def test_proc_v4_picks_lowest_metric_gateway():
|
||||
assert gateway.parse_proc_route_v4(PROC_V4) == "192.168.1.1" # eth0, metric 100
|
||||
|
||||
|
||||
def test_proc_v4_none_without_default_route():
|
||||
assert gateway.parse_proc_route_v4(PROC_V4.splitlines()[0] + "\n") is None
|
||||
|
||||
|
||||
def test_proc_v6_skips_reject_and_non_default():
|
||||
assert gateway.parse_proc_route_v6(PROC_V6) == "fe80::1"
|
||||
|
||||
|
||||
def test_macos():
|
||||
assert gateway.parse_macos_route(MACOS) == "192.168.1.1"
|
||||
assert gateway.parse_macos_route("route: writing to routing socket: not in table") is None
|
||||
|
||||
|
||||
def test_windows_v4_lowest_metric():
|
||||
assert gateway.parse_windows_route(WIN_V4, "ipv4") == "192.168.1.1"
|
||||
|
||||
|
||||
def test_windows_v6():
|
||||
assert gateway.parse_windows_route(WIN_V6, "ipv6") == "fe80::1"
|
||||
|
||||
|
||||
def test_default_gateway_never_raises_on_parser_error(monkeypatch):
|
||||
"""default_gateway catches all exceptions; never raises even if parser fails."""
|
||||
monkeypatch.setattr("sys.platform", "linux")
|
||||
monkeypatch.setattr(gateway, "parse_proc_route_v4", lambda _: (_ for _ in ()).throw(struct.error))
|
||||
result = asyncio.run(gateway.default_gateway("ipv4"))
|
||||
assert result is None
|
||||
@@ -20,7 +20,7 @@ def test_handle_cmd_sends_command():
|
||||
import hbdclass
|
||||
|
||||
ctx = {
|
||||
"config": {"watchhosts": [], "dyndnshosts": []},
|
||||
"config": {"watchhosts": []},
|
||||
"hbdclass": hbdclass,
|
||||
"log": dummy_noop,
|
||||
"email": dummy_noop,
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
"""Connectivity probe in the standalone scripts/hbc_mini.py (stdlib-only single-file client)."""
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
_PATH = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "hbc_mini.py"
|
||||
_SPEC = importlib.util.spec_from_file_location("hbc_mini", _PATH)
|
||||
mini = importlib.util.module_from_spec(_SPEC)
|
||||
sys.modules["hbc_mini"] = mini # dataclasses resolve their module by name
|
||||
_SPEC.loader.exec_module(mini)
|
||||
|
||||
PROC_V4 = (
|
||||
"Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n"
|
||||
"wlan0\t00000000\t0100A8C0\t0003\t0\t0\t600\t00000000\t0\t0\t0\n"
|
||||
"eth0\t00000000\t0101A8C0\t0003\t0\t0\t100\t00000000\t0\t0\t0\n"
|
||||
"wlan0\t0000A8C0\t00000000\t0001\t0\t0\t600\t00FFFFFF\t0\t0\t0\n"
|
||||
)
|
||||
PROC_V6 = (
|
||||
"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "
|
||||
"fe800000000000000000000000000001 00000400 00000001 00000000 00000003 wlan0\n"
|
||||
"00000000000000000000000000000000 00 00000000000000000000000000000000 00 "
|
||||
"00000000000000000000000000000000 ffffffff 00000001 00000000 00200200 lo\n"
|
||||
"fe800000000000000000000000000000 40 00000000000000000000000000000000 00 "
|
||||
"00000000000000000000000000000000 00000100 00000001 00000000 00000001 wlan0\n"
|
||||
)
|
||||
ROUTE_GET = """ route to: 0.0.0.0
|
||||
destination: 0.0.0.0
|
||||
mask: 0.0.0.0
|
||||
gateway: 192.0.2.1
|
||||
fib: 0
|
||||
interface: vtnet0
|
||||
"""
|
||||
WIN_V4 = """IPv4 Route Table
|
||||
Active Routes:
|
||||
Network Destination Netmask Gateway Interface Metric
|
||||
0.0.0.0 0.0.0.0 192.168.1.1 192.168.1.50 25
|
||||
0.0.0.0 0.0.0.0 10.0.0.1 10.0.0.9 50
|
||||
127.0.0.0 255.0.0.0 On-link 127.0.0.1 331
|
||||
"""
|
||||
WIN_V6 = """IPv6 Route Table
|
||||
Active Routes:
|
||||
If Metric Network Destination Gateway
|
||||
12 281 ::/0 fe80::1
|
||||
1 331 ::1/128 On-link
|
||||
"""
|
||||
LINUX_PING = """PING 1.1.1.1 (1.1.1.1) 56(84) bytes of data.
|
||||
64 bytes from 1.1.1.1: icmp_seq=1 ttl=57 time=11.2 ms
|
||||
|
||||
--- 1.1.1.1 ping statistics ---
|
||||
1 packets transmitted, 1 received, 0% packet loss, time 0ms
|
||||
rtt min/avg/max/mdev = 11.234/11.234/11.234/0.000 ms
|
||||
"""
|
||||
FREEBSD_PING = """PING 1.1.1.1 (1.1.1.1): 56 data bytes
|
||||
64 bytes from 1.1.1.1: icmp_seq=0 ttl=59 time=4.006 ms
|
||||
|
||||
--- 1.1.1.1 ping statistics ---
|
||||
1 packets transmitted, 1 packets received, 0.0% packet loss, 1 packets out of wait time
|
||||
round-trip min/avg/max/stddev = 4.006/4.006/4.006/0.000 ms
|
||||
"""
|
||||
WIN_PING = """Pinging 1.1.1.1 with 32 bytes of data:
|
||||
Reply from 1.1.1.1: bytes=32 time=12ms TTL=57
|
||||
|
||||
Ping statistics for 1.1.1.1:
|
||||
Packets: Sent = 1, Received = 1, Lost = 0 (0% loss),
|
||||
"""
|
||||
LOST_PING = """--- 1.1.1.1 ping statistics ---
|
||||
1 packets transmitted, 0 received, 100% packet loss, time 0ms
|
||||
"""
|
||||
|
||||
|
||||
# ---- gateway parsers ---------------------------------------------------------------
|
||||
|
||||
def test_proc_v4_lowest_metric_wins():
|
||||
assert mini._parse_proc_route_v4(PROC_V4) == "192.168.1.1" # eth0, metric 100
|
||||
|
||||
|
||||
def test_proc_v4_none_without_default_route():
|
||||
assert mini._parse_proc_route_v4(PROC_V4.splitlines()[0] + "\n") is None
|
||||
|
||||
|
||||
def test_proc_v6_skips_reject_and_non_default():
|
||||
assert mini._parse_proc_route_v6(PROC_V6) == "fe80::1"
|
||||
|
||||
|
||||
def test_route_get_bsd_and_macos():
|
||||
assert mini._parse_route_get(ROUTE_GET) == "192.0.2.1"
|
||||
assert mini._parse_route_get("route: writing to routing socket: not in table") is None
|
||||
|
||||
|
||||
def test_windows_routes():
|
||||
assert mini._parse_windows_route(WIN_V4, "ipv4") == "192.168.1.1"
|
||||
assert mini._parse_windows_route(WIN_V6, "ipv6") == "fe80::1"
|
||||
|
||||
|
||||
def test_default_gateway_never_raises(monkeypatch):
|
||||
monkeypatch.setattr(sys, "platform", "linux")
|
||||
monkeypatch.setattr(mini, "_parse_proc_route_v4", lambda t: 1 / 0)
|
||||
assert asyncio.run(mini._default_gateway("ipv4")) is None
|
||||
|
||||
|
||||
# ---- ping ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("platform,family,expected", [
|
||||
("linux", "ipv4", ["ping", "-c", "1", "-W", "2", "-4", "h"]),
|
||||
("linux", "ipv6", ["ping", "-c", "1", "-W", "2", "-6", "h"]),
|
||||
("freebsd15", "ipv4", ["ping", "-c", "1", "-W", "2000", "h"]),
|
||||
("freebsd15", "ipv6", ["ping6", "-c", "1", "h"]),
|
||||
("darwin", "ipv4", ["ping", "-c", "1", "-W", "2000", "h"]),
|
||||
("darwin", "ipv6", ["ping6", "-c", "1", "h"]),
|
||||
("win32", "ipv4", ["ping", "-n", "1", "-w", "2000", "-4", "h"]),
|
||||
("win32", "ipv6", ["ping", "-n", "1", "-w", "2000", "-6", "h"]),
|
||||
])
|
||||
def test_probe_ping_command(monkeypatch, platform, family, expected):
|
||||
monkeypatch.setattr(sys, "platform", platform)
|
||||
assert mini._probe_ping_command("h", family) == expected
|
||||
|
||||
|
||||
def test_parse_ping_rtt():
|
||||
assert mini._parse_ping_rtt(LINUX_PING) == 11.234
|
||||
assert mini._parse_ping_rtt(FREEBSD_PING) == 4.006
|
||||
assert mini._parse_ping_rtt(WIN_PING) == 12.0
|
||||
assert mini._parse_ping_rtt(LOST_PING) is None
|
||||
assert mini._parse_ping_rtt("ping: connect: Network is unreachable") is None
|
||||
|
||||
|
||||
# ---- probe state machine ------------------------------------------------------------
|
||||
|
||||
class Net:
|
||||
"""Scriptable network: addr None = no route, 'unavailable' = family unsupported."""
|
||||
|
||||
def __init__(self):
|
||||
self.addr = {"ipv4": "192.0.2.5", "ipv6": "2001:db8::5"}
|
||||
self.gw = {"ipv4": "192.0.2.1", "ipv6": "fe80::1"}
|
||||
self.rtt = {"ipv4": 10.0, "ipv6": 20.0} # None = ping fails, 'unavailable' = no ping binary
|
||||
self.pings = []
|
||||
self.now = 1000.0
|
||||
|
||||
def local(self, fam, target):
|
||||
a = self.addr[fam]
|
||||
if a == "unavailable":
|
||||
raise mini._ProbeUnavailable("no such family")
|
||||
return a
|
||||
|
||||
async def ping(self, target, fam):
|
||||
self.pings.append(fam)
|
||||
r = self.rtt[fam]
|
||||
if r == "unavailable":
|
||||
raise mini._PingUnavailable("no ping")
|
||||
return r
|
||||
|
||||
async def gateway(self, fam):
|
||||
return self.gw[fam]
|
||||
|
||||
def clock(self):
|
||||
return self.now
|
||||
|
||||
|
||||
def make(net, **kw):
|
||||
c = mini._Connectivity(local_address_fn=net.local, ping_fn=net.ping, gateway_fn=net.gateway,
|
||||
clock=net.clock, **kw)
|
||||
c._cheap_gateway = True
|
||||
return c
|
||||
|
||||
|
||||
def refresh(c):
|
||||
asyncio.run(c.refresh())
|
||||
|
||||
|
||||
def test_present_with_addr_gateway_rtt():
|
||||
net = Net()
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
s = c.snapshot()["ipv4"]
|
||||
assert (s.present, s.addr, s.gateway, s.rtt) == (True, "192.0.2.5", "192.0.2.1", 10.0)
|
||||
assert c.flags() == {"p4": True, "p6": True}
|
||||
assert c.as_metric()["ipv6"]["gateway"] == "fe80::1"
|
||||
|
||||
|
||||
def test_no_route_is_immediately_absent():
|
||||
net = Net()
|
||||
net.addr["ipv6"] = None
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
assert c.flags() == {"p4": True, "p6": False}
|
||||
assert net.pings == ["ipv4"]
|
||||
assert c.snapshot()["ipv6"].addr is None
|
||||
|
||||
|
||||
def test_ping_is_debounced_and_rate_limited():
|
||||
net = Net()
|
||||
c = make(net, probe_interval=60.0, retry_interval=10.0)
|
||||
refresh(c)
|
||||
refresh(c)
|
||||
assert net.pings == ["ipv4", "ipv6"]
|
||||
net.now += 60
|
||||
net.rtt["ipv6"] = None
|
||||
refresh(c)
|
||||
assert c.flags()["p6"] is True
|
||||
net.now += 10
|
||||
refresh(c)
|
||||
assert c.flags()["p6"] is False
|
||||
|
||||
|
||||
def test_first_probe_failing_once_is_unknown():
|
||||
net = Net()
|
||||
net.rtt["ipv4"] = None
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
assert "p4" not in c.flags() and c.flags()["p6"] is True
|
||||
|
||||
|
||||
def test_recovers_on_first_success():
|
||||
net = Net()
|
||||
net.rtt["ipv6"] = None
|
||||
c = make(net, retry_interval=10.0)
|
||||
refresh(c)
|
||||
net.now += 10
|
||||
refresh(c)
|
||||
assert c.flags()["p6"] is False
|
||||
net.rtt["ipv6"] = 25.0
|
||||
net.now += 10
|
||||
refresh(c)
|
||||
assert c.flags()["p6"] is True
|
||||
|
||||
|
||||
def test_address_change_invalidates_cache():
|
||||
net = Net()
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
net.addr["ipv4"] = "198.51.100.7"
|
||||
refresh(c)
|
||||
assert net.pings.count("ipv4") == 2
|
||||
assert c.snapshot()["ipv4"].addr == "198.51.100.7"
|
||||
|
||||
|
||||
def test_gateway_change_invalidates_cache():
|
||||
net = Net()
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
net.gw["ipv4"] = "192.0.2.254"
|
||||
refresh(c)
|
||||
assert net.pings.count("ipv4") == 2
|
||||
|
||||
|
||||
def test_route_loss_then_return_pings_immediately():
|
||||
net = Net()
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
net.addr["ipv6"] = None
|
||||
refresh(c)
|
||||
net.addr["ipv6"] = "2001:db8::5"
|
||||
refresh(c)
|
||||
assert net.pings.count("ipv6") == 2 and c.flags()["p6"] is True
|
||||
|
||||
|
||||
def test_missing_ping_binary_leaves_presence_unknown_and_stops_retrying():
|
||||
net = Net()
|
||||
net.rtt["ipv4"] = "unavailable"
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
net.now += 600
|
||||
refresh(c)
|
||||
assert "p4" not in c.flags()
|
||||
assert net.pings.count("ipv4") == 1
|
||||
|
||||
|
||||
def test_ping_less_family_stays_unknown_after_route_returns():
|
||||
net = Net()
|
||||
net.rtt["ipv4"] = "unavailable"
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
assert "p4" not in c.flags()
|
||||
net.addr["ipv4"] = None
|
||||
refresh(c)
|
||||
assert c.flags()["p4"] is False # no route is definitive
|
||||
net.addr["ipv4"] = "192.0.2.5"
|
||||
refresh(c)
|
||||
assert "p4" not in c.flags() and net.pings.count("ipv4") == 1
|
||||
|
||||
|
||||
def test_unsupported_family_is_unknown():
|
||||
net = Net()
|
||||
net.addr["ipv6"] = "unavailable"
|
||||
c = make(net)
|
||||
refresh(c)
|
||||
assert "p6" not in c.flags() and c.flags()["p4"] is True
|
||||
|
||||
|
||||
def test_from_config():
|
||||
c = mini._Connectivity.from_config({"probe_v4": "9.9.9.9", "probe_interval": 30})
|
||||
assert c.probes["ipv4"] == "9.9.9.9"
|
||||
assert c.probes["ipv6"] == "2606:4700:4700::1111"
|
||||
assert c.probe_interval == 30.0
|
||||
|
||||
|
||||
def test_local_address_on_this_host_never_raises():
|
||||
# Real UDP connect(): either an address string, None (no route) or _ProbeUnavailable.
|
||||
for fam, target in (("ipv4", "1.1.1.1"), ("ipv6", "2606:4700:4700::1111")):
|
||||
try:
|
||||
r = mini._local_address(fam, target)
|
||||
except mini._ProbeUnavailable:
|
||||
continue
|
||||
assert r is None or isinstance(r, str)
|
||||
|
||||
|
||||
# ---- heartbeat message and plugin wiring ---------------------------------------------
|
||||
|
||||
class _Conn:
|
||||
ackcount = 3
|
||||
rtts = [0.0, 12.5]
|
||||
|
||||
|
||||
class _FlagsProbe:
|
||||
def __init__(self, flags):
|
||||
self._flags = flags
|
||||
|
||||
def flags(self):
|
||||
return self._flags
|
||||
|
||||
|
||||
def test_heartbeat_without_probe_is_unchanged():
|
||||
assert mini._build_heartbeat_msg(_Conn(), 60) == {"acks": 3, "rtt": 12.5, "interval": 60}
|
||||
|
||||
|
||||
def test_heartbeat_carries_presence_flags():
|
||||
msg = mini._build_heartbeat_msg(_Conn(), 60, _FlagsProbe({"p4": True, "p6": False}))
|
||||
assert msg == {"acks": 3, "rtt": 12.5, "interval": 60, "p4": True, "p6": False}
|
||||
|
||||
|
||||
def test_heartbeat_omits_unknown_flags():
|
||||
msg = mini._build_heartbeat_msg(_Conn(), 60, _FlagsProbe({}))
|
||||
assert "p4" not in msg and "p6" not in msg
|
||||
|
||||
|
||||
def test_flags_survive_the_wire_encoding():
|
||||
msg = mini._build_heartbeat_msg(_Conn(), 60, _FlagsProbe({"p4": True, "p6": False}))
|
||||
out = mini._stodict(mini._dicttos("HTB", dict(msg, name="h", id=1)))
|
||||
assert out["p4"] == 1 and out["p6"] == 0
|
||||
|
||||
|
||||
class _FakeProbe:
|
||||
def as_metric(self):
|
||||
return {"ipv4": {"present": True, "addr": "192.0.2.5", "gateway": "192.0.2.1", "rtt": 9.0},
|
||||
"ipv6": {"present": False, "addr": None, "gateway": None, "rtt": None}}
|
||||
|
||||
|
||||
def test_plugin_reports_connectivity_alongside_interfaces(monkeypatch):
|
||||
monkeypatch.setattr(mini, "_probe", _FakeProbe())
|
||||
monkeypatch.setattr(mini.platform, "system", lambda: "Linux")
|
||||
plugin = mini.NetworkMonitorPlugin({})
|
||||
assert asyncio.run(plugin.initialize()) is True
|
||||
data = asyncio.run(plugin._collect_metrics())
|
||||
assert data["connectivity"]["ipv4"]["present"] is True
|
||||
assert data["connectivity"]["ipv6"]["present"] is False
|
||||
|
||||
|
||||
def test_plugin_probe_only_mode_on_non_linux(monkeypatch):
|
||||
monkeypatch.setattr(mini, "_probe", _FakeProbe())
|
||||
monkeypatch.setattr(mini.platform, "system", lambda: "FreeBSD")
|
||||
plugin = mini.NetworkMonitorPlugin({})
|
||||
assert asyncio.run(plugin.initialize()) is True
|
||||
assert list(asyncio.run(plugin._collect_metrics())) == ["connectivity"]
|
||||
|
||||
|
||||
def test_plugin_still_skips_on_non_linux_without_probe(monkeypatch):
|
||||
monkeypatch.setattr(mini, "_probe", None)
|
||||
monkeypatch.setattr(mini.platform, "system", lambda: "FreeBSD")
|
||||
plugin = mini.NetworkMonitorPlugin({})
|
||||
assert asyncio.run(plugin.initialize()) is False
|
||||
assert plugin.skip_reason
|
||||
|
||||
|
||||
def test_plugin_omits_connectivity_without_probe(monkeypatch):
|
||||
monkeypatch.setattr(mini, "_probe", None)
|
||||
monkeypatch.setattr(mini.platform, "system", lambda: "Linux")
|
||||
plugin = mini.NetworkMonitorPlugin({})
|
||||
assert asyncio.run(plugin.initialize()) is True
|
||||
assert "connectivity" not in asyncio.run(plugin._collect_metrics())
|
||||
@@ -0,0 +1,120 @@
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from hbd.server import hbdclass
|
||||
from hbd.server.hbdclass import Connection as C
|
||||
|
||||
|
||||
def _host(name):
|
||||
hbdclass.Host.hosts.pop(name, None)
|
||||
return hbdclass.Host(name)
|
||||
|
||||
|
||||
def _conn(host, afam, state):
|
||||
addr = "192.0.2.1" if afam == "IPv4" else "2001:db8::1"
|
||||
conn, _ = host.conndata(0, addr, None, 1000.0)
|
||||
conn.state = state
|
||||
return conn
|
||||
|
||||
|
||||
CASES = [
|
||||
# v4 state, v6 state, presence flags, expected status
|
||||
(C.UP, C.UP, {}, "up"),
|
||||
(C.UP, C.OVERDUE, {}, "simplex"), # both present (legacy) => simplex
|
||||
(C.UP, C.OVERDUE, {"IPv6": True}, "simplex"),
|
||||
(C.UP, C.OVERDUE, {"IPv6": False}, "up"), # v6 not present => not monitored
|
||||
(C.OVERDUE, C.OVERDUE, {}, "overdue"),
|
||||
(C.OVERDUE, C.ABSENT, {"IPv6": False}, "overdue"), # only present family is overdue
|
||||
(C.UP, C.ABSENT, {"IPv6": False}, "up"),
|
||||
(C.DOWN, C.UP, {}, "down"),
|
||||
(C.UNKNOWN, C.UNKNOWN, {}, "unknown"),
|
||||
(C.OVERDUE, C.UNKNOWN, {}, "overdue"),
|
||||
(C.UP, C.UNKNOWN, {}, "up"),
|
||||
(C.OVERDUE, C.UP, {"IPv6": False}, "simplex"), # HTB arriving => present despite flag
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("s4,s6,flags,expected", CASES)
|
||||
def test_status_matrix(s4, s6, flags, expected):
|
||||
h = _host("status-matrix")
|
||||
_conn(h, "IPv4", s4)
|
||||
_conn(h, "IPv6", s6)
|
||||
for afam, present in flags.items():
|
||||
h.set_presence(afam, present, time.time())
|
||||
assert h.status() == expected
|
||||
|
||||
|
||||
def test_status_single_family_host():
|
||||
h = _host("status-v4only")
|
||||
_conn(h, "IPv4", C.UP)
|
||||
assert h.status() == "up"
|
||||
h.connections["IPv4"].state = C.OVERDUE
|
||||
assert h.status() == "overdue"
|
||||
|
||||
|
||||
def test_status_no_connections_is_unknown():
|
||||
assert _host("status-empty").status() == "unknown"
|
||||
|
||||
|
||||
def test_family_present_rules():
|
||||
h = _host("fp")
|
||||
v6 = _conn(h, "IPv6", C.OVERDUE)
|
||||
assert h.family_present("IPv6") is True # never reported => present
|
||||
h.set_presence("IPv6", False, 1.0)
|
||||
assert h.family_present("IPv6") is False
|
||||
v6.state = C.UP # real traffic overrides the flag
|
||||
assert h.family_present("IPv6") is True
|
||||
assert h.flag_present("IPv6") is False
|
||||
|
||||
|
||||
def test_stateinfo_and_statedict_expose_status():
|
||||
h = _host("status-exposed")
|
||||
_conn(h, "IPv4", C.UP)
|
||||
assert h.stateinfo()["status"] == "up"
|
||||
assert h.statedict()["status"] == "up"
|
||||
|
||||
|
||||
def test_absent_connection_statedict_is_neutral():
|
||||
h = _host("absent-dict")
|
||||
conn = _conn(h, "IPv6", C.ABSENT)
|
||||
conn.rtts.append(42.0)
|
||||
d = conn.statedict()
|
||||
assert d["state"] == "absent" and d["rtt"] == "" and d["rttstate"] == "absent"
|
||||
|
||||
|
||||
def test_presence_survives_pickle_and_old_pickles_get_default():
|
||||
import pickle
|
||||
h = _host("pickled")
|
||||
h.set_presence("IPv6", False, 5.0)
|
||||
h2 = pickle.loads(pickle.dumps(h))
|
||||
assert h2.presence == {"IPv6": {"present": False, "ts": 5.0}}
|
||||
del h2.__dict__["presence"] # simulate a pre-feature pickle
|
||||
h2.fixup()
|
||||
assert h2.presence == {}
|
||||
|
||||
|
||||
def _all_absent(name, s4, s6=None):
|
||||
h = _host(name)
|
||||
_conn(h, "IPv4", s4)
|
||||
h.set_presence("IPv4", False, time.time())
|
||||
if s6 is not None:
|
||||
_conn(h, "IPv6", s6)
|
||||
h.set_presence("IPv6", False, time.time())
|
||||
return h
|
||||
|
||||
|
||||
def test_all_absent_single_family_overdue_is_overdue():
|
||||
assert _all_absent("abs-single", C.OVERDUE).status() == "overdue"
|
||||
|
||||
|
||||
def test_all_absent_dual_stack_one_absent_one_overdue_is_overdue():
|
||||
assert _all_absent("abs-dual-od", C.ABSENT, C.OVERDUE).status() == "overdue"
|
||||
|
||||
|
||||
def test_all_absent_dual_stack_v6_up_is_up():
|
||||
assert _all_absent("abs-dual-up", C.ABSENT, C.UP).status() == "up"
|
||||
|
||||
|
||||
def test_all_absent_all_unknown_is_unknown():
|
||||
assert _all_absent("abs-unknown", C.UNKNOWN, C.UNKNOWN).status() == "unknown"
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Tests for the config read/write API helpers in http.py."""
|
||||
import pytest
|
||||
from hbd.server import http
|
||||
|
||||
|
||||
def test_mask_config_for_api_masks_user_passwords():
|
||||
config = {
|
||||
"hbd_port": 50004,
|
||||
"interval": 20,
|
||||
"users": {
|
||||
"alice": {"full_name": "Alice", "admin": True, "password": "pbkdf2:sha256:abc"},
|
||||
},
|
||||
"oauth": {},
|
||||
}
|
||||
result = http._mask_config_for_api(config)
|
||||
assert result["users"]["alice"]["password"] == "•••"
|
||||
assert result["users"]["alice"]["full_name"] == "Alice"
|
||||
|
||||
|
||||
def test_mask_config_for_api_masks_oauth_client_secret():
|
||||
config = {
|
||||
"hbd_port": 50004,
|
||||
"interval": 20,
|
||||
"users": {},
|
||||
"oauth": {
|
||||
"gitea": {"type": "gitea", "url": "https://git.example.com",
|
||||
"client_id": "cid", "client_secret": "verysecret"},
|
||||
},
|
||||
}
|
||||
result = http._mask_config_for_api(config)
|
||||
assert result["oauth"]["gitea"]["client_secret"] == "•••"
|
||||
assert result["oauth"]["gitea"]["client_id"] == "cid"
|
||||
|
||||
|
||||
def test_mask_config_for_api_includes_server_keys():
|
||||
config = {"hbd_port": 50004, "interval": 20, "users": {}, "oauth": {}}
|
||||
result = http._mask_config_for_api(config)
|
||||
assert result["server"]["hbd_port"] == 50004
|
||||
assert result["server"]["interval"] == 20
|
||||
|
||||
|
||||
def test_mask_config_for_api_no_password_in_users_leaves_no_key():
|
||||
config = {
|
||||
"hbd_port": 50004,
|
||||
"users": {"bob": {"full_name": "Bob", "admin": False}},
|
||||
"oauth": {},
|
||||
}
|
||||
result = http._mask_config_for_api(config)
|
||||
assert "password" not in result["users"]["bob"]
|
||||
|
||||
|
||||
# ---- configio integration for write path ----
|
||||
|
||||
def test_write_path_applies_server_section(tmp_path):
|
||||
cfg = tmp_path / ".hb.yaml"
|
||||
cfg.write_text("hbd_port: 50004\ninterval: 20\nusers: {}\n")
|
||||
from hbd.server import configio
|
||||
data = configio.read_roundtrip(str(cfg))
|
||||
configio.apply_structured_section(data, "server", {"interval": 60})
|
||||
configio.write_config(str(cfg), data)
|
||||
data2 = configio.read_roundtrip(str(cfg))
|
||||
assert data2["interval"] == 60
|
||||
assert data2["hbd_port"] == 50004 # unchanged
|
||||
|
||||
|
||||
def test_write_path_applies_yaml_section(tmp_path):
|
||||
cfg = tmp_path / ".hb.yaml"
|
||||
cfg.write_text(
|
||||
"hbd_port: 50004\nnotification_channels:\n old_ch:\n type: email\n"
|
||||
)
|
||||
from hbd.server import configio
|
||||
data = configio.read_roundtrip(str(cfg))
|
||||
configio.apply_yaml_section(data, "notification_channels", "new_ch:\n type: pushover\n")
|
||||
configio.write_config(str(cfg), data)
|
||||
data2 = configio.read_roundtrip(str(cfg))
|
||||
assert "new_ch" in data2["notification_channels"]
|
||||
assert "old_ch" not in data2["notification_channels"]
|
||||
|
||||
|
||||
def test_write_path_hashes_plaintext_password(tmp_path):
|
||||
cfg = tmp_path / ".hb.yaml"
|
||||
cfg.write_text("hbd_port: 50004\nusers:\n alice:\n full_name: Alice\n admin: true\n password: pbkdf2:sha256:old\n")
|
||||
from hbd.server import configio
|
||||
from hbd.server import users as users_mod
|
||||
data = configio.read_roundtrip(str(cfg))
|
||||
# Simulate what the POST handler does: hash plaintext password
|
||||
new_users = {"alice": {"full_name": "Alice", "admin": True, "password": "newplaintext"}}
|
||||
for username, attrs in new_users.items():
|
||||
pw = attrs.get("password", "")
|
||||
if pw and not pw.startswith("pbkdf2:"):
|
||||
attrs["password"] = users_mod.hash_password(pw)
|
||||
configio.apply_structured_section(data, "users", new_users)
|
||||
configio.write_config(str(cfg), data)
|
||||
data2 = configio.read_roundtrip(str(cfg))
|
||||
assert data2["users"]["alice"]["password"].startswith("pbkdf2:")
|
||||
assert data2["users"]["alice"]["password"] != "newplaintext"
|
||||
|
||||
|
||||
def test_rollback_restores_backup(tmp_path):
|
||||
cfg = tmp_path / ".hb.yaml"
|
||||
cfg.write_text("hbd_port: 50004\ninterval: 20\n")
|
||||
from hbd.server import configio
|
||||
# Make a change to create a backup
|
||||
data = configio.read_roundtrip(str(cfg))
|
||||
data["interval"] = 99
|
||||
configio.write_config(str(cfg), data)
|
||||
backups = configio.list_backups(str(cfg))
|
||||
assert len(backups) == 1
|
||||
# Read the backup and write it back (simulating rollback)
|
||||
backup_data = configio.read_roundtrip(backups[0])
|
||||
configio.write_config(str(cfg), backup_data)
|
||||
restored = configio.read_roundtrip(str(cfg))
|
||||
assert restored["interval"] == 20
|
||||
|
||||
|
||||
def test_write_path_preserves_masked_password(tmp_path):
|
||||
"""The "•••" sentinel must preserve the existing hash, not write "•••" to disk."""
|
||||
cfg = tmp_path / ".hb.yaml"
|
||||
original_hash = "pbkdf2:sha256:original_hash"
|
||||
cfg.write_text(
|
||||
f"hbd_port: 50004\nusers:\n alice:\n full_name: Alice\n admin: true\n password: {original_hash}\n"
|
||||
)
|
||||
from hbd.server import configio
|
||||
from hbd.server import users as users_mod
|
||||
data = configio.read_roundtrip(str(cfg))
|
||||
# Simulate what api_config_post does when client sends "•••" back
|
||||
existing_users = data.get("users") or {}
|
||||
users_payload = {"alice": {"full_name": "Alice", "admin": True, "password": "•••"}}
|
||||
for username, attrs in users_payload.items():
|
||||
pw = attrs.get("password", "")
|
||||
if pw and pw != "•••" and not pw.startswith("pbkdf2:"):
|
||||
attrs["password"] = users_mod.hash_password(pw)
|
||||
elif not pw or pw == "•••":
|
||||
existing_hash = (existing_users.get(username) or {}).get("password", "")
|
||||
if existing_hash:
|
||||
attrs["password"] = existing_hash
|
||||
else:
|
||||
attrs.pop("password", None)
|
||||
configio.apply_structured_section(data, "users", users_payload)
|
||||
configio.write_config(str(cfg), data)
|
||||
data2 = configio.read_roundtrip(str(cfg))
|
||||
assert data2["users"]["alice"]["password"] == original_hash, (
|
||||
f"Expected original hash preserved, got: {data2['users']['alice']['password']!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_write_path_preserves_oauth_client_secret(tmp_path):
|
||||
"""The "•••" sentinel for oauth client_secret must preserve the existing secret."""
|
||||
cfg = tmp_path / ".hb.yaml"
|
||||
original_secret = "real_client_secret_value"
|
||||
cfg.write_text(
|
||||
f"hbd_port: 50004\noauth:\n gitea:\n type: gitea\n url: https://git.example.com\n"
|
||||
f" client_id: cid123\n client_secret: {original_secret}\n"
|
||||
)
|
||||
from hbd.server import configio
|
||||
data = configio.read_roundtrip(str(cfg))
|
||||
# Simulate what api_config_post does when client sends "•••" back for client_secret
|
||||
existing_oauth = data.get("oauth") or {}
|
||||
new_oauth = {"gitea": {"type": "gitea", "url": "https://git.example.com", "client_id": "cid123", "client_secret": "•••"}}
|
||||
for name, attrs in new_oauth.items():
|
||||
cs = attrs.get("client_secret", "")
|
||||
if not cs or cs == "•••":
|
||||
existing_cs = (existing_oauth.get(name) or {}).get("client_secret", "")
|
||||
if existing_cs:
|
||||
attrs["client_secret"] = existing_cs
|
||||
else:
|
||||
attrs.pop("client_secret", None)
|
||||
data["oauth"] = new_oauth
|
||||
configio.write_config(str(cfg), data)
|
||||
data2 = configio.read_roundtrip(str(cfg))
|
||||
assert data2["oauth"]["gitea"]["client_secret"] == original_secret, (
|
||||
f"Expected original secret preserved, got: {data2['oauth']['gitea']['client_secret']!r}"
|
||||
)
|
||||
|
||||
|
||||
# ---- threshold form payload shape ----
|
||||
|
||||
def test_build_threshold_configs_new_shape_with_owner():
|
||||
form = {
|
||||
"servers": {
|
||||
"owner": "alice",
|
||||
"metrics": {"cpu_monitor.load_15min": {"operator": ">", "warning": 4.0,
|
||||
"critical": 8.0, "enabled": True}},
|
||||
},
|
||||
}
|
||||
result = http._build_threshold_configs_from_form(form)
|
||||
assert result["servers"]["owner"] == "alice"
|
||||
assert result["servers"]["thresholds"]["cpu_monitor"]["load_15min"]["warning"] == 4.0
|
||||
|
||||
|
||||
def test_build_threshold_configs_empty_owner_means_global():
|
||||
form = {"servers": {"owner": "", "metrics": {"rtt": {"warning": 100.0}}}}
|
||||
result = http._build_threshold_configs_from_form(form)
|
||||
assert "owner" not in result["servers"]
|
||||
|
||||
|
||||
def test_build_threshold_configs_ignores_entries_without_metrics():
|
||||
assert http._build_threshold_configs_from_form({"bad": {"owner": "x"}}) == {}
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Tests for _build_host_info helper in http.py."""
|
||||
from unittest.mock import MagicMock
|
||||
from hbd.server.http import _build_host_info
|
||||
|
||||
|
||||
class _FakeConn:
|
||||
def __init__(self, lastbeat):
|
||||
self.lastbeat = lastbeat
|
||||
|
||||
|
||||
class _FakeHost:
|
||||
def __init__(self, name="myhost", owner=None, managers=None,
|
||||
connections=None, os_data=None, plugin_data=None):
|
||||
self.name = name
|
||||
self.owner = owner
|
||||
self.managers = managers or []
|
||||
self.connections = connections or {}
|
||||
self._os_data = os_data
|
||||
self.plugin_data = plugin_data or {}
|
||||
|
||||
def get_latest_plugin_data(self, plugin_name):
|
||||
if plugin_name == "os_info" and self._os_data is not None:
|
||||
return (1234567890.0, self._os_data)
|
||||
return None
|
||||
|
||||
|
||||
def test_build_host_info_basic_fields():
|
||||
host = _FakeHost(owner="alice", managers=["bob", "carol"])
|
||||
result = _build_host_info(host)
|
||||
assert result["owner"] == "alice"
|
||||
assert result["managers"] == ["bob", "carol"]
|
||||
assert result["hbc_version"] is None
|
||||
assert result["hbc_type"] is None
|
||||
assert result["last_packet"] is None
|
||||
assert result["thresholds"] is None
|
||||
|
||||
|
||||
def test_build_host_info_no_owner():
|
||||
host = _FakeHost()
|
||||
result = _build_host_info(host)
|
||||
assert result["owner"] is None
|
||||
assert result["managers"] == []
|
||||
|
||||
|
||||
def test_build_host_info_reads_hbc_from_os_info():
|
||||
host = _FakeHost(os_data={"hbc_version": "5.3.0", "hbc_type": "full"})
|
||||
result = _build_host_info(host)
|
||||
assert result["hbc_version"] == "5.3.0"
|
||||
assert result["hbc_type"] == "full"
|
||||
|
||||
|
||||
def test_build_host_info_hbc_none_when_no_os_info():
|
||||
host = _FakeHost(os_data=None)
|
||||
result = _build_host_info(host)
|
||||
assert result["hbc_version"] is None
|
||||
assert result["hbc_type"] is None
|
||||
|
||||
|
||||
def test_build_host_info_last_packet_is_max_lastbeat():
|
||||
host = _FakeHost(connections={
|
||||
"IPv4": _FakeConn(1000.0),
|
||||
"IPv6": _FakeConn(2000.0),
|
||||
})
|
||||
result = _build_host_info(host)
|
||||
assert result["last_packet"] == 2000.0
|
||||
|
||||
|
||||
def test_build_host_info_last_packet_none_when_no_connections():
|
||||
host = _FakeHost(connections={})
|
||||
result = _build_host_info(host)
|
||||
assert result["last_packet"] is None
|
||||
|
||||
|
||||
def test_build_host_info_thresholds_none_without_checker():
|
||||
host = _FakeHost()
|
||||
result = _build_host_info(host, threshold_checker=None)
|
||||
assert result["thresholds"] is None
|
||||
|
||||
|
||||
def test_build_host_info_thresholds_sorted_by_metric():
|
||||
from hbd.server.threshold import ThresholdConfig
|
||||
tc_cpu = ThresholdConfig("cpu_monitor.cpu_percent", warning=80.0, critical=95.0)
|
||||
tc_mem = ThresholdConfig("memory_monitor.memory_percent", warning=85.0, critical=98.0)
|
||||
|
||||
checker = MagicMock()
|
||||
checker.get_thresholds_for_host.return_value = {
|
||||
"memory_monitor.memory_percent": tc_mem,
|
||||
"cpu_monitor.cpu_percent": tc_cpu,
|
||||
}
|
||||
|
||||
host = _FakeHost()
|
||||
result = _build_host_info(host, threshold_checker=checker)
|
||||
|
||||
assert result["thresholds"] is not None
|
||||
assert len(result["thresholds"]) == 2
|
||||
assert result["thresholds"][0]["metric"] == "cpu_monitor.cpu_percent"
|
||||
assert result["thresholds"][0]["warning"] == 80.0
|
||||
assert result["thresholds"][0]["critical"] == 95.0
|
||||
assert result["thresholds"][0]["operator"] == ">"
|
||||
assert result["thresholds"][1]["metric"] == "memory_monitor.memory_percent"
|
||||
|
||||
|
||||
def test_build_host_info_thresholds_empty_list_when_no_thresholds():
|
||||
checker = MagicMock()
|
||||
checker.get_thresholds_for_host.return_value = {}
|
||||
host = _FakeHost()
|
||||
result = _build_host_info(host, threshold_checker=checker)
|
||||
assert result["thresholds"] == []
|
||||
|
||||
|
||||
def test_build_host_info_threshold_null_warning_critical():
|
||||
from hbd.server.threshold import ThresholdConfig
|
||||
tc = ThresholdConfig("rtt.myhost", warning=None, critical=500.0)
|
||||
checker = MagicMock()
|
||||
checker.get_thresholds_for_host.return_value = {"rtt.myhost": tc}
|
||||
host = _FakeHost()
|
||||
result = _build_host_info(host, threshold_checker=checker)
|
||||
assert result["thresholds"][0]["warning"] is None
|
||||
assert result["thresholds"][0]["critical"] == 500.0
|
||||
|
||||
|
||||
def test_build_host_info_nagios_operator_serialized():
|
||||
from hbd.server.threshold import ThresholdConfig
|
||||
tc = ThresholdConfig("nagios_runner.check_http", operator="nagios")
|
||||
checker = MagicMock()
|
||||
checker.get_thresholds_for_host.return_value = {"nagios_runner.check_http": tc}
|
||||
host = _FakeHost()
|
||||
result = _build_host_info(host, threshold_checker=checker)
|
||||
assert result["thresholds"][0]["operator"] == "nagios"
|
||||
|
||||
|
||||
def test_build_host_info_covers_suffix_matched_metrics():
|
||||
"""memory_monitor.percent threshold covers swap_percent via suffix match."""
|
||||
from hbd.server.threshold import ThresholdConfig
|
||||
tc_pct = ThresholdConfig("memory_monitor.percent", warning=85.0, critical=95.0)
|
||||
checker = MagicMock()
|
||||
checker.get_thresholds_for_host.return_value = {"memory_monitor.percent": tc_pct}
|
||||
|
||||
host = _FakeHost(
|
||||
connections={},
|
||||
os_data=None,
|
||||
)
|
||||
# Simulate plugin_data with both percent and swap_percent fields
|
||||
host.plugin_data = {
|
||||
"memory_monitor": [(1234567890.0, {
|
||||
"percent": 80.0,
|
||||
"swap_percent": 25.0,
|
||||
"available_mb": 2000,
|
||||
})]
|
||||
}
|
||||
|
||||
result = _build_host_info(host, threshold_checker=checker)
|
||||
assert result["thresholds"] is not None
|
||||
t = result["thresholds"][0]
|
||||
assert t["metric"] == "memory_monitor.percent"
|
||||
assert t["covers"] == ["memory_monitor.swap_percent"]
|
||||
|
||||
|
||||
def test_build_host_info_covers_empty_when_exact_matches_only():
|
||||
"""No covers when all plugin fields match their threshold exactly."""
|
||||
from hbd.server.threshold import ThresholdConfig
|
||||
tc_pct = ThresholdConfig("memory_monitor.percent", warning=85.0, critical=95.0)
|
||||
checker = MagicMock()
|
||||
checker.get_thresholds_for_host.return_value = {"memory_monitor.percent": tc_pct}
|
||||
|
||||
host = _FakeHost()
|
||||
host.plugin_data = {
|
||||
"memory_monitor": [(1234567890.0, {"percent": 80.0})]
|
||||
}
|
||||
|
||||
result = _build_host_info(host, threshold_checker=checker)
|
||||
t = result["thresholds"][0]
|
||||
assert t["covers"] == []
|
||||
|
||||
|
||||
class _FakeConnFull:
|
||||
def __init__(self, afam, addr, state, rtt, statetime, lastbeat, rtts=None):
|
||||
self.afam = afam
|
||||
self.addr = addr
|
||||
self.state = state
|
||||
self.rtts = rtts if rtts is not None else [rtt]
|
||||
self.statetime = statetime
|
||||
self.lastbeat = lastbeat
|
||||
|
||||
|
||||
def test_build_host_info_includes_connections():
|
||||
host = _FakeHost(connections={
|
||||
"IPv4": _FakeConnFull("IPv4", "10.0.0.5", "up", 14.2, 1000.0, 2000.0),
|
||||
"IPv6": _FakeConnFull("IPv6", "fd00::5", "overdue", 0, 1500.0, 1900.0),
|
||||
})
|
||||
result = _build_host_info(host)
|
||||
conns = {c["family"]: c for c in result["connections"]}
|
||||
assert conns["IPv4"] == {"family": "IPv4", "addr": "10.0.0.5", "state": "up",
|
||||
"rtt": 14.2, "statetime": 1000.0, "lastbeat": 2000.0}
|
||||
assert conns["IPv6"]["state"] == "overdue"
|
||||
|
||||
|
||||
def test_build_host_info_connections_empty():
|
||||
host = _FakeHost()
|
||||
result = _build_host_info(host)
|
||||
assert result["connections"] == []
|
||||
|
||||
|
||||
def test_build_host_info_connection_rtt_uses_latest_sample():
|
||||
"""rtt should reflect the most recent sample, not the oldest retained one."""
|
||||
host = _FakeHost(connections={
|
||||
"IPv4": _FakeConnFull("IPv4", "10.0.0.5", "up", None, 1000.0, 2000.0,
|
||||
rtts=[100.0, 50.0, 14.2]),
|
||||
})
|
||||
result = _build_host_info(host)
|
||||
conns = {c["family"]: c for c in result["connections"]}
|
||||
assert conns["IPv4"]["rtt"] == 14.2
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Tests for the plugin-accordion visibility filter in http.py."""
|
||||
from hbd.server.http import _visible_plugin_names
|
||||
|
||||
|
||||
def test_visible_plugin_names_excludes_rtt_history_keys():
|
||||
plugin_data = {
|
||||
"cpu_monitor": [],
|
||||
"rtt_ipv4": [],
|
||||
"rtt_ipv6": [],
|
||||
"network_monitor": [],
|
||||
}
|
||||
result = _visible_plugin_names(plugin_data)
|
||||
assert sorted(result) == ["cpu_monitor", "network_monitor"]
|
||||
|
||||
|
||||
def test_visible_plugin_names_empty_when_no_plugins():
|
||||
assert _visible_plugin_names({}) == []
|
||||
|
||||
|
||||
def test_visible_plugin_names_keeps_names_not_prefixed_with_rtt_():
|
||||
plugin_data = {"nagios_runner": [], "os_info": []}
|
||||
result = _visible_plugin_names(plugin_data)
|
||||
assert sorted(result) == ["nagios_runner", "os_info"]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user