Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f4dd7cb1f | ||
|
|
ae3f2fc70f | ||
|
|
4414967bdc | ||
|
|
e3b0e5041f | ||
|
|
92982c0ca7 | ||
|
|
00fdbcaa5b | ||
|
|
600e68b509 | ||
|
|
b26e853a1c | ||
|
|
36adad5b3b | ||
|
|
7591268a89 | ||
|
|
7afc06a329 | ||
|
|
a3f303e6ba | ||
|
|
16c2922bea | ||
|
|
0343eb4a24 | ||
|
|
fb5570155b | ||
|
|
8d268adf70 | ||
|
|
5d9f161706 | ||
|
|
067feacd59 | ||
|
|
c564fafe73 | ||
|
|
8eeae48964 | ||
|
|
880dc0e33f | ||
|
|
f06b2ef9e8 | ||
|
|
447b9574a3 | ||
|
|
e98cca22f7 | ||
|
|
5fe069a1a2 | ||
|
|
ca2868a888 | ||
|
|
ddb1b2b875 | ||
|
|
39c4d45bd7 | ||
|
|
07fefab861 | ||
|
|
6b4524b30d | ||
|
|
a149a1e285 | ||
|
|
dd939ab86e | ||
|
|
5c6f462081 | ||
|
|
98b50dbbf4 | ||
|
|
960822918a | ||
|
|
a376ebba8d | ||
|
|
f603ef9bd2 | ||
|
|
731545dd0e | ||
|
|
356c77b0b8 | ||
|
|
6282077fe0 | ||
|
|
ddd857173b | ||
|
|
f46f725d12 | ||
|
|
3da6976b53 | ||
|
|
3a0c48e32b | ||
|
|
cf6e19704f | ||
|
|
b0addd7c67 | ||
|
|
32680d34a4 | ||
|
|
a7abdcb5c5 | ||
|
|
7bab15ae52 | ||
|
|
e0443293e9 | ||
|
|
39670f4e63 | ||
|
|
2e88ee2269 | ||
|
|
2ef7d473c3 | ||
|
|
862a9cdea0 | ||
|
|
9351938b15 | ||
|
|
b6ef2fe065 | ||
|
|
d5d2f066b3 | ||
|
|
d9563392c3 | ||
|
|
5f090b9d96 | ||
|
|
3cc1d92eb4 | ||
|
|
2ddba203df | ||
|
|
8a1f412d1d | ||
|
|
40c44f53f1 | ||
|
|
a6fe8546a8 | ||
|
|
e56660454d | ||
|
|
9cbf0ecb13 | ||
|
|
313bbd37ac | ||
|
|
f7320644f3 | ||
|
|
76e11b92f2 | ||
|
|
d39c0da5fe | ||
|
|
832b9d04d8 | ||
|
|
44d5f15a67 | ||
|
|
37b8e35a26 | ||
|
|
fa317a3b78 | ||
|
|
8729fe7038 | ||
|
|
f4231dd5f3 | ||
|
|
c47576637f | ||
|
|
2b9523ec28 | ||
|
|
610ad0af30 | ||
|
|
69b5b410ed | ||
|
|
8b2b0fd9d0 | ||
|
|
756b2323be | ||
|
|
6e7156b42d | ||
|
|
928035df50 | ||
|
|
0f90be659e | ||
|
|
4160e34a96 | ||
|
|
6430d2ddf3 | ||
|
|
4b87a90e76 | ||
|
|
450814daca | ||
|
|
e7786ac5da | ||
|
|
fed71d97d6 | ||
|
|
ba96da9622 | ||
|
|
7f17ddc2ff | ||
|
|
7750c5a303 | ||
|
|
e58530df7d | ||
|
|
fe7143759c | ||
|
|
236b40cfe4 | ||
|
|
4e5bafd26c | ||
|
|
817ae064af | ||
|
|
a00282913b | ||
|
|
d699a29fa9 | ||
|
|
4ce7eacfdd | ||
|
|
1cefc2676e | ||
|
|
668a135e53 | ||
|
|
59e256a042 | ||
|
|
708508157f | ||
|
|
f67fa9baff | ||
|
|
588eb2a792 | ||
|
|
b907343e36 | ||
|
|
e50a3996ae | ||
|
|
e1056a0365 | ||
|
|
1dbe0f8e64 | ||
|
|
12e8812070 | ||
|
|
9b5d8ac9b1 | ||
|
|
500d256d76 | ||
|
|
a7a45bf8c3 | ||
|
|
3e9b052f71 | ||
|
|
7444262985 | ||
|
|
3401cc0dbb | ||
|
|
ab0132a38d | ||
|
|
9e389736f8 | ||
|
|
b64a2a9313 | ||
|
|
a52744a448 | ||
|
|
5e2b04b811 | ||
|
|
8e07b09d7e | ||
|
|
653e018e4f | ||
|
|
c7326da7d9 | ||
|
|
0426a75d8c | ||
|
|
539f25d877 | ||
|
|
3e3099fc6d | ||
|
|
c9f15a3f1c | ||
|
|
6e396ad760 | ||
|
|
2800de0b4a | ||
|
|
15f7e6a64d | ||
|
|
9768d13b88 | ||
|
|
8640d731aa | ||
|
|
de81751e59 | ||
|
|
60c692cefc | ||
|
|
9a0baf3c78 | ||
|
|
55bdb9593a | ||
|
|
2009626fb4 | ||
|
|
18769afd37 | ||
|
|
31db5cf35e | ||
|
|
326f53f23d | ||
|
|
4f9bc8c868 | ||
|
|
259b4a3594 | ||
|
|
8646f68957 | ||
|
|
a4a6c1e3d9 | ||
|
|
0e8250362e | ||
|
|
2f5da9fc5e | ||
|
|
87aeec5999 | ||
|
|
f24500a6b5 | ||
|
|
a7bb183222 | ||
|
|
8207cd7b5f | ||
|
|
11f1eefa8c | ||
|
|
62f496e9f8 | ||
|
|
aef9e7769b | ||
|
|
58c2b9d996 | ||
|
|
2e8bcb630d | ||
|
|
338711181b | ||
|
|
43487f17e7 | ||
|
|
40205bf5c7 | ||
|
|
b95f1a5bb7 | ||
|
|
12f7eb722b | ||
|
|
217bba1b76 | ||
|
|
967e05ed74 | ||
|
|
c20245b0ab | ||
|
|
b9db0c552e | ||
|
|
05045bafa2 | ||
|
|
39f1b5de30 | ||
|
|
b06de6fdd3 | ||
|
|
940d0af35e | ||
|
|
d6d31aa2e3 | ||
|
|
76edfe7577 | ||
|
|
d190029728 | ||
|
|
b8307e7a9d | ||
|
|
a2fdf091f5 | ||
|
|
1914e6f28e | ||
|
|
82cbce9615 | ||
|
|
dbb779b013 | ||
|
|
ca908ee967 | ||
|
|
73c697b6c5 | ||
|
|
3e2357380b | ||
|
|
cc4a103bae | ||
|
|
53fb10fdf5 | ||
|
|
2df2ad18c9 | ||
|
|
b81a0d2a6c | ||
|
|
1a19088cfe | ||
|
|
172f6e950f | ||
|
|
4349ae217a | ||
|
|
b3aa7b585f | ||
|
|
88a3c09b51 | ||
|
|
0504402a8a | ||
|
|
ca58c18802 | ||
|
|
1ddc4b8132 | ||
|
|
5e1720ed32 | ||
|
|
77f127fe60 | ||
|
|
54fbd8d73d | ||
|
|
7ab17e26e2 | ||
|
|
28f5fa951c | ||
|
|
37f1c58969 | ||
|
|
f006077a71 | ||
|
|
d9fc8d632f | ||
|
|
f640574e4f | ||
|
|
9a19424279 | ||
|
|
ca8ba84e65 | ||
|
|
f3d08d1c9e | ||
|
|
1e4263b793 | ||
|
|
e931acb9f5 | ||
|
|
018409e71d | ||
|
|
1824f637b4 | ||
|
|
a534c06b26 | ||
|
|
d7b5c97a4e | ||
|
|
ae447ac4a6 | ||
|
|
d44ce3d124 | ||
|
|
b1985d0eb2 | ||
|
|
de778f680f | ||
|
|
d7b368c7c6 | ||
|
|
e790663f9f | ||
|
|
475319e248 | ||
|
|
ca5ef384a8 | ||
|
|
c93dbdc0f4 | ||
|
|
3a546a1e5c | ||
|
|
74c89d098c | ||
|
|
3301dbfe34 | ||
|
|
d00d903e7d | ||
|
|
babb5d61aa | ||
|
|
11d1c718b3 | ||
|
|
a99b6b54c7 | ||
|
|
8da3d550eb | ||
|
|
a76d0fc840 | ||
|
|
94cbb31c48 | ||
|
|
ae60844a8a | ||
|
|
49fa310361 | ||
|
|
28e2180f7b | ||
|
|
ce0590f015 | ||
|
|
f50acca509 | ||
|
|
72fc82b91f | ||
|
|
46f8c32c0b | ||
|
|
691f62aa69 | ||
|
|
cffc9805f9 | ||
|
|
917d6a401b | ||
|
|
2bd3a9beb6 | ||
|
|
5523c60866 | ||
|
|
ab37ac7194 | ||
|
|
f811a19d80 | ||
|
|
6239825f43 | ||
|
|
b56245bb23 | ||
|
|
331c4e804d | ||
|
|
9fd945a481 | ||
|
|
26df08eeff | ||
|
|
5819dd6b25 | ||
|
|
6fb67f8615 | ||
|
|
e70ae6f176 | ||
|
|
a77f6d380c |
@@ -10,36 +10,48 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
# - name: Set up Python
|
|
||||||
# uses: actions/setup-python@v5
|
|
||||||
# with:
|
|
||||||
# python-version: '3.11'
|
|
||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
# Use a generic run step for FreeBSD if actions/setup-python
|
|
||||||
# fails in restricted environments.
|
|
||||||
run: |
|
run: |
|
||||||
python3 --version
|
python3 --version
|
||||||
python3 -m ensurepip --upgrade
|
python3 -m ensurepip --upgrade
|
||||||
|
|
||||||
- name: Install build tools
|
- name: Install build tools
|
||||||
run: |
|
run: |
|
||||||
python3 -m pip install --upgrade pip
|
python3 -m venv .venv
|
||||||
python3 -m pip install build twine
|
.venv/bin/pip install --upgrade pip
|
||||||
|
.venv/bin/pip install build twine
|
||||||
|
|
||||||
- name: Build package
|
- name: Build package
|
||||||
run: python3 -m build
|
run: .venv/bin/python -m build
|
||||||
|
|
||||||
- name: Extract version from tag
|
- name: Extract version from tag
|
||||||
id: get_version
|
id: get_version
|
||||||
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
||||||
|
- name: Generate changelog
|
||||||
|
id: changelog
|
||||||
|
run: |
|
||||||
|
PREV_TAG=$(git tag --sort=-version:refname | grep -m 1 -v "^${GITHUB_REF#refs/tags/}$")
|
||||||
|
if [ -n "$PREV_TAG" ]; then
|
||||||
|
CHANGELOG=$(git log --pretty=format:"- %s" "${PREV_TAG}..HEAD")
|
||||||
|
else
|
||||||
|
CHANGELOG="Initial release"
|
||||||
|
fi
|
||||||
|
# Write multiline to output
|
||||||
|
{
|
||||||
|
echo "CHANGELOG<<EOF"
|
||||||
|
echo "$CHANGELOG"
|
||||||
|
echo "EOF"
|
||||||
|
} >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
- name: Upload to Gitea PyPI registry
|
- name: Upload to Gitea PyPI registry
|
||||||
env:
|
env:
|
||||||
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
|
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
|
||||||
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
|
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
python3 -m twine upload --repository-url https://git.wrede.ca/api/packages/andreas/pypi dist/*
|
.venv/bin/python3 -m twine upload --repository-url https://git.wrede.ca/api/packages/andreas/pypi dist/*
|
||||||
|
|
||||||
- name: Create release
|
- name: Create release
|
||||||
uses: actions/gitea-release-action@v1
|
uses: actions/gitea-release-action@v1
|
||||||
@@ -48,4 +60,4 @@ jobs:
|
|||||||
dist/*.whl
|
dist/*.whl
|
||||||
dist/*.tar.gz
|
dist/*.tar.gz
|
||||||
title: "Release ${{ steps.get_version.outputs.VERSION }}"
|
title: "Release ${{ steps.get_version.outputs.VERSION }}"
|
||||||
body: "Release version ${{ steps.get_version.outputs.VERSION }}"
|
body: "${{ steps.changelog.outputs.CHANGELOG }}"
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ __pycache__/
|
|||||||
*.pyo
|
*.pyo
|
||||||
.flake8
|
.flake8
|
||||||
.venv/
|
.venv/
|
||||||
|
.continue/
|
||||||
test/
|
test/
|
||||||
build/
|
build/
|
||||||
dist/
|
dist/
|
||||||
@@ -12,3 +13,6 @@ dist/
|
|||||||
ssl/
|
ssl/
|
||||||
uv.lock
|
uv.lock
|
||||||
.hb.yaml
|
.hb.yaml
|
||||||
|
.superpowers/
|
||||||
|
rndc-key
|
||||||
|
docs/superpowers/
|
||||||
|
|||||||
+525
@@ -0,0 +1,525 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to this project are documented here, organized by release.
|
||||||
|
|
||||||
|
## [5.4.0]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- gate remote command execution behind allow_remote_command
|
||||||
|
- flapping detection suppresses notification storms
|
||||||
|
- remove log section from live dashboard (moved to /log)
|
||||||
|
- dedicated /log page with journal-backed history and nav item
|
||||||
|
- GET /api/0/log — paged, filtered events journal API
|
||||||
|
- eventlog writes to dedicated events journal; init/backfill/close wiring
|
||||||
|
- events journal read path with filters and backward pagination
|
||||||
|
- events journal write primitives (log_event, backfill, get_events_journal)
|
||||||
|
- Live Dashboard redesign — 6 columns, hover details, last-alert column
|
||||||
|
- extend the record-row design system to Host Overview, Alerts, and About
|
||||||
|
- unified record-row redesign of the settings page
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- harden events log — non-dict ring entries, off-loop journal reads, no default-config singleton
|
||||||
|
- skip journal scheduling when the event loop is not running
|
||||||
|
- dedup log rows between API seed and websocket tail
|
||||||
|
- journal startup/shutdown lifecycle events (fired outside journal lifetime)
|
||||||
|
- drop typing names not yet used from journal.py import
|
||||||
|
- settings toolbar sticks below the site nav instead of scrolling away
|
||||||
|
- settings page not scrollable — restore html/body overflow override
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.12]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- retire per-channel private flag on profile page
|
||||||
|
- settings UI renders per-user edit rights and owner controls
|
||||||
|
- settings page accessible to all authenticated users
|
||||||
|
- per-user filtering of settings sections
|
||||||
|
- scoped non-admin saves through POST /api/0/config
|
||||||
|
- threshold form payload carries per-config owner
|
||||||
|
- channel ownership rule — owner-presence means private, admin promote/demote
|
||||||
|
- scoped threshold-config merge for non-admin saves
|
||||||
|
- scoped host merge for non-admin config saves
|
||||||
|
- ownership visibility helpers for config entities
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- users loop clobbered requesting username in settings filtering
|
||||||
|
- type annotations for mypy parity with master
|
||||||
|
- don't re-announce boot/shutdown on SIGHUP restart
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.11]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- add Windows hbc client with PyInstaller spec and NSSM install script
|
||||||
|
- clear alerts for individual plugin metrics that disappear between samples
|
||||||
|
- show alerts for all hosts on Alerts page, not just watched
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- declare plugin interval so stale data waits two full intervals
|
||||||
|
- cap event buffer and replay only recent messages on dashboard connect
|
||||||
|
- strip plugin timers when pickling Host to prevent save failure
|
||||||
|
- correct zero-safe pathconf checks and connectivity prefix match
|
||||||
|
- address security vulnerabilities from audit
|
||||||
|
- don't purge connectivity/rtt alerts in purge_stale_alerts
|
||||||
|
- restore connectivity alerts for overdue/unknown/down hosts on startup
|
||||||
|
- clear plugin data and timers on connection UP transition
|
||||||
|
- restore host link from Dashboard to Host Overview
|
||||||
|
- don't set stale timer until two plugin samples establish real interval
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.10]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- clear stale plugin data and persist OAuth users to config
|
||||||
|
- auto-scale CPU history graph Y axis
|
||||||
|
- add CPU usage history graph to CPU Monitor section
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- remove bak file in bumpminor.sh
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.9]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- auto-update CHANGELOG and README in bumpminor.sh
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.8]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Wiki home page with overview and getting started guide
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Release workflow: use `GITHUB_REF`/`GITHUB_OUTPUT` (Gitea Actions uses GitHub-compatible variable names)
|
||||||
|
- Release workflow: replace `head -1` with `grep -m 1` to avoid SIGPIPE (exit 141) in changelog step
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.7]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Dark mode with light/dark/auto theme setting
|
||||||
|
- UNKNOWN level filter in Log of Events
|
||||||
|
- Per-metric grace period input in threshold settings
|
||||||
|
- Replace Dynamic DNS YAML editor with a web form
|
||||||
|
- Sort hosts, thresholds, and channels alphabetically on settings page
|
||||||
|
- Suppress alerts for unwatched hosts
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Preserve log message order when replaying history on connect
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.6]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- MIT license
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Correct ZFS pool status threshold operator and add per-metric grace
|
||||||
|
- Normalize email and domain fields
|
||||||
|
- Move dependencies back under `[project]` in pyproject.toml
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.4]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Run full reload after HTTP config publish, not just `config.reload()`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.3]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Replace YAML threshold editor with a form-based UI
|
||||||
|
- Replace multi-select fields with dual-panel picker on settings page
|
||||||
|
- Nav bar button to publish pending config changes
|
||||||
|
- Host, level, and message filters in Log of Events
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Remove container max-width; stop stretching inputs on settings page
|
||||||
|
|
||||||
|
### Removed
|
||||||
|
- Legacy `dyndnshosts`/`drophosts` config keys
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.2]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Retry DNS resolution indefinitely; add `-4`/`-6` address-family flags to `hbc` and `hbc_mini`
|
||||||
|
- Replace YAML hosts editor with form-based CRUD table
|
||||||
|
- Replace YAML notification channel editor with form-based UI
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Support list-valued `threshold_config` in hosts table
|
||||||
|
- Derive hosts threshold config list from config file keys
|
||||||
|
- Replace channel checkboxes in Users table with multi-select
|
||||||
|
- Support plugin-level `enabled: false` in threshold config
|
||||||
|
- Always populate glance strip for all hosts on page load
|
||||||
|
- Fetch host info on initial page load
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.1]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Host info section in Host Overview (fetched and rendered on card expand)
|
||||||
|
- `GET /api/0/hosts/{hostname}/info` endpoint
|
||||||
|
- Show suffix-matched metric coverage in host info threshold table
|
||||||
|
- Move `hbc_version` and `hbc_type` out of `os_info` into the host info section
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Correct `THRESHOLD_DEFAULTS` metric keys and add missing defaults
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.3.0]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Profile page self-service: change identity, password, and notification channels
|
||||||
|
- Settings page editor with form sections, YAML editors, stage/publish/rollback workflow
|
||||||
|
- Config read API: `GET /api/0/config`, `/section/{name}`, `/backups`
|
||||||
|
- Config write API: `POST /api/0/config`, `POST /api/0/config/rollback`
|
||||||
|
- `configio` module for comment-preserving YAML round-trip writes
|
||||||
|
- Multi-provider OAuth2 login page and generic provider routes
|
||||||
|
- Log login/logout events to the event log with auth source
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- ZFS monitor alerts dropped on restart with wildcard pool thresholds
|
||||||
|
- Preserve OAuth users across config reload
|
||||||
|
- Config API error handling, consistent 403 messages, deduplicated key lists
|
||||||
|
- Validate password body type; coerce `notification_channels` to strings in profile API
|
||||||
|
- Preserve OAuth `client_secret` on roundtrip; harden rollback path validation
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.2.6]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Alerts host-filter field with URL query parameter and notify URL
|
||||||
|
- Optional logo on Gitea OAuth login button
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Show human-readable duration in re-notification messages
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.2.5]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Alert CRITICAL on degraded or suspended ZFS pools (ONLINE=OK, DEGRADED=WARNING, all else=CRITICAL)
|
||||||
|
- Sign in with Gitea button on login page with OAuth2 redirect/callback routes
|
||||||
|
- OAuth2 CSRF state management
|
||||||
|
- Host owner shown in glance strip for admin users
|
||||||
|
- C port of `hbc_mini` (single-file client in `scripts/c/`)
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Use `base_url` config for OAuth redirect URI to handle reverse proxy deployments
|
||||||
|
- Preserve OAuth users across config reload
|
||||||
|
- Escape HTML in login page error display
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.2.4]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- `hbc`/`hbc_mini`: `owner` config field included in `os_info`; server applies to host record
|
||||||
|
- Server requests InfoPlugin refresh when a host has no plugin data
|
||||||
|
- Event log stores structured dicts; filter by user
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Strip `_status_code` suffix from displayed metric names in threshold alerts
|
||||||
|
- Use plain URL in Mattermost plugin metrics link
|
||||||
|
- Fall back to `default_owner` when `os_info` has no owner
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.2.3]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- `hbc`/`hbc_mini`: log name and version at startup
|
||||||
|
- Show metric name inline with hostname in alerts and notifications
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Send shutdown message only if a boot message was previously sent; suppress both on restart
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.2.2]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Retry connection on network error instead of permanently dropping it
|
||||||
|
- Silence `aiohttp.access` log; strip plugin prefix in alerts UI
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.2.1]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Threshold and logging improvements
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.2.0]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- `nagios` operator for direct exit-code severity mapping
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Always show `THRESHOLD_DEFAULTS` in Settings threshold config
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.21]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- `nagios_runner` improvements and alerts page fixes
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.20]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Generic threshold matching for `nagios_runner` with `{check_name}` display support
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Reduce default hysteresis from 10% to 2%
|
||||||
|
- Show recovery threshold in alerts UI
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.19]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Exclude ZFS ARC from `memory_percent`
|
||||||
|
- Add `uptime_seconds` to `cpu_monitor`
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Send boot/shutdown message on the first open connection, not blindly on the first in list
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.18]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Fetch-based Update/Delete buttons with toast notifications on Host Overview
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Settings thresholds show correct per-config metrics; miscellaneous `hbc` fixes
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.17]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Owner Update/Delete buttons on Host Overview; purge stale alerts on reload
|
||||||
|
- Retry `AsyncConnection.open()` indefinitely; drop IPv6 only on early startup failure
|
||||||
|
- Alert pie chart in the nav bar
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Make Alerts page scrollable
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.16]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Generic `ping_monitor` thresholds; round RTT to nearest ms
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.15]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Link hostnames in Live Dashboard to Host Overview
|
||||||
|
- Threshold Configurations section on settings page
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Suppress notifications on alert de-escalation (e.g. CRITICAL→WARNING)
|
||||||
|
- Suppress recover messages for down durations under 4 seconds
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.14]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- ZFS pool renderer in Host Overview
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.13]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- ZFS monitor plugin
|
||||||
|
- Host-level watch flag to suppress notifications
|
||||||
|
- Filter Live Dashboard and Host Overview by owner/manager
|
||||||
|
- Composable `threshold_config` list for per-host threshold layering
|
||||||
|
- Restart on SIGHUP in `hbc` and `hbc_mini`
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Mask `api_password` and `access_token` in settings page
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.12]
|
||||||
|
|
||||||
|
Internal release — no user-visible changes.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.11]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Install under Docker
|
||||||
|
- Clean up install script
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.10]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Synchronize version in `hbc_mini`
|
||||||
|
- Install script no longer overwrites itself
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.9]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Install `hbc_mini` via package or install script
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.8]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Track `hbc` type and version
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Nav bar position
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.7]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- `hbc_mini`: single-file heartbeat client
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Drop dead connections on protocol error
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.6]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Simplify event log usage; fix argument handling
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.5]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Update `hbc` via `hb_install.sh` instead of code patching
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.4]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Redesign Plugin Metrics page as Host Overview
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.3]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Validate absolute command paths at `nagios_runner` init
|
||||||
|
- Async subprocess in `nagios_runner` with stderr capture and signal handling
|
||||||
|
- `skip_reason` field on `Plugin`; surface in `PluginLoader` init messaging
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Use `shlex.split()` for `nagios_runner` path validation to handle quoted paths
|
||||||
|
- Reconfigure logging to syslog after `daemonize()`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.2]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Plugin config lookup shadowed by `CLIENT_DEFAULTS` plugins key
|
||||||
|
- Apply grace period to all threshold alerts before logging/notifying
|
||||||
|
- RECOVER routing: use consistent level name and route via alerted channel
|
||||||
|
- Early reminder notifications and lost recovery notifications
|
||||||
|
- Non-alerting of overdue hosts
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Swiss clock widget in the UI
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.1]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- SMS and Matrix notification channels
|
||||||
|
- CLI commands `stop`, `restart`, and `reload` for `hbd`
|
||||||
|
- WebSocket endpoint at `http://.../ws`
|
||||||
|
- Mobile HTML pages
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Profile not updating
|
||||||
|
- Sortable columns in tables
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.1.0]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Ping monitor plugin
|
||||||
|
- Persist state to pickle file; restart timers on server restart
|
||||||
|
- SIGHUP config reload for `hbd`
|
||||||
|
- Renotify on CRITICAL only; persistent user sessions
|
||||||
|
- RTT count threshold
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- Bogus notification on new clients
|
||||||
|
- Show "overdue" in alerts instead of null
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.0.12]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- User management and settings page
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.0.10]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Publish package to Gitea PyPI registry
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.0.9]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Use `SO_TIMESTAMP` for RTT measurement (Linux, FreeBSD, macOS)
|
||||||
|
- Persist state to pickle file; restart timers on restart
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## [5.0.6]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Major codebase refactoring: restructured into client/server components
|
||||||
|
- Per-client threshold configuration
|
||||||
|
- Display and acknowledge alerts in the UI
|
||||||
|
- Proper `hbc` termination; `hbd` config reloadable at runtime
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
1. Don't assume. Don't hide confusion. Surface tradeoffs.
|
|
||||||
2. Minimum code that solves the problem. Nothing speculative.
|
|
||||||
3. Touch only what you must. Clean up only your own mess.
|
|
||||||
4. Define success criteria. Loop until verified.
|
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
1. Don't assume. Don't hide confusion. Surface tradeoffs.
|
||||||
|
2. Minimum code that solves the problem. Nothing speculative.
|
||||||
|
3. Touch only what you must. Clean up only your own mess.
|
||||||
|
4. Define success criteria. Loop until verified.
|
||||||
|
|
||||||
|
This project's hbd daemon is deployed and in production on host w02
|
||||||
|
under user 'nagios', which you can reach via ssh nagios@w02.
|
||||||
|
The hbc client is used on 30 hosts in multiple networks.
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
# Heartbeat
|
||||||
|
|
||||||
|
Heartbeat is a lightweight host monitoring system built around a simple idea: each machine you want to monitor runs a small client (`hbc`) that sends a UDP "heartbeat" packet to a central server (`hbd`) on a regular interval. If a heartbeat stops arriving, you get notified. Alongside reachability, clients can ship system metrics — CPU, memory, disk, network — and the server will alert you when any of those cross a threshold.
|
||||||
|
|
||||||
|
## How it works
|
||||||
|
|
||||||
|
```
|
||||||
|
[ monitored host ] [ your server ]
|
||||||
|
┌─────────────┐ UDP 50003 ┌────────────────────────┐
|
||||||
|
│ hbc │ ────────────> │ hbd │
|
||||||
|
│ │ │ host state tracking │
|
||||||
|
│ plugins: │ <──────────── │ threshold alerting │
|
||||||
|
│ cpu, mem, │ ACK / CMD │ notifications │
|
||||||
|
│ disk, ... │ │ web dashboard + API │
|
||||||
|
└─────────────┘ └────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
- **hbd** — the server daemon. Tracks which hosts are alive, evaluates metric thresholds, fires notifications, serves the web dashboard and REST API.
|
||||||
|
- **hbc** — the client. Sends heartbeats and plugin data over UDP. Runs on any Linux/BSD/macOS host.
|
||||||
|
- **hbc_mini** — a zero-dependency single-file alternative (`hbc_mini.py` or `hbc_mini.c`) for hosts where you can't install Python packages.
|
||||||
|
|
||||||
|
Notifications can go to Pushover, email, Mattermost, Matrix, Signal, or VoIP.ms SMS. The dashboard shows host connectivity, RTT graphs, active alerts, and per-host plugin metrics in real time via WebSocket.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Getting started
|
||||||
|
|
||||||
|
This tutorial sets up a server on one machine and a client on a second machine. You'll end up with a working dashboard and your first host being monitored.
|
||||||
|
|
||||||
|
### 1. Install the server
|
||||||
|
|
||||||
|
On the machine that will run `hbd`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://git.wrede.ca/andreas/heartbeat.git
|
||||||
|
cd heartbeat
|
||||||
|
python3 -m venv .venv
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install .
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify the install:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbd --help
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Create a server config
|
||||||
|
|
||||||
|
Create `~/.hb.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
hb_port: 50003 # UDP port — clients send heartbeats here
|
||||||
|
hbd_port: 50004 # HTTP port — web dashboard and API
|
||||||
|
ws_port: 50005 # WebSocket port — live dashboard updates
|
||||||
|
|
||||||
|
interval: 20 # Expected heartbeat interval (seconds)
|
||||||
|
grace: 2 # Seconds of slack before a host is considered overdue
|
||||||
|
|
||||||
|
pickfile: ~/.hb.pick
|
||||||
|
pidfile: ~/.hb.pid
|
||||||
|
logfile: ~/.hb.log
|
||||||
|
```
|
||||||
|
|
||||||
|
That's enough to get started. No hosts, no users, no notifications needed yet — the server will accept any client that connects.
|
||||||
|
|
||||||
|
### 3. Start the server
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbd serve -c ~/.hb.yaml -f -v
|
||||||
|
```
|
||||||
|
|
||||||
|
`-f` keeps it in the foreground so you can watch the log. You should see:
|
||||||
|
|
||||||
|
```
|
||||||
|
Heartbeat daemon starting on UDP :50003, HTTP :50004, WS :50005
|
||||||
|
```
|
||||||
|
|
||||||
|
Open `http://your-server:50004/live` in a browser. The dashboard is empty for now.
|
||||||
|
|
||||||
|
### 4. Install the client on a host to monitor
|
||||||
|
|
||||||
|
On the machine you want to monitor (must be able to reach the server on UDP 50003):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install hbd # or: copy scripts/hbc_mini.py if you can't install packages
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Quick start — no config file
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbc your-server.example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
Within a few seconds the server log will show the host checking in, and it will appear on the dashboard.
|
||||||
|
|
||||||
|
#### With a config file
|
||||||
|
|
||||||
|
Create `~/.hbc.yaml` on the client host:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
hb_port: 50003
|
||||||
|
interval: 10 # Send a heartbeat every 10 seconds
|
||||||
|
|
||||||
|
plugins:
|
||||||
|
cpu_monitor:
|
||||||
|
interval: 60
|
||||||
|
memory_monitor:
|
||||||
|
interval: 60
|
||||||
|
disk_monitor:
|
||||||
|
interval: 60
|
||||||
|
```
|
||||||
|
|
||||||
|
Then start the client:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbc -c ~/.hbc.yaml your-server.example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
Send a boot message at startup so the server logs when the host came up:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbc -b -c ~/.hbc.yaml your-server.example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
Run as a daemon (logs go to syslog):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbc -d -b -c ~/.hbc.yaml your-server.example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. View the dashboard
|
||||||
|
|
||||||
|
Open `http://your-server:50004/live`. You'll see the monitored host, its last heartbeat time, and RTT. Click the host name to see plugin metrics.
|
||||||
|
|
||||||
|
Navigate to `/plugins/<hostname>` for CPU, memory, and disk graphs.
|
||||||
|
|
||||||
|
### 6. Add a notification channel (optional)
|
||||||
|
|
||||||
|
Edit `~/.hb.yaml` on the server:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notification_channels:
|
||||||
|
pushover_ops:
|
||||||
|
type: pushover
|
||||||
|
token: YOUR_APP_TOKEN
|
||||||
|
user: YOUR_USER_KEY
|
||||||
|
|
||||||
|
users:
|
||||||
|
alice:
|
||||||
|
password: pbkdf2:sha256:... # generate: hbd passwd alice
|
||||||
|
admin: true
|
||||||
|
notification_channels: [pushover_ops]
|
||||||
|
|
||||||
|
default_owner: alice
|
||||||
|
```
|
||||||
|
|
||||||
|
Generate the password hash:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbd passwd alice
|
||||||
|
```
|
||||||
|
|
||||||
|
Paste the output into the config, then reload:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbd reload
|
||||||
|
```
|
||||||
|
|
||||||
|
Test the channel:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
hbd notify
|
||||||
|
```
|
||||||
|
|
||||||
|
### 7. Set a threshold alert (optional)
|
||||||
|
|
||||||
|
Add to `~/.hb.yaml`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
thresholds:
|
||||||
|
cpu_monitor:
|
||||||
|
cpu_percent:
|
||||||
|
warning: 80.0
|
||||||
|
critical: 90.0
|
||||||
|
disk_monitor:
|
||||||
|
partitions:
|
||||||
|
/:
|
||||||
|
percent:
|
||||||
|
warning: 80.0
|
||||||
|
critical: 90.0
|
||||||
|
```
|
||||||
|
|
||||||
|
Reload: `hbd reload`. The server will now alert when a monitored host crosses these values.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What's next
|
||||||
|
|
||||||
|
| Topic | Where to look |
|
||||||
|
|---|---|
|
||||||
|
| Full server config reference | [README — Server](https://git.wrede.ca/andreas/heartbeat/src/branch/master/README.md#server-hbd) |
|
||||||
|
| Client options and all plugins | [README — Client](https://git.wrede.ca/andreas/heartbeat/src/branch/master/README.md#client-hbc) |
|
||||||
|
| Threshold alerting details | [THRESHOLD_ALERTING.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/THRESHOLD_ALERTING.md) |
|
||||||
|
| Notification channels | [NOTIFICATIONS.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/NOTIFICATIONS.md) |
|
||||||
|
| User accounts and roles | [USERS.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/USERS.md) |
|
||||||
|
| Writing a custom plugin | [PLUGIN_DEVELOPMENT.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/PLUGIN_DEVELOPMENT.md) |
|
||||||
|
| Nagios check integration | [NAGIOS_INTEGRATION.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/NAGIOS_INTEGRATION.md) |
|
||||||
|
| REST API | [HTTP_API.md](https://git.wrede.ca/andreas/heartbeat/src/branch/master/docs/HTTP_API.md) |
|
||||||
|
| Zero-dependency client | [README — hbc_mini](https://git.wrede.ca/andreas/heartbeat/src/branch/master/README.md#hbc_mini--zero-dependency-client) |
|
||||||
+21
@@ -0,0 +1,21 @@
|
|||||||
|
# MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2002 - 2026 Andreas Wrede
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
# Dark Mode
|
||||||
|
|
||||||
|
Every page in the Heartbeat web UI supports light mode, dark mode, and automatic (follows the OS/browser setting). Each user picks their preference independently; it is stored in the browser and takes effect immediately without a page reload.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Choosing a theme
|
||||||
|
|
||||||
|
Open your profile page (`/profile`) and scroll to the **Appearance** section. Click one of the three buttons:
|
||||||
|
|
||||||
|
| Button | Behaviour |
|
||||||
|
|--------|-----------|
|
||||||
|
| **Auto** | Follows the OS or browser dark-mode preference. Updates live if the system setting changes. |
|
||||||
|
| **Light** | Always light, regardless of system setting. |
|
||||||
|
| **Dark** | Always dark, regardless of system setting. |
|
||||||
|
|
||||||
|
The preference is stored in `localStorage` under the key `hbd_theme` and applies to the current browser only. Clearing browser storage resets it to **Auto**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation notes
|
||||||
|
|
||||||
|
### No flash of unstyled content
|
||||||
|
|
||||||
|
A small synchronous `<script>` runs at the very top of `<head>`, before any CSS is parsed, and sets `data-theme="dark"` on `<html>` when the stored preference (or the system setting in auto mode) calls for dark. Because it runs before paint, there is no visible flicker on page load.
|
||||||
|
|
||||||
|
### CSS custom properties
|
||||||
|
|
||||||
|
All colours are expressed as CSS custom properties defined in `head.html`:
|
||||||
|
|
||||||
|
```
|
||||||
|
:root — light-mode values (default)
|
||||||
|
html[data-theme="dark"] — dark-mode overrides
|
||||||
|
```
|
||||||
|
|
||||||
|
Key variables:
|
||||||
|
|
||||||
|
| Variable | Purpose |
|
||||||
|
|----------|---------|
|
||||||
|
| `--bg` | Page background |
|
||||||
|
| `--surface` | Card / panel background |
|
||||||
|
| `--surface-2` / `--surface-3` | Slightly lighter/darker surfaces (table rows, hover states) |
|
||||||
|
| `--text` / `--text-sec` / `--text-muted` | Primary, secondary, muted text |
|
||||||
|
| `--border` / `--border-2`…`4` | Border shades from prominent to faint |
|
||||||
|
| `--link` | Hyperlink and interactive-element colour |
|
||||||
|
| `--nav-bg` | Navigation bar background |
|
||||||
|
| `--input-bg` / `--input-border` | Form control colours |
|
||||||
|
| `--shadow` / `--shadow-sm` | Box-shadow alphas |
|
||||||
|
|
||||||
|
A single global rule in `head.html` themes all `<input>`, `<select>`, and `<textarea>` elements across every page at once:
|
||||||
|
|
||||||
|
```css
|
||||||
|
html[data-theme="dark"] input:not([type=checkbox]):not([type=radio]),
|
||||||
|
html[data-theme="dark"] select,
|
||||||
|
html[data-theme="dark"] textarea { … }
|
||||||
|
```
|
||||||
|
|
||||||
|
Each page template adds its own `html[data-theme="dark"]` block for page-specific elements (cards, tables, badges, etc.).
|
||||||
|
|
||||||
|
### Auto-mode live updates
|
||||||
|
|
||||||
|
A `matchMedia` change listener in `head.html` updates `data-theme` whenever the OS preference changes, so users in **Auto** mode see the theme switch without reloading.
|
||||||
|
|
||||||
|
### Semantic colours are unchanged
|
||||||
|
|
||||||
|
Alert colours (red for critical, orange for warning, green for ok) and status indicators are intentionally left as fixed values — they are semantic signals, not surface colours, and look correct on both light and dark backgrounds.
|
||||||
@@ -53,6 +53,17 @@ See [User Management](USERS.md) for full authentication documentation.
|
|||||||
|--------|------|-------------|------|
|
|--------|------|-------------|------|
|
||||||
| `GET` | `/api/0/users` | List all users | Admin |
|
| `GET` | `/api/0/users` | List all users | Admin |
|
||||||
| `GET` | `/api/0/users/me` | Own profile | Authenticated |
|
| `GET` | `/api/0/users/me` | Own profile | Authenticated |
|
||||||
|
| `PUT` | `/api/0/users/me` | Update own profile | Authenticated |
|
||||||
|
|
||||||
|
### Notification Channels
|
||||||
|
|
||||||
|
| Method | Path | Description | Role |
|
||||||
|
|--------|------|-------------|------|
|
||||||
|
| `GET` | `/api/0/notification_channel_types` | Channel type schemas | Authenticated |
|
||||||
|
| `GET` | `/api/0/notification_channels` | List visible channels | Authenticated |
|
||||||
|
| `POST` | `/api/0/notification_channels` | Create a channel | Authenticated |
|
||||||
|
| `PUT` | `/api/0/notification_channels/{name}` | Update a channel | Owner or Admin |
|
||||||
|
| `DELETE` | `/api/0/notification_channels/{name}` | Delete a channel | Owner or Admin |
|
||||||
|
|
||||||
### Host Management
|
### Host Management
|
||||||
|
|
||||||
@@ -203,6 +214,101 @@ Changes take effect immediately but are not written back to the config file. Upd
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Notification Channel Endpoints
|
||||||
|
|
||||||
|
Channels are visible to all users by default. Channels marked `private: true` are only visible to their owner. Admins see all channels.
|
||||||
|
|
||||||
|
#### GET /api/0/notification_channel_types
|
||||||
|
Return the schema for every supported notifier type. Used by the web UI to dynamically render the channel creation form.
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"pushover": {
|
||||||
|
"label": "Pushover",
|
||||||
|
"fields": [
|
||||||
|
{"key": "token", "label": "App token", "type": "secret", "required": true},
|
||||||
|
{"key": "user", "label": "User key", "type": "secret", "required": true},
|
||||||
|
{"key": "sound", "label": "Sound", "type": "text", "required": false}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"email": { "label": "E-mail", "fields": [ ... ] },
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### GET /api/0/notification_channels
|
||||||
|
List channels visible to the current user (public channels + own private channels). Admins receive all channels.
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "pushover_ops",
|
||||||
|
"type": "pushover",
|
||||||
|
"type_label": "Pushover",
|
||||||
|
"owner": null,
|
||||||
|
"private": false,
|
||||||
|
"min_level": "WARNING",
|
||||||
|
"fields": [
|
||||||
|
{"key": "token", "label": "App token", "value": "•••", "sensitive": true},
|
||||||
|
{"key": "user", "label": "User key", "value": "•••", "sensitive": true}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
Sensitive fields (`type: "secret"`) are always returned as `"•••"`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### POST /api/0/notification_channels
|
||||||
|
Create a new channel. The creating user becomes the channel's `owner`.
|
||||||
|
|
||||||
|
**Request body:**
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "my_pushover",
|
||||||
|
"type": "pushover",
|
||||||
|
"token": "app-token",
|
||||||
|
"user": "user-key",
|
||||||
|
"min_level": "WARNING",
|
||||||
|
"private": true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response:** `{"ok": true, "name": "my_pushover"}`
|
||||||
|
|
||||||
|
**Status codes:** `200 OK`, `400` (missing required field or unknown type), `409` (name already exists)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### PUT /api/0/notification_channels/{name}
|
||||||
|
Update an existing channel. Only the channel owner or an admin may update it.
|
||||||
|
|
||||||
|
Secret fields sent as `"•••"` are preserved from the existing config (same pattern as OAuth secrets in the admin config editor).
|
||||||
|
|
||||||
|
**Request body:** same shape as POST, `name` ignored (taken from URL).
|
||||||
|
|
||||||
|
**Response:** `{"ok": true}`
|
||||||
|
|
||||||
|
**Status codes:** `200 OK`, `403 Forbidden`, `404 Not Found`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### DELETE /api/0/notification_channels/{name}
|
||||||
|
Delete a channel. Only the channel owner or an admin may delete it.
|
||||||
|
|
||||||
|
**Response:** `{"ok": true}`
|
||||||
|
|
||||||
|
**Status codes:** `200 OK`, `403 Forbidden`, `404 Not Found`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
### Alert Endpoints
|
### Alert Endpoints
|
||||||
|
|
||||||
#### GET /api/0/hosts/{hostname}/alerts
|
#### GET /api/0/hosts/{hostname}/alerts
|
||||||
|
|||||||
@@ -406,6 +406,21 @@ Potential improvements for future versions:
|
|||||||
- Journal file encryption
|
- Journal file encryption
|
||||||
- Signed journal entries
|
- Signed journal entries
|
||||||
|
|
||||||
|
## Events journal
|
||||||
|
|
||||||
|
Alert/connectivity events shown on the `/log` page are written to a second,
|
||||||
|
dedicated journal (one JSON object per line, same rotation mechanism):
|
||||||
|
|
||||||
|
| Key | Default | Meaning |
|
||||||
|
|---|---|---|
|
||||||
|
| `events_journal_file` | `events.journal` | Filename inside `journal_dir` |
|
||||||
|
| `events_journal_max_size` | 10 MB | Rotation threshold |
|
||||||
|
| `events_journal_max_backups` | 10 | Rotated files kept |
|
||||||
|
|
||||||
|
`journal_dir` and `journal_enabled` are shared with the message journal.
|
||||||
|
On startup, if the events journal file is empty, it is seeded once from the
|
||||||
|
pickled in-memory message ring so history survives the upgrade.
|
||||||
|
|
||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
- [Configuration Guide](../hbd/config.py) - Full configuration options
|
- [Configuration Guide](../hbd/config.py) - Full configuration options
|
||||||
|
|||||||
@@ -104,11 +104,6 @@ The `nagios_runner` plugin collects:
|
|||||||
- `{name}_{metric}_min` - Minimum value (if present)
|
- `{name}_{metric}_min` - Minimum value (if present)
|
||||||
- `{name}_{metric}_max` - Maximum value (if present)
|
- `{name}_{metric}_max` - Maximum value (if present)
|
||||||
|
|
||||||
**Overall:**
|
|
||||||
- `overall_status` - Worst status from all commands
|
|
||||||
- `overall_status_code` - Worst status code
|
|
||||||
- `plugin_count` - Number of Nagios plugins executed
|
|
||||||
|
|
||||||
## Configuration Options
|
## Configuration Options
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
|
|||||||
+78
-7
@@ -9,6 +9,7 @@ Notifications are dispatched to the **owner and managers** of a host, each via t
|
|||||||
```
|
```
|
||||||
Alert event (udp.py / threshold.py)
|
Alert event (udp.py / threshold.py)
|
||||||
└─ notify.send_notification(host_name, Notification)
|
└─ notify.send_notification(host_name, Notification)
|
||||||
|
├─ flap.observe(host, service, level) → pass | trip | suppress
|
||||||
├─ look up host.owner + host.managers
|
├─ look up host.owner + host.managers
|
||||||
├─ for each user → user.notification_channels
|
├─ for each user → user.notification_channels
|
||||||
└─ for each channel → _dispatch_to_channel (filtered by min_level)
|
└─ for each channel → _dispatch_to_channel (filtered by min_level)
|
||||||
@@ -19,6 +20,7 @@ Every notification carries:
|
|||||||
- **body** — detail message (metric value, threshold, duration)
|
- **body** — detail message (metric value, threshold, duration)
|
||||||
- **url** — link to the plugin metrics page (`{base_url}/plugins#{hostname}`)
|
- **url** — link to the plugin metrics page (`{base_url}/plugins#{hostname}`)
|
||||||
- **level** — `RECOVER | WARNING | CRITICAL | INFO`
|
- **level** — `RECOVER | WARNING | CRITICAL | INFO`
|
||||||
|
- **service** — flap-detection key within the host (empty = the host itself)
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
@@ -30,9 +32,18 @@ Set `base_url` so notification links point to your hbd instance:
|
|||||||
base_url: https://hbd.example.com
|
base_url: https://hbd.example.com
|
||||||
```
|
```
|
||||||
|
|
||||||
### Global channel definitions
|
### Channel definitions
|
||||||
|
|
||||||
Define channels once; reference them by name from user configs:
|
Channels are defined under `notification_channels`. Each entry specifies a delivery type and its credentials. Ownership is the single visibility signal:
|
||||||
|
|
||||||
|
| Field | Default | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| `owner` | *(absent)* | Owning username. Present = private to that user; absent = global. |
|
||||||
|
| `min_level` | `WARNING` | Minimum alert level this channel receives. |
|
||||||
|
|
||||||
|
(The former `private` flag is retired; leftover `private` keys are ignored and dropped on the next edit.)
|
||||||
|
|
||||||
|
**Global channels** (no `owner`; set in the config file or by an admin) can be selected by all users but edited only by admins:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
notification_channels:
|
notification_channels:
|
||||||
@@ -41,7 +52,7 @@ notification_channels:
|
|||||||
type: pushover
|
type: pushover
|
||||||
token: your-app-token
|
token: your-app-token
|
||||||
user: your-user-key
|
user: your-user-key
|
||||||
min_level: WARNING # optional, default: WARNING
|
min_level: WARNING
|
||||||
|
|
||||||
email_ops:
|
email_ops:
|
||||||
type: email
|
type: email
|
||||||
@@ -58,14 +69,14 @@ notification_channels:
|
|||||||
homeserver: https://matrix.example.org
|
homeserver: https://matrix.example.org
|
||||||
access_token: syt_xxx
|
access_token: syt_xxx
|
||||||
room_id: "!abc:matrix.example.org"
|
room_id: "!abc:matrix.example.org"
|
||||||
min_level: CRITICAL # only send critical alerts to this room
|
min_level: CRITICAL
|
||||||
|
|
||||||
sms_oncall:
|
sms_oncall:
|
||||||
type: sms_voipms
|
type: sms_voipms
|
||||||
api_user: me@example.com
|
api_user: me@example.com
|
||||||
api_password: secret
|
api_password: secret
|
||||||
did: "5551234567" # your voip.ms DID number
|
did: "5551234567"
|
||||||
dst: "5559876543" # destination number
|
dst: "5559876543"
|
||||||
min_level: CRITICAL
|
min_level: CRITICAL
|
||||||
|
|
||||||
signal_ops:
|
signal_ops:
|
||||||
@@ -82,9 +93,31 @@ notification_channels:
|
|||||||
username: heartbeat-bot
|
username: heartbeat-bot
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**User-created channels** are written by authenticated users through the API, their profile page, or the settings page. They carry an `owner` field and are private to that user:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
notification_channels:
|
||||||
|
|
||||||
|
alice_personal:
|
||||||
|
type: pushover
|
||||||
|
token: personal-token
|
||||||
|
user: personal-key
|
||||||
|
owner: alice # private to alice
|
||||||
|
```
|
||||||
|
|
||||||
|
### Channel visibility
|
||||||
|
|
||||||
|
| Channel | Who can see / select it | Who can edit it |
|
||||||
|
|---|---|---|
|
||||||
|
| No `owner` (global) | All users | Admins |
|
||||||
|
| `owner` set (private) | Only the `owner` | The owner |
|
||||||
|
| Any channel | Admins always see everything | Admins |
|
||||||
|
|
||||||
|
Admins can **promote** a private channel to global by clearing its owner on the settings page, or **demote** a global channel by assigning an owner.
|
||||||
|
|
||||||
### Users with notification channels
|
### Users with notification channels
|
||||||
|
|
||||||
Each user lists which global channels they receive notifications on:
|
Each user lists which channels they receive notifications on. Users can manage their own selection from the profile page:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
users:
|
users:
|
||||||
@@ -237,6 +270,43 @@ min_level: WARNING
|
|||||||
|
|
||||||
Reminder notifications (re-notify) are sent only for CRITICAL level alerts.
|
Reminder notifications (re-notify) are sent only for CRITICAL level alerts.
|
||||||
|
|
||||||
|
## Flap detection
|
||||||
|
|
||||||
|
A check that toggles between OK and alerting produces a notification per swing. Flap
|
||||||
|
detection silences it after the first few.
|
||||||
|
|
||||||
|
A **`(host, service)`** pair is flapping once it exceeds `flap_count` WARNING/CRITICAL
|
||||||
|
notifications within `flap_interval` minutes. Threshold alerts key on their metric path,
|
||||||
|
so a flapping disk check does not silence an unrelated CPU alert; connectivity, boot and
|
||||||
|
shutdown events key on the host itself.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
flap_count: 5 # notifications within the window that trip flapping (0 disables)
|
||||||
|
flap_interval: 10 # minutes — both the counting window and the quiet window
|
||||||
|
```
|
||||||
|
|
||||||
|
Lifecycle:
|
||||||
|
|
||||||
|
| Event | Effect |
|
||||||
|
|---|---|
|
||||||
|
| Alerts 1..`flap_count` within the window | Delivered normally |
|
||||||
|
| Alert `flap_count + 1` | Delivered with ` Now flapping!! No more messages!` appended to the body |
|
||||||
|
| Every notification after that | Dropped — including RECOVER and INFO |
|
||||||
|
| RECOVER while flapping | Dropped, and starts the `flap_interval` quiet window |
|
||||||
|
| WARNING/CRITICAL during the quiet window | Restarts the quiet window; still flapping |
|
||||||
|
| Quiet window elapses | Flapping ends **silently** — no notification |
|
||||||
|
|
||||||
|
Only outbound notifications are suppressed. `notify.eventlog` keeps recording every event,
|
||||||
|
so the journal and the `/log` page retain the full history of the flap.
|
||||||
|
|
||||||
|
Flapping pairs appear in each host's `stateinfo()` under `flapping` (a list of service
|
||||||
|
keys; `""` means the host itself) and render as an amber **flapping** badge next to the
|
||||||
|
host name on the live dashboard, with the affected services in its tooltip.
|
||||||
|
|
||||||
|
State lives at module level in `hbd/server/flap.py` and is never pickled — a server restart
|
||||||
|
starts every check with a clean slate. Hosts with `watch: false` never reach the
|
||||||
|
notification path, so they never flap.
|
||||||
|
|
||||||
## API reference
|
## API reference
|
||||||
|
|
||||||
### `send_notification(host_name, notif) -> dict`
|
### `send_notification(host_name, notif) -> dict`
|
||||||
@@ -270,6 +340,7 @@ Called once at startup from `main.py`. Pass the running asyncio event loop so Ma
|
|||||||
- Check that the host has an `owner` or `managers` set
|
- Check that the host has an `owner` or `managers` set
|
||||||
- Check that users have `notification_channels` listed
|
- Check that users have `notification_channels` listed
|
||||||
- Check that the channel names in user config match keys under `notification_channels:`
|
- Check that the channel names in user config match keys under `notification_channels:`
|
||||||
|
- If a user can't select a channel, check whether it has an `owner` other than that user
|
||||||
|
|
||||||
**min_level filtering too aggressive:**
|
**min_level filtering too aggressive:**
|
||||||
- Default is `WARNING` — both WARNING and CRITICAL are sent
|
- Default is `WARNING` — both WARNING and CRITICAL are sent
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ This guide explains how to create custom plugins for the Heartbeat monitoring sy
|
|||||||
- [Plugin Types](#plugin-types)
|
- [Plugin Types](#plugin-types)
|
||||||
- [Creating a Plugin](#creating-a-plugin)
|
- [Creating a Plugin](#creating-a-plugin)
|
||||||
- [Plugin Lifecycle](#plugin-lifecycle)
|
- [Plugin Lifecycle](#plugin-lifecycle)
|
||||||
|
- [Server-initiated InfoPlugin refresh](#server-initiated-infoplugin-refresh)
|
||||||
- [Configuration](#configuration)
|
- [Configuration](#configuration)
|
||||||
- [Best Practices](#best-practices)
|
- [Best Practices](#best-practices)
|
||||||
- [Examples](#examples)
|
- [Examples](#examples)
|
||||||
@@ -250,6 +251,28 @@ Understanding the plugin lifecycle helps you implement plugins correctly:
|
|||||||
└─> Plugin releases resources, closes connections
|
└─> Plugin releases resources, closes connections
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Server-initiated InfoPlugin refresh
|
||||||
|
|
||||||
|
When a heartbeat packet arrives from a host the server has no plugin data for (e.g. after a server restart), the server sets `request_update = 1` in the ACK reply. The client detects this flag and immediately re-runs all InfoPlugins — clearing their cached results first — then resends the data as PLG messages.
|
||||||
|
|
||||||
|
This means InfoPlugin data will always reach the server as soon as possible without requiring a client restart. No action is needed from plugin authors: the framework handles cache invalidation and re-collection automatically.
|
||||||
|
|
||||||
|
The lifecycle for this case looks like:
|
||||||
|
|
||||||
|
```
|
||||||
|
Server restarts, host reconnects
|
||||||
|
└─> hbd receives HTB with no existing plugin_data for host
|
||||||
|
└─> hbd sets request_update=1 in ACK
|
||||||
|
|
||||||
|
Client receives ACK
|
||||||
|
└─> Detects request_update flag
|
||||||
|
└─> Clears _cache on every registered InfoPlugin
|
||||||
|
└─> Calls collect() on each InfoPlugin
|
||||||
|
└─> Sends fresh PLG messages to server
|
||||||
|
```
|
||||||
|
|
||||||
|
If you write an `InfoPlugin` with side effects in `_collect_info()` (opening connections, writing files, etc.), be aware it may be called more than once per client session when this mechanism triggers.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
### Plugin-Specific Configuration
|
### Plugin-Specific Configuration
|
||||||
|
|||||||
+224
-65
@@ -256,6 +256,56 @@ disk_monitor:
|
|||||||
operator: "<"
|
operator: "<"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### ZFS Monitor
|
||||||
|
|
||||||
|
ZFS pool health is checked automatically for every pool. A pool in any state
|
||||||
|
other than `ONLINE` (e.g. `DEGRADED`, `SUSPENDED`, `FAULTED`, `UNAVAIL`) raises
|
||||||
|
a **CRITICAL** alert by default — no configuration required.
|
||||||
|
|
||||||
|
The default threshold is equivalent to:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
zfs_monitor:
|
||||||
|
pools:
|
||||||
|
'*':
|
||||||
|
status:
|
||||||
|
warning: 1
|
||||||
|
critical: 2
|
||||||
|
operator: ">"
|
||||||
|
hysteresis: 0.0
|
||||||
|
display: "ZFS pool {pool_name} is {health}"
|
||||||
|
```
|
||||||
|
|
||||||
|
`'*'` matches every pool on the host. The notification message includes the pool
|
||||||
|
name and its current health string, e.g. `ZFS pool tank is DEGRADED`.
|
||||||
|
|
||||||
|
**Override for specific pools** — named pool entries take priority over `'*'`:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
zfs_monitor:
|
||||||
|
pools:
|
||||||
|
# Suppress health alerts for a scratch pool (not mission-critical)
|
||||||
|
scratch:
|
||||||
|
status:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
# Capacity threshold for a specific pool
|
||||||
|
tank:
|
||||||
|
capacity:
|
||||||
|
warning: 75.0
|
||||||
|
critical: 90.0
|
||||||
|
operator: ">"
|
||||||
|
hysteresis: 0.05
|
||||||
|
```
|
||||||
|
|
||||||
|
**Alert state paths** follow the pattern `zfs_monitor.<pool_name>.status`,
|
||||||
|
so acknowledgements and silences target individual pools:
|
||||||
|
|
||||||
|
```
|
||||||
|
zfs_monitor.tank.status
|
||||||
|
zfs_monitor.backup.status
|
||||||
|
```
|
||||||
|
|
||||||
### Network Monitor
|
### Network Monitor
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
@@ -814,34 +864,32 @@ Planned features:
|
|||||||
|
|
||||||
## Multi-Threshold Configuration
|
## Multi-Threshold Configuration
|
||||||
|
|
||||||
**New in version 2.0**: Support for multiple named threshold configurations with per-host mapping.
|
Support for multiple named threshold configurations with per-host mapping and composable layering.
|
||||||
|
|
||||||
### Overview
|
### Overview
|
||||||
|
|
||||||
The multi-threshold feature allows you to:
|
The multi-threshold feature allows you to:
|
||||||
- Define multiple sets of threshold configurations
|
- Define multiple named threshold configurations
|
||||||
- Map different hosts to different threshold sets
|
- Assign one or more configurations to each host
|
||||||
|
- Compose configurations by layering — each named config's overrides are applied in order on top of the defaults
|
||||||
- Use different sensitivity levels for different environments
|
- Use different sensitivity levels for different environments
|
||||||
- Maintain a default configuration for unmapped hosts
|
|
||||||
|
|
||||||
### Configuration Structure
|
### Configuration Structure
|
||||||
|
|
||||||
|
Named configurations are defined under `threshold_configs`. Each host selects which ones to use via `threshold_config` in the `hosts` section (a string for a single config, or a list to layer multiple):
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# Optional: Set the default configuration name (defaults to "default")
|
# Optional: set the default configuration name (defaults to "default")
|
||||||
default_threshold_config: "default"
|
default_threshold_config: "default"
|
||||||
|
|
||||||
# Define multiple named threshold configurations
|
|
||||||
threshold_configs:
|
threshold_configs:
|
||||||
# Configuration name 1
|
|
||||||
default:
|
default:
|
||||||
thresholds:
|
thresholds:
|
||||||
# Standard threshold definitions
|
|
||||||
cpu_monitor:
|
cpu_monitor:
|
||||||
cpu_percent:
|
cpu_percent:
|
||||||
warning: 80.0
|
warning: 80.0
|
||||||
critical: 90.0
|
critical: 90.0
|
||||||
|
|
||||||
# Configuration name 2
|
|
||||||
high_sensitivity:
|
high_sensitivity:
|
||||||
thresholds:
|
thresholds:
|
||||||
cpu_monitor:
|
cpu_monitor:
|
||||||
@@ -849,7 +897,6 @@ threshold_configs:
|
|||||||
warning: 60.0
|
warning: 60.0
|
||||||
critical: 75.0
|
critical: 75.0
|
||||||
|
|
||||||
# Configuration name 3
|
|
||||||
low_sensitivity:
|
low_sensitivity:
|
||||||
thresholds:
|
thresholds:
|
||||||
cpu_monitor:
|
cpu_monitor:
|
||||||
@@ -857,14 +904,77 @@ threshold_configs:
|
|||||||
warning: 90.0
|
warning: 90.0
|
||||||
critical: 95.0
|
critical: 95.0
|
||||||
|
|
||||||
# Map specific hosts to specific configurations
|
hosts:
|
||||||
host_threshold_mapping:
|
prod-web-01:
|
||||||
prod-web-01: high_sensitivity
|
threshold_config: high_sensitivity # single config
|
||||||
prod-web-02: high_sensitivity
|
|
||||||
dev-server-01: low_sensitivity
|
dev-server-01:
|
||||||
# Unmapped hosts use default_threshold_config
|
threshold_config: low_sensitivity
|
||||||
|
|
||||||
|
# Hosts with no threshold_config use default_threshold_config
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Composable Configurations (list form)
|
||||||
|
|
||||||
|
`threshold_config` can be a list. Configs are applied **left to right**: the defaults are the base, then each named config's overrides are layered on top. Later entries in the list win on any metric they define.
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
threshold_configs:
|
||||||
|
default:
|
||||||
|
thresholds:
|
||||||
|
cpu_monitor:
|
||||||
|
cpu_percent: {warning: 80, critical: 90}
|
||||||
|
memory_monitor:
|
||||||
|
memory_percent: {warning: 85, critical: 95}
|
||||||
|
disk_monitor:
|
||||||
|
partitions:
|
||||||
|
/:
|
||||||
|
percent: {warning: 80, critical: 90}
|
||||||
|
|
||||||
|
# Tighter CPU limits for busy servers
|
||||||
|
high_cpu_load:
|
||||||
|
thresholds:
|
||||||
|
cpu_monitor:
|
||||||
|
cpu_percent: {warning: 60, critical: 75}
|
||||||
|
|
||||||
|
# Tighter disk limits for data-heavy servers
|
||||||
|
busy_disk:
|
||||||
|
thresholds:
|
||||||
|
disk_monitor:
|
||||||
|
partitions:
|
||||||
|
/:
|
||||||
|
percent: {warning: 70, critical: 85}
|
||||||
|
|
||||||
|
hosts:
|
||||||
|
# Gets default thresholds only
|
||||||
|
web-01:
|
||||||
|
threshold_config: default
|
||||||
|
|
||||||
|
# Gets tighter CPU limits, default memory and disk
|
||||||
|
build-server:
|
||||||
|
threshold_config: high_cpu_load
|
||||||
|
|
||||||
|
# Layers both: tighter CPU AND tighter disk, default memory
|
||||||
|
db-01:
|
||||||
|
threshold_config: [high_cpu_load, busy_disk]
|
||||||
|
|
||||||
|
# Three layers: busy_disk overrides high_cpu_load if they conflict
|
||||||
|
storage-01:
|
||||||
|
threshold_config: [default, high_cpu_load, busy_disk]
|
||||||
|
```
|
||||||
|
|
||||||
|
**How layering works:**
|
||||||
|
|
||||||
|
Starting from the `default` thresholds:
|
||||||
|
|
||||||
|
| Layer | Applied config | Effect |
|
||||||
|
|-------|---------------|--------|
|
||||||
|
| Base | `default` | all default thresholds |
|
||||||
|
| +1 | `high_cpu_load` | cpu_percent overridden to 60/75 |
|
||||||
|
| +2 | `busy_disk` | disk percent overridden to 70/85; cpu_percent stays at 60/75 |
|
||||||
|
|
||||||
|
Each named config only overrides the metrics it explicitly defines. Metrics not mentioned in a config inherit from the layers beneath.
|
||||||
|
|
||||||
### Use Cases
|
### Use Cases
|
||||||
|
|
||||||
#### 1. Environment-Based Thresholds
|
#### 1. Environment-Based Thresholds
|
||||||
@@ -887,11 +997,15 @@ threshold_configs:
|
|||||||
warning: 90.0 # More relaxed for dev
|
warning: 90.0 # More relaxed for dev
|
||||||
critical: 98.0
|
critical: 98.0
|
||||||
|
|
||||||
host_threshold_mapping:
|
hosts:
|
||||||
prod-web-01: production
|
prod-web-01:
|
||||||
prod-web-02: production
|
threshold_config: production
|
||||||
dev-web-01: development
|
prod-web-02:
|
||||||
dev-web-02: development
|
threshold_config: production
|
||||||
|
dev-web-01:
|
||||||
|
threshold_config: development
|
||||||
|
dev-web-02:
|
||||||
|
threshold_config: development
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 2. Server Role-Based Thresholds
|
#### 2. Server Role-Based Thresholds
|
||||||
@@ -914,7 +1028,7 @@ threshold_configs:
|
|||||||
warning: 70.0
|
warning: 70.0
|
||||||
critical: 85.0
|
critical: 85.0
|
||||||
memory_monitor:
|
memory_monitor:
|
||||||
percent:
|
memory_percent:
|
||||||
warning: 90.0 # Databases can use high memory
|
warning: 90.0 # Databases can use high memory
|
||||||
critical: 97.0
|
critical: 97.0
|
||||||
disk_monitor:
|
disk_monitor:
|
||||||
@@ -927,17 +1041,23 @@ threshold_configs:
|
|||||||
cache:
|
cache:
|
||||||
thresholds:
|
thresholds:
|
||||||
memory_monitor:
|
memory_monitor:
|
||||||
percent:
|
memory_percent:
|
||||||
warning: 95.0 # Redis/Memcached can use very high memory
|
warning: 95.0 # Redis/Memcached can use very high memory
|
||||||
critical: 99.0
|
critical: 99.0
|
||||||
|
|
||||||
host_threshold_mapping:
|
hosts:
|
||||||
web-01: webserver
|
web-01:
|
||||||
web-02: webserver
|
threshold_config: webserver
|
||||||
db-01: database
|
web-02:
|
||||||
db-02: database
|
threshold_config: webserver
|
||||||
redis-01: cache
|
db-01:
|
||||||
memcached-01: cache
|
threshold_config: database
|
||||||
|
db-02:
|
||||||
|
threshold_config: database
|
||||||
|
redis-01:
|
||||||
|
threshold_config: cache
|
||||||
|
memcached-01:
|
||||||
|
threshold_config: cache
|
||||||
```
|
```
|
||||||
|
|
||||||
#### 3. Sensitivity Levels
|
#### 3. Sensitivity Levels
|
||||||
@@ -952,7 +1072,7 @@ threshold_configs:
|
|||||||
partitions:
|
partitions:
|
||||||
/:
|
/:
|
||||||
percent:
|
percent:
|
||||||
warning: 70.0 # Very sensitive
|
warning: 70.0
|
||||||
critical: 80.0
|
critical: 80.0
|
||||||
hysteresis: 0.15
|
hysteresis: 0.15
|
||||||
|
|
||||||
@@ -976,52 +1096,91 @@ threshold_configs:
|
|||||||
critical: 98.0
|
critical: 98.0
|
||||||
hysteresis: 0.05
|
hysteresis: 0.05
|
||||||
|
|
||||||
host_threshold_mapping:
|
hosts:
|
||||||
payment-gateway: critical
|
payment-gateway:
|
||||||
auth-server: critical
|
threshold_config: critical
|
||||||
web-01: standard
|
auth-server:
|
||||||
web-02: standard
|
threshold_config: critical
|
||||||
test-server: relaxed
|
web-01:
|
||||||
|
threshold_config: standard
|
||||||
|
web-02:
|
||||||
|
threshold_config: standard
|
||||||
|
test-server:
|
||||||
|
threshold_config: relaxed
|
||||||
```
|
```
|
||||||
|
|
||||||
### Backward Compatibility
|
#### 4. Composable Profiles
|
||||||
|
|
||||||
The legacy single threshold configuration is fully supported:
|
Build host-specific thresholds by combining small, focused configs:
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
# Old format - still works
|
|
||||||
thresholds:
|
|
||||||
cpu_monitor:
|
|
||||||
cpu_percent:
|
|
||||||
warning: 80.0
|
|
||||||
critical: 90.0
|
|
||||||
```
|
|
||||||
|
|
||||||
This is equivalent to:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# New format
|
|
||||||
threshold_configs:
|
threshold_configs:
|
||||||
|
# Baseline — everything at default levels
|
||||||
default:
|
default:
|
||||||
thresholds:
|
thresholds:
|
||||||
cpu_monitor:
|
cpu_monitor:
|
||||||
cpu_percent:
|
cpu_percent: {warning: 80, critical: 90}
|
||||||
warning: 80.0
|
memory_monitor:
|
||||||
critical: 90.0
|
memory_percent: {warning: 85, critical: 95}
|
||||||
```
|
|
||||||
|
|
||||||
|
# Overlay: tighter CPU only
|
||||||
|
tight_cpu:
|
||||||
|
thresholds:
|
||||||
|
cpu_monitor:
|
||||||
|
cpu_percent: {warning: 60, critical: 75}
|
||||||
|
|
||||||
|
# Overlay: tighter memory only
|
||||||
|
tight_memory:
|
||||||
|
thresholds:
|
||||||
|
memory_monitor:
|
||||||
|
memory_percent: {warning: 70, critical: 85}
|
||||||
|
|
||||||
|
# Overlay: extra disk partition for database servers
|
||||||
|
db_disk:
|
||||||
|
thresholds:
|
||||||
|
disk_monitor:
|
||||||
|
partitions:
|
||||||
|
/var/lib/postgresql:
|
||||||
|
percent: {warning: 75, critical: 88}
|
||||||
|
|
||||||
|
hosts:
|
||||||
|
# Plain web server
|
||||||
|
web-01:
|
||||||
|
threshold_config: default
|
||||||
|
|
||||||
|
# Build server: tight CPU, default memory and disk
|
||||||
|
build-01:
|
||||||
|
threshold_config: tight_cpu
|
||||||
|
|
||||||
|
# Database: tight CPU + tight memory + extra disk partition
|
||||||
|
db-01:
|
||||||
|
threshold_config: [tight_cpu, tight_memory, db_disk]
|
||||||
|
|
||||||
|
# Replica database: tight memory + extra disk, normal CPU
|
||||||
|
db-02:
|
||||||
|
threshold_config: [tight_memory, db_disk]
|
||||||
|
```
|
||||||
### Configuration Priority
|
### Configuration Priority
|
||||||
|
|
||||||
1. **Host-specific mapping**: If host is in `host_threshold_mapping`, use that config
|
1. **Host `threshold_config` (list)**: Layer each named config's overrides left-to-right on top of the defaults
|
||||||
2. **Default config**: Use `default_threshold_config`
|
2. **Host `threshold_config` (string)**: Use that single named config directly
|
||||||
3. **First alphabetically**: If default not found, use first config alphabetically
|
3. **`host_threshold_mapping`** (legacy): Same as above, string only
|
||||||
4. **Legacy fallback**: If `threshold_configs` not present, use `thresholds`
|
4. **`default_threshold_config`**: Used for hosts with no mapping
|
||||||
|
5. **First alphabetically**: If the default config is not found, use the first config alphabetically
|
||||||
|
6. **Legacy `thresholds` section**: Used when `threshold_configs` is absent entirely
|
||||||
|
|
||||||
### Example: Complete Multi-Threshold Setup
|
### Backward Compatibility
|
||||||
|
|
||||||
See `hbd/config_multi_threshold_example.yaml` for a complete example with:
|
The legacy `host_threshold_mapping` top-level key and the flat `thresholds` section are still fully supported:
|
||||||
- 4 named configurations (default, high_sensitivity, low_sensitivity, database)
|
|
||||||
- Host-to-config mappings for production, development, and test systems
|
```yaml
|
||||||
- Specialized database server thresholds
|
# Still works — equivalent to hosts: {prod-web-01: {threshold_config: high_sensitivity}}
|
||||||
- Custom display messages with plugin data
|
host_threshold_mapping:
|
||||||
|
prod-web-01: high_sensitivity
|
||||||
|
|
||||||
|
# Still works — equivalent to threshold_configs: {default: {thresholds: ...}}
|
||||||
|
thresholds:
|
||||||
|
cpu_monitor:
|
||||||
|
cpu_percent: {warning: 80, critical: 90}
|
||||||
|
```
|
||||||
|
|
||||||
|
|||||||
+54
-1
@@ -19,6 +19,15 @@ Users are defined in the server config file. Each host can have an **owner**, ze
|
|||||||
|
|
||||||
`admin` is a flag on the user, not a per-host role. An admin user has owner-level access on every host without being listed as owner/manager/monitor.
|
`admin` is a flag on the user, not a per-host role. An admin user has owner-level access on every host without being listed as owner/manager/monitor.
|
||||||
|
|
||||||
|
### Settings page access
|
||||||
|
|
||||||
|
All authenticated users may open `/settings`. Admins see every section; non-admins see only **Notification Channels**, **Hosts**, and **Threshold Configurations**, filtered to global items plus what they own or manage:
|
||||||
|
|
||||||
|
- **Owners** may add hosts, delete their hosts, edit host settings, and change access lists (managers/monitors, ownership transfer).
|
||||||
|
- **Managers** may edit host settings (watch, dyndns, channel and threshold assignments) but not access lists, and may not delete hosts.
|
||||||
|
- Anyone may create private notification channels and threshold configs (owned by them) and assign them — or global ones — to their hosts.
|
||||||
|
- Admins promote a private channel/threshold config to global by clearing its owner, or demote by assigning one.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
@@ -36,7 +45,7 @@ users:
|
|||||||
bob:
|
bob:
|
||||||
full_name: Bob Smith
|
full_name: Bob Smith
|
||||||
password: pbkdf2:sha256:...
|
password: pbkdf2:sha256:...
|
||||||
notification_channels: [pushover_standard]
|
notification_channels: [pushover_standard] # channels bob has selected
|
||||||
|
|
||||||
carol:
|
carol:
|
||||||
full_name: Carol Jones
|
full_name: Carol Jones
|
||||||
@@ -46,6 +55,24 @@ default_owner: andreas # owns hosts with no explicit owner
|
|||||||
# falls back to the first admin user if omitted
|
# falls back to the first admin user if omitted
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Client-declared host ownership
|
||||||
|
|
||||||
|
A host can declare its own owner directly in the hbc or hbc_mini client configuration. This is useful for hosts that are not listed in the server config, or during initial setup before a server-side config entry has been created.
|
||||||
|
|
||||||
|
**`~/.hbc.yaml`** (hbc):
|
||||||
|
```yaml
|
||||||
|
owner: andreas
|
||||||
|
```
|
||||||
|
|
||||||
|
**`~/.hbc.json`** (hbc_mini):
|
||||||
|
```json
|
||||||
|
{ "owner": "andreas" }
|
||||||
|
```
|
||||||
|
|
||||||
|
When set, the value is included in the `os_info` plugin data sent to the server. The server applies it as `host.owner` the first time `os_info` arrives, provided no owner has been configured server-side for that host. Server-configured ownership always takes precedence.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
### Assigning roles to hosts
|
### Assigning roles to hosts
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
@@ -170,6 +197,32 @@ Return the currently authenticated user's profile.
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
#### PUT /api/0/users/me
|
||||||
|
Update the current user's profile. All fields are optional — send only what you want to change.
|
||||||
|
|
||||||
|
**Update display name and avatar:**
|
||||||
|
```json
|
||||||
|
{ "full_name": "Carol Jones", "avatar": "/avatars/carol.png" }
|
||||||
|
```
|
||||||
|
|
||||||
|
**Change notification channel selection:**
|
||||||
|
```json
|
||||||
|
{ "notification_channels": ["pushover_ops", "email_ops"] }
|
||||||
|
```
|
||||||
|
Only channels visible to the user (global + own) are accepted; others are silently dropped.
|
||||||
|
|
||||||
|
**Change password:**
|
||||||
|
```json
|
||||||
|
{ "password": { "current": "oldpass", "new": "newpass" } }
|
||||||
|
```
|
||||||
|
Requires the correct current password. New password is hashed before storage.
|
||||||
|
|
||||||
|
**Response:** `{"ok": true}`
|
||||||
|
|
||||||
|
**Status codes:** `200 OK`, `400` (missing/invalid field), `401` (unauthenticated), `403` (wrong current password)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
### Host Access
|
### Host Access
|
||||||
|
|
||||||
#### GET /api/0/hosts/{hostname}/access
|
#### GET /api/0/hosts/{hostname}/access
|
||||||
|
|||||||
@@ -1,602 +0,0 @@
|
|||||||
# Plugin Error Checking Implementation Plan
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
|
||||||
|
|
||||||
**Goal:** Improve plugin error checking in hbc, especially for nagios_runner, and fix logger messages silently discarded in daemon mode.
|
|
||||||
|
|
||||||
**Architecture:** Three focused changes across three files: (1) `hbd/client/plugin.py` gains a `skip_reason` attribute on Plugin and updated PluginLoader messaging; (2) `hbd/client/plugins/nagios_runner.py` gains async subprocess execution, stderr capture, signal-killed process handling, and init-time command path validation; (3) `hbd/client/main.py` gains proper post-fork logging reconfiguration to syslog.
|
|
||||||
|
|
||||||
**Tech Stack:** Python 3.11+, asyncio, `logging.handlers.SysLogHandler`, pytest
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## File Map
|
|
||||||
|
|
||||||
| Action | Path | What changes |
|
|
||||||
|---|---|---|
|
|
||||||
| Modify | `hbd/client/plugin.py` | `Plugin.__init__` gains `skip_reason`; `PluginLoader` checks it |
|
|
||||||
| Modify | `hbd/client/plugins/nagios_runner.py` | async subprocess, stderr, signal codes, init validation, `skip_reason` |
|
|
||||||
| Modify | `hbd/client/main.py` | `_reconfigure_logging_for_daemon()` helper; remove redundant syslog calls |
|
|
||||||
| Create | `tests/test_plugin.py` | PluginLoader messaging tests |
|
|
||||||
| Create | `tests/test_nagios_runner.py` | NagiosRunnerPlugin behaviour tests |
|
|
||||||
|
|
||||||
Run tests throughout with:
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_plugin.py tests/test_nagios_runner.py -v
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 1: Plugin.skip_reason + PluginLoader messaging
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `hbd/client/plugin.py:40-48` (Plugin.__init__)
|
|
||||||
- Modify: `hbd/client/plugin.py:369-381` (PluginLoader.load_from_directory)
|
|
||||||
- Create: `tests/test_plugin.py`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing tests**
|
|
||||||
|
|
||||||
Create `tests/test_plugin.py`:
|
|
||||||
|
|
||||||
```python
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
import textwrap
|
|
||||||
|
|
||||||
from hbd.client.plugin import Plugin, PluginLoader, PluginRegistry
|
|
||||||
|
|
||||||
|
|
||||||
def test_plugin_skip_reason_defaults_none(tmp_path):
|
|
||||||
plugin_code = textwrap.dedent("""
|
|
||||||
from hbd.client.plugin import MonitorPlugin
|
|
||||||
|
|
||||||
class MinimalPlugin(MonitorPlugin):
|
|
||||||
name = "minimal"
|
|
||||||
version = "1.0.0"
|
|
||||||
interval = 60
|
|
||||||
|
|
||||||
async def initialize(self):
|
|
||||||
return True
|
|
||||||
|
|
||||||
async def _collect_metrics(self):
|
|
||||||
return {}
|
|
||||||
""")
|
|
||||||
(tmp_path / "minimal.py").write_text(plugin_code)
|
|
||||||
registry = PluginRegistry()
|
|
||||||
loader = PluginLoader(registry)
|
|
||||||
asyncio.run(loader.load_from_directory(tmp_path))
|
|
||||||
plugin = registry.get("minimal")
|
|
||||||
assert plugin is not None
|
|
||||||
assert plugin.skip_reason is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_loader_logs_info_when_skip_reason_set(tmp_path, caplog):
|
|
||||||
plugin_code = textwrap.dedent("""
|
|
||||||
from hbd.client.plugin import MonitorPlugin
|
|
||||||
|
|
||||||
class SkippablePlugin(MonitorPlugin):
|
|
||||||
name = "skippable"
|
|
||||||
version = "1.0.0"
|
|
||||||
interval = 60
|
|
||||||
|
|
||||||
async def initialize(self):
|
|
||||||
self.skip_reason = "not configured in yaml"
|
|
||||||
return False
|
|
||||||
|
|
||||||
async def _collect_metrics(self):
|
|
||||||
return {}
|
|
||||||
""")
|
|
||||||
(tmp_path / "skippable.py").write_text(plugin_code)
|
|
||||||
registry = PluginRegistry()
|
|
||||||
loader = PluginLoader(registry)
|
|
||||||
|
|
||||||
with caplog.at_level(logging.INFO, logger="plugin.loader"):
|
|
||||||
count = asyncio.run(loader.load_from_directory(tmp_path))
|
|
||||||
|
|
||||||
assert count == 0
|
|
||||||
assert any("skipped: not configured in yaml" in r.message for r in caplog.records)
|
|
||||||
assert not any("failed initialization" in r.message for r in caplog.records)
|
|
||||||
|
|
||||||
|
|
||||||
def test_loader_logs_warning_when_no_skip_reason(tmp_path, caplog):
|
|
||||||
plugin_code = textwrap.dedent("""
|
|
||||||
from hbd.client.plugin import MonitorPlugin
|
|
||||||
|
|
||||||
class FailPlugin(MonitorPlugin):
|
|
||||||
name = "fail"
|
|
||||||
version = "1.0.0"
|
|
||||||
interval = 60
|
|
||||||
|
|
||||||
async def initialize(self):
|
|
||||||
return False
|
|
||||||
|
|
||||||
async def _collect_metrics(self):
|
|
||||||
return {}
|
|
||||||
""")
|
|
||||||
(tmp_path / "fail_plugin.py").write_text(plugin_code)
|
|
||||||
registry = PluginRegistry()
|
|
||||||
loader = PluginLoader(registry)
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING, logger="plugin.loader"):
|
|
||||||
count = asyncio.run(loader.load_from_directory(tmp_path))
|
|
||||||
|
|
||||||
assert count == 0
|
|
||||||
assert any("failed initialization" in r.message for r in caplog.records)
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run tests to verify they fail**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_plugin.py -v
|
|
||||||
```
|
|
||||||
Expected: `test_plugin_skip_reason_defaults_none` FAILS (attribute missing), others may error.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Add `skip_reason` to `Plugin.__init__`**
|
|
||||||
|
|
||||||
In `hbd/client/plugin.py`, in `Plugin.__init__` (around line 46), add one line:
|
|
||||||
|
|
||||||
```python
|
|
||||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
|
||||||
self.config = config or {}
|
|
||||||
self.logger = logging.getLogger(f"plugin.{self.name}")
|
|
||||||
self._initialized = False
|
|
||||||
self.skip_reason: Optional[str] = None
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Update PluginLoader messaging**
|
|
||||||
|
|
||||||
In `hbd/client/plugin.py`, replace the `if not initialized:` block (around line 372):
|
|
||||||
|
|
||||||
```python
|
|
||||||
if not initialized:
|
|
||||||
if plugin.skip_reason:
|
|
||||||
self.logger.info(
|
|
||||||
f"Plugin {plugin.name} skipped: {plugin.skip_reason}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self.logger.warning(
|
|
||||||
f"Plugin {plugin.name} failed initialization, skipping"
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 5: Run tests to verify they pass**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_plugin.py -v
|
|
||||||
```
|
|
||||||
Expected: all 3 tests PASS.
|
|
||||||
|
|
||||||
- [ ] **Step 6: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add hbd/client/plugin.py tests/test_plugin.py
|
|
||||||
git commit -m "feat: add skip_reason to Plugin; improve PluginLoader init messaging"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 2: NagiosRunnerPlugin — skip_reason when no commands
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `hbd/client/plugins/nagios_runner.py:88-105` (initialize)
|
|
||||||
- Modify: `tests/test_nagios_runner.py` (create)
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing test**
|
|
||||||
|
|
||||||
Create `tests/test_nagios_runner.py`:
|
|
||||||
|
|
||||||
```python
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import stat
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from hbd.client.plugins.nagios_runner import (
|
|
||||||
NagiosRunnerPlugin,
|
|
||||||
NAGIOS_OK,
|
|
||||||
NAGIOS_WARNING,
|
|
||||||
NAGIOS_CRITICAL,
|
|
||||||
NAGIOS_UNKNOWN,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_no_commands_sets_skip_reason():
|
|
||||||
plugin = NagiosRunnerPlugin(config={"commands": []})
|
|
||||||
result = asyncio.run(plugin.initialize())
|
|
||||||
assert result is False
|
|
||||||
assert plugin.skip_reason is not None
|
|
||||||
assert "nagios_runner.commands" in plugin.skip_reason
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run test to verify it fails**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_nagios_runner.py::test_no_commands_sets_skip_reason -v
|
|
||||||
```
|
|
||||||
Expected: FAIL — `plugin.skip_reason` is `None`.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Set skip_reason in NagiosRunnerPlugin.initialize()**
|
|
||||||
|
|
||||||
In `hbd/client/plugins/nagios_runner.py`, replace the early-return block in `initialize()` (around line 96):
|
|
||||||
|
|
||||||
```python
|
|
||||||
if not self.commands:
|
|
||||||
self.skip_reason = "no commands configured (add nagios_runner.commands to config)"
|
|
||||||
self.logger.info("No Nagios commands configured")
|
|
||||||
return False
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run test to verify it passes**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_nagios_runner.py::test_no_commands_sets_skip_reason -v
|
|
||||||
```
|
|
||||||
Expected: PASS.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add hbd/client/plugins/nagios_runner.py tests/test_nagios_runner.py
|
|
||||||
git commit -m "feat: set skip_reason on nagios_runner when no commands configured"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 3: NagiosRunnerPlugin — async subprocess, stderr capture, negative return codes
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `hbd/client/plugins/nagios_runner.py` (imports + `_run_nagios_plugin`)
|
|
||||||
- Modify: `tests/test_nagios_runner.py`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing tests**
|
|
||||||
|
|
||||||
Append to `tests/test_nagios_runner.py`:
|
|
||||||
|
|
||||||
```python
|
|
||||||
def test_stderr_used_when_stdout_empty(tmp_path):
|
|
||||||
script = tmp_path / "check_err.sh"
|
|
||||||
script.write_text("#!/bin/sh\necho 'error from stderr' >&2\nexit 2\n")
|
|
||||||
script.chmod(script.stat().st_mode | stat.S_IEXEC)
|
|
||||||
|
|
||||||
config = {"commands": [{"name": "t", "command": str(script)}], "timeout": 5}
|
|
||||||
plugin = NagiosRunnerPlugin(config=config)
|
|
||||||
asyncio.run(plugin.initialize())
|
|
||||||
data = asyncio.run(plugin._collect_metrics())
|
|
||||||
|
|
||||||
assert "error from stderr" in data["t_output"]
|
|
||||||
assert data["t_status_code"] == NAGIOS_CRITICAL
|
|
||||||
|
|
||||||
|
|
||||||
def test_stderr_appended_when_both_present(tmp_path):
|
|
||||||
script = tmp_path / "check_both.sh"
|
|
||||||
script.write_text("#!/bin/sh\necho 'OK - all good'\necho 'extra detail' >&2\nexit 0\n")
|
|
||||||
script.chmod(script.stat().st_mode | stat.S_IEXEC)
|
|
||||||
|
|
||||||
config = {"commands": [{"name": "t", "command": str(script)}], "timeout": 5}
|
|
||||||
plugin = NagiosRunnerPlugin(config=config)
|
|
||||||
asyncio.run(plugin.initialize())
|
|
||||||
data = asyncio.run(plugin._collect_metrics())
|
|
||||||
|
|
||||||
assert "OK - all good" in data["t_output"]
|
|
||||||
assert "extra detail" in data["t_output"]
|
|
||||||
assert data["t_status_code"] == NAGIOS_OK
|
|
||||||
|
|
||||||
|
|
||||||
def test_negative_returncode_maps_to_unknown():
|
|
||||||
# kill -9 $$ kills the shell itself; asyncio sees returncode -9
|
|
||||||
config = {"commands": [{"name": "t", "command": "kill -9 $$"}], "timeout": 5}
|
|
||||||
plugin = NagiosRunnerPlugin(config=config)
|
|
||||||
asyncio.run(plugin.initialize())
|
|
||||||
data = asyncio.run(plugin._collect_metrics())
|
|
||||||
|
|
||||||
assert data["t_status_code"] == NAGIOS_UNKNOWN
|
|
||||||
assert "signal" in data["t_output"].lower()
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run tests to verify they fail**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_nagios_runner.py::test_stderr_used_when_stdout_empty \
|
|
||||||
tests/test_nagios_runner.py::test_stderr_appended_when_both_present \
|
|
||||||
tests/test_nagios_runner.py::test_negative_returncode_maps_to_unknown -v
|
|
||||||
```
|
|
||||||
Expected: all FAIL — current implementation ignores stderr and doesn't handle negative codes.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Update imports in nagios_runner.py**
|
|
||||||
|
|
||||||
Replace the import block at the top of `hbd/client/plugins/nagios_runner.py`:
|
|
||||||
|
|
||||||
```python
|
|
||||||
import asyncio
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
|
||||||
|
|
||||||
from hbd.client.plugin import MonitorPlugin
|
|
||||||
```
|
|
||||||
|
|
||||||
(Remove `import subprocess`; add `import asyncio` and `import os`.)
|
|
||||||
|
|
||||||
- [ ] **Step 4: Upgrade collection log level from DEBUG to INFO**
|
|
||||||
|
|
||||||
In `hbd/client/plugins/nagios_runner.py`, in `_collect_metrics()`, change the debug log (around line 144) so results are visible at INFO level:
|
|
||||||
|
|
||||||
```python
|
|
||||||
self.logger.info(
|
|
||||||
f"Executed {name}: {STATUS_NAMES.get(status_code, 'UNKNOWN')} - {output[:50]}"
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 5: Replace `_run_nagios_plugin` with async implementation**
|
|
||||||
|
|
||||||
Replace the entire `_run_nagios_plugin` method in `hbd/client/plugins/nagios_runner.py`:
|
|
||||||
|
|
||||||
```python
|
|
||||||
async def _run_nagios_plugin(
|
|
||||||
self,
|
|
||||||
command: str
|
|
||||||
) -> Tuple[int, str, Dict[str, Any]]:
|
|
||||||
"""Execute a Nagios plugin and parse its output."""
|
|
||||||
try:
|
|
||||||
proc = await asyncio.create_subprocess_shell(
|
|
||||||
command,
|
|
||||||
stdout=asyncio.subprocess.PIPE,
|
|
||||||
stderr=asyncio.subprocess.PIPE,
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
|
||||||
proc.communicate(), timeout=self.timeout
|
|
||||||
)
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
proc.kill()
|
|
||||||
await proc.communicate()
|
|
||||||
self.logger.error(f"Command timed out: {command}")
|
|
||||||
return NAGIOS_UNKNOWN, f"Command timed out after {self.timeout}s", {}
|
|
||||||
|
|
||||||
status_code = proc.returncode
|
|
||||||
|
|
||||||
if status_code < 0:
|
|
||||||
return NAGIOS_UNKNOWN, f"Process killed by signal {-status_code}", {}
|
|
||||||
|
|
||||||
if status_code > 3:
|
|
||||||
status_code = NAGIOS_UNKNOWN
|
|
||||||
|
|
||||||
stdout = stdout_bytes.decode(errors="replace").strip()
|
|
||||||
stderr = stderr_bytes.decode(errors="replace").strip()
|
|
||||||
|
|
||||||
# Parse perfdata from stdout before mixing in stderr
|
|
||||||
perfdata = self._parse_perfdata(stdout)
|
|
||||||
|
|
||||||
# Build status message
|
|
||||||
status_part = stdout.split('|')[0].strip() if '|' in stdout else stdout
|
|
||||||
|
|
||||||
if not stdout and stderr:
|
|
||||||
output_msg = stderr
|
|
||||||
elif stdout and stderr:
|
|
||||||
output_msg = f"{status_part} [stderr: {stderr}]"
|
|
||||||
else:
|
|
||||||
output_msg = status_part
|
|
||||||
|
|
||||||
return status_code, output_msg, perfdata
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
self.logger.error(f"Error executing command: {e}")
|
|
||||||
return NAGIOS_UNKNOWN, f"Execution error: {str(e)}", {}
|
|
||||||
```
|
|
||||||
|
|
||||||
Also remove the now-unused `self.shell` line from `__init__` (the `shell` config key is no longer used since `create_subprocess_shell` always uses a shell):
|
|
||||||
|
|
||||||
In `NagiosRunnerPlugin.__init__`, remove:
|
|
||||||
```python
|
|
||||||
self.shell: bool = config.get("shell", True) if config else True
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 6: Run tests to verify they pass**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_nagios_runner.py -v
|
|
||||||
```
|
|
||||||
Expected: all tests PASS including the 3 new ones.
|
|
||||||
|
|
||||||
- [ ] **Step 7: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add hbd/client/plugins/nagios_runner.py tests/test_nagios_runner.py
|
|
||||||
git commit -m "feat: async subprocess in nagios_runner with stderr capture and signal handling"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 4: NagiosRunnerPlugin — command path validation at init
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `hbd/client/plugins/nagios_runner.py` (initialize)
|
|
||||||
- Modify: `tests/test_nagios_runner.py`
|
|
||||||
|
|
||||||
- [ ] **Step 1: Write failing tests**
|
|
||||||
|
|
||||||
Append to `tests/test_nagios_runner.py`:
|
|
||||||
|
|
||||||
```python
|
|
||||||
def test_absolute_path_not_found_warns(caplog):
|
|
||||||
fake_cmd = "/nonexistent_hbc_test_path/check_something"
|
|
||||||
config = {"commands": [{"name": "t", "command": fake_cmd}]}
|
|
||||||
plugin = NagiosRunnerPlugin(config=config)
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING, logger="plugin.nagios_runner"):
|
|
||||||
asyncio.run(plugin.initialize())
|
|
||||||
|
|
||||||
assert any("not found" in r.message for r in caplog.records)
|
|
||||||
|
|
||||||
|
|
||||||
def test_absolute_path_not_executable_warns(caplog, tmp_path):
|
|
||||||
non_exec = tmp_path / "check_test"
|
|
||||||
non_exec.write_text("#!/bin/sh\necho OK\n")
|
|
||||||
non_exec.chmod(0o644) # readable but not executable
|
|
||||||
|
|
||||||
config = {"commands": [{"name": "t", "command": str(non_exec)}]}
|
|
||||||
plugin = NagiosRunnerPlugin(config=config)
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING, logger="plugin.nagios_runner"):
|
|
||||||
asyncio.run(plugin.initialize())
|
|
||||||
|
|
||||||
assert any("not executable" in r.message for r in caplog.records)
|
|
||||||
|
|
||||||
|
|
||||||
def test_relative_path_not_checked(caplog):
|
|
||||||
# Relative paths (resolved via PATH) must not generate warnings
|
|
||||||
config = {"commands": [{"name": "t", "command": "echo OK"}]}
|
|
||||||
plugin = NagiosRunnerPlugin(config=config)
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING, logger="plugin.nagios_runner"):
|
|
||||||
asyncio.run(plugin.initialize())
|
|
||||||
|
|
||||||
assert not any(
|
|
||||||
"not found" in r.message or "not executable" in r.message
|
|
||||||
for r in caplog.records
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Run tests to verify they fail**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_nagios_runner.py::test_absolute_path_not_found_warns \
|
|
||||||
tests/test_nagios_runner.py::test_absolute_path_not_executable_warns \
|
|
||||||
tests/test_nagios_runner.py::test_relative_path_not_checked -v
|
|
||||||
```
|
|
||||||
Expected: `test_absolute_path_not_found_warns` and `test_absolute_path_not_executable_warns` FAIL (no warnings logged); `test_relative_path_not_checked` may pass.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Add command path validation to `initialize()`**
|
|
||||||
|
|
||||||
In `hbd/client/plugins/nagios_runner.py`, extend `initialize()` by adding validation after the existing "log each command" loop (after line 103, before `return True`):
|
|
||||||
|
|
||||||
```python
|
|
||||||
# Validate absolute command paths early
|
|
||||||
for cmd_config in self.commands:
|
|
||||||
name = cmd_config.get("name", "unnamed")
|
|
||||||
command = cmd_config.get("command", "")
|
|
||||||
if not command:
|
|
||||||
continue
|
|
||||||
exe = command.split()[0]
|
|
||||||
if os.path.isabs(exe):
|
|
||||||
if not os.path.isfile(exe):
|
|
||||||
self.logger.warning(
|
|
||||||
f"Command '{name}': executable not found: {exe}"
|
|
||||||
)
|
|
||||||
elif not os.access(exe, os.X_OK):
|
|
||||||
self.logger.warning(
|
|
||||||
f"Command '{name}': executable not executable: {exe}"
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 4: Run full test suite to verify all pass**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_plugin.py tests/test_nagios_runner.py -v
|
|
||||||
```
|
|
||||||
Expected: all tests PASS.
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add hbd/client/plugins/nagios_runner.py tests/test_nagios_runner.py
|
|
||||||
git commit -m "feat: validate absolute command paths at nagios_runner init"
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Task 5: Daemon mode logging — route to syslog after fork
|
|
||||||
|
|
||||||
**Files:**
|
|
||||||
- Modify: `hbd/client/main.py` (new helper + updated daemon block)
|
|
||||||
|
|
||||||
No automated test for daemonization itself (fork behaviour is hard to unit-test). Manual verification steps are provided below.
|
|
||||||
|
|
||||||
- [ ] **Step 1: Add `_reconfigure_logging_for_daemon` helper**
|
|
||||||
|
|
||||||
In `hbd/client/main.py`, add this function just before `def build_parser()` (around line 589):
|
|
||||||
|
|
||||||
```python
|
|
||||||
def _reconfigure_logging_for_daemon(log_level: int) -> None:
|
|
||||||
"""Replace StreamHandlers (now writing to /dev/null) with a SysLogHandler."""
|
|
||||||
from logging.handlers import SysLogHandler
|
|
||||||
|
|
||||||
root = logging.getLogger()
|
|
||||||
for handler in root.handlers[:]:
|
|
||||||
root.removeHandler(handler)
|
|
||||||
handler.close()
|
|
||||||
|
|
||||||
try:
|
|
||||||
syslog_handler = SysLogHandler(
|
|
||||||
address="/dev/log",
|
|
||||||
facility=SysLogHandler.LOG_DAEMON,
|
|
||||||
)
|
|
||||||
except OSError:
|
|
||||||
syslog_handler = SysLogHandler(
|
|
||||||
address=("localhost", 514),
|
|
||||||
facility=SysLogHandler.LOG_DAEMON,
|
|
||||||
)
|
|
||||||
# Attach the fallback first so the warning reaches syslog
|
|
||||||
syslog_handler.setFormatter(
|
|
||||||
logging.Formatter("hbc[%(process)d]: %(name)s %(levelname)s: %(message)s")
|
|
||||||
)
|
|
||||||
root.addHandler(syslog_handler)
|
|
||||||
root.setLevel(log_level)
|
|
||||||
logging.warning("/dev/log not found, using syslog UDP localhost:514")
|
|
||||||
return
|
|
||||||
|
|
||||||
syslog_handler.setFormatter(
|
|
||||||
logging.Formatter("hbc[%(process)d]: %(name)s %(levelname)s: %(message)s")
|
|
||||||
)
|
|
||||||
root.addHandler(syslog_handler)
|
|
||||||
root.setLevel(log_level)
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 2: Update the daemon block in `main()`**
|
|
||||||
|
|
||||||
In `hbd/client/main.py`, replace the entire `if args.daemon:` block (lines 664–675):
|
|
||||||
|
|
||||||
```python
|
|
||||||
if args.daemon:
|
|
||||||
print("Daemonizing...")
|
|
||||||
daemonize()
|
|
||||||
_reconfigure_logging_for_daemon(log_level)
|
|
||||||
logging.info(f"hbc starting, sending heartbeat to {', '.join(args.hosts)}")
|
|
||||||
```
|
|
||||||
|
|
||||||
This removes the `import syslog`, `syslog.openlog()`, and `syslog.syslog()` calls (now handled by the logging system) and removes the no-op second `logging.basicConfig()` call.
|
|
||||||
|
|
||||||
- [ ] **Step 3: Run existing test suite to confirm no regressions**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python -m pytest tests/test_plugin.py tests/test_nagios_runner.py -v
|
|
||||||
```
|
|
||||||
Expected: all tests still PASS.
|
|
||||||
|
|
||||||
- [ ] **Step 4: Manual smoke test — verify syslog output in daemon mode**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# In one terminal, tail syslog
|
|
||||||
sudo journalctl -f -t hbc
|
|
||||||
|
|
||||||
# In another terminal, start hbc in daemon mode (replace HOST with a real or dummy host)
|
|
||||||
python -m hbd.client.main -d -v localhost
|
|
||||||
|
|
||||||
# Expected in journalctl output:
|
|
||||||
# hbc[<pid>]: hbc.main INFO: Starting hbc for <hostname> -> ['localhost']
|
|
||||||
# hbc[<pid>]: hbc.main INFO: hbc starting, sending heartbeat to localhost
|
|
||||||
# hbc[<pid>]: plugin.loader INFO: ...
|
|
||||||
|
|
||||||
# Stop the daemon
|
|
||||||
pkill -f "hbd.client.main"
|
|
||||||
```
|
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git add hbd/client/main.py
|
|
||||||
git commit -m "fix: reconfigure logging to syslog after daemonize() instead of no-op basicConfig"
|
|
||||||
```
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
# Plugin Error Checking & Daemon Logging — Design Spec
|
|
||||||
|
|
||||||
**Date:** 2026-04-25
|
|
||||||
**Scope:** hbc client — daemon mode logging, nagios_runner plugin robustness, PluginLoader messaging
|
|
||||||
**Files affected:** `hbd/client/main.py`, `hbd/client/plugins/nagios_runner.py`, `hbd/client/plugin.py`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 1. Daemon Mode Logging
|
|
||||||
|
|
||||||
### Problem
|
|
||||||
In `main()`, `logging.basicConfig()` is called before `daemonize()` (establishing a StreamHandler to stderr), then called again after `daemonize()`. The second call is a no-op — Python ignores `basicConfig()` when handlers are already configured. After daemonization, stderr is redirected to `/dev/null`, so all subsequent log output is silently discarded.
|
|
||||||
|
|
||||||
The existing `syslog.openlog()` / `syslog.syslog()` calls (lines 666–668) write a single startup message but do not integrate with the `logging` system, so plugin and connection log messages never reach syslog.
|
|
||||||
|
|
||||||
### Fix
|
|
||||||
After `daemonize()`, explicitly reconfigure the root logger:
|
|
||||||
|
|
||||||
1. Remove all existing handlers (they now write to `/dev/null`).
|
|
||||||
2. Add `logging.handlers.SysLogHandler(address='/dev/log', facility=LOG_DAEMON)`.
|
|
||||||
3. Set formatter: `hbc[%(process)d]: %(name)s %(levelname)s: %(message)s`
|
|
||||||
4. Preserve the `log_level` already determined from `-v`/`-x` CLI flags.
|
|
||||||
|
|
||||||
Remove the redundant `syslog.openlog()` / `syslog.syslog()` calls — the logging system handles routing.
|
|
||||||
|
|
||||||
**Fallback:** If `/dev/log` does not exist (containers, some BSDs), fall back to `SysLogHandler(address=('localhost', 514))`. Log one warning (to stderr, before handlers are replaced) so the operator knows.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. Nagios Runner Improvements
|
|
||||||
|
|
||||||
### 2a — Async Subprocess
|
|
||||||
`_run_nagios_plugin()` is declared `async def` but calls `subprocess.run()` synchronously, blocking the event loop for the full command duration.
|
|
||||||
|
|
||||||
**Fix:** Replace with `asyncio.create_subprocess_shell()` + `await proc.communicate()`. Enforce timeout with `asyncio.wait_for(..., timeout=self.timeout)` and catch `asyncio.TimeoutError`.
|
|
||||||
|
|
||||||
### 2b — Stderr Capture
|
|
||||||
Subprocess stderr is currently discarded (`capture_output=True` only captures stdout in the sync call; stderr content is lost).
|
|
||||||
|
|
||||||
**Fix:** Pass `stderr=asyncio.subprocess.PIPE` to `create_subprocess_shell`. After `communicate()`, if stdout is empty but stderr has content, use stderr as the output message. If both have content, append stderr to the output for visibility.
|
|
||||||
|
|
||||||
### 2c — Negative Return Codes
|
|
||||||
A negative `returncode` means the process was killed by a signal (SIGKILL, OOM, etc.). The current code treats these as-is, which may produce unexpected status values.
|
|
||||||
|
|
||||||
**Fix:** If `returncode < 0`, map to `NAGIOS_UNKNOWN` with message `"Process killed by signal {-returncode}"`.
|
|
||||||
|
|
||||||
### 2d — Command Path Validation at Init
|
|
||||||
`initialize()` currently only checks that the commands list is non-empty.
|
|
||||||
|
|
||||||
**Fix:** For each command entry during `initialize()`:
|
|
||||||
- Warn and skip the entry if `name` or `command` is missing.
|
|
||||||
- Extract the executable (first whitespace-delimited token of the command string).
|
|
||||||
- If the executable is an absolute path, check `os.path.isfile()` and `os.access(..., os.X_OK)`. Log a `WARNING` if either check fails.
|
|
||||||
- Commands with relative paths or shell builtins are not checked (they may be on PATH) — just noted.
|
|
||||||
- Validation warns only; all original entries in `self.commands` are retained and still attempted at collection time (where the existing missing-name/command guard already skips them). The plugin initializes successfully as long as the commands list is non-empty.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 3. PluginLoader Messaging
|
|
||||||
|
|
||||||
### Problem
|
|
||||||
When `initialize()` returns `False`, the loader always logs:
|
|
||||||
> `WARNING: Plugin X failed initialization, skipping`
|
|
||||||
|
|
||||||
This is alarming when the real reason is simply "no commands configured". There is no API to distinguish "not configured" from "genuinely broken".
|
|
||||||
|
|
||||||
### Fix
|
|
||||||
Add an optional `skip_reason` attribute to `Plugin.__init__()` (defaults to `None`).
|
|
||||||
|
|
||||||
In `PluginLoader.load_from_directory()`, after `initialize()` returns `False`:
|
|
||||||
- If `plugin.skip_reason` is set → `logger.info(f"Plugin {plugin.name} skipped: {plugin.skip_reason}")`
|
|
||||||
- If `plugin.skip_reason` is `None` → `logger.warning(f"Plugin {plugin.name} failed initialization, skipping")` (existing behaviour)
|
|
||||||
|
|
||||||
In `NagiosRunnerPlugin.initialize()`, when no commands are configured:
|
|
||||||
```python
|
|
||||||
self.skip_reason = "no commands configured (add nagios_runner.commands to config)"
|
|
||||||
return False
|
|
||||||
```
|
|
||||||
|
|
||||||
Genuine failures (exceptions) continue to go through the existing `except` block in the loader, logging at `ERROR` with traceback — unchanged.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Decisions
|
|
||||||
|
|
||||||
| Topic | Decision |
|
|
||||||
|---|---|
|
|
||||||
| Daemon log destination | syslog only (LOG_DAEMON facility) |
|
|
||||||
| Syslog fallback | localhost:514 UDP if `/dev/log` absent |
|
|
||||||
| Nagios result log level | INFO for all statuses (OK/WARNING/CRITICAL/UNKNOWN) |
|
|
||||||
| Invalid command handling at init | Warn and continue; still attempt at collection time |
|
|
||||||
| PluginLoader API change | `skip_reason` attribute on Plugin base class, checked by loader |
|
|
||||||
+1
-1
@@ -14,4 +14,4 @@ Install options:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
__all__ = ["__version__"]
|
__all__ = ["__version__"]
|
||||||
__version__ = "5.1.9"
|
__version__ = "5.4.0"
|
||||||
|
|||||||
@@ -16,6 +16,12 @@ CLIENT_DEFAULTS = {
|
|||||||
"hb_port": 50003, # Port where hbd servers listen
|
"hb_port": 50003, # Port where hbd servers listen
|
||||||
"interval": 10, # Heartbeat interval in seconds
|
"interval": 10, # Heartbeat interval in seconds
|
||||||
|
|
||||||
|
# Host identity
|
||||||
|
"owner": None, # Optional username to set as this host's owner on the server
|
||||||
|
|
||||||
|
# Security
|
||||||
|
"allow_remote_command": False, # Execute shell commands received in CMD packets from the server
|
||||||
|
|
||||||
# Runtime flags
|
# Runtime flags
|
||||||
"foreground": False,
|
"foreground": False,
|
||||||
"verbose": False,
|
"verbose": False,
|
||||||
|
|||||||
+157
-58
@@ -21,6 +21,7 @@ from typing import Dict, List, Optional
|
|||||||
# Import protocol and config
|
# Import protocol and config
|
||||||
from .config import load_config
|
from .config import load_config
|
||||||
from ..common.proto import dicttos, stodict
|
from ..common.proto import dicttos, stodict
|
||||||
|
from .. import __version__
|
||||||
|
|
||||||
# Import plugin system
|
# Import plugin system
|
||||||
from .plugin import PluginRegistry, PluginLoader, InfoPlugin, MonitorPlugin
|
from .plugin import PluginRegistry, PluginLoader, InfoPlugin, MonitorPlugin
|
||||||
@@ -36,6 +37,10 @@ dorestart = False
|
|||||||
shutdown_event: Optional[asyncio.Event] = None
|
shutdown_event: Optional[asyncio.Event] = None
|
||||||
active_tasks: List[asyncio.Task] = []
|
active_tasks: List[asyncio.Task] = []
|
||||||
|
|
||||||
|
# Set from config in async_main. Off by default: a CMD packet is an unauthenticated
|
||||||
|
# UDP datagram, so executing one must be opted into per host.
|
||||||
|
allow_remote_command = False
|
||||||
|
|
||||||
|
|
||||||
class AsyncConnection:
|
class AsyncConnection:
|
||||||
"""Async UDP connection to a heartbeat server."""
|
"""Async UDP connection to a heartbeat server."""
|
||||||
@@ -56,6 +61,9 @@ class AsyncConnection:
|
|||||||
self.transport: Optional[asyncio.DatagramTransport] = None
|
self.transport: Optional[asyncio.DatagramTransport] = None
|
||||||
self.protocol: Optional[asyncio.DatagramProtocol] = None
|
self.protocol: Optional[asyncio.DatagramProtocol] = None
|
||||||
self._dead = False
|
self._dead = False
|
||||||
|
self._ever_opened = False
|
||||||
|
self._open_fail_count = 0 # consecutive failures before first success
|
||||||
|
self.request_info_event: asyncio.Event = asyncio.Event()
|
||||||
|
|
||||||
self.logger = logging.getLogger(f"hbc.conn.{addr}")
|
self.logger = logging.getLogger(f"hbc.conn.{addr}")
|
||||||
|
|
||||||
@@ -73,6 +81,7 @@ class AsyncConnection:
|
|||||||
lambda: HeartbeatProtocol(self),
|
lambda: HeartbeatProtocol(self),
|
||||||
family=self.af
|
family=self.af
|
||||||
)
|
)
|
||||||
|
self._ever_opened = True
|
||||||
self.logger.debug(f"Opened connection to {self.addr}:{self.port}")
|
self.logger.debug(f"Opened connection to {self.addr}:{self.port}")
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -134,6 +143,9 @@ class AsyncConnection:
|
|||||||
|
|
||||||
self.ackcount += 1
|
self.ackcount += 1
|
||||||
self.logger.debug(f"ACK received, RTT: {rtt:.1f}ms")
|
self.logger.debug(f"ACK received, RTT: {rtt:.1f}ms")
|
||||||
|
if msg.get("request_update"):
|
||||||
|
self.logger.info("server requested plugin info refresh")
|
||||||
|
self.request_info_event.set()
|
||||||
|
|
||||||
|
|
||||||
class HeartbeatProtocol(asyncio.DatagramProtocol):
|
class HeartbeatProtocol(asyncio.DatagramProtocol):
|
||||||
@@ -169,14 +181,13 @@ class HeartbeatProtocol(asyncio.DatagramProtocol):
|
|||||||
self.logger.error(f"Error processing datagram: {e}", exc_info=True)
|
self.logger.error(f"Error processing datagram: {e}", exc_info=True)
|
||||||
|
|
||||||
def error_received(self, exc):
|
def error_received(self, exc):
|
||||||
"""Handle protocol errors."""
|
"""Handle protocol errors — close transport so the heartbeat sender retries."""
|
||||||
self.logger.warning(f"Protocol error on {self.connection.addr}: {exc} — dropping connection")
|
self.logger.warning(f"Protocol error on {self.connection.addr}: {exc} — will retry")
|
||||||
self.connection._dead = True
|
|
||||||
self.connection.close()
|
self.connection.close()
|
||||||
|
|
||||||
|
|
||||||
async def handle_command(conn: AsyncConnection, msg: dict):
|
async def handle_command(conn: AsyncConnection, msg: dict):
|
||||||
"""Execute a command received from server."""
|
"""Execute a command received from server, if allow_remote_command is set."""
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
cmd = msg.get("cmd", "")
|
cmd = msg.get("cmd", "")
|
||||||
@@ -184,6 +195,15 @@ async def handle_command(conn: AsyncConnection, msg: dict):
|
|||||||
return
|
return
|
||||||
|
|
||||||
logger = logging.getLogger("hbc.command")
|
logger = logging.getLogger("hbc.command")
|
||||||
|
|
||||||
|
if not allow_remote_command:
|
||||||
|
logger.warning(f"Refused command (allow_remote_command is false): {cmd}")
|
||||||
|
await conn.sendto({
|
||||||
|
"service": "command",
|
||||||
|
"msg": "Refused: allow_remote_command is false",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
|
||||||
logger.info(f"Executing command: {cmd}")
|
logger.info(f"Executing command: {cmd}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -262,15 +282,51 @@ async def handle_update(conn: AsyncConnection, _msg: dict): # pyright: ignore[r
|
|||||||
|
|
||||||
|
|
||||||
async def heartbeat_sender(conn: AsyncConnection, interval: int):
|
async def heartbeat_sender(conn: AsyncConnection, interval: int):
|
||||||
"""Send periodic heartbeats.
|
"""Send periodic heartbeats, retrying the connection if it is not open.
|
||||||
|
|
||||||
|
IPv6 connections that fail to open before their first successful send are
|
||||||
|
dropped after IPV6_EARLY_FAIL_LIMIT attempts so that a network without IPv6
|
||||||
|
does not keep a dead sender alive. IPv4 connections are retried indefinitely.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
conn: Connection to send on
|
conn: Connection to send on
|
||||||
interval: Heartbeat interval in seconds
|
interval: Heartbeat interval in seconds
|
||||||
"""
|
"""
|
||||||
logger = logging.getLogger("hbc.heartbeat")
|
logger = logging.getLogger("hbc.heartbeat")
|
||||||
|
IPV6_EARLY_FAIL_LIMIT = 3
|
||||||
|
|
||||||
|
while running and not conn._dead:
|
||||||
|
# Ensure transport is open before attempting to send.
|
||||||
|
if not conn.transport:
|
||||||
|
opened = await conn.open()
|
||||||
|
if opened:
|
||||||
|
conn._open_fail_count = 0
|
||||||
|
else:
|
||||||
|
conn._open_fail_count += 1
|
||||||
|
# Drop an IPv6 connection that has never come up within the
|
||||||
|
# first few attempts — it is likely unavailable on this network.
|
||||||
|
if (not conn._ever_opened
|
||||||
|
and conn.af == socket.AF_INET6
|
||||||
|
and conn._open_fail_count >= IPV6_EARLY_FAIL_LIMIT):
|
||||||
|
logger.warning(
|
||||||
|
f"IPv6 connection to {conn.addr} unreachable after "
|
||||||
|
f"{conn._open_fail_count} attempts, disabling"
|
||||||
|
)
|
||||||
|
conn._dead = True
|
||||||
|
break
|
||||||
|
# Retry after the normal interval; IPv4 retries forever.
|
||||||
|
try:
|
||||||
|
if shutdown_event:
|
||||||
|
await asyncio.wait_for(shutdown_event.wait(), timeout=interval)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
await asyncio.sleep(interval)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
continue
|
||||||
|
|
||||||
while running:
|
|
||||||
try:
|
try:
|
||||||
msg = {
|
msg = {
|
||||||
"acks": conn.ackcount,
|
"acks": conn.ackcount,
|
||||||
@@ -279,19 +335,16 @@ async def heartbeat_sender(conn: AsyncConnection, interval: int):
|
|||||||
}
|
}
|
||||||
await conn.sendto(msg, "HTB")
|
await conn.sendto(msg, "HTB")
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error sending heartbeat: {e}", exc_info=True)
|
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
logger.debug("Heartbeat sender cancelled")
|
logger.debug("Heartbeat sender cancelled")
|
||||||
raise
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error sending heartbeat: {e}", exc_info=True)
|
||||||
|
|
||||||
# Wait for next interval or shutdown event
|
# Wait for next interval or shutdown event
|
||||||
try:
|
try:
|
||||||
if shutdown_event:
|
if shutdown_event:
|
||||||
await asyncio.wait_for(
|
await asyncio.wait_for(shutdown_event.wait(), timeout=interval)
|
||||||
shutdown_event.wait(),
|
|
||||||
timeout=interval
|
|
||||||
)
|
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
await asyncio.sleep(interval)
|
await asyncio.sleep(interval)
|
||||||
@@ -302,6 +355,27 @@ async def heartbeat_sender(conn: AsyncConnection, interval: int):
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
async def _info_plugin_refresh_loop(conn: AsyncConnection, info_plugins: List):
|
||||||
|
"""Wait for server requests to re-send InfoPlugin data."""
|
||||||
|
logger = logging.getLogger("hbc.plugins")
|
||||||
|
while running:
|
||||||
|
await conn.request_info_event.wait()
|
||||||
|
if not running:
|
||||||
|
break
|
||||||
|
conn.request_info_event.clear()
|
||||||
|
logger.info("refreshing InfoPlugins on server request")
|
||||||
|
for plugin in info_plugins:
|
||||||
|
plugin._cache = None
|
||||||
|
try:
|
||||||
|
data = await plugin.collect()
|
||||||
|
if data:
|
||||||
|
await conn.sendto(
|
||||||
|
{"plugin": plugin.name, **data, "_interval": plugin.interval}, "PLG")
|
||||||
|
logger.info(f"Resent {plugin.name} data")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error re-collecting {plugin.name}: {e}", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
async def plugin_collector(conn: AsyncConnection, registry: PluginRegistry):
|
async def plugin_collector(conn: AsyncConnection, registry: PluginRegistry):
|
||||||
"""Collect and send plugin data.
|
"""Collect and send plugin data.
|
||||||
|
|
||||||
@@ -317,8 +391,8 @@ async def plugin_collector(conn: AsyncConnection, registry: PluginRegistry):
|
|||||||
try:
|
try:
|
||||||
data = await plugin.collect()
|
data = await plugin.collect()
|
||||||
if data:
|
if data:
|
||||||
# Create PLG message with plugin name
|
# Create PLG message with plugin name and declared interval
|
||||||
plugin_msg = {"plugin": plugin.name, **data}
|
plugin_msg = {"plugin": plugin.name, **data, "_interval": plugin.interval}
|
||||||
await conn.sendto(plugin_msg, "PLG")
|
await conn.sendto(plugin_msg, "PLG")
|
||||||
logger.info(f"Sent {plugin.name} data")
|
logger.info(f"Sent {plugin.name} data")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -333,24 +407,21 @@ async def plugin_collector(conn: AsyncConnection, registry: PluginRegistry):
|
|||||||
for plugin in monitor_plugins:
|
for plugin in monitor_plugins:
|
||||||
by_interval[plugin.interval].append(plugin)
|
by_interval[plugin.interval].append(plugin)
|
||||||
|
|
||||||
# Create tasks for each interval
|
# Create tasks for each interval; always include the info-refresh watcher
|
||||||
tasks = []
|
tasks = [asyncio.create_task(_info_plugin_refresh_loop(conn, info_plugins))]
|
||||||
for interval, plugins in by_interval.items():
|
for interval, plugins in by_interval.items():
|
||||||
task = asyncio.create_task(
|
tasks.append(asyncio.create_task(
|
||||||
plugin_collector_interval(conn, plugins, interval)
|
plugin_collector_interval(conn, plugins, interval)
|
||||||
)
|
))
|
||||||
tasks.append(task)
|
|
||||||
|
|
||||||
# Wait for all tasks
|
try:
|
||||||
if tasks:
|
await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
try:
|
except asyncio.CancelledError:
|
||||||
await asyncio.gather(*tasks, return_exceptions=True)
|
logger.debug("Plugin collector cancelled, cancelling sub-tasks")
|
||||||
except asyncio.CancelledError:
|
for task in tasks:
|
||||||
logger.debug("Plugin collector cancelled, cancelling sub-tasks")
|
if not task.done():
|
||||||
for task in tasks:
|
task.cancel()
|
||||||
if not task.done():
|
raise
|
||||||
task.cancel()
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
async def plugin_collector_interval(
|
async def plugin_collector_interval(
|
||||||
@@ -373,7 +444,7 @@ async def plugin_collector_interval(
|
|||||||
data = await plugin.collect()
|
data = await plugin.collect()
|
||||||
if data:
|
if data:
|
||||||
# Don't use encode_plugin_data - create dict directly
|
# Don't use encode_plugin_data - create dict directly
|
||||||
plugin_msg = {"plugin": plugin.name, **data}
|
plugin_msg = {"plugin": plugin.name, **data, "_interval": plugin.interval}
|
||||||
await conn.sendto(plugin_msg, "PLG")
|
await conn.sendto(plugin_msg, "PLG")
|
||||||
logger.debug(f"Sent {plugin.name} data")
|
logger.debug(f"Sent {plugin.name} data")
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
@@ -427,16 +498,14 @@ async def cleanup(connections: List[AsyncConnection]):
|
|||||||
logger = logging.getLogger("hbc.cleanup")
|
logger = logging.getLogger("hbc.cleanup")
|
||||||
logger.info("Cleaning up connections")
|
logger.info("Cleaning up connections")
|
||||||
|
|
||||||
for conn in connections:
|
target = next((c for c in connections if c.transport), connections[0] if connections else None)
|
||||||
|
# A SIGHUP restart is not a host shutdown, so don't announce one.
|
||||||
|
if target and send_shutdown and not dorestart:
|
||||||
try:
|
try:
|
||||||
msg = {
|
await target.sendto({"shutdown": 1, "acks": target.ackcount})
|
||||||
"shutdown": 1,
|
|
||||||
"acks": conn.ackcount
|
|
||||||
}
|
|
||||||
await conn.sendto(msg)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error sending shutdown: {e}")
|
logger.error(f"Error sending shutdown: {e}")
|
||||||
|
for conn in connections:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
# Give messages time to send
|
# Give messages time to send
|
||||||
@@ -445,11 +514,12 @@ async def cleanup(connections: List[AsyncConnection]):
|
|||||||
|
|
||||||
async def async_main(args, config):
|
async def async_main(args, config):
|
||||||
"""Async main function."""
|
"""Async main function."""
|
||||||
global running, shutdown_event, active_tasks
|
global running, shutdown_event, active_tasks, send_shutdown, allow_remote_command
|
||||||
|
|
||||||
# Create shutdown event
|
# Create shutdown event
|
||||||
shutdown_event = asyncio.Event()
|
shutdown_event = asyncio.Event()
|
||||||
active_tasks = []
|
active_tasks = []
|
||||||
|
allow_remote_command = bool(config.get("allow_remote_command", False))
|
||||||
|
|
||||||
logger = logging.getLogger("hbc.main")
|
logger = logging.getLogger("hbc.main")
|
||||||
|
|
||||||
@@ -462,47 +532,63 @@ async def async_main(args, config):
|
|||||||
hb_port = config.get("hb_port", PORT)
|
hb_port = config.get("hb_port", PORT)
|
||||||
interval = config.get("interval", INTERVAL)
|
interval = config.get("interval", INTERVAL)
|
||||||
|
|
||||||
logger.info(f"Starting hbc for {iam} -> {hb_hosts}")
|
logger.info(f"hbc {__version__} on {iam} -> {hb_hosts} port={hb_port}, interval={interval}s")
|
||||||
logger.info(f"Port: {hb_port}, Interval: {interval}s")
|
if allow_remote_command:
|
||||||
|
logger.warning("allow_remote_command is true — CMD packets from the server will be executed")
|
||||||
|
|
||||||
|
af_filter = (socket.AF_INET if getattr(args, "ipv4_only", False)
|
||||||
|
else socket.AF_INET6 if getattr(args, "ipv6_only", False)
|
||||||
|
else 0)
|
||||||
|
|
||||||
# Create connections
|
# Create connections
|
||||||
connections = []
|
connections = []
|
||||||
conn_id = 1
|
conn_id = 1
|
||||||
|
_retry_delay = 5
|
||||||
|
|
||||||
for host in hb_hosts:
|
while running and not connections:
|
||||||
try:
|
for host in hb_hosts:
|
||||||
addrs = socket.getaddrinfo(host, hb_port, 0, 0, socket.SOL_UDP)
|
try:
|
||||||
except socket.gaierror as e:
|
addrs = socket.getaddrinfo(host, hb_port, af_filter, 0, socket.SOL_UDP)
|
||||||
logger.error(f"Cannot resolve {host}: {e}")
|
except socket.gaierror as e:
|
||||||
continue
|
logger.warning(f"Cannot resolve {host}: {e} — retrying in {_retry_delay}s")
|
||||||
|
continue
|
||||||
for addr_info in addrs:
|
for addr_info in addrs:
|
||||||
af = addr_info[0]
|
af = addr_info[0]
|
||||||
addr = addr_info[4][0]
|
addr = addr_info[4][0]
|
||||||
|
conn = AsyncConnection(conn_id, addr, hb_port, af, iam)
|
||||||
conn = AsyncConnection(conn_id, addr, hb_port, af, iam)
|
if not await conn.open():
|
||||||
if await conn.open():
|
logger.warning(f"Initial open to {addr} failed, heartbeat sender will retry")
|
||||||
connections.append(conn)
|
connections.append(conn)
|
||||||
conn_id += 1
|
conn_id += 1
|
||||||
|
if not connections:
|
||||||
|
try:
|
||||||
|
if shutdown_event:
|
||||||
|
await asyncio.wait_for(shutdown_event.wait(), timeout=_retry_delay)
|
||||||
|
else:
|
||||||
|
await asyncio.sleep(_retry_delay)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
_retry_delay = min(_retry_delay * 2, 60)
|
||||||
|
|
||||||
if not connections:
|
if not connections:
|
||||||
logger.error("No connections established")
|
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
logger.info(f"Created {len(connections)} connections")
|
logger.info(f"Created {len(connections)} connections")
|
||||||
|
|
||||||
# Send boot/message if requested
|
# Send boot/message if requested
|
||||||
|
send_shutdown = False
|
||||||
if args.boot or args.message:
|
if args.boot or args.message:
|
||||||
boot_msg = {}
|
boot_msg = {}
|
||||||
if args.boot:
|
if args.boot:
|
||||||
boot_msg["boot"] = 1
|
boot_msg["boot"] = 1
|
||||||
|
send_shutdown = True
|
||||||
if args.message:
|
if args.message:
|
||||||
boot_msg["service"] = "service"
|
boot_msg["service"] = "service"
|
||||||
boot_msg["msg"] = args.message
|
boot_msg["msg"] = args.message
|
||||||
|
|
||||||
boot_msg["acks"] = 0
|
boot_msg["acks"] = 0
|
||||||
for conn in connections:
|
target = next((c for c in connections if c.transport), connections[0])
|
||||||
await conn.sendto(boot_msg)
|
await target.sendto(boot_msg)
|
||||||
|
|
||||||
if args.message and not args.daemon:
|
if args.message and not args.daemon:
|
||||||
# Message-only mode
|
# Message-only mode
|
||||||
@@ -525,6 +611,13 @@ async def async_main(args, config):
|
|||||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||||
loop.add_signal_handler(sig, stop)
|
loop.add_signal_handler(sig, stop)
|
||||||
|
|
||||||
|
def _sighup():
|
||||||
|
global dorestart
|
||||||
|
dorestart = True
|
||||||
|
stop()
|
||||||
|
|
||||||
|
loop.add_signal_handler(signal.SIGHUP, _sighup)
|
||||||
|
|
||||||
# Start async tasks
|
# Start async tasks
|
||||||
# Heartbeat senders (one per connection)
|
# Heartbeat senders (one per connection)
|
||||||
for conn in connections:
|
for conn in connections:
|
||||||
@@ -662,6 +755,9 @@ def build_parser():
|
|||||||
default=0,
|
default=0,
|
||||||
help="Increase debug level"
|
help="Increase debug level"
|
||||||
)
|
)
|
||||||
|
af_group = parser.add_mutually_exclusive_group()
|
||||||
|
af_group.add_argument("-4", dest="ipv4_only", action="store_true", help="Use IPv4 only")
|
||||||
|
af_group.add_argument("-6", dest="ipv6_only", action="store_true", help="Use IPv6 only")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"hosts",
|
"hosts",
|
||||||
nargs="+",
|
nargs="+",
|
||||||
@@ -695,7 +791,7 @@ def main(argv=None):
|
|||||||
|
|
||||||
# Daemonize if requested
|
# Daemonize if requested
|
||||||
if args.daemon:
|
if args.daemon:
|
||||||
print("Daemonizing...")
|
logging.info("Daemonizing...")
|
||||||
daemonize()
|
daemonize()
|
||||||
_reconfigure_logging_for_daemon(log_level)
|
_reconfigure_logging_for_daemon(log_level)
|
||||||
logging.info(f"hbc starting, sending heartbeat to {', '.join(args.hosts)}")
|
logging.info(f"hbc starting, sending heartbeat to {', '.join(args.hosts)}")
|
||||||
@@ -713,7 +809,10 @@ def main(argv=None):
|
|||||||
# Handle restart
|
# Handle restart
|
||||||
if dorestart:
|
if dorestart:
|
||||||
logging.info("Restarting...")
|
logging.info("Restarting...")
|
||||||
os.execv(sys.argv[0], sys.argv)
|
# Drop -b/--boot so the re-exec'd process doesn't re-announce a boot;
|
||||||
|
# a SIGHUP restart is not a host reboot.
|
||||||
|
restart_argv = [sys.argv[0]] + [a for a in sys.argv[1:] if a not in ("-b", "--boot")]
|
||||||
|
os.execv(restart_argv[0], restart_argv)
|
||||||
|
|
||||||
sys.exit(exit_code)
|
sys.exit(exit_code)
|
||||||
|
|
||||||
|
|||||||
@@ -364,7 +364,10 @@ class PluginLoader:
|
|||||||
|
|
||||||
# Instantiate plugin with config — check plugins subdict first,
|
# Instantiate plugin with config — check plugins subdict first,
|
||||||
# then top-level keys (e.g. nagios_runner: ... at root of config).
|
# then top-level keys (e.g. nagios_runner: ... at root of config).
|
||||||
plugin_instance_config = plugins_subconfig.get(obj.name) or raw_config.get(obj.name, {})
|
plugin_instance_config = dict(plugins_subconfig.get(obj.name) or raw_config.get(obj.name) or {})
|
||||||
|
# Propagate top-level owner so os_info (and any future plugin) can report it.
|
||||||
|
if "owner" in raw_config and "owner" not in plugin_instance_config:
|
||||||
|
plugin_instance_config["owner"] = raw_config["owner"]
|
||||||
plugin = obj(config=plugin_instance_config)
|
plugin = obj(config=plugin_instance_config)
|
||||||
|
|
||||||
# Initialize plugin
|
# Initialize plugin
|
||||||
|
|||||||
@@ -119,6 +119,13 @@ class CPUMonitorPlugin(MonitorPlugin):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.logger.debug(f"Could not get CPU times: {e}")
|
self.logger.debug(f"Could not get CPU times: {e}")
|
||||||
|
|
||||||
|
# Uptime in seconds
|
||||||
|
try:
|
||||||
|
import time
|
||||||
|
data["uptime_seconds"] = int(time.time() - self.psutil.boot_time())
|
||||||
|
except Exception as e:
|
||||||
|
self.logger.debug(f"Could not get uptime: {e}")
|
||||||
|
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Collected CPU metrics: {data.get('cpu_percent', 'N/A')}% usage"
|
f"Collected CPU metrics: {data.get('cpu_percent', 'N/A')}% usage"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -127,7 +127,7 @@ class FilesystemInfoPlugin(InfoPlugin):
|
|||||||
try:
|
try:
|
||||||
# Maximum filename length
|
# Maximum filename length
|
||||||
max_name = os.pathconf(partition.mountpoint, 'PC_NAME_MAX')
|
max_name = os.pathconf(partition.mountpoint, 'PC_NAME_MAX')
|
||||||
if max_name:
|
if max_name is not None:
|
||||||
fs_info['maxfile'] = max_name
|
fs_info['maxfile'] = max_name
|
||||||
except (OSError, ValueError):
|
except (OSError, ValueError):
|
||||||
pass
|
pass
|
||||||
@@ -135,7 +135,7 @@ class FilesystemInfoPlugin(InfoPlugin):
|
|||||||
try:
|
try:
|
||||||
# Maximum path length
|
# Maximum path length
|
||||||
max_path = os.pathconf(partition.mountpoint, 'PC_PATH_MAX')
|
max_path = os.pathconf(partition.mountpoint, 'PC_PATH_MAX')
|
||||||
if max_path:
|
if max_path is not None:
|
||||||
fs_info['maxpath'] = max_path
|
fs_info['maxpath'] = max_path
|
||||||
except (OSError, ValueError):
|
except (OSError, ValueError):
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -14,6 +14,24 @@ except ImportError:
|
|||||||
|
|
||||||
from hbd.client.plugin import MonitorPlugin
|
from hbd.client.plugin import MonitorPlugin
|
||||||
|
|
||||||
|
|
||||||
|
def _zfs_arc_bytes() -> int:
|
||||||
|
"""Return current ZFS ARC size in bytes, or 0 if ZFS is not present.
|
||||||
|
|
||||||
|
ZFS ARC is reclaimable but is not included in MemAvailable by the Linux
|
||||||
|
kernel (it is not in SReclaimable), so it would otherwise be counted as
|
||||||
|
used memory.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with open("/proc/spl/kstat/zfs/arcstats") as fh:
|
||||||
|
for line in fh:
|
||||||
|
parts = line.split()
|
||||||
|
if len(parts) >= 3 and parts[0] == "size":
|
||||||
|
return int(parts[2])
|
||||||
|
except (OSError, ValueError):
|
||||||
|
pass
|
||||||
|
return 0
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -101,11 +119,21 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
|
|
||||||
# Virtual (physical) memory statistics
|
# Virtual (physical) memory statistics
|
||||||
vmem = psutil.virtual_memory()
|
vmem = psutil.virtual_memory()
|
||||||
|
|
||||||
|
# psutil's available already excludes page cache / file buffers
|
||||||
|
# (uses MemAvailable on Linux). Add ZFS ARC on top because the kernel
|
||||||
|
# does not include it in SReclaimable / MemAvailable even though it is
|
||||||
|
# reclaimable.
|
||||||
|
arc_bytes = _zfs_arc_bytes()
|
||||||
|
available = min(vmem.available + arc_bytes, vmem.total)
|
||||||
|
used = vmem.total - available
|
||||||
|
percent = round(used / vmem.total * 100, 1) if vmem.total else 0.0
|
||||||
|
|
||||||
metrics['memory_total'] = vmem.total
|
metrics['memory_total'] = vmem.total
|
||||||
metrics['memory_available'] = vmem.available
|
metrics['memory_available'] = available
|
||||||
metrics['memory_used'] = vmem.used
|
metrics['memory_used'] = used
|
||||||
metrics['memory_free'] = vmem.free
|
metrics['memory_free'] = vmem.free
|
||||||
metrics['memory_percent'] = vmem.percent
|
metrics['memory_percent'] = percent
|
||||||
|
|
||||||
# Platform-specific memory details
|
# Platform-specific memory details
|
||||||
if hasattr(vmem, 'active'):
|
if hasattr(vmem, 'active'):
|
||||||
|
|||||||
@@ -31,16 +31,13 @@ from hbd.client.plugin import MonitorPlugin
|
|||||||
|
|
||||||
|
|
||||||
# Nagios exit codes
|
# Nagios exit codes
|
||||||
NAGIOS_OK = 0
|
|
||||||
NAGIOS_WARNING = 1
|
|
||||||
NAGIOS_CRITICAL = 2
|
|
||||||
NAGIOS_UNKNOWN = 3
|
NAGIOS_UNKNOWN = 3
|
||||||
|
|
||||||
STATUS_NAMES = {
|
STATUS_NAMES = {
|
||||||
NAGIOS_OK: "OK",
|
0: "OK",
|
||||||
NAGIOS_WARNING: "WARNING",
|
1: "WARNING",
|
||||||
NAGIOS_CRITICAL: "CRITICAL",
|
2: "CRITICAL",
|
||||||
NAGIOS_UNKNOWN: "UNKNOWN"
|
3: "UNKNOWN",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -129,9 +126,6 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
"""
|
"""
|
||||||
results = {}
|
results = {}
|
||||||
|
|
||||||
# Track overall status (worst status wins)
|
|
||||||
worst_status = NAGIOS_OK
|
|
||||||
|
|
||||||
for cmd_config in self.commands:
|
for cmd_config in self.commands:
|
||||||
name = cmd_config.get("name")
|
name = cmd_config.get("name")
|
||||||
command = cmd_config.get("command")
|
command = cmd_config.get("command")
|
||||||
@@ -149,10 +143,6 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
results[f"{name}_status_code"] = status_code
|
results[f"{name}_status_code"] = status_code
|
||||||
results[f"{name}_output"] = output
|
results[f"{name}_output"] = output
|
||||||
|
|
||||||
# Track worst status
|
|
||||||
if status_code > worst_status:
|
|
||||||
worst_status = status_code
|
|
||||||
|
|
||||||
# Parse and add performance data
|
# Parse and add performance data
|
||||||
if perfdata:
|
if perfdata:
|
||||||
for metric_name, metric_value in perfdata.items():
|
for metric_name, metric_value in perfdata.items():
|
||||||
@@ -167,12 +157,6 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
results[f"{name}_status"] = "ERROR"
|
results[f"{name}_status"] = "ERROR"
|
||||||
results[f"{name}_status_code"] = NAGIOS_UNKNOWN
|
results[f"{name}_status_code"] = NAGIOS_UNKNOWN
|
||||||
results[f"{name}_output"] = str(e)
|
results[f"{name}_output"] = str(e)
|
||||||
worst_status = NAGIOS_UNKNOWN
|
|
||||||
|
|
||||||
# Add overall status
|
|
||||||
results["overall_status"] = STATUS_NAMES.get(worst_status, "UNKNOWN")
|
|
||||||
results["overall_status_code"] = worst_status
|
|
||||||
results["plugin_count"] = len(self.commands)
|
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|||||||
@@ -62,6 +62,9 @@ class OSInfoPlugin(InfoPlugin):
|
|||||||
"hbc_version": hbc_version,
|
"hbc_version": hbc_version,
|
||||||
"hbc_type": "full",
|
"hbc_type": "full",
|
||||||
}
|
}
|
||||||
|
if self.config.get("owner"):
|
||||||
|
self.logger.debug(f"Adding owner from config: {self.config['owner']}")
|
||||||
|
data["owner"] = self.config["owner"]
|
||||||
|
|
||||||
# Add Linux-specific distribution info
|
# Add Linux-specific distribution info
|
||||||
if platform.system() == "Linux":
|
if platform.system() == "Linux":
|
||||||
|
|||||||
@@ -13,12 +13,8 @@ plugins:
|
|||||||
count: 3 # ICMP packets per ping run (default 3)
|
count: 3 # ICMP packets per ping run (default 3)
|
||||||
timeout: 5 # seconds before a host is considered unreachable (default 5)
|
timeout: 5 # seconds before a host is considered unreachable (default 5)
|
||||||
hosts:
|
hosts:
|
||||||
8.8.8.8:
|
- 8.8.8.8
|
||||||
warning: 20.0 # ms
|
- 192.168.1.1
|
||||||
critical: 100.0 # ms
|
|
||||||
192.168.1.1:
|
|
||||||
warning: 5.0
|
|
||||||
critical: 20.0
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Reported metrics per host (metric key uses the hostname with dots/colons replaced
|
Reported metrics per host (metric key uses the hostname with dots/colons replaced
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
"""
|
||||||
|
ZFS pool monitoring plugin for Heartbeat.
|
||||||
|
|
||||||
|
Collects per-pool health, capacity, and cumulative I/O statistics via zpool(8).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from hbd.client.plugin import MonitorPlugin
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _int(s: str) -> Optional[int]:
|
||||||
|
try:
|
||||||
|
return int(s.strip().rstrip("KMGTkBkmgt%x"))
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _float(s: str) -> Optional[float]:
|
||||||
|
try:
|
||||||
|
return float(s.strip().rstrip("%x"))
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class ZFSMonitorPlugin(MonitorPlugin):
|
||||||
|
"""Monitor ZFS pool health, capacity, and I/O statistics.
|
||||||
|
|
||||||
|
Collects per pool:
|
||||||
|
- health: ONLINE, DEGRADED, FAULTED, etc.
|
||||||
|
- size / alloc / free: total, allocated and free bytes
|
||||||
|
- capacity: percentage used (0-100)
|
||||||
|
- frag: fragmentation percentage
|
||||||
|
- dedup: deduplication ratio
|
||||||
|
- read_ops / write_ops: cumulative I/O operations since last boot/clear
|
||||||
|
- read_bw / write_bw: cumulative bytes transferred since last boot/clear
|
||||||
|
|
||||||
|
Configuration:
|
||||||
|
interval: collection interval in seconds (default: 300)
|
||||||
|
pools: list of pool names to monitor (default: all)
|
||||||
|
"""
|
||||||
|
|
||||||
|
name = "zfs_monitor"
|
||||||
|
description = "ZFS pool health, capacity, and I/O statistics"
|
||||||
|
interval = 300
|
||||||
|
|
||||||
|
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||||
|
super().__init__(config)
|
||||||
|
self.interval = self.config.get("interval", 300)
|
||||||
|
self._pools_filter: Optional[List[str]] = self.config.get("pools", None)
|
||||||
|
|
||||||
|
async def initialize(self) -> bool:
|
||||||
|
if not shutil.which("zpool"):
|
||||||
|
self.skip_reason = "zpool not found"
|
||||||
|
return False
|
||||||
|
logger.info("ZFS monitor initialized (interval: %ds)", self.interval)
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def _run(self, *args: str) -> List[str]:
|
||||||
|
"""Run a command and return its stdout lines, or [] on error."""
|
||||||
|
try:
|
||||||
|
proc = await asyncio.create_subprocess_exec(
|
||||||
|
*args,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=15)
|
||||||
|
return stdout.decode(errors="replace").splitlines()
|
||||||
|
except (FileNotFoundError, asyncio.TimeoutError) as exc:
|
||||||
|
logger.warning("zfs_monitor: %s: %s", args[0], exc)
|
||||||
|
return []
|
||||||
|
|
||||||
|
async def _zpool_list(self) -> Dict[str, Dict]:
|
||||||
|
"""Return per-pool health and capacity from `zpool list`."""
|
||||||
|
lines = await self._run(
|
||||||
|
"zpool", "list", "-H", "-p",
|
||||||
|
"-o", "name,health,size,alloc,free,cap,frag,dedup",
|
||||||
|
)
|
||||||
|
pools: Dict[str, Dict] = {}
|
||||||
|
for line in lines:
|
||||||
|
parts = line.split("\t")
|
||||||
|
if len(parts) < 8:
|
||||||
|
continue
|
||||||
|
name = parts[0].strip()
|
||||||
|
if self._pools_filter and name not in self._pools_filter:
|
||||||
|
continue
|
||||||
|
health = parts[1].strip()
|
||||||
|
if health == "ONLINE":
|
||||||
|
status = 0
|
||||||
|
elif health in ("DEGRADED", "ONLINE with errors"):
|
||||||
|
status = 1
|
||||||
|
elif health in ("FAULTED", "OFFLINE", "UNAVAIL"):
|
||||||
|
status = 2
|
||||||
|
else:
|
||||||
|
status = 3 # unknown status
|
||||||
|
pools[name] = {
|
||||||
|
"health": health,
|
||||||
|
"status": status,
|
||||||
|
"size": _int(parts[2]),
|
||||||
|
"alloc": _int(parts[3]),
|
||||||
|
"free": _int(parts[4]),
|
||||||
|
"capacity": _float(parts[5]),
|
||||||
|
"frag": _float(parts[6]),
|
||||||
|
"dedup": _float(parts[7]),
|
||||||
|
}
|
||||||
|
return pools
|
||||||
|
|
||||||
|
async def _zpool_iostat(self) -> Dict[str, Dict]:
|
||||||
|
"""Return per-pool cumulative I/O counters from `zpool iostat`."""
|
||||||
|
lines = await self._run("zpool", "iostat", "-H", "-p")
|
||||||
|
io: Dict[str, Dict] = {}
|
||||||
|
for line in lines:
|
||||||
|
parts = line.split("\t")
|
||||||
|
if len(parts) < 7:
|
||||||
|
continue
|
||||||
|
name = parts[0].strip()
|
||||||
|
if not name or name.startswith(" "):
|
||||||
|
continue
|
||||||
|
io[name] = {
|
||||||
|
"read_ops": _int(parts[3]),
|
||||||
|
"write_ops": _int(parts[4]),
|
||||||
|
"read_bw": _int(parts[5]),
|
||||||
|
"write_bw": _int(parts[6]),
|
||||||
|
}
|
||||||
|
return io
|
||||||
|
|
||||||
|
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||||
|
pools, io = await asyncio.gather(self._zpool_list(), self._zpool_iostat())
|
||||||
|
for name, stats in io.items():
|
||||||
|
if name in pools:
|
||||||
|
pools[name].update(stats)
|
||||||
|
return {"pools": pools}
|
||||||
|
|
||||||
|
|
||||||
|
plugin = ZFSMonitorPlugin
|
||||||
@@ -134,6 +134,31 @@ thresholds:
|
|||||||
hysteresis: 0.1
|
hysteresis: 0.1
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------------
|
||||||
|
# ZFS Monitor Thresholds
|
||||||
|
# ----------------------------------------------------------------------------
|
||||||
|
zfs_monitor:
|
||||||
|
# Pool health check — built-in default; shown here for reference/override.
|
||||||
|
# status is 0 (ONLINE) or 1 (DEGRADED) or 2 (SUSPENDED, FAULTED, UNAVAIL…).
|
||||||
|
# Use '*' to apply the same rule to every pool, or name a specific pool.
|
||||||
|
pools:
|
||||||
|
'*':
|
||||||
|
status:
|
||||||
|
warning: 1 # Alert WARNING when pool is DEGRADED
|
||||||
|
critical: 2 # Alert CRITICAL when pool is SUSPENDED/FAULTED/UNAVAIL
|
||||||
|
operator: ">="
|
||||||
|
hysteresis: 0.0 # No hysteresis — a degraded pool is always alerting
|
||||||
|
grace: 0 # Fire immediately — don't wait for a second collection
|
||||||
|
display: "ZFS pool {pool_name} is {health}"
|
||||||
|
|
||||||
|
# Per-pool capacity thresholds (optional; add pools you care about)
|
||||||
|
# tank:
|
||||||
|
# capacity:
|
||||||
|
# warning: 75.0 # Warn at 75% used
|
||||||
|
# critical: 90.0 # Critical at 90% used
|
||||||
|
# operator: ">"
|
||||||
|
# hysteresis: 0.05
|
||||||
|
|
||||||
# ----------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------
|
||||||
# Network Monitor Thresholds
|
# Network Monitor Thresholds
|
||||||
# ----------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------
|
||||||
|
|||||||
+51
-35
@@ -27,21 +27,26 @@ SERVER_DEFAULTS = {
|
|||||||
|
|
||||||
# Monitoring settings
|
# Monitoring settings
|
||||||
"interval": 20, # Expected heartbeat interval (for server checks)
|
"interval": 20, # Expected heartbeat interval (for server checks)
|
||||||
"grace": 2, # Grace multiplier (interval * grace = timeout)
|
"grace": 2, # Grace period (extra seconds before notifying after a missed heartbeat)
|
||||||
"threshold_renotify_interval": 3600, # Seconds between threshold re-notifications
|
"threshold_renotify_interval": 3600, # Seconds between threshold re-notifications
|
||||||
|
|
||||||
|
# Flap detection (0 in either key disables it)
|
||||||
|
"flap_count": 5, # Warning/critical notifications within flap_interval that trip flapping
|
||||||
|
"flap_interval": 10, # Minutes: the counting window, and the quiet window after an OK
|
||||||
|
|
||||||
# User management
|
# User management
|
||||||
"users": {}, # username -> {full_name, avatar, password, admin, notification_channels}
|
"users": {}, # username -> {full_name, avatar, password, admin, notification_channels}
|
||||||
"default_owner": None, # Username that owns hosts with no explicit owner
|
"default_owner": None, # Username that owns hosts with no explicit owner
|
||||||
|
|
||||||
|
# OAuth2 providers
|
||||||
|
"oauth": {}, # oauth.gitea.{url,client_id,client_secret}
|
||||||
|
|
||||||
# Host management
|
# Host management
|
||||||
"hosts": {}, # Unified host definitions
|
"hosts": {}, # Unified host definitions
|
||||||
"dyndnshosts": [], # Hosts with dynamic DNS (legacy)
|
"dyndomains": ["example.org"], # Domains to update via nsupdate when a host with dyndns: true is updated
|
||||||
"drophosts": [], # Hosts to ignore
|
|
||||||
"dyndomains": ["wrede.org"],
|
|
||||||
|
|
||||||
# DNS updates
|
# DNS updates
|
||||||
"nsupdate_bin": "/usr/bin/nsupdate",
|
"nsupdate_bin": "/usr/bin/nsupdate", # Path to nsupdate binary
|
||||||
|
|
||||||
# WebSocket settings
|
# WebSocket settings
|
||||||
"ws_port": 50005,
|
"ws_port": 50005,
|
||||||
@@ -76,9 +81,13 @@ THRESHOLD_DEFAULTS = {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
'memory_monitor': {
|
'memory_monitor': {
|
||||||
'percent': {
|
'memory_percent': {
|
||||||
'warning': 85.0,
|
'warning': 85.0,
|
||||||
'critical': 95.0
|
'critical': 95.0
|
||||||
|
},
|
||||||
|
'swap_percent': {
|
||||||
|
'warning': 40.0,
|
||||||
|
'critical': 75.0
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
'disk_monitor': {
|
'disk_monitor': {
|
||||||
@@ -95,7 +104,31 @@ THRESHOLD_DEFAULTS = {
|
|||||||
'warning': 200,
|
'warning': 200,
|
||||||
'critical': 250.0,
|
'critical': 250.0,
|
||||||
'count': 3 # Optional: number of consecutive breaches before alerting
|
'count': 3 # Optional: number of consecutive breaches before alerting
|
||||||
}
|
},
|
||||||
|
'nagios_runner': {
|
||||||
|
'status_code': {
|
||||||
|
'display': '{check_name} {output}',
|
||||||
|
'operator': "nagios"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'zfs_monitor': {
|
||||||
|
'pools': {
|
||||||
|
'*': {
|
||||||
|
'status': {
|
||||||
|
'warning': 1,
|
||||||
|
'critical': 2,
|
||||||
|
'operator': '>=',
|
||||||
|
'hysteresis': 0.0,
|
||||||
|
'grace': 0,
|
||||||
|
'display': 'ZFS pool {pool_name} is {health}'
|
||||||
|
},
|
||||||
|
'capacity': {
|
||||||
|
'warning': 80.0,
|
||||||
|
'critical': 90.0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -219,43 +252,26 @@ def get_watchhosts(config):
|
|||||||
"""Extract watched hostnames from config (hosts with watch: true).
|
"""Extract watched hostnames from config (hosts with watch: true).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of hostnames to watch
|
# List of hostnames to watch
|
||||||
"""
|
"""
|
||||||
watchhosts = []
|
watchhosts = []
|
||||||
hosts_config = config.get("hosts", {})
|
hosts_config = config.get("hosts", {})
|
||||||
if isinstance(hosts_config, dict):
|
if isinstance(hosts_config, dict):
|
||||||
for host_name, host_attrs in hosts_config.items():
|
for host_name, host_attrs in hosts_config.items():
|
||||||
if isinstance(host_attrs, dict) and host_attrs.get("watch", False):
|
if isinstance(host_attrs, dict) and host_attrs.get("watch", True):
|
||||||
watchhosts.append(host_name)
|
watchhosts.append(host_name)
|
||||||
return watchhosts
|
return watchhosts
|
||||||
|
|
||||||
|
|
||||||
def get_dyndnshosts(config):
|
def get_dyndnshosts(config):
|
||||||
"""Extract dyndnshosts from config, supporting both new and legacy formats.
|
"""Return hostnames that have a dyndns setting in the hosts section."""
|
||||||
|
hosts_config = config.get("hosts", {})
|
||||||
Args:
|
if not isinstance(hosts_config, dict):
|
||||||
config: Configuration dictionary
|
return []
|
||||||
|
return [
|
||||||
Returns:
|
name for name, attrs in hosts_config.items()
|
||||||
List of hostnames with dynamic DNS
|
if isinstance(attrs, dict) and attrs.get("dyndns")
|
||||||
"""
|
]
|
||||||
dyndnshosts = []
|
|
||||||
|
|
||||||
# New format: hosts section with dyndns attribute
|
|
||||||
if "hosts" in config:
|
|
||||||
hosts_config = config["hosts"]
|
|
||||||
if isinstance(hosts_config, dict):
|
|
||||||
for host_name, host_attrs in hosts_config.items():
|
|
||||||
if isinstance(host_attrs, dict) and host_attrs.get("dyndns", False):
|
|
||||||
dyndnshosts.append(host_name)
|
|
||||||
|
|
||||||
# Legacy format: dyndnshosts list/set
|
|
||||||
if "dyndnshosts" in config:
|
|
||||||
legacy_dyndnshosts = config.get("dyndnshosts", [])
|
|
||||||
if isinstance(legacy_dyndnshosts, (list, set)):
|
|
||||||
dyndnshosts.extend(legacy_dyndnshosts)
|
|
||||||
|
|
||||||
return list(set(dyndnshosts)) # Remove duplicates
|
|
||||||
|
|
||||||
|
|
||||||
def get_host_config(config, hostname):
|
def get_host_config(config, hostname):
|
||||||
@@ -303,7 +319,7 @@ def get_host_access(config, hostname) -> dict:
|
|||||||
"""
|
"""
|
||||||
host_cfg = get_host_config(config, hostname)
|
host_cfg = get_host_config(config, hostname)
|
||||||
|
|
||||||
owner = host_cfg.get("owner") or get_default_owner(config)
|
owner = host_cfg.get("owner") # or get_default_owner(config)
|
||||||
|
|
||||||
managers = host_cfg.get("managers", [])
|
managers = host_cfg.get("managers", [])
|
||||||
if isinstance(managers, str):
|
if isinstance(managers, str):
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
"""Ownership rules for config-file entities (hosts, channels, threshold configs).
|
||||||
|
|
||||||
|
Rule: an entry whose config dict has a non-empty ``owner`` is private to that
|
||||||
|
owner — visible, usable, and editable only by the owner (and admins). An
|
||||||
|
entry with no ``owner`` is global: usable by everyone, editable by admins only.
|
||||||
|
|
||||||
|
The scoped-merge helpers implement the non-admin save path for
|
||||||
|
``POST /api/0/config``: the caller's visible subset of a section is replaced
|
||||||
|
by the submitted payload; everything else is preserved untouched.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
|
||||||
|
class ScopedMergeError(Exception):
|
||||||
|
"""A non-admin payload violated an ownership rule; message names the entry."""
|
||||||
|
|
||||||
|
|
||||||
|
def is_global(cfg: Any) -> bool:
|
||||||
|
"""True when *cfg* has no owner (usable by everyone)."""
|
||||||
|
return not (isinstance(cfg, dict) and cfg.get("owner"))
|
||||||
|
|
||||||
|
|
||||||
|
def user_can_use(cfg: Any, username: str) -> bool:
|
||||||
|
"""True when *username* may use/see this entry (global or own)."""
|
||||||
|
return is_global(cfg) or cfg.get("owner") == username
|
||||||
|
|
||||||
|
|
||||||
|
def _as_list(value: Any) -> list:
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if isinstance(value, str):
|
||||||
|
return [value]
|
||||||
|
return list(value)
|
||||||
|
|
||||||
|
|
||||||
|
def user_hosts(hosts_cfg: Any, username: str) -> Dict[str, Any]:
|
||||||
|
"""Subset of *hosts_cfg* where *username* is owner or manager."""
|
||||||
|
result = {}
|
||||||
|
for name, cfg in (hosts_cfg or {}).items():
|
||||||
|
if not isinstance(cfg, dict):
|
||||||
|
continue
|
||||||
|
if cfg.get("owner") == username or username in _as_list(cfg.get("managers")):
|
||||||
|
result[name] = cfg
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def user_channels(channels_cfg: Any, username: str) -> Dict[str, Any]:
|
||||||
|
"""Subset of channels usable by *username*: global + own."""
|
||||||
|
return {
|
||||||
|
name: cfg for name, cfg in (channels_cfg or {}).items()
|
||||||
|
if isinstance(cfg, dict) and user_can_use(cfg, username)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def user_threshold_configs(threshold_cfgs: Any, username: str) -> Dict[str, Any]:
|
||||||
|
"""Subset of threshold configs usable by *username*: global + own."""
|
||||||
|
return {
|
||||||
|
name: cfg for name, cfg in (threshold_cfgs or {}).items()
|
||||||
|
if isinstance(cfg, dict) and user_can_use(cfg, username)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_HOST_SETTING_KEYS = ("watch", "dyndns", "notification_channels", "threshold_config")
|
||||||
|
_HOST_ACCESS_KEYS = ("owner", "managers", "monitors")
|
||||||
|
_HOST_ALLOWED_KEYS = frozenset(_HOST_SETTING_KEYS) | frozenset(_HOST_ACCESS_KEYS)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_added_assignments(host_name: str, old_cfg: Any, new_cfg: dict,
|
||||||
|
key: str, usable: set) -> None:
|
||||||
|
"""Every value under *key* not already on the host must be in *usable*."""
|
||||||
|
old_vals = set(_as_list((old_cfg or {}).get(key)))
|
||||||
|
for v in _as_list(new_cfg.get(key)):
|
||||||
|
if v not in old_vals and v not in usable:
|
||||||
|
raise ScopedMergeError(
|
||||||
|
f"host {host_name!r}: {key} entry {v!r} is not available to you")
|
||||||
|
|
||||||
|
|
||||||
|
def merge_hosts_scoped(existing: Any, payload: Any, username: str,
|
||||||
|
channels_cfg: Any, threshold_cfgs: Any) -> Dict[str, Any]:
|
||||||
|
"""Return a new hosts section with *username*'s visible subset replaced by *payload*.
|
||||||
|
|
||||||
|
Hosts the user cannot see are preserved untouched. Visible hosts missing
|
||||||
|
from the payload are deleted (owners only). Managers may change setting
|
||||||
|
keys but not access keys and may not delete. New hosts get their owner
|
||||||
|
forced to *username*. Newly added channel/threshold assignments must be
|
||||||
|
global or owned by the user. Violations raise ScopedMergeError.
|
||||||
|
"""
|
||||||
|
existing = existing or {}
|
||||||
|
payload = payload or {}
|
||||||
|
visible = user_hosts(existing, username)
|
||||||
|
usable_channels = set(user_channels(channels_cfg, username))
|
||||||
|
usable_tcs = set(user_threshold_configs(threshold_cfgs, username)) | {"default"}
|
||||||
|
|
||||||
|
result: Dict[str, Any] = {n: c for n, c in existing.items() if n not in visible}
|
||||||
|
|
||||||
|
for name, entry in payload.items():
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
raise ScopedMergeError(f"host {name!r}: invalid entry")
|
||||||
|
unknown = set(entry) - _HOST_ALLOWED_KEYS
|
||||||
|
if unknown:
|
||||||
|
raise ScopedMergeError(
|
||||||
|
f"host {name!r}: fields not permitted: {', '.join(sorted(unknown))}")
|
||||||
|
if name in existing and name not in visible:
|
||||||
|
raise ScopedMergeError(f"host {name!r}: you are not an owner or manager")
|
||||||
|
|
||||||
|
old = visible.get(name)
|
||||||
|
if old is None:
|
||||||
|
new_cfg = dict(entry)
|
||||||
|
new_cfg["owner"] = username
|
||||||
|
else:
|
||||||
|
new_cfg = dict(old)
|
||||||
|
for key in _HOST_SETTING_KEYS:
|
||||||
|
if key in entry:
|
||||||
|
new_cfg[key] = entry[key]
|
||||||
|
else:
|
||||||
|
new_cfg.pop(key, None)
|
||||||
|
if old.get("owner") == username:
|
||||||
|
for key in _HOST_ACCESS_KEYS:
|
||||||
|
if key in entry:
|
||||||
|
new_cfg[key] = entry[key]
|
||||||
|
else:
|
||||||
|
new_cfg.pop(key, None)
|
||||||
|
else:
|
||||||
|
if "owner" in entry and (entry.get("owner") or None) != (old.get("owner") or None):
|
||||||
|
raise ScopedMergeError(
|
||||||
|
f"host {name!r}: only the owner may change ownership")
|
||||||
|
for key in ("managers", "monitors"):
|
||||||
|
if key in entry and set(_as_list(entry[key])) != set(_as_list(old.get(key))):
|
||||||
|
raise ScopedMergeError(
|
||||||
|
f"host {name!r}: only the owner may change {key}")
|
||||||
|
|
||||||
|
_check_added_assignments(name, old, new_cfg, "notification_channels", usable_channels)
|
||||||
|
_check_added_assignments(name, old, new_cfg, "threshold_config", usable_tcs)
|
||||||
|
result[name] = new_cfg
|
||||||
|
|
||||||
|
for name, old in visible.items():
|
||||||
|
if name not in payload and old.get("owner") != username:
|
||||||
|
raise ScopedMergeError(f"host {name!r}: only the owner may delete a host")
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def merge_threshold_configs_scoped(existing: Any, payload: Any,
|
||||||
|
username: str) -> Dict[str, Any]:
|
||||||
|
"""Return a new threshold_configs section with the user's own configs
|
||||||
|
replaced by *payload*.
|
||||||
|
|
||||||
|
Global and foreign-owned configs are preserved and may not appear in the
|
||||||
|
payload ('default' included). Own configs missing from the payload are
|
||||||
|
deleted. Every payload entry gets its owner forced to *username*.
|
||||||
|
"""
|
||||||
|
existing = existing or {}
|
||||||
|
payload = payload or {}
|
||||||
|
result: Dict[str, Any] = {
|
||||||
|
n: c for n, c in existing.items()
|
||||||
|
if not (isinstance(c, dict) and c.get("owner") == username)
|
||||||
|
}
|
||||||
|
for name, entry in payload.items():
|
||||||
|
if name == "default":
|
||||||
|
raise ScopedMergeError("threshold config 'default' is global and admin-managed")
|
||||||
|
old = existing.get(name)
|
||||||
|
if old is not None and (not isinstance(old, dict) or old.get("owner") != username):
|
||||||
|
raise ScopedMergeError(f"threshold config {name!r}: not owned by you")
|
||||||
|
new_cfg = dict(entry) if isinstance(entry, dict) else {}
|
||||||
|
new_cfg["owner"] = username
|
||||||
|
result[name] = new_cfg
|
||||||
|
return result
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
"""YAML round-trip read/write for .hb.yaml, with backup and atomic writes."""
|
||||||
|
|
||||||
|
import glob
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from ruamel.yaml import YAML
|
||||||
|
|
||||||
|
_write_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _make_yaml() -> YAML:
|
||||||
|
y = YAML()
|
||||||
|
y.preserve_quotes = True
|
||||||
|
return y
|
||||||
|
|
||||||
|
# Top-level keys managed by the 'server' logical section
|
||||||
|
_SERVER_KEYS = [
|
||||||
|
"hbd_port", "hbd_host", "ws_port", "wss_port", "hb_port",
|
||||||
|
"interval", "grace", "base_url", "threshold_renotify_interval",
|
||||||
|
"flap_count", "flap_interval",
|
||||||
|
"logfile", "pidfile", "pickfile", "journal_enabled", "journal_dir",
|
||||||
|
"journal_max_size", "journal_max_backups", "default_owner",
|
||||||
|
"default_threshold_config",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Top-level keys managed by the 'dns' logical section
|
||||||
|
_DNS_KEYS = ["nsupdate_bin", "rndc_key", "dyndomains"]
|
||||||
|
|
||||||
|
|
||||||
|
def read_roundtrip(path: str):
|
||||||
|
"""Load .hb.yaml with ruamel.yaml, preserving comments and ordering."""
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
return _make_yaml().load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def write_config(path: str, data) -> None:
|
||||||
|
"""Backup current file then atomically write data.
|
||||||
|
|
||||||
|
Backup naming: {path}.bak.YYYYMMDD-HHMMSS
|
||||||
|
Rotation: keep the 10 most recent backups, delete older ones.
|
||||||
|
Atomic write: write to {path}.tmp then os.replace({path}.tmp, path).
|
||||||
|
Acquires _write_lock for the full backup+write sequence.
|
||||||
|
"""
|
||||||
|
with _write_lock:
|
||||||
|
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||||
|
backup_path = f"{path}.bak.{ts}"
|
||||||
|
n = 0
|
||||||
|
while os.path.exists(backup_path):
|
||||||
|
n += 1
|
||||||
|
backup_path = f"{path}.bak.{ts}-{n}"
|
||||||
|
orig_mode = None
|
||||||
|
if os.path.exists(path):
|
||||||
|
orig_mode = os.stat(path).st_mode
|
||||||
|
with open(path, "rb") as src, open(backup_path, "wb") as dst:
|
||||||
|
dst.write(src.read())
|
||||||
|
os.chmod(backup_path, orig_mode)
|
||||||
|
backups = sorted(glob.glob(f"{path}.bak.*"), reverse=True)
|
||||||
|
for old in backups[10:]:
|
||||||
|
os.unlink(old)
|
||||||
|
tmp = f"{path}.tmp"
|
||||||
|
try:
|
||||||
|
with open(tmp, "w", encoding="utf-8") as f:
|
||||||
|
_make_yaml().dump(data, f)
|
||||||
|
if orig_mode is not None:
|
||||||
|
os.chmod(tmp, orig_mode)
|
||||||
|
os.replace(tmp, path)
|
||||||
|
except Exception:
|
||||||
|
try:
|
||||||
|
os.unlink(tmp)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def list_backups(path: str) -> list:
|
||||||
|
"""Return backup paths sorted newest-first."""
|
||||||
|
return sorted(glob.glob(f"{path}.bak.*"), reverse=True)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_structured_section(data, section: str, values: dict) -> None:
|
||||||
|
"""Merge a dict of scalar/list values into data for the named logical section.
|
||||||
|
|
||||||
|
For 'server': updates each known key individually, preserving comments on
|
||||||
|
unchanged keys. For 'users': replaces the entire users dict.
|
||||||
|
"""
|
||||||
|
if section == "server":
|
||||||
|
for key in _SERVER_KEYS:
|
||||||
|
if key in values:
|
||||||
|
data[key] = values[key]
|
||||||
|
elif section == "dns":
|
||||||
|
for key in _DNS_KEYS:
|
||||||
|
if key in values:
|
||||||
|
data[key] = values[key]
|
||||||
|
else:
|
||||||
|
data.pop(key, None)
|
||||||
|
elif section == "users":
|
||||||
|
data["users"] = values
|
||||||
|
elif section == "hosts":
|
||||||
|
data["hosts"] = values
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unknown structured section: {section!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def apply_channel(data, name: str, channel_cfg: dict) -> None:
|
||||||
|
"""Insert or replace a single notification channel entry, preserving others."""
|
||||||
|
if not data.get("notification_channels"):
|
||||||
|
data["notification_channels"] = {}
|
||||||
|
data["notification_channels"][name] = channel_cfg
|
||||||
|
|
||||||
|
|
||||||
|
def delete_channel(data, name: str) -> None:
|
||||||
|
"""Remove a notification channel by name. No-op if not found."""
|
||||||
|
nc = data.get("notification_channels") or {}
|
||||||
|
nc.pop(name, None)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_yaml_section(data, section: str, yaml_text: str) -> None:
|
||||||
|
"""Replace the named logical section by parsing yaml_text."""
|
||||||
|
parsed = _make_yaml().load(yaml_text)
|
||||||
|
if section == "notification_channels":
|
||||||
|
data["notification_channels"] = parsed
|
||||||
|
elif section == "thresholds":
|
||||||
|
data["threshold_configs"] = parsed
|
||||||
|
elif section == "hosts":
|
||||||
|
data["hosts"] = parsed
|
||||||
|
elif section == "dns":
|
||||||
|
if parsed:
|
||||||
|
for key in _DNS_KEYS:
|
||||||
|
if key in parsed:
|
||||||
|
data[key] = parsed[key]
|
||||||
|
else:
|
||||||
|
for key in _DNS_KEYS:
|
||||||
|
data.pop(key, None)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unknown YAML section: {section!r}")
|
||||||
+18
-15
@@ -4,6 +4,9 @@ from __future__ import annotations
|
|||||||
from subprocess import Popen, PIPE, STDOUT
|
from subprocess import Popen, PIPE, STDOUT
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def create_nsupdate_payload(
|
def create_nsupdate_payload(
|
||||||
@@ -123,7 +126,6 @@ async def dns_update_worker(
|
|||||||
pass
|
pass
|
||||||
continue
|
continue
|
||||||
|
|
||||||
m = f"changed address to {addr}"
|
|
||||||
for dyndomain in cfg.get("dyndomains", []):
|
for dyndomain in cfg.get("dyndomains", []):
|
||||||
err = await loop.run_in_executor(
|
err = await loop.run_in_executor(
|
||||||
None,
|
None,
|
||||||
@@ -135,28 +137,29 @@ async def dns_update_worker(
|
|||||||
cfg.get("rndc_key", "/etc/dhcpc/rndc-key"),
|
cfg.get("rndc_key", "/etc/dhcpc/rndc-key"),
|
||||||
)
|
)
|
||||||
if err:
|
if err:
|
||||||
m += f", DNS update failed: {err}"
|
m = f"DNS update failed for {addr} ({dyndomain}): {err}"
|
||||||
logger.error("DNS update failed for %s: %s", name, err)
|
logger.error("DNS update failed for %s: %s", name, err)
|
||||||
|
if log:
|
||||||
|
try:
|
||||||
|
await loop.run_in_executor(None, log, name, "ERROR", m)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
else:
|
else:
|
||||||
m += ", DNS updated."
|
m = f"DNS updated {name}.dy.{dyndomain} → {addr}"
|
||||||
|
if log:
|
||||||
|
try:
|
||||||
|
await loop.run_in_executor(None, log, name, "INFO", m)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not cfg.get("dyndomains"):
|
||||||
|
logger.warning("DNS update triggered for %s but no dyndomains configured", name)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
dnsq.task_done()
|
dnsq.task_done()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if log:
|
|
||||||
try:
|
|
||||||
await loop.run_in_executor(None, log, name, m)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if log:
|
|
||||||
try:
|
|
||||||
await loop.run_in_executor(None, log, None, "dns_update_worker exiting")
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def start_dns_worker(
|
def start_dns_worker(
|
||||||
hbdclass,
|
hbdclass,
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
"""Flap detection: silence checks that toggle faster than they are useful.
|
||||||
|
|
||||||
|
A ``(host, service)`` pair is *flapping* once it exceeds ``flap_count``
|
||||||
|
WARNING/CRITICAL notifications within ``flap_interval`` minutes. The
|
||||||
|
notification that trips the threshold is delivered with ``FLAP_MARKER``
|
||||||
|
appended; every notification for that pair afterwards is dropped. The state
|
||||||
|
clears silently ``flap_interval`` minutes after a RECOVER, provided no further
|
||||||
|
WARNING/CRITICAL arrived in the meantime.
|
||||||
|
|
||||||
|
Only outbound notifications are affected — ``notify.eventlog`` keeps recording
|
||||||
|
every event, so the journal and ``/log`` retain the full history of the flap.
|
||||||
|
|
||||||
|
State lives here at module level rather than on ``Host`` so it is never
|
||||||
|
pickled: a restart starts every check with a clean slate.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
FLAP_MARKER = "Now flapping!! No more messages!"
|
||||||
|
|
||||||
|
# Actions returned by observe()
|
||||||
|
PASS = "pass" # deliver unchanged
|
||||||
|
TRIP = "trip" # deliver with FLAP_MARKER appended
|
||||||
|
SUPPRESS = "suppress" # drop
|
||||||
|
|
||||||
|
_count = 0 # flap_count: alerts within the window needed to trip
|
||||||
|
_window = 0.0 # flap_interval in seconds
|
||||||
|
|
||||||
|
# {(host, service): {"events": [ts, ...], "flapping": bool, "ok_since": float|None}}
|
||||||
|
_state: dict = {}
|
||||||
|
|
||||||
|
|
||||||
|
def setup(cfg) -> None:
|
||||||
|
"""Read flap_count / flap_interval from *cfg* (also called on reload)."""
|
||||||
|
global _count, _window
|
||||||
|
_count = int(cfg.get("flap_count", 0) or 0)
|
||||||
|
_window = float(cfg.get("flap_interval", 0) or 0) * 60.0
|
||||||
|
|
||||||
|
|
||||||
|
def _enabled() -> bool:
|
||||||
|
return _count > 0 and _window > 0
|
||||||
|
|
||||||
|
|
||||||
|
def _label(key) -> str:
|
||||||
|
host, service = key
|
||||||
|
return f"{host}/{service}" if service else host
|
||||||
|
|
||||||
|
|
||||||
|
def _sweep(st: dict, now: float) -> bool:
|
||||||
|
"""Clear the flapping flag once the post-RECOVER quiet window has elapsed."""
|
||||||
|
if st["flapping"] and st["ok_since"] is not None and now - st["ok_since"] >= _window:
|
||||||
|
st["flapping"] = False
|
||||||
|
st["ok_since"] = None
|
||||||
|
st["events"].clear()
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def observe(host: str, service: str, level: str) -> str:
|
||||||
|
"""Record a notification for *host*/*service* and return what to do with it.
|
||||||
|
|
||||||
|
Returns PASS, TRIP or SUPPRESS.
|
||||||
|
"""
|
||||||
|
if not _enabled():
|
||||||
|
return PASS
|
||||||
|
|
||||||
|
now = time.time()
|
||||||
|
key = (host or "", service or "")
|
||||||
|
level = (level or "").upper()
|
||||||
|
st = _state.get(key)
|
||||||
|
if st is not None and _sweep(st, now):
|
||||||
|
logger.info("flapping cleared for %s", _label(key))
|
||||||
|
|
||||||
|
if level in ("WARNING", "CRITICAL"):
|
||||||
|
if st is None:
|
||||||
|
st = _state[key] = {"events": [], "flapping": False, "ok_since": None}
|
||||||
|
# An alert inside the quiet window means it never settled.
|
||||||
|
st["ok_since"] = None
|
||||||
|
st["events"] = [t for t in st["events"] if now - t < _window]
|
||||||
|
st["events"].append(now)
|
||||||
|
if st["flapping"]:
|
||||||
|
return SUPPRESS
|
||||||
|
if len(st["events"]) > _count:
|
||||||
|
st["flapping"] = True
|
||||||
|
logger.info(
|
||||||
|
"flapping detected for %s (%d alerts in %.0f min)",
|
||||||
|
_label(key), len(st["events"]), _window / 60,
|
||||||
|
)
|
||||||
|
return TRIP
|
||||||
|
return PASS
|
||||||
|
|
||||||
|
if st is None or not st["flapping"]:
|
||||||
|
return PASS
|
||||||
|
if level == "RECOVER":
|
||||||
|
# Starts the quiet window; the state ends silently when it elapses.
|
||||||
|
st["ok_since"] = now
|
||||||
|
return SUPPRESS
|
||||||
|
|
||||||
|
|
||||||
|
def flapping_services(host: str) -> list:
|
||||||
|
"""Return the services of *host* that are currently flapping.
|
||||||
|
|
||||||
|
An empty string in the result means the host itself (connectivity, boot,
|
||||||
|
shutdown) rather than a named service.
|
||||||
|
"""
|
||||||
|
if not _enabled():
|
||||||
|
return []
|
||||||
|
now = time.time()
|
||||||
|
flapping = []
|
||||||
|
for (h, service), st in _state.items():
|
||||||
|
if h != host:
|
||||||
|
continue
|
||||||
|
_sweep(st, now)
|
||||||
|
if st["flapping"]:
|
||||||
|
flapping.append(service)
|
||||||
|
return sorted(flapping)
|
||||||
+47
-2
@@ -95,7 +95,7 @@ class Connection:
|
|||||||
if not Null:
|
if not Null:
|
||||||
d["addr"] = self.addr
|
d["addr"] = self.addr
|
||||||
if self.rtts[-1]:
|
if self.rtts[-1]:
|
||||||
d["rtt"] = "%0.1f" % self.rtts[-1]
|
d["rtt"] = "%d" % round(self.rtts[-1])
|
||||||
elif self.state == Connection.UNKNOWN:
|
elif self.state == Connection.UNKNOWN:
|
||||||
d["rtt"] = ""
|
d["rtt"] = ""
|
||||||
else:
|
else:
|
||||||
@@ -297,13 +297,24 @@ class Host:
|
|||||||
self.plugin_retention = 100 # Keep last N samples per plugin
|
self.plugin_retention = 100 # Keep last N samples per plugin
|
||||||
# Alert state tracking: {metric_path: AlertState}
|
# Alert state tracking: {metric_path: AlertState}
|
||||||
self.alert_states = {}
|
self.alert_states = {}
|
||||||
|
# Stale-data timers: {plugin_name: asyncio.TimerHandle}
|
||||||
|
self.plugin_timers = {}
|
||||||
# User access control
|
# User access control
|
||||||
self.owner: str | None = None # username of owner
|
self.owner: str | None = None # username of owner
|
||||||
self.managers: list = [] # usernames with manager role
|
self.managers: list = [] # usernames with manager role
|
||||||
self.monitors: list = [] # usernames with monitor role
|
self.monitors: list = [] # usernames with monitor role
|
||||||
|
|
||||||
|
def __getstate__(self):
|
||||||
|
"""Prepare Host for pickling by excluding non-serializable timer objects."""
|
||||||
|
state = self.__dict__.copy()
|
||||||
|
# asyncio TimerHandles (and their lambda callbacks) can't be pickled.
|
||||||
|
# They're recreated when the next PLG arrives after unpickling.
|
||||||
|
state['plugin_timers'] = {}
|
||||||
|
return state
|
||||||
|
|
||||||
def statedict(self):
|
def statedict(self):
|
||||||
d = {}
|
d = {}
|
||||||
|
d["raw_name"] = self.name
|
||||||
d["name"] = self.name
|
d["name"] = self.name
|
||||||
if self.dyn:
|
if self.dyn:
|
||||||
d["name"] += "*"
|
d["name"] += "*"
|
||||||
@@ -364,7 +375,7 @@ class Host:
|
|||||||
def stateinfo(self):
|
def stateinfo(self):
|
||||||
ddict = {}
|
ddict = {}
|
||||||
for d in self.__dict__:
|
for d in self.__dict__:
|
||||||
if d in ["alert_states", "plugin_data"]:
|
if d in ["alert_states", "plugin_data", "plugin_timers"]:
|
||||||
continue
|
continue
|
||||||
if d == "connections":
|
if d == "connections":
|
||||||
cl = []
|
cl = []
|
||||||
@@ -417,6 +428,10 @@ class Host:
|
|||||||
ddict["alert_critical_unacked"] = critical_unacked
|
ddict["alert_critical_unacked"] = critical_unacked
|
||||||
ddict["alert_critical_acked"] = critical_acked
|
ddict["alert_critical_acked"] = critical_acked
|
||||||
|
|
||||||
|
# Flap detection state (module-level in flap.py, never pickled)
|
||||||
|
from . import flap
|
||||||
|
ddict["flapping"] = flap.flapping_services(self.name)
|
||||||
|
|
||||||
# User access
|
# User access
|
||||||
ddict["owner"] = getattr(self, "owner", None)
|
ddict["owner"] = getattr(self, "owner", None)
|
||||||
ddict["managers"] = list(getattr(self, "managers", []))
|
ddict["managers"] = list(getattr(self, "managers", []))
|
||||||
@@ -482,6 +497,8 @@ class Host:
|
|||||||
self.managers = []
|
self.managers = []
|
||||||
if not hasattr(self, "monitors"):
|
if not hasattr(self, "monitors"):
|
||||||
self.monitors = []
|
self.monitors = []
|
||||||
|
if not hasattr(self, "plugin_timers"):
|
||||||
|
self.plugin_timers = {}
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -541,6 +558,34 @@ class Host:
|
|||||||
"""
|
"""
|
||||||
return self.plugin_data
|
return self.plugin_data
|
||||||
|
|
||||||
|
def reset_plugin_timer(self, plugin_name, timeout_seconds, callback):
|
||||||
|
"""Reset the stale-data timer for a plugin.
|
||||||
|
|
||||||
|
If no new PLG data arrives within timeout_seconds, callback(host, plugin_name)
|
||||||
|
is called so the caller can clear history and alerts.
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
existing = self.plugin_timers.get(plugin_name)
|
||||||
|
if existing and not existing.cancelled():
|
||||||
|
existing.cancel()
|
||||||
|
|
||||||
|
async def _fire():
|
||||||
|
await callback(self, plugin_name)
|
||||||
|
|
||||||
|
try:
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
self.plugin_timers[plugin_name] = loop.call_later(
|
||||||
|
timeout_seconds, lambda: asyncio.create_task(_fire())
|
||||||
|
)
|
||||||
|
except RuntimeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def cancel_plugin_timer(self, plugin_name):
|
||||||
|
"""Cancel the stale timer for a plugin, if any."""
|
||||||
|
handle = self.plugin_timers.pop(plugin_name, None)
|
||||||
|
if handle and not handle.cancelled():
|
||||||
|
handle.cancel()
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# User-role helpers
|
# User-role helpers
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|||||||
+970
-15
File diff suppressed because it is too large
Load Diff
+133
-1
@@ -11,7 +11,7 @@ import os
|
|||||||
import asyncio
|
import asyncio
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Dict, Any, Optional
|
from typing import Dict, Any, Optional, Callable, Iterable, List, Tuple, Union
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -153,6 +153,31 @@ class MessageJournal:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error writing to journal: {e}")
|
logger.error(f"Error writing to journal: {e}")
|
||||||
|
|
||||||
|
async def log_event(self, event: Dict[str, Any]):
|
||||||
|
"""Write a caller-provided dict verbatim as one JSONL line (with rotation)."""
|
||||||
|
if not self.enabled or not self._initialized:
|
||||||
|
return
|
||||||
|
async with self._lock:
|
||||||
|
try:
|
||||||
|
line = json.dumps(event, separators=(',', ':')) + '\n'
|
||||||
|
nbytes = len(line.encode('utf-8'))
|
||||||
|
if self._current_size + nbytes > self.max_size:
|
||||||
|
await self._rotate()
|
||||||
|
if self._file_handle:
|
||||||
|
self._file_handle.write(line)
|
||||||
|
self._file_handle.flush()
|
||||||
|
self._current_size += nbytes
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Error writing event to journal: {e}")
|
||||||
|
|
||||||
|
async def backfill(self, events: Iterable[Dict[str, Any]]):
|
||||||
|
"""One-time seed: write *events* only if the journal file is currently empty."""
|
||||||
|
if not self.enabled or not self._initialized or self._current_size > 0:
|
||||||
|
return
|
||||||
|
for ev in events:
|
||||||
|
if isinstance(ev, dict):
|
||||||
|
await self.log_event(ev)
|
||||||
|
|
||||||
async def _rotate(self):
|
async def _rotate(self):
|
||||||
"""
|
"""
|
||||||
Rotate the journal file.
|
Rotate the journal file.
|
||||||
@@ -309,6 +334,88 @@ class MessageJournal:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_journal_events(journal_dir: Union[str, Path], journal_file: str) -> Iterable[Dict[str, Any]]:
|
||||||
|
"""Yield event dicts from the journal files, newest event first.
|
||||||
|
|
||||||
|
Reads the current file, then rotated backups newest-to-oldest (backup
|
||||||
|
names embed rotation timestamps, so filename sort is chronological).
|
||||||
|
"""
|
||||||
|
dirp = Path(journal_dir)
|
||||||
|
files = [dirp / journal_file]
|
||||||
|
files.extend(sorted(dirp.glob(journal_file + '.*'), reverse=True))
|
||||||
|
for f in files:
|
||||||
|
if not f.is_file():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
lines = f.read_text(encoding='utf-8', errors='replace').splitlines()
|
||||||
|
except OSError as e:
|
||||||
|
logger.warning(f"Cannot read journal file {f}: {e}")
|
||||||
|
continue
|
||||||
|
for line in reversed(lines):
|
||||||
|
try:
|
||||||
|
ev = json.loads(line)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if isinstance(ev, dict):
|
||||||
|
yield ev
|
||||||
|
|
||||||
|
|
||||||
|
def filter_events(
|
||||||
|
events: Iterable[Dict[str, Any]],
|
||||||
|
limit: int = 100,
|
||||||
|
before: Optional[float] = None,
|
||||||
|
host: Optional[str] = None,
|
||||||
|
level: Optional[str] = None,
|
||||||
|
q: Optional[str] = None,
|
||||||
|
predicate: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
||||||
|
) -> Tuple[List[Dict[str, Any]], bool]:
|
||||||
|
"""Filter an iterable of event dicts already ordered newest-first.
|
||||||
|
|
||||||
|
Returns (events, more): up to *limit* matching events, and whether at
|
||||||
|
least one further matching event exists beyond the limit.
|
||||||
|
"""
|
||||||
|
host_l = host.lower() if host else None
|
||||||
|
level_l = level.lower() if level else None
|
||||||
|
q_l = q.lower() if q else None
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for ev in events:
|
||||||
|
if not isinstance(ev, dict):
|
||||||
|
continue
|
||||||
|
ts = ev.get('ts')
|
||||||
|
if before is not None and (not isinstance(ts, (int, float)) or ts >= before):
|
||||||
|
continue
|
||||||
|
if host_l and host_l not in str(ev.get('host') or '').lower():
|
||||||
|
continue
|
||||||
|
if level_l and str(ev.get('level') or '').lower() != level_l:
|
||||||
|
continue
|
||||||
|
if q_l and q_l not in str(ev.get('message') or '').lower():
|
||||||
|
continue
|
||||||
|
if predicate is not None and not predicate(ev):
|
||||||
|
continue
|
||||||
|
if len(out) >= limit:
|
||||||
|
return out, True
|
||||||
|
out.append(ev)
|
||||||
|
return out, False
|
||||||
|
|
||||||
|
|
||||||
|
def read_events(
|
||||||
|
journal_dir: Union[str, Path],
|
||||||
|
journal_file: str = 'events.journal',
|
||||||
|
*,
|
||||||
|
limit: int = 100,
|
||||||
|
before: Optional[float] = None,
|
||||||
|
host: Optional[str] = None,
|
||||||
|
level: Optional[str] = None,
|
||||||
|
q: Optional[str] = None,
|
||||||
|
predicate: Optional[Callable[[Dict[str, Any]], bool]] = None,
|
||||||
|
) -> Tuple[List[Dict[str, Any]], bool]:
|
||||||
|
"""Read filtered events newest-first from the events journal files."""
|
||||||
|
return filter_events(
|
||||||
|
_iter_journal_events(journal_dir, journal_file),
|
||||||
|
limit=limit, before=before, host=host, level=level, q=q, predicate=predicate,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# Global journal instance
|
# Global journal instance
|
||||||
_journal_instance: Optional[MessageJournal] = None
|
_journal_instance: Optional[MessageJournal] = None
|
||||||
|
|
||||||
@@ -340,3 +447,28 @@ async def log_message(msg: Dict[str, Any], addr: tuple, timestamp: Optional[floa
|
|||||||
"""
|
"""
|
||||||
journal = get_journal()
|
journal = get_journal()
|
||||||
await journal.log_message(msg, addr, timestamp)
|
await journal.log_message(msg, addr, timestamp)
|
||||||
|
|
||||||
|
|
||||||
|
# Global events journal instance (human-readable event log, written by notify.eventlog)
|
||||||
|
_events_journal_instance: Optional[MessageJournal] = None
|
||||||
|
|
||||||
|
|
||||||
|
def get_events_journal(config: Optional[Dict[str, Any]] = None) -> MessageJournal:
|
||||||
|
"""Get or create the global events journal instance.
|
||||||
|
|
||||||
|
Uses the events_journal_* config keys; shares journal_dir and
|
||||||
|
journal_enabled with the raw-datagram journal.
|
||||||
|
"""
|
||||||
|
global _events_journal_instance
|
||||||
|
if _events_journal_instance is None:
|
||||||
|
cfg = config or {}
|
||||||
|
_events_journal_instance = MessageJournal(
|
||||||
|
{
|
||||||
|
'journal_dir': cfg.get('journal_dir', '/var/log/heartbeat'),
|
||||||
|
'journal_file': cfg.get('events_journal_file', 'events.journal'),
|
||||||
|
'journal_max_size': cfg.get('events_journal_max_size', 10 * 1024 * 1024),
|
||||||
|
'journal_max_backups': cfg.get('events_journal_max_backups', 10),
|
||||||
|
'journal_enabled': cfg.get('journal_enabled', True),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return _events_journal_instance
|
||||||
|
|||||||
+43
-12
@@ -78,9 +78,7 @@ async def reload_configuration(config_obj, config_path, components):
|
|||||||
True if reload succeeded, False otherwise
|
True if reload succeeded, False otherwise
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
logger.info("=" * 60)
|
|
||||||
logger.info("Starting configuration reload...")
|
logger.info("Starting configuration reload...")
|
||||||
logger.info("=" * 60)
|
|
||||||
|
|
||||||
# Reload config file
|
# Reload config file
|
||||||
new_config = await config_obj.reload(config_path)
|
new_config = await config_obj.reload(config_path)
|
||||||
@@ -101,9 +99,10 @@ async def reload_configuration(config_obj, config_path, components):
|
|||||||
access = config_mod.get_host_access(new_config, hostname)
|
access = config_mod.get_host_access(new_config, hostname)
|
||||||
host.apply_access(access["owner"], access["managers"], access["monitors"])
|
host.apply_access(access["owner"], access["managers"], access["monitors"])
|
||||||
|
|
||||||
# Reload threshold checker
|
# Reload threshold checker and prune alerts orphaned by the new config
|
||||||
if 'threshold_checker' in components:
|
if 'threshold_checker' in components:
|
||||||
components['threshold_checker'].reload(new_config)
|
components['threshold_checker'].reload(new_config)
|
||||||
|
components['threshold_checker'].purge_stale_alerts(hbdclass)
|
||||||
|
|
||||||
# Note: Changes to the following require restart:
|
# Note: Changes to the following require restart:
|
||||||
# - hb_port, hbd_port, ws_port (already bound)
|
# - hb_port, hbd_port, ws_port (already bound)
|
||||||
@@ -114,13 +113,11 @@ async def reload_configuration(config_obj, config_path, components):
|
|||||||
# These are reloadable and effective immediately:
|
# These are reloadable and effective immediately:
|
||||||
# - notification_channels
|
# - notification_channels
|
||||||
# - threshold_configs
|
# - threshold_configs
|
||||||
# - hosts (watchhosts, dyndnshosts, notification_channels)
|
# - hosts (watchhosts, dyndns, notification_channels)
|
||||||
# - grace period (used on next heartbeat)
|
# - grace period (used on next heartbeat)
|
||||||
# - debug/verbose flags (used on next message)
|
# - debug/verbose flags (used on next message)
|
||||||
|
|
||||||
logger.info("=" * 60)
|
|
||||||
logger.info("Configuration reload completed successfully")
|
logger.info("Configuration reload completed successfully")
|
||||||
logger.info("=" * 60)
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -168,6 +165,15 @@ async def _run_async(config, config_path=None):
|
|||||||
msg_journal = journal_mod.get_journal(config)
|
msg_journal = journal_mod.get_journal(config)
|
||||||
await msg_journal.initialize()
|
await msg_journal.initialize()
|
||||||
|
|
||||||
|
# Initialize events journal (human-readable event log for the /log page)
|
||||||
|
events_journal = journal_mod.get_events_journal(config)
|
||||||
|
await events_journal.initialize()
|
||||||
|
if data.msgs:
|
||||||
|
# One-time seed on upgrade: only writes when the journal file is empty
|
||||||
|
await events_journal.backfill(data.msgs)
|
||||||
|
|
||||||
|
notify_mod.eventlog(None, "INFO", f"hbd version {__version__} starting up")
|
||||||
|
|
||||||
# Initialize threshold checker
|
# Initialize threshold checker
|
||||||
threshold_checker = threshold_mod.ThresholdChecker(
|
threshold_checker = threshold_mod.ThresholdChecker(
|
||||||
config=config,
|
config=config,
|
||||||
@@ -241,6 +247,13 @@ async def _run_async(config, config_path=None):
|
|||||||
)
|
)
|
||||||
udp.restore_connection_timers(hbdclass, restore_ctx)
|
udp.restore_connection_timers(hbdclass, restore_ctx)
|
||||||
|
|
||||||
|
# Drop alert states that no longer have a matching threshold (stale after
|
||||||
|
# upgrade or config change between runs).
|
||||||
|
threshold_checker.purge_stale_alerts(hbdclass)
|
||||||
|
|
||||||
|
async def _http_reload_callback():
|
||||||
|
await reload_configuration(config, config_path, components)
|
||||||
|
|
||||||
# HTTP server (asyncio-based via aiohttp)
|
# HTTP server (asyncio-based via aiohttp)
|
||||||
try:
|
try:
|
||||||
http_task = asyncio.create_task(
|
http_task = asyncio.create_task(
|
||||||
@@ -250,9 +263,11 @@ async def _run_async(config, config_path=None):
|
|||||||
config=config,
|
config=config,
|
||||||
hbdclass=hbdclass,
|
hbdclass=hbdclass,
|
||||||
tcss=None,
|
tcss=None,
|
||||||
|
threshold_checker=threshold_checker,
|
||||||
verbose=config.get("verbose", False),
|
verbose=config.get("verbose", False),
|
||||||
get_now=lambda: time.time(),
|
get_now=lambda: time.time(),
|
||||||
VER="",
|
VER="",
|
||||||
|
reload_callback=_http_reload_callback,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
@@ -373,6 +388,26 @@ async def _run_async(config, config_path=None):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Error closing message journal: %s", e)
|
logger.warning("Error closing message journal: %s", e)
|
||||||
|
|
||||||
|
# Journal the shutdown event now; run()'s shutdown eventlog fires after this journal closes
|
||||||
|
try:
|
||||||
|
await events_journal.log_event(
|
||||||
|
{
|
||||||
|
"ts": time.time(),
|
||||||
|
"host": None,
|
||||||
|
"level": "INFO",
|
||||||
|
"service": None,
|
||||||
|
"message": f"hbd version {__version__} shutdown",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("Could not journal shutdown event: %s", e)
|
||||||
|
|
||||||
|
# Close events journal
|
||||||
|
try:
|
||||||
|
await events_journal.close()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Error closing events journal: %s", e)
|
||||||
|
|
||||||
# Signal DNS worker to exit and await it
|
# Signal DNS worker to exit and await it
|
||||||
try:
|
try:
|
||||||
if "dns_task" in locals() and dns_task:
|
if "dns_task" in locals() and dns_task:
|
||||||
@@ -416,7 +451,6 @@ def load_pickled_hosts(config, hbdclass):
|
|||||||
pickfile = config.get("pickfile", "hbd.pickle")
|
pickfile = config.get("pickfile", "hbd.pickle")
|
||||||
dyndnshosts = config_mod.get_dyndnshosts(config)
|
dyndnshosts = config_mod.get_dyndnshosts(config)
|
||||||
watchhosts = config_mod.get_watchhosts(config)
|
watchhosts = config_mod.get_watchhosts(config)
|
||||||
drophosts = config.get("drophosts", [])
|
|
||||||
if 1 and os.path.exists(pickfile):
|
if 1 and os.path.exists(pickfile):
|
||||||
if config.get("verbose", False):
|
if config.get("verbose", False):
|
||||||
logger.info("opening pickls %s", pickfile)
|
logger.info("opening pickls %s", pickfile)
|
||||||
@@ -442,9 +476,6 @@ def load_pickled_hosts(config, hbdclass):
|
|||||||
hbdclass.Host.hosts[h].apply_access(
|
hbdclass.Host.hosts[h].apply_access(
|
||||||
access["owner"], access["managers"], access["monitors"]
|
access["owner"], access["managers"], access["monitors"]
|
||||||
)
|
)
|
||||||
for h in drophosts:
|
|
||||||
if h in hbdclass.Host.hosts:
|
|
||||||
del hbdclass.Host.hosts[h]
|
|
||||||
if config.get("verbose", False):
|
if config.get("verbose", False):
|
||||||
logger.info("%s pickled hosts loaded", len(hbdclass.Host.hosts))
|
logger.info("%s pickled hosts loaded", len(hbdclass.Host.hosts))
|
||||||
else:
|
else:
|
||||||
@@ -469,6 +500,8 @@ def run(config, config_path=None):
|
|||||||
if config.get("debug", 0) > 0:
|
if config.get("debug", 0) > 0:
|
||||||
log_level = logging.DEBUG
|
log_level = logging.DEBUG
|
||||||
logging.basicConfig(level=log_level)
|
logging.basicConfig(level=log_level)
|
||||||
|
if not config.get("debug", 0):
|
||||||
|
logging.getLogger("aiohttp.access").propagate = False
|
||||||
load_pickled_hosts(config, hbdclass)
|
load_pickled_hosts(config, hbdclass)
|
||||||
|
|
||||||
notify_mod.initlog(logfile=config.get("logfile", "messages.log"))
|
notify_mod.initlog(logfile=config.get("logfile", "messages.log"))
|
||||||
@@ -483,8 +516,6 @@ def run(config, config_path=None):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Failed to write pidfile %s: %s", pidfile, e)
|
logger.warning("Failed to write pidfile %s: %s", pidfile, e)
|
||||||
|
|
||||||
eventlog(None, "INFO", f"hbd version {__version__} starting up")
|
|
||||||
|
|
||||||
if config_path:
|
if config_path:
|
||||||
logger.info(f"Config file: {config_path} (reload with SIGHUP)")
|
logger.info(f"Config file: {config_path} (reload with SIGHUP)")
|
||||||
else:
|
else:
|
||||||
|
|||||||
+59
-7
@@ -25,6 +25,7 @@ from dataclasses import dataclass, field
|
|||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from . import data
|
from . import data
|
||||||
|
from . import flap as flap_mod
|
||||||
from . import ws as ws_mod
|
from . import ws as ws_mod
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -33,6 +34,7 @@ msg_to_websockets = ws_mod.broadcast
|
|||||||
|
|
||||||
# Module-level state set via setup()
|
# Module-level state set via setup()
|
||||||
_config: dict = {}
|
_config: dict = {}
|
||||||
|
_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||||
|
|
||||||
# Tracks which channels fired a WARNING/CRITICAL per host.
|
# Tracks which channels fired a WARNING/CRITICAL per host.
|
||||||
# {host_name: set of channel_names} — used to route RECOVER to the same channels.
|
# {host_name: set of channel_names} — used to route RECOVER to the same channels.
|
||||||
@@ -62,6 +64,7 @@ class Notification:
|
|||||||
body: str # detail message
|
body: str # detail message
|
||||||
level: str # RECOVER | WARNING | CRITICAL | INFO
|
level: str # RECOVER | WARNING | CRITICAL | INFO
|
||||||
url: str = "" # link to plugin metrics page
|
url: str = "" # link to plugin metrics page
|
||||||
|
service: str = "" # flap-detection key within the host ("" = the host itself)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -70,14 +73,18 @@ class Notification:
|
|||||||
|
|
||||||
def setup(cfg: dict, loop: Optional[asyncio.AbstractEventLoop] = None):
|
def setup(cfg: dict, loop: Optional[asyncio.AbstractEventLoop] = None):
|
||||||
"""Initialize notifier from configuration dict."""
|
"""Initialize notifier from configuration dict."""
|
||||||
global _config
|
global _config, _loop
|
||||||
_config = dict(cfg)
|
_config = dict(cfg)
|
||||||
|
flap_mod.setup(_config)
|
||||||
|
if loop is not None:
|
||||||
|
_loop = loop
|
||||||
|
|
||||||
|
|
||||||
def reload_config(cfg: dict):
|
def reload_config(cfg: dict):
|
||||||
"""Reload notification configuration on SIGHUP."""
|
"""Reload notification configuration on SIGHUP."""
|
||||||
global _config
|
global _config
|
||||||
_config = dict(cfg)
|
_config = dict(cfg)
|
||||||
|
flap_mod.setup(_config)
|
||||||
logger.info("Notification configuration reloaded")
|
logger.info("Notification configuration reloaded")
|
||||||
|
|
||||||
|
|
||||||
@@ -106,11 +113,23 @@ def closelog():
|
|||||||
|
|
||||||
def eventlog(host, lvl, m, service=None):
|
def eventlog(host, lvl, m, service=None):
|
||||||
ts = time.time()
|
ts = time.time()
|
||||||
|
msg = {
|
||||||
|
"ts": ts,
|
||||||
|
"host": host or None,
|
||||||
|
"level": lvl,
|
||||||
|
"service": service,
|
||||||
|
"message": m,
|
||||||
|
}
|
||||||
|
data.msgs.append(msg)
|
||||||
|
# Cap the in-memory buffer so it doesn't grow without bound; this list is
|
||||||
|
# replayed to every dashboard client on connect and persisted in the pickle.
|
||||||
|
cap = _config.get("msg_buffer_size", 500)
|
||||||
|
if cap and len(data.msgs) > cap:
|
||||||
|
del data.msgs[:-cap]
|
||||||
s = f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))} {lvl} "
|
s = f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))} {lvl} "
|
||||||
if host:
|
if host:
|
||||||
s += f"{host} "
|
s += f"{host} "
|
||||||
s += m
|
s += m
|
||||||
data.msgs.append(s)
|
|
||||||
logger.info(s)
|
logger.info(s)
|
||||||
if logf:
|
if logf:
|
||||||
try:
|
try:
|
||||||
@@ -118,7 +137,22 @@ def eventlog(host, lvl, m, service=None):
|
|||||||
logf.flush()
|
logf.flush()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("failed to write to logfile: %s", e)
|
logger.warning("failed to write to logfile: %s", e)
|
||||||
msg_to_websockets("message", s)
|
msg_to_websockets("message", msg)
|
||||||
|
_journal_event(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def _journal_event(msg: dict):
|
||||||
|
"""Schedule an async write of *msg* to the events journal (no-op without a loop)."""
|
||||||
|
if _loop is None or not _loop.is_running():
|
||||||
|
return
|
||||||
|
from . import journal as journal_mod
|
||||||
|
ej = journal_mod._events_journal_instance
|
||||||
|
if ej is None or not ej.enabled:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
asyncio.run_coroutine_threadsafe(ej.log_event(msg), _loop)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("failed to schedule events journal write: %s", e)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -133,10 +167,14 @@ def _send_pushover(channel_cfg: dict, notif: Notification) -> bool:
|
|||||||
if not token or not user:
|
if not token or not user:
|
||||||
logger.warning("pushover: missing token or user")
|
logger.warning("pushover: missing token or user")
|
||||||
return False
|
return False
|
||||||
params: dict = {"token": token, "user": user, "title": notif.title, "message": notif.body}
|
body = "%s: %s" % (notif.title, notif.body)
|
||||||
|
title = ""
|
||||||
|
params: dict = {"token": token, "user": user, "title": title, "message": body}
|
||||||
|
if channel_cfg.get("sound"):
|
||||||
|
params["sound"] = channel_cfg["sound"]
|
||||||
if notif.url:
|
if notif.url:
|
||||||
params["url"] = notif.url
|
params["url"] = notif.url
|
||||||
params["url_title"] = "Plugin metrics"
|
params["url_title"] = "Heartbeat"
|
||||||
conn = http.client.HTTPSConnection("api.pushover.net:443")
|
conn = http.client.HTTPSConnection("api.pushover.net:443")
|
||||||
try:
|
try:
|
||||||
conn.request(
|
conn.request(
|
||||||
@@ -209,7 +247,7 @@ def _send_mattermost(channel_cfg: dict, notif: Notification) -> bool:
|
|||||||
return False
|
return False
|
||||||
text = f"**{notif.title}**\n{notif.body}"
|
text = f"**{notif.title}**\n{notif.body}"
|
||||||
if notif.url:
|
if notif.url:
|
||||||
text += f"\n[Plugin metrics]({notif.url})"
|
text += f"\n[Plugin metrics] {notif.url}"
|
||||||
ses = {"url": host, "scheme": "http", "basepath": "/api/v4", "port": 8065}
|
ses = {"url": host, "scheme": "http", "basepath": "/api/v4", "port": 8065}
|
||||||
mm = Driver(ses)
|
mm = Driver(ses)
|
||||||
payload: dict = {"text": text, "channel": channel, "username": channel_cfg.get("username", "hbd")}
|
payload: dict = {"text": text, "channel": channel, "username": channel_cfg.get("username", "hbd")}
|
||||||
@@ -357,6 +395,9 @@ _TIMEOUT = 15 # seconds per channel send
|
|||||||
|
|
||||||
async def _dispatch_to_channel(channel_name: str, channel_cfg: dict, notif: Notification) -> bool:
|
async def _dispatch_to_channel(channel_name: str, channel_cfg: dict, notif: Notification) -> bool:
|
||||||
"""Send *notif* to a single named channel, honouring min_level."""
|
"""Send *notif* to a single named channel, honouring min_level."""
|
||||||
|
# Strip ownership metadata — notifier drivers only need delivery credentials.
|
||||||
|
channel_cfg = {k: v for k, v in channel_cfg.items() if k not in ("owner", "private")}
|
||||||
|
|
||||||
level = notif.level.upper()
|
level = notif.level.upper()
|
||||||
if level != "RECOVER":
|
if level != "RECOVER":
|
||||||
min_level = channel_cfg.get("min_level", "WARNING").upper()
|
min_level = channel_cfg.get("min_level", "WARNING").upper()
|
||||||
@@ -392,7 +433,7 @@ def _build_url(host_name: str) -> str:
|
|||||||
base_url = _config.get("base_url", "").rstrip("/")
|
base_url = _config.get("base_url", "").rstrip("/")
|
||||||
if not base_url:
|
if not base_url:
|
||||||
return ""
|
return ""
|
||||||
return f"{base_url}/plugins#{host_name}"
|
return f"{base_url}/alerts?filter={host_name}"
|
||||||
|
|
||||||
|
|
||||||
async def send_notification(host_name: str, notif: Notification) -> dict:
|
async def send_notification(host_name: str, notif: Notification) -> dict:
|
||||||
@@ -402,11 +443,22 @@ async def send_notification(host_name: str, notif: Notification) -> dict:
|
|||||||
notification_channels, and dispatches. Silently does nothing if
|
notification_channels, and dispatches. Silently does nothing if
|
||||||
no users are configured.
|
no users are configured.
|
||||||
|
|
||||||
|
Flap detection runs first: once *host_name*/*notif.service* is flapping the
|
||||||
|
notification is dropped, and the one that trips the state carries
|
||||||
|
``flap.FLAP_MARKER``.
|
||||||
|
|
||||||
Returns a dict of {channel_name: bool} results.
|
Returns a dict of {channel_name: bool} results.
|
||||||
"""
|
"""
|
||||||
from . import users as users_mod
|
from . import users as users_mod
|
||||||
from . import hbdclass
|
from . import hbdclass
|
||||||
|
|
||||||
|
action = flap_mod.observe(host_name, notif.service, notif.level)
|
||||||
|
if action == flap_mod.SUPPRESS:
|
||||||
|
logger.debug("flapping: suppressed %s notification for %s", notif.level, host_name)
|
||||||
|
return {}
|
||||||
|
if action == flap_mod.TRIP:
|
||||||
|
notif.body = f"{notif.body} {flap_mod.FLAP_MARKER}"
|
||||||
|
|
||||||
if not users_mod.users_enabled():
|
if not users_mod.users_enabled():
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,254 @@
|
|||||||
|
"""OAuth2 provider support.
|
||||||
|
|
||||||
|
Config shape (in ~/.hb.yaml):
|
||||||
|
|
||||||
|
oauth:
|
||||||
|
my-gitea: # route slug → /login/oauth/my-gitea
|
||||||
|
type: gitea # "gitea" | "github" | "nextcloud"
|
||||||
|
# omit type to default to "gitea"
|
||||||
|
url: https://git.example.com # required for gitea and nextcloud
|
||||||
|
client_id: <client-id>
|
||||||
|
client_secret: <client-secret>
|
||||||
|
label: "Work Gitea" # optional display name on login button
|
||||||
|
logo: https://example.com/logo.png # optional logo URL
|
||||||
|
|
||||||
|
github:
|
||||||
|
type: github
|
||||||
|
client_id: <client-id>
|
||||||
|
client_secret: <client-secret>
|
||||||
|
|
||||||
|
nextcloud:
|
||||||
|
type: nextcloud
|
||||||
|
url: https://cloud.example.com
|
||||||
|
client_id: <client-id>
|
||||||
|
client_secret: <client-secret>
|
||||||
|
|
||||||
|
Register the OAuth app with each provider and set the redirect URI to:
|
||||||
|
https://<hbd-host>/login/oauth/<name>/callback
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import secrets
|
||||||
|
import time
|
||||||
|
import urllib.parse
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
STATE_TTL = 600 # 10 minutes
|
||||||
|
|
||||||
|
# state_token -> expiry timestamp
|
||||||
|
_states: dict[str, float] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def make_state() -> str:
|
||||||
|
"""Generate a CSRF state token, store it with TTL, and return it."""
|
||||||
|
_purge_states()
|
||||||
|
token = secrets.token_hex(32)
|
||||||
|
_states[token] = time.time() + STATE_TTL
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def validate_state(state: str) -> bool:
|
||||||
|
"""Return True if *state* is known and unexpired; always removes it."""
|
||||||
|
expiry = _states.pop(state, None)
|
||||||
|
if expiry is None:
|
||||||
|
return False
|
||||||
|
return time.time() < expiry
|
||||||
|
|
||||||
|
|
||||||
|
def _purge_states() -> None:
|
||||||
|
"""Remove all expired CSRF state tokens from the in-memory store."""
|
||||||
|
now = time.time()
|
||||||
|
expired = [k for k, exp in list(_states.items()) if exp < now]
|
||||||
|
for k in expired:
|
||||||
|
del _states[k]
|
||||||
|
|
||||||
|
|
||||||
|
class OAuthError(Exception):
|
||||||
|
"""Raised when the OAuth2 flow fails for any reason."""
|
||||||
|
|
||||||
|
|
||||||
|
PROVIDER_DEFS: dict = {
|
||||||
|
"gitea": {
|
||||||
|
"authorize_url_tmpl": "{url}/login/oauth/authorize",
|
||||||
|
"token_url_tmpl": "{url}/login/oauth/access_token",
|
||||||
|
"profile_url_tmpl": "{url}/api/v1/user",
|
||||||
|
"scope": "user:email",
|
||||||
|
"field_map": {"username": "login", "full_name": "full_name", "avatar": "avatar_url"},
|
||||||
|
"profile_data_path": [],
|
||||||
|
"requires_url": True,
|
||||||
|
"default_label": "Gitea",
|
||||||
|
},
|
||||||
|
"github": {
|
||||||
|
"authorize_url_tmpl": "https://github.com/login/oauth/authorize",
|
||||||
|
"token_url_tmpl": "https://github.com/login/oauth/access_token",
|
||||||
|
"profile_url_tmpl": "https://api.github.com/user",
|
||||||
|
"scope": "read:user",
|
||||||
|
"field_map": {"username": "login", "full_name": "name", "avatar": "avatar_url"},
|
||||||
|
"profile_data_path": [],
|
||||||
|
"requires_url": False,
|
||||||
|
"default_label": "GitHub",
|
||||||
|
},
|
||||||
|
"nextcloud": {
|
||||||
|
"authorize_url_tmpl": "{url}/apps/oauth2/authorize",
|
||||||
|
"token_url_tmpl": "{url}/apps/oauth2/api/v1/token",
|
||||||
|
"profile_url_tmpl": "{url}/ocs/v2.php/cloud/user?format=json",
|
||||||
|
"scope": "",
|
||||||
|
"field_map": {"username": "id", "full_name": "display-name", "avatar": None},
|
||||||
|
"profile_data_path": ["ocs", "data"],
|
||||||
|
"requires_url": True,
|
||||||
|
"default_label": "Nextcloud",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ResolvedProvider:
|
||||||
|
"""A fully resolved OAuth2 provider instance, ready to use."""
|
||||||
|
name: str
|
||||||
|
type: str
|
||||||
|
label: str
|
||||||
|
logo: str
|
||||||
|
authorize_url: str
|
||||||
|
token_url: str
|
||||||
|
profile_url: str
|
||||||
|
scope: str
|
||||||
|
client_id: str
|
||||||
|
client_secret: str
|
||||||
|
field_map: dict
|
||||||
|
profile_data_path: list
|
||||||
|
|
||||||
|
|
||||||
|
def get_providers(config: dict) -> list[ResolvedProvider]:
|
||||||
|
"""Return a ResolvedProvider for every valid entry in config['oauth'].
|
||||||
|
|
||||||
|
Entries with missing required fields or unknown types are skipped with
|
||||||
|
a warning log. Order follows config declaration order.
|
||||||
|
"""
|
||||||
|
result = []
|
||||||
|
oauth_cfg = config.get("oauth", {})
|
||||||
|
if not isinstance(oauth_cfg, dict):
|
||||||
|
return result
|
||||||
|
for name, entry in oauth_cfg.items():
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
provider_type = entry.get("type", "gitea")
|
||||||
|
defn = PROVIDER_DEFS.get(provider_type)
|
||||||
|
if defn is None:
|
||||||
|
logger.warning("OAuth: unknown provider type %r for %r, skipping", provider_type, name)
|
||||||
|
continue
|
||||||
|
client_id = entry.get("client_id", "")
|
||||||
|
client_secret = entry.get("client_secret", "")
|
||||||
|
if not client_id or not client_secret:
|
||||||
|
logger.warning("OAuth: %r missing client_id or client_secret, skipping", name)
|
||||||
|
continue
|
||||||
|
url = entry.get("url", "").rstrip("/")
|
||||||
|
if defn["requires_url"] and not url:
|
||||||
|
logger.warning("OAuth: %r requires url but none configured, skipping", name)
|
||||||
|
continue
|
||||||
|
label = entry.get("label") or defn["default_label"]
|
||||||
|
logo = entry.get("logo", "")
|
||||||
|
result.append(ResolvedProvider(
|
||||||
|
name=name,
|
||||||
|
type=provider_type,
|
||||||
|
label=label,
|
||||||
|
logo=logo,
|
||||||
|
authorize_url=defn["authorize_url_tmpl"].format(url=url),
|
||||||
|
token_url=defn["token_url_tmpl"].format(url=url),
|
||||||
|
profile_url=defn["profile_url_tmpl"].format(url=url),
|
||||||
|
scope=defn["scope"],
|
||||||
|
client_id=client_id,
|
||||||
|
client_secret=client_secret,
|
||||||
|
field_map=dict(defn["field_map"]),
|
||||||
|
profile_data_path=list(defn["profile_data_path"]),
|
||||||
|
))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def is_enabled(config: dict) -> bool:
|
||||||
|
"""Return True when at least one OAuth provider is fully configured."""
|
||||||
|
return bool(get_providers(config))
|
||||||
|
|
||||||
|
|
||||||
|
def build_auth_url(provider: ResolvedProvider, state: str, redirect_uri: str) -> str:
|
||||||
|
"""Return the provider's OAuth2 authorization URL to redirect the browser to."""
|
||||||
|
params: dict = {
|
||||||
|
"client_id": provider.client_id,
|
||||||
|
"redirect_uri": redirect_uri,
|
||||||
|
"response_type": "code",
|
||||||
|
"state": state,
|
||||||
|
}
|
||||||
|
if provider.scope:
|
||||||
|
params["scope"] = provider.scope
|
||||||
|
return f"{provider.authorize_url}?{urllib.parse.urlencode(params)}"
|
||||||
|
|
||||||
|
|
||||||
|
async def exchange_code(provider: ResolvedProvider, code: str, redirect_uri: str) -> str:
|
||||||
|
"""Exchange an authorization *code* for an access token.
|
||||||
|
|
||||||
|
Returns the access token string. Raises OAuthError on any failure.
|
||||||
|
"""
|
||||||
|
payload = {
|
||||||
|
"client_id": provider.client_id,
|
||||||
|
"client_secret": provider.client_secret,
|
||||||
|
"code": code,
|
||||||
|
"grant_type": "authorization_code",
|
||||||
|
"redirect_uri": redirect_uri,
|
||||||
|
}
|
||||||
|
timeout = aiohttp.ClientTimeout(total=10)
|
||||||
|
try:
|
||||||
|
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||||
|
async with session.post(
|
||||||
|
provider.token_url,
|
||||||
|
json=payload,
|
||||||
|
headers={"Accept": "application/json"},
|
||||||
|
) as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
text = await resp.text()
|
||||||
|
raise OAuthError(f"Token exchange failed ({resp.status}): {text}")
|
||||||
|
data = await resp.json()
|
||||||
|
token = data.get("access_token")
|
||||||
|
if not token:
|
||||||
|
raise OAuthError(f"No access_token in response: {data}")
|
||||||
|
except aiohttp.ClientError as exc:
|
||||||
|
raise OAuthError(f"Token exchange network error: {exc}") from exc
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_user(provider: ResolvedProvider, token: str) -> dict:
|
||||||
|
"""Fetch the authenticated user's profile from the provider.
|
||||||
|
|
||||||
|
Returns a dict with keys: login, full_name, avatar_url.
|
||||||
|
Raises OAuthError on any failure.
|
||||||
|
"""
|
||||||
|
timeout = aiohttp.ClientTimeout(total=10)
|
||||||
|
try:
|
||||||
|
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||||
|
async with session.get(
|
||||||
|
provider.profile_url,
|
||||||
|
headers={
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
"Accept": "application/json",
|
||||||
|
},
|
||||||
|
) as resp:
|
||||||
|
if resp.status != 200:
|
||||||
|
text = await resp.text()
|
||||||
|
raise OAuthError(f"User fetch failed ({resp.status}): {text}")
|
||||||
|
data = await resp.json()
|
||||||
|
except aiohttp.ClientError as exc:
|
||||||
|
raise OAuthError(f"User fetch network error: {exc}") from exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
for key in provider.profile_data_path:
|
||||||
|
data = data.get(key, {})
|
||||||
|
avatar_field = provider.field_map.get("avatar")
|
||||||
|
return {
|
||||||
|
"login": data.get(provider.field_map["username"], ""),
|
||||||
|
"full_name": data.get(provider.field_map["full_name"], ""),
|
||||||
|
"avatar_url": data.get(avatar_field, "") if avatar_field else "",
|
||||||
|
}
|
||||||
|
except AttributeError:
|
||||||
|
raise OAuthError(f"Unexpected profile response structure from {provider.type}")
|
||||||
+253
-38
@@ -21,19 +21,73 @@ editable bool Reserved for future use — currently always False
|
|||||||
sensitive bool True when the raw value must never be shown
|
sensitive bool True when the raw value must never be shown
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from . import config_access
|
||||||
|
|
||||||
# Credential field names that should always be masked.
|
# Credential field names that should always be masked.
|
||||||
_SECRET_KEYS = frozenset({
|
_SECRET_KEYS = frozenset({
|
||||||
"password", "token", "user_key", "api_key", "secret",
|
"password", "token", "user_key", "api_key", "secret",
|
||||||
"smtp_password", "smtp_user",
|
"smtp_password", "smtp_user", "api_password", "access_token",
|
||||||
})
|
})
|
||||||
|
|
||||||
_CHANNEL_TYPE_LABELS = {
|
CHANNEL_TYPE_SCHEMAS = {
|
||||||
"pushover": "Pushover",
|
"pushover": {
|
||||||
"email": "E-mail",
|
"label": "Pushover",
|
||||||
"signal": "Signal",
|
"fields": [
|
||||||
"mattermost": "Mattermost",
|
{"key": "token", "label": "App token", "type": "secret", "required": True},
|
||||||
|
{"key": "user", "label": "User key", "type": "secret", "required": True},
|
||||||
|
{"key": "sound", "label": "Sound", "type": "text", "required": False},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"email": {
|
||||||
|
"label": "E-mail",
|
||||||
|
"fields": [
|
||||||
|
{"key": "recipients", "label": "Recipients (comma-separated)", "type": "list", "required": True},
|
||||||
|
{"key": "sender", "label": "From address", "type": "text", "required": True},
|
||||||
|
{"key": "smtp_server", "label": "SMTP server", "type": "text", "required": True},
|
||||||
|
{"key": "smtp_port", "label": "SMTP port", "type": "port", "required": False},
|
||||||
|
{"key": "smtp_user", "label": "SMTP username", "type": "text", "required": False},
|
||||||
|
{"key": "smtp_password", "label": "SMTP password", "type": "secret", "required": False},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"signal": {
|
||||||
|
"label": "Signal",
|
||||||
|
"fields": [
|
||||||
|
{"key": "user", "label": "Sender number", "type": "text", "required": True},
|
||||||
|
{"key": "recipient", "label": "Recipient number", "type": "text", "required": True},
|
||||||
|
{"key": "cli_path", "label": "signal-cli path", "type": "text", "required": False},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"matrix": {
|
||||||
|
"label": "Matrix",
|
||||||
|
"fields": [
|
||||||
|
{"key": "homeserver", "label": "Homeserver URL", "type": "text", "required": True},
|
||||||
|
{"key": "access_token", "label": "Access token", "type": "secret", "required": True},
|
||||||
|
{"key": "room_id", "label": "Room ID", "type": "text", "required": True},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"sms_voipms": {
|
||||||
|
"label": "SMS (voip.ms)",
|
||||||
|
"fields": [
|
||||||
|
{"key": "api_user", "label": "API username", "type": "text", "required": True},
|
||||||
|
{"key": "api_password", "label": "API password", "type": "secret", "required": True},
|
||||||
|
{"key": "did", "label": "DID (from)", "type": "text", "required": True},
|
||||||
|
{"key": "dst", "label": "Destination", "type": "text", "required": True},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"mattermost": {
|
||||||
|
"label": "Mattermost",
|
||||||
|
"fields": [
|
||||||
|
{"key": "host", "label": "Host", "type": "text", "required": True},
|
||||||
|
{"key": "token", "label": "Webhook token", "type": "secret", "required": True},
|
||||||
|
{"key": "channel", "label": "Channel", "type": "text", "required": True},
|
||||||
|
{"key": "username", "label": "Bot username", "type": "text", "required": False},
|
||||||
|
{"key": "icon", "label": "Icon URL", "type": "text", "required": False},
|
||||||
|
],
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_CHANNEL_TYPE_LABELS = {k: v["label"] for k, v in CHANNEL_TYPE_SCHEMAS.items()}
|
||||||
|
|
||||||
|
|
||||||
def _mask(value):
|
def _mask(value):
|
||||||
"""Return a masked placeholder for sensitive values."""
|
"""Return a masked placeholder for sensitive values."""
|
||||||
@@ -88,9 +142,14 @@ def _sanitize_channel(name, cfg):
|
|||||||
# Public API
|
# Public API
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def get_settings_sections(config: dict) -> list:
|
def get_settings_sections(config: dict, threshold_checker=None, user=None) -> list:
|
||||||
"""Return ordered list of setting sections for the settings page.
|
"""Return ordered list of setting sections for the settings page.
|
||||||
|
|
||||||
|
*user* is an object with ``username``/``admin`` attributes, or None for
|
||||||
|
the unauthenticated (admin-equivalent) view. Non-admins get only the
|
||||||
|
channels/hosts/thresholds sections, filtered to global items plus what
|
||||||
|
they own or manage.
|
||||||
|
|
||||||
Each section:
|
Each section:
|
||||||
{
|
{
|
||||||
"title": str,
|
"title": str,
|
||||||
@@ -110,6 +169,9 @@ def get_settings_sections(config: dict) -> list:
|
|||||||
"sensitive": bool,
|
"sensitive": bool,
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
|
is_admin = user is None or getattr(user, "admin", False)
|
||||||
|
username: str = getattr(user, "username", "") or ""
|
||||||
|
|
||||||
def field(key, label, ftype, description="", editable=False, sensitive=False):
|
def field(key, label, ftype, description="", editable=False, sensitive=False):
|
||||||
raw = config.get(key)
|
raw = config.get(key)
|
||||||
if sensitive:
|
if sensitive:
|
||||||
@@ -143,14 +205,17 @@ def get_settings_sections(config: dict) -> list:
|
|||||||
}
|
}
|
||||||
|
|
||||||
# ---- Notification channels (complex, built separately) ----------------
|
# ---- Notification channels (complex, built separately) ----------------
|
||||||
|
_METADATA_KEYS = {"type", "owner", "private", "min_level"}
|
||||||
notif_channels = []
|
notif_channels = []
|
||||||
for ch_name, ch_cfg in (config.get("notification_channels") or {}).items():
|
for ch_name, ch_cfg in sorted((config.get("notification_channels") or {}).items()):
|
||||||
if not isinstance(ch_cfg, dict):
|
if not isinstance(ch_cfg, dict):
|
||||||
continue
|
continue
|
||||||
|
if not is_admin and not config_access.user_can_use(ch_cfg, username):
|
||||||
|
continue
|
||||||
ch_type = ch_cfg.get("type", "")
|
ch_type = ch_cfg.get("type", "")
|
||||||
fields = []
|
fields = []
|
||||||
for k, v in ch_cfg.items():
|
for k, v in ch_cfg.items():
|
||||||
if k == "type":
|
if k in _METADATA_KEYS:
|
||||||
continue
|
continue
|
||||||
sensitive = k in _SECRET_KEYS
|
sensitive = k in _SECRET_KEYS
|
||||||
fields.append({
|
fields.append({
|
||||||
@@ -165,60 +230,140 @@ def get_settings_sections(config: dict) -> list:
|
|||||||
"name": ch_name,
|
"name": ch_name,
|
||||||
"type": ch_type,
|
"type": ch_type,
|
||||||
"type_label": _CHANNEL_TYPE_LABELS.get(ch_type, ch_type.title()),
|
"type_label": _CHANNEL_TYPE_LABELS.get(ch_type, ch_type.title()),
|
||||||
|
"owner": ch_cfg.get("owner"),
|
||||||
|
"editable": is_admin or ch_cfg.get("owner") == username,
|
||||||
|
"min_level": ch_cfg.get("min_level", "WARNING"),
|
||||||
"fields": fields,
|
"fields": fields,
|
||||||
})
|
})
|
||||||
|
|
||||||
# ---- Users (show metadata only, never password hashes) ----------------
|
# ---- Users (show metadata only, never password hashes) ----------------
|
||||||
users_list = []
|
users_list = []
|
||||||
for username, attrs in (config.get("users") or {}).items():
|
for uname, attrs in (config.get("users") or {}).items():
|
||||||
if not isinstance(attrs, dict):
|
if not isinstance(attrs, dict):
|
||||||
continue
|
continue
|
||||||
users_list.append({
|
users_list.append({
|
||||||
"username": username,
|
"username": uname,
|
||||||
"full_name": attrs.get("full_name", ""),
|
"full_name": attrs.get("full_name", ""),
|
||||||
"admin": bool(attrs.get("admin", False)),
|
"admin": bool(attrs.get("admin", False)),
|
||||||
"avatar": attrs.get("avatar", ""),
|
"avatar": attrs.get("avatar", ""),
|
||||||
"notification_channels": attrs.get("notification_channels", []),
|
"notification_channels": attrs.get("notification_channels", []),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# ---- Threshold configurations -----------------------------------------
|
||||||
|
def _tc_to_row(tc):
|
||||||
|
return {
|
||||||
|
"metric": tc.metric_path,
|
||||||
|
"operator": tc.operator.value,
|
||||||
|
"warning": tc.warning,
|
||||||
|
"critical": tc.critical,
|
||||||
|
"hysteresis": tc.hysteresis,
|
||||||
|
"count": tc.count,
|
||||||
|
"enabled": tc.enabled,
|
||||||
|
"display": tc.display or "",
|
||||||
|
"grace": tc.grace,
|
||||||
|
}
|
||||||
|
|
||||||
|
threshold_config_list = []
|
||||||
|
raw_threshold_cfgs = config.get("threshold_configs") or {}
|
||||||
|
if threshold_checker is not None:
|
||||||
|
if threshold_checker.threshold_configs:
|
||||||
|
for cfg_name, cfg_metrics in sorted(threshold_checker.threshold_configs.items()):
|
||||||
|
raw_cfg = raw_threshold_cfgs.get(cfg_name)
|
||||||
|
tc_owner = raw_cfg.get("owner") if isinstance(raw_cfg, dict) else None
|
||||||
|
if not is_admin and tc_owner and tc_owner != username:
|
||||||
|
continue
|
||||||
|
# For the default config use the merged effective set;
|
||||||
|
# for named overrides use only the explicitly defined metrics
|
||||||
|
# (threshold_raw_configs) so inherited defaults are not repeated.
|
||||||
|
if cfg_name == "default":
|
||||||
|
display_metrics = cfg_metrics
|
||||||
|
else:
|
||||||
|
display_metrics = threshold_checker.threshold_raw_configs.get(cfg_name, cfg_metrics)
|
||||||
|
metrics = sorted(
|
||||||
|
[_tc_to_row(tc) for tc in display_metrics.values()],
|
||||||
|
key=lambda m: m["metric"],
|
||||||
|
)
|
||||||
|
threshold_config_list.append({
|
||||||
|
"name": cfg_name,
|
||||||
|
"metrics": metrics,
|
||||||
|
"owner": tc_owner,
|
||||||
|
"editable": is_admin or (tc_owner is not None and tc_owner == username),
|
||||||
|
})
|
||||||
|
elif threshold_checker.thresholds:
|
||||||
|
metrics = sorted(
|
||||||
|
[_tc_to_row(tc) for tc in threshold_checker.thresholds.values()],
|
||||||
|
key=lambda m: m["metric"],
|
||||||
|
)
|
||||||
|
threshold_config_list.append({"name": "default", "metrics": metrics,
|
||||||
|
"owner": None, "editable": is_admin})
|
||||||
|
|
||||||
# ---- Hosts summary ----------------------------------------------------
|
# ---- Hosts summary ----------------------------------------------------
|
||||||
hosts_list = []
|
hosts_list = []
|
||||||
for hname, hcfg in (config.get("hosts") or {}).items():
|
for hname, hcfg in sorted((config.get("hosts") or {}).items()):
|
||||||
if not isinstance(hcfg, dict):
|
if not isinstance(hcfg, dict):
|
||||||
continue
|
continue
|
||||||
|
managers = hcfg.get("managers", [])
|
||||||
|
if isinstance(managers, str):
|
||||||
|
managers = [managers]
|
||||||
|
if not is_admin and hcfg.get("owner") != username and username not in managers:
|
||||||
|
continue
|
||||||
hosts_list.append({
|
hosts_list.append({
|
||||||
"name": hname,
|
"name": hname,
|
||||||
"watch": bool(hcfg.get("watch", False)),
|
"watch": bool(hcfg.get("watch", True)),
|
||||||
"dyndns": bool(hcfg.get("dyndns", False)),
|
"dyndns": bool(hcfg.get("dyndns", False)),
|
||||||
"owner": hcfg.get("owner", ""),
|
"owner": hcfg.get("owner", ""),
|
||||||
"managers": hcfg.get("managers", []),
|
"is_owner": is_admin or hcfg.get("owner") == username,
|
||||||
|
"managers": managers,
|
||||||
"monitors": hcfg.get("monitors", []),
|
"monitors": hcfg.get("monitors", []),
|
||||||
"threshold_config": hcfg.get("threshold_config", ""),
|
"threshold_configs": (
|
||||||
|
list(v) if isinstance(v := hcfg.get("threshold_config"), list)
|
||||||
|
else ([v] if v else [])
|
||||||
|
),
|
||||||
"notification_channels": hcfg.get("notification_channels", []),
|
"notification_channels": hcfg.get("notification_channels", []),
|
||||||
})
|
})
|
||||||
|
|
||||||
return [
|
# ---- OAuth providers -------------------------------------------------------
|
||||||
|
oauth_providers = []
|
||||||
|
for pname, pattrs in (config.get("oauth") or {}).items():
|
||||||
|
if not isinstance(pattrs, dict):
|
||||||
|
continue
|
||||||
|
cs = pattrs.get("client_secret", "")
|
||||||
|
oauth_providers.append({
|
||||||
|
"name": pname,
|
||||||
|
"type": pattrs.get("type", "gitea"),
|
||||||
|
"url": pattrs.get("url", ""),
|
||||||
|
"client_id": pattrs.get("client_id", ""),
|
||||||
|
"client_secret": "•••" if cs else "",
|
||||||
|
"label": pattrs.get("label", ""),
|
||||||
|
"logo": pattrs.get("logo", ""),
|
||||||
|
})
|
||||||
|
|
||||||
|
sections: list = [
|
||||||
{
|
{
|
||||||
"id": "network",
|
"id": "network",
|
||||||
"title": "Network",
|
"title": "Network",
|
||||||
"description": "Ports and bind addresses for all server sockets.",
|
"description": "Ports and bind addresses for all server sockets.",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": "server",
|
||||||
"fields": [
|
"fields": [
|
||||||
field("hb_port", "Heartbeat UDP port", "port",
|
field("hb_port", "Heartbeat UDP port", "port",
|
||||||
"UDP port the server listens on for heartbeat datagrams."),
|
"UDP port the server listens on for heartbeat datagrams.", editable=True),
|
||||||
field("hbd_host", "HTTP bind address", "text",
|
field("hbd_host", "HTTP bind address", "text",
|
||||||
"Interface to bind the HTTP server to. Empty = all interfaces."),
|
"Interface to bind the HTTP server to. Empty = all interfaces.", editable=True),
|
||||||
field("hbd_port", "HTTP API port", "port",
|
field("hbd_port", "HTTP API port", "port",
|
||||||
"TCP port for the HTTP API and web UI."),
|
"TCP port for the HTTP API and web UI.", editable=True),
|
||||||
field("ws_port", "WebSocket port", "port",
|
field("ws_port", "WebSocket port", "port",
|
||||||
"TCP port for the plain WebSocket server."),
|
"TCP port for the plain WebSocket server.", editable=True),
|
||||||
field("wss_port", "Secure WebSocket port", "port",
|
field("wss_port", "Secure WebSocket port", "port",
|
||||||
"TCP port for WSS (TLS WebSocket). Leave empty to disable."),
|
"TCP port for WSS (TLS WebSocket). Leave empty to disable.", editable=True),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "tls",
|
"id": "tls",
|
||||||
"title": "TLS / WebSocket Security",
|
"title": "TLS / WebSocket Security",
|
||||||
"description": "Certificate paths used when wss_port is set.",
|
"description": "Certificate paths used when wss_port is set.",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": None,
|
||||||
"fields": [
|
"fields": [
|
||||||
field("cert_path", "Certificate directory", "path",
|
field("cert_path", "Certificate directory", "path",
|
||||||
"Directory containing the TLS certificate and key files."),
|
"Directory containing the TLS certificate and key files."),
|
||||||
@@ -232,73 +377,103 @@ def get_settings_sections(config: dict) -> list:
|
|||||||
"id": "monitoring",
|
"id": "monitoring",
|
||||||
"title": "Monitoring",
|
"title": "Monitoring",
|
||||||
"description": "Heartbeat timing and alert re-notification behaviour.",
|
"description": "Heartbeat timing and alert re-notification behaviour.",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": "server",
|
||||||
"fields": [
|
"fields": [
|
||||||
field("interval", "Heartbeat interval", "duration",
|
field("interval", "Heartbeat interval", "duration",
|
||||||
"Expected time between heartbeat messages from each client."),
|
"Expected time between heartbeat messages from each client.", editable=True),
|
||||||
field("grace", "Grace multiplier", "number",
|
field("grace", "Grace period", "number",
|
||||||
"A host is marked overdue after interval × grace seconds of silence."),
|
"Extra seconds to wait after a missed heartbeat before sending notifications.", editable=True),
|
||||||
field("threshold_renotify_interval", "Re-notify interval", "duration",
|
field("threshold_renotify_interval", "Re-notify interval", "duration",
|
||||||
"How often to re-send notifications for ongoing threshold alerts."),
|
"How often to re-send notifications for ongoing threshold alerts.", editable=True),
|
||||||
|
field("flap_count", "Flap count", "number",
|
||||||
|
"Warning/critical notifications within the flap interval that mark a "
|
||||||
|
"service as flapping and silence it. 0 disables flap detection.", editable=True),
|
||||||
|
field("flap_interval", "Flap interval", "number",
|
||||||
|
"Minutes: the window flap count is measured over, and how long a service "
|
||||||
|
"must stay OK before flapping ends.", editable=True),
|
||||||
field("autosave_interval", "Autosave interval", "duration",
|
field("autosave_interval", "Autosave interval", "duration",
|
||||||
"How often the server saves its state to disk."),
|
"How often the server saves its state to disk."),
|
||||||
|
field("base_url", "Base URL", "text",
|
||||||
|
"Base URL for notification links.", editable=True),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "persistence",
|
"id": "persistence",
|
||||||
"title": "Persistence & Logging",
|
"title": "Persistence & Logging",
|
||||||
"description": "State file and event log settings.",
|
"description": "State file and event log settings.",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": "server",
|
||||||
"fields": [
|
"fields": [
|
||||||
field("pickfile", "State file", "path",
|
field("pickfile", "State file", "path",
|
||||||
"Path to the pickle file used to persist host state across restarts."),
|
"Path to the pickle file used to persist host state across restarts.", editable=True),
|
||||||
field("logfile", "Event log", "path",
|
field("logfile", "Event log", "path",
|
||||||
"Path to the event log file."),
|
"Path to the event log file.", editable=True),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "journal",
|
"id": "journal",
|
||||||
"title": "Message Journal",
|
"title": "Message Journal",
|
||||||
"description": "All received heartbeat and plugin messages are journalled here.",
|
"description": "All received heartbeat and plugin messages are journalled here.",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": "server",
|
||||||
"fields": [
|
"fields": [
|
||||||
field("journal_enabled", "Enabled", "boolean",
|
field("journal_enabled", "Enabled", "boolean",
|
||||||
"Turn journalling on or off."),
|
"Turn journalling on or off.", editable=True),
|
||||||
field("journal_dir", "Journal directory","path",
|
field("journal_dir", "Journal directory","path",
|
||||||
"Directory where journal files are written."),
|
"Directory where journal files are written.", editable=True),
|
||||||
field("journal_file", "Journal filename", "text",
|
field("journal_file", "Journal filename", "text",
|
||||||
"Base filename for the journal (rotated copies get a numeric suffix)."),
|
"Base filename for the journal (rotated copies get a numeric suffix)."),
|
||||||
field("journal_max_size", "Max file size", "size",
|
field("journal_max_size", "Max file size", "size",
|
||||||
"Rotate the journal when it exceeds this size."),
|
"Rotate the journal when it exceeds this size.", editable=True),
|
||||||
field("journal_max_backups", "Backup count", "number",
|
field("journal_max_backups", "Backup count", "number",
|
||||||
"Number of rotated journal files to keep."),
|
"Number of rotated journal files to keep.", editable=True),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "dns",
|
"id": "dns",
|
||||||
"title": "Dynamic DNS",
|
"title": "Dynamic DNS",
|
||||||
"description": "nsupdate-based DNS registration for dynamic hosts.",
|
"description": "nsupdate-based DNS registration via nsupdate(8).",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": "dns",
|
||||||
"fields": [
|
"fields": [
|
||||||
field("nsupdate_bin", "nsupdate binary", "path",
|
field("nsupdate_bin", "nsupdate binary", "path",
|
||||||
"Full path to the nsupdate executable."),
|
"Path to the nsupdate binary.", editable=True),
|
||||||
field("dyndomains", "Dynamic domains", "list",
|
field("rndc_key", "RNDC key file", "path",
|
||||||
"DNS zones managed by nsupdate for dynamic hosts."),
|
"Path to the rndc key file used to authenticate DNS updates.", editable=True),
|
||||||
field("drophosts", "Drop hosts", "list",
|
field("dyndomains", "Dynamic domains", "list",
|
||||||
"Hostnames to silently ignore — no state, no alerts."),
|
"Domains updated via nsupdate when a host with dyndns: true reports in.",
|
||||||
|
editable=True),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "users",
|
"id": "users",
|
||||||
"title": "Users",
|
"title": "Users",
|
||||||
"description": "Accounts defined in the config file. Password hashes are never shown.",
|
"description": "Accounts defined in the config file. Password hashes are never shown.",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": "users",
|
||||||
"users": users_list,
|
"users": users_list,
|
||||||
"fields": [
|
"fields": [
|
||||||
field("default_owner", "Default owner", "text",
|
field("default_owner", "Default owner", "text",
|
||||||
"Username that owns hosts with no explicit owner. "
|
"Username that owns hosts with no explicit owner. "
|
||||||
"Falls back to the first admin user."),
|
"Falls back to the first admin user.", editable=True),
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "oauth",
|
||||||
|
"title": "OAuth Providers",
|
||||||
|
"description": "OAuth2 login providers. Client secrets are masked.",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": "oauth",
|
||||||
|
"providers": oauth_providers,
|
||||||
|
"fields": [],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "channels",
|
"id": "channels",
|
||||||
"title": "Notification Channels",
|
"title": "Notification Channels",
|
||||||
"description": "Named notification providers. Credentials are masked.",
|
"description": "Named notification providers. Credentials are masked.",
|
||||||
|
"section_mode": "channels",
|
||||||
|
"api_section": "notification_channels",
|
||||||
"channels": notif_channels,
|
"channels": notif_channels,
|
||||||
"fields": [
|
"fields": [
|
||||||
field("default_notification_channels", "Default channels", "list",
|
field("default_notification_channels", "Default channels", "list",
|
||||||
@@ -309,13 +484,29 @@ def get_settings_sections(config: dict) -> list:
|
|||||||
"id": "hosts",
|
"id": "hosts",
|
||||||
"title": "Hosts",
|
"title": "Hosts",
|
||||||
"description": "Host definitions loaded from the config file.",
|
"description": "Host definitions loaded from the config file.",
|
||||||
|
"section_mode": "hosts",
|
||||||
|
"api_section": "hosts",
|
||||||
"hosts": hosts_list,
|
"hosts": hosts_list,
|
||||||
"fields": [],
|
"fields": [],
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "thresholds",
|
||||||
|
"title": "Threshold Configurations",
|
||||||
|
"description": "Named alert threshold sets. Each defines warning/critical levels per metric.",
|
||||||
|
"section_mode": "thresholds",
|
||||||
|
"api_section": "thresholds",
|
||||||
|
"threshold_configs": threshold_config_list,
|
||||||
|
"fields": [
|
||||||
|
field("default_threshold_config", "Default config", "text",
|
||||||
|
"Threshold config used for hosts with no explicit mapping.", editable=True),
|
||||||
|
],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "runtime",
|
"id": "runtime",
|
||||||
"title": "Runtime",
|
"title": "Runtime",
|
||||||
"description": "Flags set at startup (require restart to change).",
|
"description": "Flags set at startup (require restart to change).",
|
||||||
|
"section_mode": "form",
|
||||||
|
"api_section": None,
|
||||||
"fields": [
|
"fields": [
|
||||||
field("foreground", "Foreground mode", "boolean",
|
field("foreground", "Foreground mode", "boolean",
|
||||||
"Run in the foreground instead of daemonising."),
|
"Run in the foreground instead of daemonising."),
|
||||||
@@ -326,3 +517,27 @@ def get_settings_sections(config: dict) -> list:
|
|||||||
],
|
],
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
if not is_admin:
|
||||||
|
sections = [s for s in sections if s["id"] in ("channels", "hosts", "thresholds")]
|
||||||
|
for s in sections:
|
||||||
|
s["fields"] = []
|
||||||
|
return sections
|
||||||
|
|
||||||
|
|
||||||
|
def get_settings_data(config: dict, threshold_checker=None, user=None) -> dict:
|
||||||
|
"""Return sections list + auxiliary data for the settings template."""
|
||||||
|
sections = get_settings_sections(config, threshold_checker=threshold_checker, user=user)
|
||||||
|
is_admin = user is None or getattr(user, "admin", False)
|
||||||
|
username: str = getattr(user, "username", "") or ""
|
||||||
|
channels = config.get("notification_channels") or {}
|
||||||
|
threshold_cfgs = config.get("threshold_configs") or {}
|
||||||
|
if not is_admin:
|
||||||
|
channels = config_access.user_channels(channels, username)
|
||||||
|
threshold_cfgs = config_access.user_threshold_configs(threshold_cfgs, username)
|
||||||
|
return {
|
||||||
|
"sections": sections,
|
||||||
|
"all_channel_names": sorted(channels.keys()),
|
||||||
|
"all_usernames": sorted((config.get("users") or {}).keys()),
|
||||||
|
"all_threshold_configs": sorted(threshold_cfgs.keys()),
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,155 @@
|
|||||||
|
/* hbd-ui.css — shared design system for hbd pages.
|
||||||
|
*
|
||||||
|
* Green-biased neutrals, teal accent, mono identifiers. Pages link this
|
||||||
|
* after static/style.css and build on the tokens + components below.
|
||||||
|
* Introduced with the settings-page redesign; see settings.html for the
|
||||||
|
* fullest use of the row/editor idiom.
|
||||||
|
*/
|
||||||
|
|
||||||
|
:root {
|
||||||
|
--st-paper: #f4f6f4;
|
||||||
|
--st-surface: #ffffff;
|
||||||
|
--st-surface-2: #eef1ee;
|
||||||
|
--st-ink: #1d2420;
|
||||||
|
--st-muted: #5f6a63;
|
||||||
|
--st-faint: #8a948d;
|
||||||
|
--st-line: #dde3de;
|
||||||
|
--st-line-soft: #e8ece8;
|
||||||
|
--st-accent: #0e7280;
|
||||||
|
--st-accent-ink: #ffffff;
|
||||||
|
--st-accent-soft: #e0eef0;
|
||||||
|
--st-ok: #3f9c5a;
|
||||||
|
--st-ok-soft: #e2f2e7;
|
||||||
|
--st-warn: #d9982a;
|
||||||
|
--st-warn-soft: #f9efdc;
|
||||||
|
--st-crit: #d64545;
|
||||||
|
--st-crit-soft: #f9e3e3;
|
||||||
|
--st-badge-owner: #ede7f6; --st-badge-owner-ink: #5e35b1;
|
||||||
|
--st-badge-mgr: #e3f2fd; --st-badge-mgr-ink: #1565c0;
|
||||||
|
--st-mono: ui-monospace, "SF Mono", "Cascadia Mono", Menlo, Consolas, monospace;
|
||||||
|
}
|
||||||
|
html[data-theme="dark"] {
|
||||||
|
--st-paper: #131816;
|
||||||
|
--st-surface: #1b211e;
|
||||||
|
--st-surface-2: #222925;
|
||||||
|
--st-ink: #e4eae6;
|
||||||
|
--st-muted: #9aa69e;
|
||||||
|
--st-faint: #6e7a72;
|
||||||
|
--st-line: #2c342f;
|
||||||
|
--st-line-soft: #252c28;
|
||||||
|
--st-accent: #4cc4d4;
|
||||||
|
--st-accent-ink: #0d2326;
|
||||||
|
--st-accent-soft: #143b40;
|
||||||
|
--st-ok: #5cba77;
|
||||||
|
--st-ok-soft: #1c2f22;
|
||||||
|
--st-warn: #e0ad4e;
|
||||||
|
--st-warn-soft: #33290f;
|
||||||
|
--st-crit: #e06c6c;
|
||||||
|
--st-crit-soft: #3a1a1a;
|
||||||
|
--st-badge-owner: #33294a; --st-badge-owner-ink: #b39ddb;
|
||||||
|
--st-badge-mgr: #1b3350; --st-badge-mgr-ink: #90caf9;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── sticky toolbar under the site nav ── */
|
||||||
|
.st-toolbar {
|
||||||
|
position: sticky; top: var(--nav-h, 48px); z-index: 90;
|
||||||
|
display: flex; align-items: center; gap: 12px; flex-wrap: wrap;
|
||||||
|
padding: 9px 20px;
|
||||||
|
background: var(--st-surface); border-bottom: 1px solid var(--st-line);
|
||||||
|
}
|
||||||
|
.st-toolbar .brand { font-family: var(--st-mono); font-size: 15px; font-weight: 700; letter-spacing: -.02em; color: var(--st-ink); }
|
||||||
|
.st-toolbar .brand .tld { color: var(--st-accent); }
|
||||||
|
.st-toolbar .hintline { font-size: 12px; color: var(--st-faint); }
|
||||||
|
.st-toolbar .spacer { flex: 1; }
|
||||||
|
|
||||||
|
/* ── ECG pulse divider ── */
|
||||||
|
.pulse { display: block; width: 100%; height: 14px; }
|
||||||
|
.pulse polyline { fill: none; stroke: var(--st-accent); stroke-width: 1.5; opacity: .55; }
|
||||||
|
|
||||||
|
/* ── buttons ── */
|
||||||
|
.btn {
|
||||||
|
font: 600 13px/1 inherit; border: 1px solid var(--st-line); border-radius: 6px;
|
||||||
|
background: var(--st-surface); color: var(--st-ink); padding: 7px 14px; cursor: pointer;
|
||||||
|
}
|
||||||
|
.btn:focus-visible, a:focus-visible { outline: 2px solid var(--st-accent); outline-offset: 2px; }
|
||||||
|
.btn.primary { background: var(--st-accent); border-color: var(--st-accent); color: var(--st-accent-ink); }
|
||||||
|
.btn.quiet { border-color: transparent; background: transparent; color: var(--st-muted); }
|
||||||
|
.btn.small { padding: 4px 10px; font-size: 12px; }
|
||||||
|
.btn.danger { color: var(--st-crit); }
|
||||||
|
.btn:disabled { opacity: .6; cursor: default; }
|
||||||
|
|
||||||
|
/* ── section headers as config keys ── */
|
||||||
|
section.st { margin-top: 26px; scroll-margin-top: calc(var(--bars-h, 94px) + 12px); }
|
||||||
|
.sec-head { display: flex; align-items: baseline; gap: 10px; margin-bottom: 8px; flex-wrap: wrap; }
|
||||||
|
.sec-head h2 { margin: 0; font-family: var(--st-mono); font-size: 15px; font-weight: 700; color: var(--st-ink); }
|
||||||
|
.sec-head h2 .colon { color: var(--st-accent); }
|
||||||
|
.sec-head .count { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-faint); }
|
||||||
|
.sec-head .sub { font-size: 12px; color: var(--st-muted); }
|
||||||
|
.sec-head .add { margin-left: auto; }
|
||||||
|
|
||||||
|
/* ── record list + row ── */
|
||||||
|
.list { background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 8px; overflow: hidden; }
|
||||||
|
.row {
|
||||||
|
display: grid; grid-template-columns: 190px minmax(0,1fr) auto;
|
||||||
|
gap: 4px 14px; align-items: center;
|
||||||
|
padding: 8px 14px; border-bottom: 1px solid var(--st-line-soft);
|
||||||
|
transition: background .12s ease;
|
||||||
|
}
|
||||||
|
@media (prefers-reduced-motion: reduce) { .row { transition: none; } }
|
||||||
|
.row:last-child { border-bottom: none; }
|
||||||
|
.row:hover { background: var(--st-surface-2); }
|
||||||
|
.row .id { font-family: var(--st-mono); font-size: 13px; font-weight: 600; color: var(--st-ink); display: flex; align-items: center; gap: 8px; min-width: 0; }
|
||||||
|
.row .id .name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.row .act { display: flex; gap: 4px; justify-content: flex-end; }
|
||||||
|
|
||||||
|
.dot { width: 8px; height: 8px; border-radius: 50%; flex: none; }
|
||||||
|
.dot.ok { background: var(--st-ok); }
|
||||||
|
.dot.warn { background: var(--st-warn); }
|
||||||
|
.dot.crit { background: var(--st-crit); }
|
||||||
|
.dot.off { background: transparent; border: 1.5px solid var(--st-faint); }
|
||||||
|
|
||||||
|
.facts { display: flex; flex-wrap: wrap; gap: 4px 6px; align-items: center; min-width: 0; }
|
||||||
|
.chip {
|
||||||
|
display: inline-flex; align-items: center; gap: 4px;
|
||||||
|
font-family: var(--st-mono); font-size: 11px; line-height: 1.3;
|
||||||
|
padding: 3px 7px; border-radius: 4px;
|
||||||
|
background: var(--st-surface-2); color: var(--st-muted);
|
||||||
|
}
|
||||||
|
.chip.owner { background: var(--st-badge-owner); color: var(--st-badge-owner-ink); }
|
||||||
|
.chip.mgr { background: var(--st-badge-mgr); color: var(--st-badge-mgr-ink); }
|
||||||
|
.chip.level { background: transparent; border: 1px solid var(--st-line); }
|
||||||
|
.chip.private { background: transparent; border: 1px dashed var(--st-badge-owner-ink); color: var(--st-badge-owner-ink); }
|
||||||
|
.chip.ok { background: var(--st-ok-soft); color: var(--st-ok); }
|
||||||
|
.chip.warn { background: var(--st-warn-soft); color: var(--st-warn); }
|
||||||
|
.chip.crit { background: var(--st-crit-soft); color: var(--st-crit); }
|
||||||
|
.chip .k { color: var(--st-faint); }
|
||||||
|
.chip .w { color: var(--st-warn); font-weight: 700; }
|
||||||
|
.chip .c { color: var(--st-crit); font-weight: 700; }
|
||||||
|
|
||||||
|
/* ── key/value rows (server groups, about page) ── */
|
||||||
|
.kv .row { grid-template-columns: 230px minmax(0,1fr); padding: 6px 14px; }
|
||||||
|
.group {
|
||||||
|
display: flex; align-items: center; gap: 10px;
|
||||||
|
padding: 7px 14px 5px; font-family: var(--st-mono); font-size: 11px; color: var(--st-faint);
|
||||||
|
background: var(--st-surface); border-bottom: 1px solid var(--st-line-soft);
|
||||||
|
}
|
||||||
|
.group .gbtn { margin-left: auto; }
|
||||||
|
.kv .val { font-family: var(--st-mono); font-size: 12.5px; color: var(--st-muted); min-width: 0; }
|
||||||
|
.kv .val a { color: var(--st-accent); text-decoration: none; }
|
||||||
|
.kv .val a:hover { text-decoration: underline; }
|
||||||
|
.kv .desc { display: block; font-size: 11px; color: var(--st-faint); margin-top: 2px; }
|
||||||
|
.val-masked { color: var(--st-faint); letter-spacing: 2px; }
|
||||||
|
|
||||||
|
.note { margin: 8px 2px 0; font-size: 12px; color: var(--st-faint); }
|
||||||
|
|
||||||
|
/* ── handheld defaults for rows ── */
|
||||||
|
@media (max-width: 760px) {
|
||||||
|
.st-toolbar { padding: 8px 10px; }
|
||||||
|
.st-toolbar .hintline { display: none; }
|
||||||
|
.row { grid-template-columns: minmax(0,1fr) auto; padding: 9px 12px; }
|
||||||
|
.row .id { grid-row: 1; grid-column: 1; }
|
||||||
|
.row .act { grid-row: 1; grid-column: 2; }
|
||||||
|
.row .facts { grid-column: 1 / -1; }
|
||||||
|
.kv .row { grid-template-columns: 1fr; }
|
||||||
|
section.st { scroll-margin-top: calc(var(--bars-h, 94px) + 50px); }
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
/* hbd-ui.js — shared helpers for pages using hbd-ui.css.
|
||||||
|
*
|
||||||
|
* Keeps sticky toolbars stuck just below the fixed site nav, whose height
|
||||||
|
* varies with viewport width and wrapping.
|
||||||
|
*/
|
||||||
|
|
||||||
|
function _setStickyOffsets() {
|
||||||
|
const nav = document.querySelector('.nav');
|
||||||
|
const tb = document.querySelector('.st-toolbar');
|
||||||
|
const navH = nav ? nav.offsetHeight : 48;
|
||||||
|
const barsH = navH + (tb ? tb.offsetHeight : 46);
|
||||||
|
document.documentElement.style.setProperty('--nav-h', navH + 'px');
|
||||||
|
document.documentElement.style.setProperty('--bars-h', barsH + 'px');
|
||||||
|
}
|
||||||
|
window.addEventListener('resize', _setStickyOffsets);
|
||||||
|
document.addEventListener('DOMContentLoaded', _setStickyOffsets);
|
||||||
@@ -185,7 +185,7 @@
|
|||||||
/* Slightly larger tap targets in tables */
|
/* Slightly larger tap targets in tables */
|
||||||
#ntable td, #ntable th {
|
#ntable td, #ntable th {
|
||||||
padding: 4px 6px !important;
|
padding: 4px 6px !important;
|
||||||
font-size: 0.82em !important;
|
font-size: 1.00em !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Cards on plugin/alerts pages */
|
/* Cards on plugin/alerts pages */
|
||||||
|
|||||||
+85
-151
@@ -1,176 +1,110 @@
|
|||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html>
|
<html>
|
||||||
{% include 'head.html' %}
|
{% include 'head.html' %}
|
||||||
|
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||||||
|
<script src="/static/hbd-ui.js"></script>
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
html, body { overflow: visible; }
|
html, body { height: auto; overflow: visible; }
|
||||||
|
body { background: var(--st-paper); padding: 60px 0 0; }
|
||||||
|
|
||||||
.container {
|
.about-main { max-width: 700px; margin: 0 auto; padding: 0 20px 60px; }
|
||||||
max-width: 700px;
|
|
||||||
margin: 0 auto;
|
.hero {
|
||||||
|
display: flex; align-items: baseline; gap: 14px; flex-wrap: wrap;
|
||||||
|
padding: 26px 2px 0;
|
||||||
}
|
}
|
||||||
|
.hero .hb-logo {
|
||||||
h1 {
|
font-family: var(--st-mono); font-size: 30px; font-weight: 700;
|
||||||
color: #333;
|
letter-spacing: -.03em; color: var(--st-ink);
|
||||||
margin-bottom: 4px;
|
|
||||||
font-size: 1.5em;
|
|
||||||
}
|
}
|
||||||
|
.hero .hb-logo .tld { color: var(--st-accent); }
|
||||||
.subtitle {
|
.hero .version { font-family: var(--st-mono); font-size: 13px; color: var(--st-accent);
|
||||||
color: #666;
|
background: var(--st-accent-soft); border-radius: 999px; padding: 3px 12px; }
|
||||||
margin-bottom: 24px;
|
.hero .tagline { flex-basis: 100%; font-size: 13px; color: var(--st-muted); }
|
||||||
font-size: 0.9em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.section {
|
|
||||||
background: #fff;
|
|
||||||
border-radius: 8px;
|
|
||||||
box-shadow: 0 1px 6px rgba(0,0,0,0.1);
|
|
||||||
padding: 20px 24px;
|
|
||||||
margin-bottom: 20px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.section h2 {
|
|
||||||
font-size: 1em;
|
|
||||||
font-weight: 700;
|
|
||||||
color: #333;
|
|
||||||
margin: 0 0 16px;
|
|
||||||
padding-bottom: 10px;
|
|
||||||
border-bottom: 1px solid #eee;
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.5px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.info-row {
|
|
||||||
display: flex;
|
|
||||||
align-items: baseline;
|
|
||||||
padding: 8px 0;
|
|
||||||
border-bottom: 1px solid #f5f5f5;
|
|
||||||
font-size: 0.9em;
|
|
||||||
}
|
|
||||||
.info-row:last-child { border-bottom: none; }
|
|
||||||
|
|
||||||
.info-label {
|
|
||||||
width: 160px;
|
|
||||||
flex-shrink: 0;
|
|
||||||
color: #666;
|
|
||||||
font-size: 0.88em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.info-value {
|
|
||||||
color: #222;
|
|
||||||
word-break: break-all;
|
|
||||||
}
|
|
||||||
|
|
||||||
.info-value a {
|
|
||||||
color: #0066cc;
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
.info-value a:hover { text-decoration: underline; }
|
|
||||||
|
|
||||||
.version-badge {
|
|
||||||
display: inline-block;
|
|
||||||
padding: 3px 12px;
|
|
||||||
background: #e8f0fe;
|
|
||||||
color: #1a73e8;
|
|
||||||
border-radius: 12px;
|
|
||||||
font-size: 0.85em;
|
|
||||||
font-weight: 600;
|
|
||||||
font-family: monospace;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hb-logo {
|
|
||||||
font-size: 2.5em;
|
|
||||||
font-weight: 700;
|
|
||||||
color: #0066cc;
|
|
||||||
letter-spacing: -1px;
|
|
||||||
margin-bottom: 6px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hb-tagline {
|
|
||||||
color: #555;
|
|
||||||
font-size: 0.95em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.logo-section {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 20px;
|
|
||||||
padding: 8px 0 4px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.logo-text { flex: 1; }
|
|
||||||
</style>
|
</style>
|
||||||
|
|
||||||
<body>
|
<body>
|
||||||
{% include 'nav.html' %}
|
{% include 'nav.html' %}
|
||||||
|
|
||||||
<div class="container">
|
<div class="st-toolbar">
|
||||||
<h1>{{ header }}</h1>
|
<span class="brand">hbd<span class="tld">·about</span></span>
|
||||||
<p class="subtitle">Heartbeat monitoring system</p>
|
<span class="hintline">heartbeat monitoring system</span>
|
||||||
|
</div>
|
||||||
|
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||||||
|
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||||||
|
</svg>
|
||||||
|
|
||||||
<div class="section">
|
<div class="about-main">
|
||||||
<div class="logo-section">
|
<div class="hero">
|
||||||
<div class="logo-text">
|
<span class="hb-logo">heart<span class="tld">beat</span></span>
|
||||||
<div class="hb-logo">Heartbeat</div>
|
<span class="version">v{{ hbd_version }}</span>
|
||||||
<div class="hb-tagline">Lightweight host monitoring over UDP</div>
|
<span class="tagline">Lightweight host monitoring over UDP</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<section class="st">
|
||||||
|
<div class="sec-head">
|
||||||
|
<h2>version<span class="colon">:</span></h2>
|
||||||
|
</div>
|
||||||
|
<div class="list kv">
|
||||||
|
<div class="row">
|
||||||
|
<span class="id"><span class="name">server</span></span>
|
||||||
|
<span class="val">{{ hbd_version }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="id"><span class="name">python</span></span>
|
||||||
|
<span class="val">{{ python_version }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="id"><span class="name">license</span></span>
|
||||||
|
<span class="val">MIT</span>
|
||||||
</div>
|
</div>
|
||||||
<span class="version-badge">v{{ hbd_version }}</span>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</section>
|
||||||
|
|
||||||
<div class="section">
|
<section class="st">
|
||||||
<h2>Version</h2>
|
<div class="sec-head">
|
||||||
<div class="info-row">
|
<h2>runtime<span class="colon">:</span></h2>
|
||||||
<span class="info-label">Server version</span>
|
|
||||||
<span class="info-value">{{ hbd_version }}</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="info-row">
|
<div class="list kv">
|
||||||
<span class="info-label">Python</span>
|
<div class="row">
|
||||||
<span class="info-value">{{ python_version }}</span>
|
<span class="id"><span class="name">host</span></span>
|
||||||
|
<span class="val">{{ server_hostname }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="id"><span class="name">started</span></span>
|
||||||
|
<span class="val">{{ start_time_str }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="id"><span class="name">uptime</span></span>
|
||||||
|
<span class="val" id="uptime-value">{{ uptime_str }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="id"><span class="name">hosts_monitored</span></span>
|
||||||
|
<span class="val">{{ host_count }}</span>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-row">
|
</section>
|
||||||
<span class="info-label">License</span>
|
|
||||||
<span class="info-value">MIT</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="section">
|
<section class="st">
|
||||||
<h2>Runtime</h2>
|
<div class="sec-head">
|
||||||
<div class="info-row">
|
<h2>contact<span class="colon">:</span></h2>
|
||||||
<span class="info-label">Host</span>
|
|
||||||
<span class="info-value">{{ server_hostname }}</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="info-row">
|
<div class="list kv">
|
||||||
<span class="info-label">Started</span>
|
<div class="row">
|
||||||
<span class="info-value">{{ start_time_str }}</span>
|
<span class="id"><span class="name">author</span></span>
|
||||||
|
<span class="val">Andreas Wrede</span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="id"><span class="name">email</span></span>
|
||||||
|
<span class="val"><a href="mailto:aew.hbd@wrede.ca">aew.hbd@wrede.ca</a></span>
|
||||||
|
</div>
|
||||||
|
<div class="row">
|
||||||
|
<span class="id"><span class="name">repository</span></span>
|
||||||
|
<span class="val"><a href="https://git.wrede.ca/andreas/heartbeat" target="_blank" rel="noopener">git.wrede.ca/andreas/heartbeat</a></span>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="info-row">
|
</section>
|
||||||
<span class="info-label">Uptime</span>
|
|
||||||
<span class="info-value" id="uptime-value">{{ uptime_str }}</span>
|
|
||||||
</div>
|
|
||||||
<div class="info-row">
|
|
||||||
<span class="info-label">Hosts monitored</span>
|
|
||||||
<span class="info-value">{{ host_count }}</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="section">
|
|
||||||
<h2>Contact & Source</h2>
|
|
||||||
<div class="info-row">
|
|
||||||
<span class="info-label">Author</span>
|
|
||||||
<span class="info-value">Andreas Wrede</span>
|
|
||||||
</div>
|
|
||||||
<div class="info-row">
|
|
||||||
<span class="info-label">Email</span>
|
|
||||||
<span class="info-value"><a href="mailto:aew@wrede.ca">aew@wrede.ca</a></span>
|
|
||||||
</div>
|
|
||||||
<div class="info-row">
|
|
||||||
<span class="info-label">Repository</span>
|
|
||||||
<span class="info-value"><a href="https://git.wrede.ca/andreas/heartbeat" target="_blank" rel="noopener">git.wrede.ca/andreas/heartbeat</a></span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
|
|||||||
+164
-427
@@ -1,523 +1,236 @@
|
|||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html>
|
<html>
|
||||||
{% include 'head.html' %}
|
{% include 'head.html' %}
|
||||||
|
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||||||
|
<script src="/static/hbd-ui.js"></script>
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
|
html, body { height: auto; overflow: visible; }
|
||||||
|
body { background: var(--st-paper); padding: 60px 0 0; }
|
||||||
|
|
||||||
.container {
|
.alerts-main { max-width: 1100px; margin: 0 auto; padding: 0 20px 60px; }
|
||||||
max-width: 1400px;
|
|
||||||
margin: 0 auto;
|
/* summary tiles */
|
||||||
|
.stats { display: flex; gap: 10px; flex-wrap: wrap; padding-top: 18px; }
|
||||||
|
.stat {
|
||||||
|
flex: 1 1 140px; display: flex; align-items: baseline; gap: 10px;
|
||||||
|
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 8px;
|
||||||
|
padding: 10px 14px;
|
||||||
}
|
}
|
||||||
|
.stat .num { font-family: var(--st-mono); font-size: 24px; font-weight: 700; font-variant-numeric: tabular-nums; }
|
||||||
|
.stat .lbl { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-faint); }
|
||||||
|
.stat.crit .num { color: var(--st-crit); }
|
||||||
|
.stat.warn .num { color: var(--st-warn); }
|
||||||
|
.stat.ok .num { color: var(--st-ok); }
|
||||||
|
|
||||||
h1 { color: #333; margin-bottom: 5px; margin-top: 15px; font-size: 1.5em; }
|
/* filter bar */
|
||||||
|
.filterbar { display: flex; gap: 6px; align-items: center; flex-wrap: wrap; margin: 14px 0 0; }
|
||||||
.subtitle {
|
.filterbar .fbtn {
|
||||||
color: #666;
|
font-family: var(--st-mono); font-size: 12px; cursor: pointer;
|
||||||
margin-bottom: 30px;
|
color: var(--st-muted); background: var(--st-surface); border: 1px solid var(--st-line);
|
||||||
|
border-radius: 999px; padding: 5px 12px;
|
||||||
}
|
}
|
||||||
|
.filterbar .fbtn.active { background: var(--st-accent-soft); border-color: var(--st-accent); color: var(--st-accent); font-weight: 600; }
|
||||||
.summary-cards {
|
.filterbar input {
|
||||||
display: flex;
|
font: 12px var(--st-mono); color: var(--st-ink);
|
||||||
flex-wrap: wrap;
|
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||||||
gap: 10px;
|
padding: 5px 12px; min-width: 170px;
|
||||||
margin-bottom: 16px;
|
|
||||||
}
|
}
|
||||||
|
.filterbar input.invalid { border-color: var(--st-crit); }
|
||||||
|
.filterbar .upd { margin-left: auto; font-size: 11.5px; color: var(--st-faint); }
|
||||||
|
|
||||||
.summary-card {
|
/* alert row specifics */
|
||||||
background: white;
|
.row.alert-crit { border-left: 3px solid var(--st-crit); padding-left: 11px; }
|
||||||
border-radius: 6px;
|
.row.alert-warn { border-left: 3px solid var(--st-warn); padding-left: 11px; }
|
||||||
padding: 6px 14px;
|
.row.acked { opacity: .55; border-left-style: dashed; }
|
||||||
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
|
.row .id a { color: inherit; text-decoration: none; }
|
||||||
display: flex;
|
.row .id a:hover { color: var(--st-accent); }
|
||||||
align-items: center;
|
.acked-chip { color: var(--st-ok); font-size: 12px; white-space: nowrap; }
|
||||||
gap: 8px;
|
|
||||||
border-left: 4px solid #ddd;
|
.empty {
|
||||||
|
padding: 36px 14px; text-align: center; color: var(--st-faint); font-size: 13px;
|
||||||
}
|
}
|
||||||
|
.empty .big { font-size: 26px; color: var(--st-ok); display: block; margin-bottom: 6px; }
|
||||||
.summary-card.critical { border-left-color: #ea1e0f; }
|
.errorbox {
|
||||||
.summary-card.warning { border-left-color: #ff9800; }
|
background: var(--st-crit-soft); color: var(--st-crit);
|
||||||
.summary-card.ok { border-left-color: #4caf50; }
|
border-radius: 8px; padding: 14px 16px; font-size: 13px;
|
||||||
|
|
||||||
.summary-number {
|
|
||||||
font-size: 1.4em;
|
|
||||||
font-weight: bold;
|
|
||||||
line-height: 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
.summary-number.critical { color: #ea1e0f; }
|
|
||||||
.summary-number.warning { color: #ff9800; }
|
|
||||||
.summary-number.ok { color: #4caf50; }
|
|
||||||
|
|
||||||
.summary-label {
|
|
||||||
color: #666;
|
|
||||||
font-size: 0.85em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.filters {
|
|
||||||
background: white;
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 15px;
|
|
||||||
margin-bottom: 20px;
|
|
||||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
|
||||||
display: flex;
|
|
||||||
gap: 15px;
|
|
||||||
align-items: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.filter-label {
|
|
||||||
font-weight: bold;
|
|
||||||
color: #555;
|
|
||||||
}
|
|
||||||
|
|
||||||
.filter-button {
|
|
||||||
padding: 8px 16px;
|
|
||||||
border: 2px solid #ddd;
|
|
||||||
background: white;
|
|
||||||
border-radius: 20px;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: all 0.2s;
|
|
||||||
font-size: 0.9em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.filter-button:hover {
|
|
||||||
border-color: #2196f3;
|
|
||||||
}
|
|
||||||
|
|
||||||
.filter-button.active {
|
|
||||||
background: #2196f3;
|
|
||||||
color: white;
|
|
||||||
border-color: #2196f3;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alerts-container {
|
|
||||||
background: white;
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 20px;
|
|
||||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-item {
|
|
||||||
border-left: 5px solid #ddd;
|
|
||||||
padding: 15px;
|
|
||||||
margin-bottom: 15px;
|
|
||||||
background: #fafafa;
|
|
||||||
border-radius: 4px;
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
align-items: center;
|
|
||||||
transition: all 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-item.acknowledged {
|
|
||||||
opacity: 0.8;
|
|
||||||
background: #f0f0f0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-item:hover {
|
|
||||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
|
||||||
transform: translateX(5px);
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-item.critical {
|
|
||||||
border-left-color: #f44336;
|
|
||||||
background: #ffebee;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-item.warning {
|
|
||||||
border-left-color: #ff9800;
|
|
||||||
background: #fff3e0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-item.unknown {
|
|
||||||
border-left-color: #9e9e9e;
|
|
||||||
background: #f5f5f5;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-main {
|
|
||||||
flex: 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-header {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 15px;
|
|
||||||
margin-bottom: 8px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-level {
|
|
||||||
padding: 4px 12px;
|
|
||||||
border-radius: 12px;
|
|
||||||
font-size: 0.75em;
|
|
||||||
font-weight: bold;
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.5px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-level.critical {
|
|
||||||
background: #f44336;
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-level.warning {
|
|
||||||
background: #ff9800;
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-level.unknown {
|
|
||||||
background: #9e9e9e;
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-hostname {
|
|
||||||
font-weight: bold;
|
|
||||||
color: #333;
|
|
||||||
font-size: 1.1em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-metric {
|
|
||||||
color: #666;
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
font-size: 0.9em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-details {
|
|
||||||
display: flex;
|
|
||||||
gap: 20px;
|
|
||||||
color: #666;
|
|
||||||
font-size: 0.9em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-value {
|
|
||||||
font-weight: bold;
|
|
||||||
color: #333;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-duration {
|
|
||||||
color: #999;
|
|
||||||
font-size: 0.85em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.alert-actions {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 8px;
|
|
||||||
margin-left: 15px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.acknowledge-btn {
|
|
||||||
padding: 8px 16px;
|
|
||||||
background: #2196f3;
|
|
||||||
color: white;
|
|
||||||
border: none;
|
|
||||||
border-radius: 4px;
|
|
||||||
cursor: pointer;
|
|
||||||
font-size: 0.85em;
|
|
||||||
transition: all 0.2s;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.acknowledge-btn:hover {
|
|
||||||
background: #1976d2;
|
|
||||||
transform: scale(1.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.acknowledge-btn:disabled {
|
|
||||||
background: #ccc;
|
|
||||||
cursor: not-allowed;
|
|
||||||
transform: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.acknowledged-badge {
|
|
||||||
padding: 4px 8px;
|
|
||||||
background: #4caf50;
|
|
||||||
color: white;
|
|
||||||
border-radius: 4px;
|
|
||||||
font-size: 0.75em;
|
|
||||||
text-align: center;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.no-alerts {
|
|
||||||
text-align: center;
|
|
||||||
padding: 60px 20px;
|
|
||||||
color: #999;
|
|
||||||
}
|
|
||||||
|
|
||||||
.no-alerts-icon {
|
|
||||||
font-size: 4em;
|
|
||||||
margin-bottom: 20px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.loading {
|
|
||||||
text-align: center;
|
|
||||||
padding: 40px;
|
|
||||||
color: #666;
|
|
||||||
}
|
|
||||||
|
|
||||||
.error {
|
|
||||||
background: #ffebee;
|
|
||||||
border-left: 4px solid #f44336;
|
|
||||||
padding: 20px;
|
|
||||||
margin: 20px 0;
|
|
||||||
border-radius: 4px;
|
|
||||||
color: #c62828;
|
|
||||||
}
|
|
||||||
|
|
||||||
.refresh-info {
|
|
||||||
text-align: center;
|
|
||||||
color: #999;
|
|
||||||
font-size: 0.85em;
|
|
||||||
margin-top: 20px;
|
|
||||||
padding-top: 20px;
|
|
||||||
border-top: 1px solid #e0e0e0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.last-update {
|
|
||||||
color: #666;
|
|
||||||
font-size: 0.9em;
|
|
||||||
text-align: right;
|
|
||||||
margin-bottom: 15px;
|
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|
||||||
<body>
|
<body>
|
||||||
{% include 'nav.html' %}
|
{% include 'nav.html' %}
|
||||||
|
|
||||||
<div class="container">
|
<div class="st-toolbar">
|
||||||
<h1>{{ header }}</h1>
|
<span class="brand">hbd<span class="tld">·alerts</span></span>
|
||||||
<p class="subtitle">Real-time monitoring alerts and threshold violations</p>
|
<span class="hintline">threshold violations and reachability — refreshes every 15 s</span>
|
||||||
|
</div>
|
||||||
|
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||||||
|
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||||||
|
</svg>
|
||||||
|
|
||||||
<div class="summary-cards" id="summary-cards">
|
<div class="alerts-main">
|
||||||
<div class="summary-card critical">
|
<div class="stats">
|
||||||
<div class="summary-label">Critical</div>
|
<div class="stat crit"><span class="num" id="critical-count">–</span><span class="lbl">critical</span></div>
|
||||||
<div class="summary-number critical" id="critical-count">-</div>
|
<div class="stat warn"><span class="num" id="warning-count">–</span><span class="lbl">warning</span></div>
|
||||||
</div>
|
<div class="stat ok"><span class="num" id="host-count">–</span><span class="lbl">hosts</span></div>
|
||||||
<div class="summary-card warning">
|
|
||||||
<div class="summary-label">Warning</div>
|
|
||||||
<div class="summary-number warning" id="warning-count">-</div>
|
|
||||||
</div>
|
|
||||||
<div class="summary-card ok">
|
|
||||||
<div class="summary-label">Total Hosts</div>
|
|
||||||
<div class="summary-number ok" id="host-count">-</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="filters">
|
<div class="filterbar">
|
||||||
<span class="filter-label">Show:</span>
|
<button class="fbtn active" onclick="filterAlerts('all', this)">all</button>
|
||||||
<button class="filter-button active" onclick="filterAlerts('all')">All</button>
|
<button class="fbtn" onclick="filterAlerts('critical', this)">critical</button>
|
||||||
<button class="filter-button" onclick="filterAlerts('critical')">Critical Only</button>
|
<button class="fbtn" onclick="filterAlerts('warning', this)">warning</button>
|
||||||
<button class="filter-button" onclick="filterAlerts('warning')">Warning Only</button>
|
<input id="host-filter" type="text" placeholder="host filter (regex)" oninput="onHostFilterInput(this)">
|
||||||
|
<span class="upd">updated <span id="last-update-time">never</span></span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="alerts-container">
|
<section class="st" id="alerts">
|
||||||
<div class="last-update">Last updated: <span id="last-update-time">Never</span></div>
|
<div class="sec-head">
|
||||||
<div id="alerts-list">
|
<h2>alerts<span class="colon">:</span></h2>
|
||||||
<div class="loading">Loading alerts...</div>
|
<span class="count" id="alert-count"></span>
|
||||||
</div>
|
</div>
|
||||||
<div class="refresh-info">
|
<div class="list" id="alerts-list">
|
||||||
Auto-refreshing every 15 seconds
|
<div class="empty">Loading alerts…</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</section>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
let currentFilter = 'all';
|
let currentFilter = 'all';
|
||||||
let allAlerts = [];
|
let allAlerts = [];
|
||||||
|
let hostFilterRe = null;
|
||||||
|
|
||||||
|
function escHtml(s) {
|
||||||
|
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');
|
||||||
|
}
|
||||||
|
|
||||||
async function loadAlerts() {
|
async function loadAlerts() {
|
||||||
try {
|
try {
|
||||||
const response = await fetch('/api/0/alerts');
|
const response = await fetch('/api/0/alerts');
|
||||||
if (!response.ok) {
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||||
throw new Error(`HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
allAlerts = data.alerts;
|
allAlerts = data.alerts;
|
||||||
|
|
||||||
// Update summary cards
|
|
||||||
document.getElementById('critical-count').textContent = data.summary.critical || 0;
|
document.getElementById('critical-count').textContent = data.summary.critical || 0;
|
||||||
document.getElementById('warning-count').textContent = data.summary.warning || 0;
|
document.getElementById('warning-count').textContent = data.summary.warning || 0;
|
||||||
document.getElementById('host-count').textContent = data.host_count || 0;
|
document.getElementById('host-count').textContent = data.host_count || 0;
|
||||||
|
|
||||||
// Update last update time
|
|
||||||
document.getElementById('last-update-time').textContent = new Date().toLocaleTimeString();
|
document.getElementById('last-update-time').textContent = new Date().toLocaleTimeString();
|
||||||
|
|
||||||
// Render alerts
|
|
||||||
renderAlerts(allAlerts);
|
renderAlerts(allAlerts);
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
document.getElementById('alerts-list').innerHTML =
|
document.getElementById('alerts-list').innerHTML =
|
||||||
`<div class="error">Failed to load alerts: ${error.message}</div>`;
|
`<div class="errorbox">Failed to load alerts: ${escHtml(error.message)}. Retrying automatically.</div>`;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderAlerts(alerts) {
|
function renderAlerts(alerts) {
|
||||||
const container = document.getElementById('alerts-list');
|
const container = document.getElementById('alerts-list');
|
||||||
|
|
||||||
// Filter alerts based on current filter
|
let filtered = alerts;
|
||||||
let filteredAlerts = alerts;
|
|
||||||
if (currentFilter !== 'all') {
|
if (currentFilter !== 'all') {
|
||||||
filteredAlerts = alerts.filter(alert =>
|
filtered = filtered.filter(a => a.level.toLowerCase() === currentFilter);
|
||||||
alert.level.toLowerCase() === currentFilter
|
}
|
||||||
);
|
if (hostFilterRe) {
|
||||||
|
filtered = filtered.filter(a => hostFilterRe.test(a.hostname));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (filteredAlerts.length === 0) {
|
document.getElementById('alert-count').textContent =
|
||||||
if (currentFilter === 'all' && alerts.length === 0) {
|
filtered.length === alerts.length ? alerts.length : `${filtered.length} of ${alerts.length}`;
|
||||||
container.innerHTML = `
|
|
||||||
<div class="no-alerts">
|
if (filtered.length === 0) {
|
||||||
<div class="no-alerts-icon">✓</div>
|
container.innerHTML = (currentFilter === 'all' && !hostFilterRe && alerts.length === 0)
|
||||||
<h2>All Systems Normal</h2>
|
? `<div class="empty"><span class="big">✓</span>All systems normal — no active alerts.</div>`
|
||||||
<p>No active alerts at this time</p>
|
: `<div class="empty">No matching alerts.</div>`;
|
||||||
</div>
|
|
||||||
`;
|
|
||||||
} else {
|
|
||||||
container.innerHTML = `
|
|
||||||
<div class="no-alerts">
|
|
||||||
<p>No ${currentFilter} alerts</p>
|
|
||||||
</div>
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let html = '';
|
container.innerHTML = filtered.map(renderAlert).join('');
|
||||||
for (const alert of filteredAlerts) {
|
|
||||||
html += renderAlert(alert);
|
|
||||||
}
|
|
||||||
container.innerHTML = html;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderAlert(alert) {
|
function renderAlert(alert) {
|
||||||
const level = alert.level.toLowerCase();
|
const level = alert.level.toLowerCase();
|
||||||
|
const cls = level === 'critical' ? 'crit' : (level === 'warning' ? 'warn' : 'level');
|
||||||
const duration = getDuration(alert.since);
|
const duration = getDuration(alert.since);
|
||||||
const acknowledged = alert.acknowledged || false;
|
const acked = alert.acknowledged || false;
|
||||||
|
|
||||||
// Use formatted message if available, otherwise build from individual fields
|
const metric = (alert.metric_path.includes('.')
|
||||||
let valueText = `Value: <span class="alert-value">${formatValue(alert.last_value)}</span>`;
|
? alert.metric_path.slice(alert.metric_path.indexOf('.') + 1)
|
||||||
|
: alert.metric_path).replace(/_status_code$/, '');
|
||||||
|
|
||||||
|
let chips = `<span class="chip ${cls}">${escHtml(alert.level)}</span>`;
|
||||||
|
chips += `<span class="chip"><span class="k">metric</span> ${escHtml(metric)}</span>`;
|
||||||
if (alert.formatted_message) {
|
if (alert.formatted_message) {
|
||||||
valueText += ` <span class="threshold-info">${alert.formatted_message}</span>`;
|
chips += `<span class="chip">${escHtml(alert.formatted_message)}</span>`;
|
||||||
} else if (alert.threshold_value !== undefined && alert.threshold_value !== null && alert.operator) {
|
|
||||||
valueText += ` <span class="threshold-info">(threshold: ${alert.operator} ${formatValue(alert.threshold_value)})</span>`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build actions section
|
|
||||||
let actionsHtml = '';
|
|
||||||
if (acknowledged) {
|
|
||||||
actionsHtml = `
|
|
||||||
<div class="alert-actions">
|
|
||||||
<div class="acknowledged-badge">✓ Acknowledged</div>
|
|
||||||
</div>
|
|
||||||
`;
|
|
||||||
} else {
|
} else {
|
||||||
actionsHtml = `
|
chips += `<span class="chip"><span class="k">value</span> ${escHtml(formatValue(alert.last_value))}</span>`;
|
||||||
<div class="alert-actions">
|
if (alert.threshold_value !== undefined && alert.threshold_value !== null && alert.operator) {
|
||||||
<button class="acknowledge-btn" onclick="acknowledgeAlert('${alert.hostname}', '${alert.metric_path}', event)">
|
chips += `<span class="chip level">${escHtml(alert.operator)} ${escHtml(formatValue(alert.threshold_value))}</span>`;
|
||||||
Acknowledge
|
}
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
`;
|
|
||||||
}
|
}
|
||||||
|
if (alert.recovery_threshold !== undefined && alert.recovery_threshold !== null) {
|
||||||
|
const recOp = (alert.operator === '>' || alert.operator === '>=') ? '<' : '>';
|
||||||
|
chips += `<span class="chip level">recovers ${recOp} ${escHtml(formatValue(alert.recovery_threshold))}</span>`;
|
||||||
|
}
|
||||||
|
chips += `<span class="chip"><span class="k">for</span> ${duration}</span>`;
|
||||||
|
|
||||||
|
const act = acked
|
||||||
|
? `<span class="acked-chip">✓ acknowledged</span>`
|
||||||
|
: `<button class="btn small" onclick="acknowledgeAlert('${escHtml(alert.hostname)}', '${escHtml(alert.metric_path)}', event)">Acknowledge</button>`;
|
||||||
|
|
||||||
return `
|
return `
|
||||||
<div class="alert-item ${level} ${acknowledged ? 'acknowledged' : ''}">
|
<div class="row alert-${cls === 'level' ? 'warn' : cls}${acked ? ' acked' : ''}">
|
||||||
<div class="alert-main">
|
<span class="id">
|
||||||
<div class="alert-header">
|
<span class="dot ${cls === 'crit' ? 'crit' : 'warn'}"></span>
|
||||||
<span class="alert-level ${level}">${alert.level}</span>
|
<a class="name" href="/plugins#${encodeURIComponent(alert.hostname)}">${escHtml(alert.hostname)}</a>
|
||||||
<span class="alert-hostname">${alert.hostname}</span>
|
</span>
|
||||||
</div>
|
<span class="facts">${chips}</span>
|
||||||
<div class="alert-metric">${alert.metric_path}</div>
|
<span class="act">${act}</span>
|
||||||
<div class="alert-details">
|
</div>`;
|
||||||
<span>${valueText}</span>
|
|
||||||
<span class="alert-duration">Active for ${duration}</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
${actionsHtml}
|
|
||||||
</div>
|
|
||||||
`;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatValue(value) {
|
function formatValue(value) {
|
||||||
if (typeof value === 'number') {
|
if (typeof value === 'number') {
|
||||||
if (value > 1000) {
|
if (value > 1000) return value.toLocaleString();
|
||||||
return value.toLocaleString();
|
|
||||||
}
|
|
||||||
return value.toFixed(2);
|
return value.toFixed(2);
|
||||||
}
|
}
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getDuration(timestamp) {
|
function getDuration(timestamp) {
|
||||||
const now = Date.now() / 1000;
|
const seconds = Math.floor(Date.now() / 1000 - timestamp);
|
||||||
const seconds = Math.floor(now - timestamp);
|
if (seconds < 60) return `${seconds}s`;
|
||||||
|
if (seconds < 3600) return `${Math.floor(seconds / 60)}m`;
|
||||||
if (seconds < 60) {
|
if (seconds < 86400) {
|
||||||
return `${seconds}s`;
|
return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
|
||||||
} else if (seconds < 3600) {
|
|
||||||
return `${Math.floor(seconds / 60)}m`;
|
|
||||||
} else if (seconds < 86400) {
|
|
||||||
const hours = Math.floor(seconds / 3600);
|
|
||||||
const minutes = Math.floor((seconds % 3600) / 60);
|
|
||||||
return `${hours}h ${minutes}m`;
|
|
||||||
} else {
|
|
||||||
const days = Math.floor(seconds / 86400);
|
|
||||||
const hours = Math.floor((seconds % 86400) / 3600);
|
|
||||||
return `${days}d ${hours}h`;
|
|
||||||
}
|
}
|
||||||
|
return `${Math.floor(seconds / 86400)}d ${Math.floor((seconds % 86400) / 3600)}h`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function filterAlerts(filter) {
|
function filterAlerts(filter, btn) {
|
||||||
currentFilter = filter;
|
currentFilter = filter;
|
||||||
|
document.querySelectorAll('.filterbar .fbtn').forEach(b => b.classList.remove('active'));
|
||||||
// Update active button
|
btn.classList.add('active');
|
||||||
document.querySelectorAll('.filter-button').forEach(btn => {
|
|
||||||
btn.classList.remove('active');
|
|
||||||
});
|
|
||||||
event.target.classList.add('active');
|
|
||||||
|
|
||||||
// Re-render with new filter
|
|
||||||
renderAlerts(allAlerts);
|
renderAlerts(allAlerts);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function acknowledgeAlert(hostname, metricPath, event) {
|
async function acknowledgeAlert(hostname, metricPath, event) {
|
||||||
// Prevent event bubbling
|
if (event) event.stopPropagation();
|
||||||
if (event) {
|
|
||||||
event.stopPropagation();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Disable the button
|
|
||||||
const button = event.target;
|
const button = event.target;
|
||||||
button.disabled = true;
|
button.disabled = true;
|
||||||
button.textContent = 'Acknowledging...';
|
button.textContent = 'Acknowledging…';
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch('/api/0/alerts/acknowledge', {
|
const response = await fetch('/api/0/alerts/acknowledge', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {'Content-Type': 'application/json'},
|
||||||
'Content-Type': 'application/json',
|
body: JSON.stringify({hostname: hostname, metric_path: metricPath}),
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
hostname: hostname,
|
|
||||||
metric_path: metricPath,
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||||
if (!response.ok) {
|
|
||||||
throw new Error(`HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
|
const a = allAlerts.find(x => x.hostname === hostname && x.metric_path === metricPath);
|
||||||
// Update the alert in our local data
|
if (a) {
|
||||||
const alert = allAlerts.find(a => a.hostname === hostname && a.metric_path === metricPath);
|
a.acknowledged = true;
|
||||||
if (alert) {
|
a.acknowledged_at = result.acknowledged_at;
|
||||||
alert.acknowledged = true;
|
|
||||||
alert.acknowledged_at = result.acknowledged_at;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Re-render alerts
|
|
||||||
renderAlerts(allAlerts);
|
renderAlerts(allAlerts);
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
alert(`Failed to acknowledge alert: ${error.message}`);
|
alert(`Failed to acknowledge alert: ${error.message}`);
|
||||||
button.disabled = false;
|
button.disabled = false;
|
||||||
@@ -525,10 +238,34 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Auto-refresh every 15 seconds
|
function onHostFilterInput(input) {
|
||||||
|
const val = input.value.trim();
|
||||||
|
if (!val) {
|
||||||
|
hostFilterRe = null;
|
||||||
|
input.classList.remove('invalid');
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
hostFilterRe = new RegExp(val, 'i');
|
||||||
|
input.classList.remove('invalid');
|
||||||
|
} catch (_) {
|
||||||
|
hostFilterRe = null;
|
||||||
|
input.classList.add('invalid');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
renderAlerts(allAlerts);
|
||||||
|
}
|
||||||
|
|
||||||
setInterval(loadAlerts, 15000);
|
setInterval(loadAlerts, 15000);
|
||||||
|
|
||||||
// Initial load
|
(function () {
|
||||||
|
const param = new URLSearchParams(window.location.search).get('filter');
|
||||||
|
if (param) {
|
||||||
|
const input = document.getElementById('host-filter');
|
||||||
|
input.value = param;
|
||||||
|
onHostFilterInput(input);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
loadAlerts();
|
loadAlerts();
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<footer>
|
<footer>
|
||||||
<div id="copyright">
|
<div id="copyright">
|
||||||
©2002-2026 <A HREF="mailto:andreas@wrede.ca">Andreas Wrede</A> All Rights Reserved.</p>
|
©2002-2026 <A HREF="mailto:aew.hbd@wrede.ca">Andreas Wrede</A> All Rights Reserved.</p>
|
||||||
</div>
|
</div>
|
||||||
</footer>
|
</footer>
|
||||||
+119
-14
@@ -5,7 +5,68 @@
|
|||||||
<link rel="icon" href="/static/images/favicon.ico" sizes="32x32" />
|
<link rel="icon" href="/static/images/favicon.ico" sizes="32x32" />
|
||||||
<title>{{ title }}</title>
|
<title>{{ title }}</title>
|
||||||
{% if extra_scripts %}<script src="{{ extra_scripts }}"></script>{% endif %}
|
{% if extra_scripts %}<script src="{{ extra_scripts }}"></script>{% endif %}
|
||||||
|
<script>
|
||||||
|
/* Apply saved theme before first paint to avoid flash */
|
||||||
|
(function() {
|
||||||
|
try {
|
||||||
|
var p = localStorage.getItem('hbd_theme') || 'auto';
|
||||||
|
var dark = p === 'dark' || (p === 'auto' && window.matchMedia('(prefers-color-scheme: dark)').matches);
|
||||||
|
if (dark) document.documentElement.setAttribute('data-theme', 'dark');
|
||||||
|
} catch(e) {}
|
||||||
|
})();
|
||||||
|
</script>
|
||||||
<style>
|
<style>
|
||||||
|
/* ── Theme variables ── */
|
||||||
|
:root {
|
||||||
|
--bg: #f5f5f5;
|
||||||
|
--surface: #ffffff;
|
||||||
|
--surface-2: #f8f8f8;
|
||||||
|
--surface-3: #f5f5f5;
|
||||||
|
--text: #222222;
|
||||||
|
--text-2: #333333;
|
||||||
|
--text-3: #555555;
|
||||||
|
--text-sec: #666666;
|
||||||
|
--text-muted: #888888;
|
||||||
|
--text-dim: #aaaaaa;
|
||||||
|
--text-ghost: #cccccc;
|
||||||
|
--border: #e0e0e0;
|
||||||
|
--border-2: #eeeeee;
|
||||||
|
--border-3: #f0f0f0;
|
||||||
|
--border-4: #f5f5f5;
|
||||||
|
--link: #0066cc;
|
||||||
|
--nav-bg: #ffffff;
|
||||||
|
--input-bg: #ffffff;
|
||||||
|
--input-border: #cccccc;
|
||||||
|
--shadow-sm: rgba(0,0,0,.08);
|
||||||
|
--shadow: rgba(0,0,0,.10);
|
||||||
|
--shadow-nav: rgba(0,0,0,.10);
|
||||||
|
}
|
||||||
|
html[data-theme="dark"] {
|
||||||
|
color-scheme: dark;
|
||||||
|
--bg: #111827;
|
||||||
|
--surface: #1f2937;
|
||||||
|
--surface-2: #283447;
|
||||||
|
--surface-3: #374151;
|
||||||
|
--text: #e5e7eb;
|
||||||
|
--text-2: #d1d5db;
|
||||||
|
--text-3: #9ca3af;
|
||||||
|
--text-sec: #9ca3af;
|
||||||
|
--text-muted: #6b7280;
|
||||||
|
--text-dim: #4b5563;
|
||||||
|
--text-ghost: #374151;
|
||||||
|
--border: #374151;
|
||||||
|
--border-2: #2d3748;
|
||||||
|
--border-3: #253040;
|
||||||
|
--border-4: #1e2a38;
|
||||||
|
--link: #60a5fa;
|
||||||
|
--nav-bg: #1f2937;
|
||||||
|
--input-bg: #283447;
|
||||||
|
--input-border: #4b5563;
|
||||||
|
--shadow-sm: rgba(0,0,0,.30);
|
||||||
|
--shadow: rgba(0,0,0,.40);
|
||||||
|
--shadow-nav: rgba(0,0,0,.40);
|
||||||
|
}
|
||||||
|
|
||||||
/* ── Reset / shared baseline ── */
|
/* ── Reset / shared baseline ── */
|
||||||
*, *::before, *::after { box-sizing: border-box; }
|
*, *::before, *::after { box-sizing: border-box; }
|
||||||
html {
|
html {
|
||||||
@@ -16,10 +77,11 @@
|
|||||||
margin: 0;
|
margin: 0;
|
||||||
padding: 10px;
|
padding: 10px;
|
||||||
padding-top: 60px;
|
padding-top: 60px;
|
||||||
background: #f5f5f5;
|
background: var(--bg);
|
||||||
|
color: var(--text);
|
||||||
}
|
}
|
||||||
h1 { font-size: 1.5em; color: #333; margin: 0 0 5px; }
|
h1 { font-size: 1.5em; color: var(--text-2); margin: 0 0 5px; }
|
||||||
h2 { font-size: 1.1em; color: #333; margin: 0 0 8px; }
|
h2 { font-size: 1.1em; color: var(--text-2); margin: 0 0 8px; }
|
||||||
p { margin: 0; }
|
p { margin: 0; }
|
||||||
|
|
||||||
/* Navigation bar — shared across all pages */
|
/* Navigation bar — shared across all pages */
|
||||||
@@ -29,9 +91,9 @@
|
|||||||
left: 0;
|
left: 0;
|
||||||
right: 0;
|
right: 0;
|
||||||
z-index: 200;
|
z-index: 200;
|
||||||
background: #fff;
|
background: var(--nav-bg);
|
||||||
padding: 6px 12px;
|
padding: 6px 12px;
|
||||||
box-shadow: 0 2px 4px rgba(0,0,0,.1);
|
box-shadow: 0 2px 4px var(--shadow-nav);
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
@@ -42,25 +104,25 @@
|
|||||||
.nav a {
|
.nav a {
|
||||||
margin-right: 20px;
|
margin-right: 20px;
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
color: #0066cc;
|
color: var(--link);
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
font-size: 0.9em;
|
font-size: 0.9em;
|
||||||
}
|
}
|
||||||
.nav a:hover { text-decoration: underline; }
|
.nav a:hover { text-decoration: underline; }
|
||||||
.nav a.active { color: #333; font-weight: bold; }
|
.nav a.active { color: var(--text-2); font-weight: bold; }
|
||||||
.nav-user {
|
.nav-user {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 8px;
|
gap: 8px;
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
color: #333;
|
color: var(--text-2);
|
||||||
font-size: 0.9em;
|
font-size: 0.9em;
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
padding: 4px 8px;
|
padding: 4px 8px;
|
||||||
border-radius: 20px;
|
border-radius: 20px;
|
||||||
transition: background 0.15s;
|
transition: background 0.15s;
|
||||||
}
|
}
|
||||||
.nav-user:hover { background: #f0f4ff; text-decoration: none; }
|
.nav-user:hover { background: var(--surface-2); text-decoration: none; }
|
||||||
.nav-username {
|
.nav-username {
|
||||||
max-width: 0;
|
max-width: 0;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
@@ -81,7 +143,7 @@
|
|||||||
.nav-initials {
|
.nav-initials {
|
||||||
width: 28px; height: 28px;
|
width: 28px; height: 28px;
|
||||||
border-radius: 50%;
|
border-radius: 50%;
|
||||||
background: #0066cc;
|
background: var(--link);
|
||||||
color: #fff;
|
color: #fff;
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
@@ -106,7 +168,7 @@
|
|||||||
.nav-hamburger span {
|
.nav-hamburger span {
|
||||||
display: block;
|
display: block;
|
||||||
height: 3px;
|
height: 3px;
|
||||||
background: #555;
|
background: var(--text-muted);
|
||||||
border-radius: 2px;
|
border-radius: 2px;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,19 +180,51 @@
|
|||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
align-items: flex-start;
|
align-items: flex-start;
|
||||||
padding-top: 8px;
|
padding-top: 8px;
|
||||||
border-top: 1px solid #eee;
|
border-top: 1px solid var(--border-2);
|
||||||
order: 3;
|
order: 3;
|
||||||
}
|
}
|
||||||
.nav-links.nav-open { display: flex; }
|
.nav-links.nav-open { display: flex; }
|
||||||
.nav-links a { margin-right: 0; padding: 6px 0; font-size: 1em; }
|
.nav-links a { margin-right: 0; padding: 6px 0; font-size: 1em; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ── Global dark-mode: inputs ── */
|
||||||
|
html[data-theme="dark"] input:not([type=checkbox]):not([type=radio]),
|
||||||
|
html[data-theme="dark"] select,
|
||||||
|
html[data-theme="dark"] textarea {
|
||||||
|
background-color: var(--input-bg);
|
||||||
|
border-color: var(--input-border);
|
||||||
|
color: var(--text);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pending config publish button */
|
||||||
|
.nav-publish-btn {
|
||||||
|
background: #e65100;
|
||||||
|
color: #fff;
|
||||||
|
border: none;
|
||||||
|
border-radius: 4px;
|
||||||
|
padding: 4px 10px;
|
||||||
|
font-size: 0.82em;
|
||||||
|
font-weight: 600;
|
||||||
|
cursor: pointer;
|
||||||
|
flex-shrink: 0;
|
||||||
|
white-space: nowrap;
|
||||||
|
margin-left: auto;
|
||||||
|
}
|
||||||
|
.nav-publish-btn:hover { background: #bf360c; }
|
||||||
|
.nav-publish-btn:disabled { opacity: 0.7; cursor: default; }
|
||||||
|
|
||||||
/* Swiss railway clock — nav */
|
/* Swiss railway clock — nav */
|
||||||
.nav-clock {
|
.nav-pie {
|
||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
line-height: 0;
|
line-height: 0;
|
||||||
margin-left: auto;
|
margin-left: auto;
|
||||||
padding: 4px 4px 4px 0;
|
padding: 4px 4px 4px 0;
|
||||||
|
}
|
||||||
|
#alert-pie { display: block; cursor: default; }
|
||||||
|
.nav-clock {
|
||||||
|
flex-shrink: 0;
|
||||||
|
line-height: 0;
|
||||||
|
padding: 4px 4px 4px 0;
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
#swiss-clock { display: block; }
|
#swiss-clock { display: block; }
|
||||||
@@ -208,7 +302,7 @@
|
|||||||
ctx.restore();
|
ctx.restore();
|
||||||
}
|
}
|
||||||
|
|
||||||
hand((m + s / 60) / 60 * Math.PI * 2 - Math.PI / 2,
|
hand((sFrac >= 58.5 ? m + 1 : m) / 60 * Math.PI * 2 - Math.PI / 2,
|
||||||
R * 0.88, -R * 0.12, SIZE * 0.027, '#222'); /* minute */
|
R * 0.88, -R * 0.12, SIZE * 0.027, '#222'); /* minute */
|
||||||
hand((h + m / 60) / 12 * Math.PI * 2 - Math.PI / 2,
|
hand((h + m / 60) / 12 * Math.PI * 2 - Math.PI / 2,
|
||||||
R * 0.58, -R * 0.12, SIZE * 0.039, '#222'); /* hour */
|
R * 0.58, -R * 0.12, SIZE * 0.039, '#222'); /* hour */
|
||||||
@@ -256,6 +350,17 @@
|
|||||||
setTimeout(clockTick, delay);
|
setTimeout(clockTick, delay);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Keep auto-theme in sync with system setting changes */
|
||||||
|
try {
|
||||||
|
window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', function(e) {
|
||||||
|
var pref = localStorage.getItem('hbd_theme') || 'auto';
|
||||||
|
if (pref === 'auto') {
|
||||||
|
if (e.matches) { document.documentElement.setAttribute('data-theme', 'dark'); }
|
||||||
|
else { document.documentElement.removeAttribute('data-theme'); }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch(e) {}
|
||||||
|
|
||||||
document.addEventListener('DOMContentLoaded', function() {
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
/* Start the shared tick loop */
|
/* Start the shared tick loop */
|
||||||
clockTick();
|
clockTick();
|
||||||
|
|||||||
+380
-538
@@ -1,566 +1,408 @@
|
|||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html>
|
<html>
|
||||||
{% include 'head.html' %}
|
{% include 'head.html' %}
|
||||||
|
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||||||
|
<script src="/static/hbd-ui.js"></script>
|
||||||
|
|
||||||
<style>
|
<style>
|
||||||
body {
|
html, body { height: auto; overflow: visible; }
|
||||||
display: flex;
|
body { background: var(--st-paper); padding: 60px 0 0; }
|
||||||
flex-direction: column;
|
|
||||||
height: 100vh;
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (max-width: 640px) {
|
.live-main { max-width: 1180px; margin: 0 auto; padding: 0 20px 70px; }
|
||||||
body {
|
|
||||||
height: auto;
|
|
||||||
min-height: 100vh;
|
|
||||||
overflow: auto;
|
|
||||||
flex-direction: column;
|
|
||||||
}
|
|
||||||
.container {
|
|
||||||
max-height: none;
|
|
||||||
overflow: visible;
|
|
||||||
}
|
|
||||||
.table-section {
|
|
||||||
max-height: 55vh;
|
|
||||||
}
|
|
||||||
.log-section {
|
|
||||||
flex: none;
|
|
||||||
max-height: 40vh;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
.container {
|
.st-toolbar input {
|
||||||
flex: 1;
|
font: 12px var(--st-mono); color: var(--st-ink);
|
||||||
min-height: 0;
|
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||||||
max-width: 1600px;
|
padding: 5px 12px; min-width: 160px;
|
||||||
width: 100%;
|
|
||||||
margin: 0 auto;
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 15px;
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
}
|
||||||
|
.live-dot { display: inline-flex; align-items: center; gap: 6px; font-size: 12px; font-weight: 600; color: var(--st-ok); }
|
||||||
|
.live-dot::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: var(--st-ok); }
|
||||||
|
.live-dot.reconnecting { color: var(--st-crit); }
|
||||||
|
.live-dot.reconnecting::before { background: var(--st-crit); }
|
||||||
|
|
||||||
h1 {
|
/* ── host rows: 6 columns ── */
|
||||||
color: #333;
|
.hrow, .hhead {
|
||||||
margin-bottom: 5px;
|
display: grid;
|
||||||
margin-top: 15px;
|
grid-template-columns: 175px 105px 70px 125px 85px minmax(0,1fr);
|
||||||
font-size: 1.5em;
|
gap: 4px 14px; align-items: center;
|
||||||
|
padding: 6px 14px; border-bottom: 1px solid var(--st-line-soft);
|
||||||
}
|
}
|
||||||
|
.hrow:last-child { border-bottom: none; }
|
||||||
h2 {
|
.hrow:hover { background: var(--st-surface-2); }
|
||||||
color: #333;
|
.hhead {
|
||||||
margin-bottom: 10px;
|
font-family: var(--st-mono); font-size: 10.5px; text-transform: uppercase; letter-spacing: .05em;
|
||||||
font-size: 1.2em;
|
color: var(--st-faint); background: var(--st-surface-2); border-radius: 8px 8px 0 0;
|
||||||
padding: 10px 15px;
|
|
||||||
background: white;
|
|
||||||
border-radius: 6px;
|
|
||||||
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
|
|
||||||
}
|
}
|
||||||
|
.hhead .num, .hrow .num { text-align: right; }
|
||||||
|
|
||||||
.subtitle {
|
.hrow.sev-warn { border-left: 3px solid var(--st-warn); padding-left: 11px; }
|
||||||
color: #666;
|
.hrow.sev-crit { border-left: 3px solid var(--st-crit); padding-left: 11px; }
|
||||||
margin-bottom: 15px;
|
|
||||||
font-size: 0.9em;
|
.host { font-family: var(--st-mono); font-size: 13px; font-weight: 400; display: flex; align-items: center; gap: 7px; min-width: 0; }
|
||||||
|
.host.watched { font-weight: 700; }
|
||||||
|
.host a { color: inherit; text-decoration: none; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.host a:hover { color: var(--st-accent); }
|
||||||
|
.dot.down { background: var(--st-crit); box-shadow: 0 0 0 3px var(--st-crit-soft); }
|
||||||
|
.dot.overdue { background: transparent; border: 2px solid var(--st-warn); }
|
||||||
|
.stale { color: var(--st-faint); font-size: 10px; border: 1px solid var(--st-line); border-radius: 4px; padding: 1px 4px; font-family: var(--st-mono); white-space: nowrap; }
|
||||||
|
.flap { color: var(--st-warn); font-size: 10px; border: 1px solid var(--st-warn); border-radius: 4px; padding: 1px 4px; font-family: var(--st-mono); white-space: nowrap; }
|
||||||
|
.chip.outline { background: transparent; border: 1px solid var(--st-line); }
|
||||||
|
|
||||||
|
.lat { font-family: var(--st-mono); font-size: 12px; color: var(--st-muted); font-variant-numeric: tabular-nums; }
|
||||||
|
.since { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-faint); font-variant-numeric: tabular-nums; }
|
||||||
|
|
||||||
|
.lastalert { font-family: var(--st-mono); font-size: 11.5px; color: var(--st-muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.lastalert .lvl { font-weight: 700; }
|
||||||
|
.lastalert .lvl.crit { color: var(--st-crit); }
|
||||||
|
.lastalert .lvl.warn { color: var(--st-warn); }
|
||||||
|
.lastalert .ago { color: var(--st-faint); }
|
||||||
|
.lastalert.quiet { color: var(--st-faint); }
|
||||||
|
|
||||||
|
/* ── hover card on the status chip ── */
|
||||||
|
.tip { position: relative; display: inline-flex; }
|
||||||
|
.tipbox {
|
||||||
|
display: none; position: absolute; z-index: 40; top: calc(100% + 6px); left: 0;
|
||||||
|
width: 340px; background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 8px;
|
||||||
|
box-shadow: 0 8px 28px var(--shadow); padding: 10px 12px;
|
||||||
}
|
}
|
||||||
|
.tip:hover .tipbox, .tip:focus-within .tipbox { display: block; }
|
||||||
|
.tipbox table { border-collapse: collapse; width: 100%; font-family: var(--st-mono); font-size: 11.5px; }
|
||||||
|
.tipbox th { text-align: left; font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: var(--st-faint); padding: 2px 6px 4px; border-bottom: 1px solid var(--st-line-soft); }
|
||||||
|
.tipbox td { padding: 3px 6px; color: var(--st-muted); }
|
||||||
|
.tipbox td:first-child { color: var(--st-faint); }
|
||||||
|
.tipbox .up { color: var(--st-ok); font-weight: 700; }
|
||||||
|
.tipbox .dn { color: var(--st-crit); font-weight: 700; }
|
||||||
|
.tipbox .ov { color: var(--st-warn); font-weight: 700; }
|
||||||
|
.tipbox .meta { margin-top: 6px; padding-top: 6px; border-top: 1px solid var(--st-line-soft); font-size: 11px; color: var(--st-faint); }
|
||||||
|
|
||||||
.content {
|
@media (max-width: 760px) {
|
||||||
display: flex;
|
.hhead { display: none; }
|
||||||
flex-direction: column;
|
.hrow { grid-template-columns: minmax(0,1fr) auto auto; padding: 8px 12px; }
|
||||||
gap: 15px;
|
.hrow .num { text-align: left; }
|
||||||
}
|
.hrow .since { display: none; }
|
||||||
|
.lastalert { grid-column: 1 / -1; }
|
||||||
.table-section {
|
.tipbox { left: auto; right: 0; }
|
||||||
background: white;
|
|
||||||
border-radius: 6px;
|
|
||||||
padding: 15px;
|
|
||||||
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
|
|
||||||
overflow-x: auto;
|
|
||||||
overflow-y: auto;
|
|
||||||
max-height: 60vh;
|
|
||||||
}
|
|
||||||
|
|
||||||
.log-section {
|
|
||||||
flex: 1;
|
|
||||||
min-height: 0;
|
|
||||||
background: white;
|
|
||||||
border-radius: 6px;
|
|
||||||
padding: 15px;
|
|
||||||
box-shadow: 0 1px 4px rgba(0,0,0,0.1);
|
|
||||||
overflow-y: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable {
|
|
||||||
border-collapse: collapse;
|
|
||||||
width: 100%;
|
|
||||||
font-size: 0.9em;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable td,
|
|
||||||
#ntable th {
|
|
||||||
border: 1px solid #e0e0e0;
|
|
||||||
text-align: left;
|
|
||||||
padding: 2px 4px;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable tr:nth-child(even) {
|
|
||||||
background-color: #fafafa;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable tr:hover {
|
|
||||||
background-color: #e3f2fd;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable tbody tr.row-warning {
|
|
||||||
background-color: #fff8c5;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable tbody tr.row-critical {
|
|
||||||
background-color: #fde8e8;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable tbody tr.row-warning:hover {
|
|
||||||
background-color: #fff0a0;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable tbody tr.row-critical:hover {
|
|
||||||
background-color: #f9c8c8;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable th {
|
|
||||||
padding: 6px 8px;
|
|
||||||
background-color: #2196f3;
|
|
||||||
color: white;
|
|
||||||
font-weight: 600;
|
|
||||||
position: sticky;
|
|
||||||
top: 0;
|
|
||||||
z-index: 10;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable
|
|
||||||
th:not(.sorttable_sorted):not(.sorttable_sorted_reverse):not(.sorttable_nosort):after {
|
|
||||||
content: " ⇅";
|
|
||||||
opacity: 0.5;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Alert count column styling */
|
|
||||||
#ntable td.alert-warning {
|
|
||||||
color: #ff9800;
|
|
||||||
font-weight: bold;
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
#ntable td.alert-critical {
|
|
||||||
color: #f44336;
|
|
||||||
font-weight: bold;
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Scrollbar styling */
|
|
||||||
.log-section::-webkit-scrollbar {
|
|
||||||
width: 8px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.log-section::-webkit-scrollbar-track {
|
|
||||||
background: #f1f1f1;
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.log-section::-webkit-scrollbar-thumb {
|
|
||||||
background: #888;
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.log-section::-webkit-scrollbar-thumb:hover {
|
|
||||||
background: #555;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Message styling */
|
|
||||||
#messages {
|
|
||||||
font-size: 0.85em;
|
|
||||||
line-height: 1.0;
|
|
||||||
}
|
|
||||||
|
|
||||||
#messages div {
|
|
||||||
padding: 5px 0;
|
|
||||||
border-bottom: 1px solid #f0f0f0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Modal for connection status messages */
|
|
||||||
.connection-modal {
|
|
||||||
display: none;
|
|
||||||
position: fixed;
|
|
||||||
z-index: 1000;
|
|
||||||
left: 0;
|
|
||||||
top: 0;
|
|
||||||
width: 100%;
|
|
||||||
height: 100%;
|
|
||||||
background-color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.connection-modal.show {
|
|
||||||
display: flex;
|
|
||||||
justify-content: center;
|
|
||||||
align-items: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.connection-modal-content {
|
|
||||||
background-color: white;
|
|
||||||
padding: 30px 40px;
|
|
||||||
border-radius: 8px;
|
|
||||||
text-align: center;
|
|
||||||
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.3);
|
|
||||||
min-width: 300px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.connection-modal-content p {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 16px;
|
|
||||||
color: #333;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* State indicators */
|
|
||||||
.state-up {
|
|
||||||
color: #4caf50;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
|
|
||||||
.state-down {
|
|
||||||
color: #f44336;
|
|
||||||
font-weight: 700;
|
|
||||||
}
|
|
||||||
|
|
||||||
.state-overdue {
|
|
||||||
color: #ff9800;
|
|
||||||
font-weight: 700;
|
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
<script type="text/javascript">
|
|
||||||
var cnt = 0;
|
|
||||||
var nTable = document;
|
|
||||||
var name_idx = {};
|
|
||||||
var c = 0;
|
|
||||||
var HBD_VERSION = "{{ hbd_version }}";
|
|
||||||
|
|
||||||
function hostNameHtml(data) {
|
|
||||||
var nameHtml = data.name;
|
|
||||||
if (!data.hbc_version || data.hbc_version !== HBD_VERSION) {
|
|
||||||
nameHtml += ' 🥀';
|
|
||||||
}
|
|
||||||
return data.dyn ? '<b>' + nameHtml + '</b>' : nameHtml;
|
|
||||||
}
|
|
||||||
|
|
||||||
function setup() {
|
|
||||||
name_idx = {};
|
|
||||||
nTable = document.getElementById("ntable");
|
|
||||||
for (var i = 0, row; (row = nTable.rows[i]); i++) {
|
|
||||||
if (i == 0) continue;
|
|
||||||
var cell = nTable.rows[i].cells[0];
|
|
||||||
var name = cell.dataset.name || cell.innerText.replace(/\s*🥀\s*$/, '').trim();
|
|
||||||
name_idx[name] = nTable.rows[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function updateRowAlert(row, data) {
|
|
||||||
var criticalUnacked = data.alert_critical_unacked || 0;
|
|
||||||
var criticalAcked = data.alert_critical_acked || 0;
|
|
||||||
var warningUnacked = data.alert_warning_unacked || 0;
|
|
||||||
var warningAcked = data.alert_warning_acked || 0;
|
|
||||||
row.classList.remove('row-warning', 'row-critical');
|
|
||||||
if (criticalUnacked > 0 || criticalAcked > 0) {
|
|
||||||
row.classList.add('row-critical');
|
|
||||||
} else if (warningUnacked > 0 || warningAcked > 0) {
|
|
||||||
row.classList.add('row-warning');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function createRow(data) {
|
|
||||||
var row = document.createElement("tr");
|
|
||||||
var c_name = document.createElement("td");
|
|
||||||
var c_warning = document.createElement("td");
|
|
||||||
c_warning.style.textAlign = "center";
|
|
||||||
c_warning.style.color = "#ff9800";
|
|
||||||
c_warning.style.fontWeight = "bold";
|
|
||||||
var c_critical = document.createElement("td");
|
|
||||||
c_critical.style.textAlign = "center";
|
|
||||||
c_critical.style.color = "#f44336";
|
|
||||||
c_critical.style.fontWeight = "bold";
|
|
||||||
var c_ipv4addr = document.createElement("td");
|
|
||||||
var c_ipv4state = document.createElement("td");
|
|
||||||
var c_ipv4latency = document.createElement("td");
|
|
||||||
c_ipv4latency.style.textAlign = "right";
|
|
||||||
var c_ipv4statets = document.createElement("td");
|
|
||||||
c_ipv4statets.style.textAlign = "right";
|
|
||||||
var c_ipv6addr = document.createElement("td");
|
|
||||||
var c_ipv6state = document.createElement("td");
|
|
||||||
var c_ipv6latency = document.createElement("td");
|
|
||||||
c_ipv6latency.style.textAlign = "right";
|
|
||||||
var c_ipv6statets = document.createElement("td");
|
|
||||||
c_ipv6statets.style.textAlign = "right";
|
|
||||||
row.appendChild(c_name);
|
|
||||||
row.appendChild(c_warning);
|
|
||||||
row.appendChild(c_critical);
|
|
||||||
row.appendChild(c_ipv4addr);
|
|
||||||
row.appendChild(c_ipv4state);
|
|
||||||
row.appendChild(c_ipv4latency);
|
|
||||||
row.appendChild(c_ipv4statets);
|
|
||||||
row.appendChild(c_ipv6addr);
|
|
||||||
row.appendChild(c_ipv6state);
|
|
||||||
row.appendChild(c_ipv6latency);
|
|
||||||
row.appendChild(c_ipv6statets);
|
|
||||||
c_name.dataset.name = data.name;
|
|
||||||
c_name.innerHTML = hostNameHtml(data);
|
|
||||||
|
|
||||||
// Set alert counts in "x/y" format (unacked/acked)
|
|
||||||
var warningUnacked = data.alert_warning_unacked || 0;
|
|
||||||
var warningAcked = data.alert_warning_acked || 0;
|
|
||||||
var criticalUnacked = data.alert_critical_unacked || 0;
|
|
||||||
var criticalAcked = data.alert_critical_acked || 0;
|
|
||||||
|
|
||||||
if (warningUnacked > 0 || warningAcked > 0) {
|
|
||||||
c_warning.innerHTML = warningAcked > 0 ? warningUnacked + "/" + warningAcked : warningUnacked;
|
|
||||||
} else {
|
|
||||||
c_warning.innerHTML = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criticalUnacked > 0 || criticalAcked > 0) {
|
|
||||||
c_critical.innerHTML = criticalAcked > 0 ? criticalUnacked + "/" + criticalAcked : criticalUnacked;
|
|
||||||
} else {
|
|
||||||
c_critical.innerHTML = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
c_ipv4addr.innerHTML = data.connections[0].addr;
|
|
||||||
c_ipv4state.innerHTML = data.connections[0].state;
|
|
||||||
if (data.connections.length > 1) {
|
|
||||||
c_ipv6addr.innerHTML = data.connections[1].addr;
|
|
||||||
c_ipv6state.innerHTML = data.connections[1].state;
|
|
||||||
}
|
|
||||||
var table = document.getElementById("ntablebody"); // find table to append to
|
|
||||||
table.appendChild(row); // append row to table
|
|
||||||
name_idx[c_name] = row;
|
|
||||||
updateRowAlert(row, data);
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatTS(ts) {
|
|
||||||
const now = new Date();
|
|
||||||
const d = new Date(ts * 1000);
|
|
||||||
|
|
||||||
const pad = n => String(n).padStart(2, '0');
|
|
||||||
const timeStr = `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
|
|
||||||
|
|
||||||
// Same calendar day → show time only
|
|
||||||
if (d.toDateString() === now.toDateString()) {
|
|
||||||
return timeStr;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Within 8 days → show "-X d hh:mm:ss"
|
|
||||||
const todayStart = new Date(now.getFullYear(), now.getMonth(), now.getDate());
|
|
||||||
const dStart = new Date(d.getFullYear(), d.getMonth(), d.getDate());
|
|
||||||
const diffDays = Math.round((todayStart - dStart) / 86400000);
|
|
||||||
if (diffDays < 8) {
|
|
||||||
return `-${diffDays}d ${timeStr}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Older → date only
|
|
||||||
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
function update_table(data) {
|
|
||||||
if (!(data.name in name_idx)) {
|
|
||||||
createRow(data);
|
|
||||||
setup();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update name cell (version indicator)
|
|
||||||
var nameCell = name_idx[data.name].cells[0];
|
|
||||||
nameCell.dataset.name = data.name;
|
|
||||||
nameCell.innerHTML = hostNameHtml(data);
|
|
||||||
|
|
||||||
// Update warning and critical counts in "x/y" format (unacked/acked)
|
|
||||||
var warningUnacked = data.alert_warning_unacked || 0;
|
|
||||||
var warningAcked = data.alert_warning_acked || 0;
|
|
||||||
var criticalUnacked = data.alert_critical_unacked || 0;
|
|
||||||
var criticalAcked = data.alert_critical_acked || 0;
|
|
||||||
|
|
||||||
if (warningUnacked > 0 || warningAcked > 0) {
|
|
||||||
name_idx[data.name].cells[1].innerHTML = warningAcked > 0 ? warningUnacked + "/" + warningAcked : warningUnacked;
|
|
||||||
} else {
|
|
||||||
name_idx[data.name].cells[1].innerHTML = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
if (criticalUnacked > 0 || criticalAcked > 0) {
|
|
||||||
name_idx[data.name].cells[2].innerHTML = criticalAcked > 0 ? criticalUnacked + "/" + criticalAcked : criticalUnacked;
|
|
||||||
} else {
|
|
||||||
name_idx[data.name].cells[2].innerHTML = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
for (var i = 0; i < data.connections.length; i++) {
|
|
||||||
// Offset by 2 for the warning/critical count columns
|
|
||||||
name_idx[data.name].cells[3 + i * 4].innerHTML = data.connections[i].addr;
|
|
||||||
name_idx[data.name].cells[6 + i * 4].innerHTML = formatTS(
|
|
||||||
data.connections[i].statetime
|
|
||||||
);
|
|
||||||
if (data.connections[i].state == "up") {
|
|
||||||
state = '<span class="state-up">up</span>';
|
|
||||||
latency = Number.parseFloat(data.connections[i].rtts[0]).toFixed(2);
|
|
||||||
} else {
|
|
||||||
if (data.connections[i].state == "unknown") {
|
|
||||||
state = "";
|
|
||||||
latency = "";
|
|
||||||
name_idx[data.name].cells[3 + i * 4].innerHTML = "";
|
|
||||||
name_idx[data.name].cells[6 + i * 4].innerHTML = "";
|
|
||||||
} else if (data.connections[i].state == "down") {
|
|
||||||
state = '<span class="state-down">down</span>';
|
|
||||||
latency = "-";
|
|
||||||
} else if (data.connections[i].state == "overdue") {
|
|
||||||
state = '<span class="state-overdue">overdue</span>';
|
|
||||||
latency = "-";
|
|
||||||
} else {
|
|
||||||
state = "<b>" + data.connections[i].state + "</b>";
|
|
||||||
latency = "-";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
name_idx[data.name].cells[4 + i * 4].innerHTML = state;
|
|
||||||
name_idx[data.name].cells[5 + i * 4].innerHTML = latency;
|
|
||||||
}
|
|
||||||
updateRowAlert(name_idx[data.name], data);
|
|
||||||
}
|
|
||||||
|
|
||||||
function WS_Connect() {
|
|
||||||
if ("WebSocket" in window) {
|
|
||||||
//N.B: subprotocol field causes chrome to error 1006
|
|
||||||
var ws_hbd = new WebSocket("{{heartbeat_ws_url}}", /* "hdb" */ );
|
|
||||||
|
|
||||||
ws_hbd.onopen = function () {
|
|
||||||
// Web Socket is connected, send data using send()
|
|
||||||
console.log("ws connect {{heartbeat_ws_url}}");
|
|
||||||
// Hide modal window if visible
|
|
||||||
var modal = document.getElementById("connectionModal");
|
|
||||||
if (modal) {
|
|
||||||
modal.classList.remove("show");
|
|
||||||
}
|
|
||||||
ws_hbd.send("heartbeat_web");
|
|
||||||
};
|
|
||||||
|
|
||||||
ws_hbd.onerror = function (event) {
|
|
||||||
console.log(event);
|
|
||||||
};
|
|
||||||
|
|
||||||
ws_hbd.onmessage = function (event) {
|
|
||||||
/* console.log(event.data); */
|
|
||||||
var state = JSON.parse(event.data);
|
|
||||||
/* console.log("State: " + state.type); */
|
|
||||||
if (state.type == "host") {
|
|
||||||
update_table(state.data);
|
|
||||||
} else if (state.type == "message") {
|
|
||||||
var msgs = document.getElementById("messages");
|
|
||||||
msgs.insertAdjacentHTML("afterbegin", "<div>" + state.data + "</div>");
|
|
||||||
}
|
|
||||||
cnt++;
|
|
||||||
};
|
|
||||||
|
|
||||||
ws_hbd.onclose = function (event) {
|
|
||||||
/* console.log(event); */
|
|
||||||
console.log("Connection is closed, reopening");
|
|
||||||
// Show modal window
|
|
||||||
var modal = document.getElementById("connectionModal");
|
|
||||||
if (modal) {
|
|
||||||
modal.classList.add("show");
|
|
||||||
}
|
|
||||||
setTimeout(function () {
|
|
||||||
WS_Connect();
|
|
||||||
}, 3000);
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
// The browser doesn't support WebSocket
|
|
||||||
console.log("WebSocket NOT supported by your Browser!");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
WS_Connect();
|
|
||||||
</script>
|
|
||||||
<body>
|
<body>
|
||||||
{% include 'nav.html' %}
|
{% include 'nav.html' %}
|
||||||
|
|
||||||
{% include 'menu.html' %}
|
<div class="st-toolbar">
|
||||||
|
<span class="brand">hbd<span class="tld">·live</span></span>
|
||||||
<div class="container">
|
<span class="hintline">every host, one line — details on hover, history in the overview</span>
|
||||||
<div>
|
<span class="spacer"></span>
|
||||||
<h1>{{ header }}</h1>
|
<input type="text" id="host-filter" placeholder="host filter (regex)">
|
||||||
<p class="subtitle">Real-time host monitoring and event log</p>
|
<span class="live-dot reconnecting" id="ws-status">connecting…</span>
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="table-section">
|
|
||||||
<table id="ntable" class="sortable">
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>Name</th>
|
|
||||||
<th style="text-align: center" title="Warning Alerts">⚠️</th>
|
|
||||||
<th style="text-align: center" title="Critical Alerts">🔴</th>
|
|
||||||
<th>IPv4 Addr</th>
|
|
||||||
<th>State</th>
|
|
||||||
<th style="text-align: right">Latency</th>
|
|
||||||
<th style="text-align: right">Last State</th>
|
|
||||||
<th>IPv6 Addr</th>
|
|
||||||
<th>State</th>
|
|
||||||
<th style="text-align: right">Latency</th>
|
|
||||||
<th style="text-align: right">Last State</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody id="ntablebody">
|
|
||||||
{% for host in hosts %}
|
|
||||||
<tr class="{% if host.alert_critical_unacked > 0 or host.alert_critical_acked > 0 %}row-critical{% elif host.alert_warning_unacked > 0 or host.alert_warning_acked > 0 %}row-warning{% endif %}">
|
|
||||||
<td data-name="{{ host.name }}">{{ host.name }}{% if not host.hbc_version or host.hbc_version != hbd_version %} 🥀{% endif %}</td>
|
|
||||||
<td style="text-align: center; color: #ff9800; font-weight: bold;">
|
|
||||||
{%- set warning_unacked = host.alert_warning_unacked -%}
|
|
||||||
{%- set warning_acked = host.alert_warning_acked -%}
|
|
||||||
{%- if warning_unacked > 0 or warning_acked > 0 -%}
|
|
||||||
{{ warning_unacked }}{% if warning_acked > 0 %}/{{ warning_acked }}{% endif %}
|
|
||||||
{%- endif -%}
|
|
||||||
</td>
|
|
||||||
<td style="text-align: center; color: #f44336; font-weight: bold;">
|
|
||||||
{%- set critical_unacked = host.alert_critical_unacked -%}
|
|
||||||
{%- set critical_acked = host.alert_critical_acked -%}
|
|
||||||
{%- if critical_unacked > 0 or critical_acked > 0 -%}
|
|
||||||
{{ critical_unacked }}{% if critical_acked > 0 %}/{{ critical_acked }}{% endif %}
|
|
||||||
{%- endif -%}
|
|
||||||
</td>
|
|
||||||
{% for conn in host.connections %}
|
|
||||||
<td>{{ conn.addr if conn.addr else '' }}</td>
|
|
||||||
<td>{{ conn.state if conn.state else '' }}</td>
|
|
||||||
<td style="text-align: right">{{ conn.latency if conn.latency else '' }}</td>
|
|
||||||
<td style="text-align: right">{{ conn.last_state_ts if conn.last_state_ts else '' }}</td>
|
|
||||||
{% endfor %}
|
|
||||||
{% if host.connections|length == 0 %}
|
|
||||||
<td></td><td></td><td></td><td></td>
|
|
||||||
<td></td><td></td><td></td><td></td>
|
|
||||||
{% elif host.connections|length == 1 %}
|
|
||||||
<td></td><td></td><td></td><td></td>
|
|
||||||
{% endif %}
|
|
||||||
</tr>
|
|
||||||
{% endfor %}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="log-section">
|
|
||||||
<h2>Log of Events</h2>
|
|
||||||
<div id="messages"></div>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
|
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||||||
|
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||||||
|
</svg>
|
||||||
|
|
||||||
{% include 'foot.html' %}
|
<div class="live-main">
|
||||||
|
<section class="st">
|
||||||
<!-- Connection status modal -->
|
<div class="sec-head">
|
||||||
<div id="connectionModal" class="connection-modal">
|
<h2>hosts<span class="colon">:</span></h2>
|
||||||
<div class="connection-modal-content">
|
<span class="count" id="host-counts"></span>
|
||||||
<p>⚠️ Connection is closed, reopening...</p>
|
<span class="sub">sorted by severity, then name</span>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="list">
|
||||||
|
<div class="hhead">
|
||||||
|
<span>host</span><span>status</span><span class="num">latency</span>
|
||||||
|
<span>alerts</span><span>since</span><span>last alert</span>
|
||||||
|
</div>
|
||||||
|
<div id="host-rows"></div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
setup();
|
var HBD_VERSION = "{{ hbd_version }}";
|
||||||
|
var INITIAL_HOSTS = {{ hosts | tojson }};
|
||||||
|
|
||||||
|
var hostData = {}; // name -> latest stateinfo payload
|
||||||
|
var lastAlert = {}; // name -> {level, message, ts}
|
||||||
|
var hostFilterRe = null;
|
||||||
|
|
||||||
|
function escHtml(s) {
|
||||||
|
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- derived per-host facts ----------------------------------------
|
||||||
|
var STATE_RANK = { down: 3, overdue: 2, up: 1 };
|
||||||
|
|
||||||
|
function worstConnection(data) {
|
||||||
|
var worst = null;
|
||||||
|
(data.connections || []).forEach(function (c) {
|
||||||
|
if (!c.state || c.state === 'unknown') return;
|
||||||
|
if (!worst || (STATE_RANK[c.state] || 0) > (STATE_RANK[worst.state] || 0)) worst = c;
|
||||||
|
});
|
||||||
|
return worst;
|
||||||
|
}
|
||||||
|
|
||||||
|
function severity(data) {
|
||||||
|
var w = worstConnection(data);
|
||||||
|
if ((data.alert_critical_unacked || 0) > 0 || (data.alert_critical_acked || 0) > 0) return 'crit';
|
||||||
|
if (w && w.state === 'down') return 'crit';
|
||||||
|
if ((data.alert_warning_unacked || 0) > 0 || (data.alert_warning_acked || 0) > 0) return 'warn';
|
||||||
|
if (w && w.state === 'overdue') return 'warn';
|
||||||
|
return 'ok';
|
||||||
|
}
|
||||||
|
var SEV_ORDER = { crit: 0, warn: 1, ok: 2 };
|
||||||
|
|
||||||
|
function relTime(ts) {
|
||||||
|
if (!ts) return '—';
|
||||||
|
var s = Math.max(0, Math.floor(Date.now() / 1000 - ts));
|
||||||
|
if (s < 60) return s + 's';
|
||||||
|
if (s < 3600) return Math.floor(s / 60) + 'm';
|
||||||
|
if (s < 86400) return Math.floor(s / 3600) + 'h ' + Math.floor((s % 3600) / 60) + 'm';
|
||||||
|
if (s < 86400 * 30) return Math.floor(s / 86400) + 'd ' + Math.floor((s % 86400) / 3600) + 'h';
|
||||||
|
return Math.floor(s / 86400) + 'd';
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtClock(ts) {
|
||||||
|
var d = new Date(ts * 1000);
|
||||||
|
function p(n) { return n < 10 ? '0' + n : '' + n; }
|
||||||
|
var t = p(d.getHours()) + ':' + p(d.getMinutes()) + ':' + p(d.getSeconds());
|
||||||
|
return (d.toDateString() === new Date().toDateString())
|
||||||
|
? t
|
||||||
|
: d.getFullYear() + '-' + p(d.getMonth() + 1) + '-' + p(d.getDate()) + ' ' + t;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- host row rendering ---------------------------------------------
|
||||||
|
function statusChip(data) {
|
||||||
|
var w = worstConnection(data);
|
||||||
|
if (!w) return '<span class="chip outline">—</span>';
|
||||||
|
var cls = w.state === 'up' ? 'ok' : (w.state === 'down' ? 'crit' : 'warn');
|
||||||
|
return '<span class="chip ' + cls + '">' + escHtml(w.state) + '</span>';
|
||||||
|
}
|
||||||
|
|
||||||
|
function tipCard(data) {
|
||||||
|
var rows = '';
|
||||||
|
(data.connections || []).forEach(function (c) {
|
||||||
|
if (!c.state || c.state === 'unknown') return;
|
||||||
|
var cls = c.state === 'up' ? 'up' : (c.state === 'down' ? 'dn' : 'ov');
|
||||||
|
var fam = (c.afam || '').replace('IPv', 'v');
|
||||||
|
var rtt = (c.state === 'up' && c.rtts && c.rtts[0]) ? Math.round(c.rtts[0]) + ' ms' : '—';
|
||||||
|
rows += '<tr><td>' + escHtml(fam) + '</td><td>' + escHtml(c.addr || '—') + '</td>'
|
||||||
|
+ '<td class="' + cls + '">' + escHtml(c.state) + '</td>'
|
||||||
|
+ '<td>' + rtt + '</td><td>' + (c.statetime ? fmtClock(c.statetime) : '—') + '</td></tr>';
|
||||||
|
});
|
||||||
|
if (!rows) rows = '<tr><td colspan="5">no connections yet</td></tr>';
|
||||||
|
var meta = [];
|
||||||
|
meta.push('agent ' + (data.hbc_version ? 'v' + escHtml(String(data.hbc_version)) : 'unknown'));
|
||||||
|
if (data.dyn) meta.push('dyndns');
|
||||||
|
if (data.watched) meta.push('watched');
|
||||||
|
if (data.interval) meta.push('interval ' + escHtml(String(data.interval)) + ' s');
|
||||||
|
return '<span class="tipbox" role="tooltip">'
|
||||||
|
+ '<table><thead><tr><th></th><th>address</th><th>state</th><th>rtt</th><th>since</th></tr></thead>'
|
||||||
|
+ '<tbody>' + rows + '</tbody></table>'
|
||||||
|
+ '<div class="meta">' + meta.join(' · ') + '</div></span>';
|
||||||
|
}
|
||||||
|
|
||||||
|
function alertChips(data) {
|
||||||
|
var cu = data.alert_critical_unacked || 0, ca = data.alert_critical_acked || 0;
|
||||||
|
var wu = data.alert_warning_unacked || 0, wa = data.alert_warning_acked || 0;
|
||||||
|
var out = '';
|
||||||
|
if (cu) out += '<span class="chip crit"><b>' + cu + '</b> crit</span> ';
|
||||||
|
if (wu) out += '<span class="chip warn"><b>' + wu + '</b> warn</span> ';
|
||||||
|
var acked = ca + wa;
|
||||||
|
if (acked) out += '<span class="chip outline">' + acked + ' ack’d</span>';
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function lastAlertHtml(name) {
|
||||||
|
var a = lastAlert[name];
|
||||||
|
if (!a) return '<span class="lastalert quiet">—</span>';
|
||||||
|
var lvl = (a.level || '').toUpperCase();
|
||||||
|
var quiet = (lvl !== 'WARNING' && lvl !== 'CRITICAL');
|
||||||
|
var lvlCls = lvl === 'CRITICAL' ? 'crit' : (lvl === 'WARNING' ? 'warn' : '');
|
||||||
|
var short = lvl === 'CRITICAL' ? 'CRIT' : (lvl === 'WARNING' ? 'WARN' : escHtml(lvl));
|
||||||
|
return '<span class="lastalert' + (quiet ? ' quiet' : '') + '" title="' + escHtml(a.message) + '">'
|
||||||
|
+ '<span class="lvl ' + lvlCls + '">' + short + '</span> '
|
||||||
|
+ escHtml(a.message)
|
||||||
|
+ (a.ts ? ' <span class="ago">· ' + relTime(a.ts) + ' ago</span>' : '')
|
||||||
|
+ '</span>';
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderRow(data) {
|
||||||
|
var name = data.name;
|
||||||
|
var sev = severity(data);
|
||||||
|
var row = document.querySelector('.hrow[data-host="' + CSS.escape(name) + '"]');
|
||||||
|
var isNew = false;
|
||||||
|
if (!row) {
|
||||||
|
row = document.createElement('div');
|
||||||
|
row.dataset.host = name;
|
||||||
|
isNew = true;
|
||||||
|
}
|
||||||
|
row.className = 'hrow' + (sev !== 'ok' ? ' sev-' + sev : '');
|
||||||
|
row.dataset.sev = sev;
|
||||||
|
|
||||||
|
var w = worstConnection(data);
|
||||||
|
var up = (data.connections || []).find(function (c) { return c.state === 'up' && c.rtts && c.rtts.length; });
|
||||||
|
var lat = up ? Math.round(up.rtts[0]) + ' ms' : '—';
|
||||||
|
var since = w ? relTime(w.statetime) : '—';
|
||||||
|
var staleTag = '';
|
||||||
|
if (!data.hbc_version) staleTag = '<span class="stale" title="agent version unknown">agent?</span>';
|
||||||
|
else if (data.hbc_version !== HBD_VERSION) {
|
||||||
|
staleTag = '<span class="stale" title="agent v' + escHtml(String(data.hbc_version)) + ' ≠ server v' + HBD_VERSION + '">v' + escHtml(String(data.hbc_version)) + '</span>';
|
||||||
|
}
|
||||||
|
var dotCls = w ? (w.state === 'up' ? sev : (w.state === 'down' ? 'down' : 'overdue')) : 'off';
|
||||||
|
var flapTag = '';
|
||||||
|
var flapping = data.flapping || [];
|
||||||
|
if (flapping.length) {
|
||||||
|
var svcs = flapping.map(function (s) { return s || 'host'; }).join(', ');
|
||||||
|
flapTag = '<span class="flap" title="notifications suppressed: ' + escHtml(svcs) + '">flapping</span>';
|
||||||
|
}
|
||||||
|
|
||||||
|
row.innerHTML =
|
||||||
|
'<span class="host' + (data.watched ? ' watched' : '') + '">'
|
||||||
|
+ '<span class="dot ' + dotCls + '"></span>'
|
||||||
|
+ '<a href="/plugins#' + encodeURIComponent(name) + '">' + escHtml(name) + '</a>' + staleTag + flapTag + '</span>'
|
||||||
|
+ '<span class="tip" tabindex="0">' + statusChip(data) + tipCard(data) + '</span>'
|
||||||
|
+ '<span class="lat num">' + lat + '</span>'
|
||||||
|
+ '<span>' + alertChips(data) + '</span>'
|
||||||
|
+ '<span class="since">' + since + '</span>'
|
||||||
|
+ lastAlertHtml(name);
|
||||||
|
|
||||||
|
if (isNew) {
|
||||||
|
document.getElementById('host-rows').appendChild(row);
|
||||||
|
sortRows();
|
||||||
|
}
|
||||||
|
applyHostFilter(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sortRows() {
|
||||||
|
var container = document.getElementById('host-rows');
|
||||||
|
var rows = [...container.children];
|
||||||
|
rows.sort(function (a, b) {
|
||||||
|
var d = (SEV_ORDER[a.dataset.sev] ?? 2) - (SEV_ORDER[b.dataset.sev] ?? 2);
|
||||||
|
return d !== 0 ? d : a.dataset.host.localeCompare(b.dataset.host);
|
||||||
|
});
|
||||||
|
rows.forEach(function (r) { container.appendChild(r); });
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateCounts() {
|
||||||
|
var total = 0, up = 0, overdue = 0, down = 0;
|
||||||
|
Object.values(hostData).forEach(function (d) {
|
||||||
|
total++;
|
||||||
|
var w = worstConnection(d);
|
||||||
|
if (!w) return;
|
||||||
|
if (w.state === 'up') up++;
|
||||||
|
else if (w.state === 'overdue') overdue++;
|
||||||
|
else if (w.state === 'down') down++;
|
||||||
|
});
|
||||||
|
var parts = [total + ' hosts', up + ' up'];
|
||||||
|
if (overdue) parts.push(overdue + ' overdue');
|
||||||
|
if (down) parts.push(down + ' down');
|
||||||
|
document.getElementById('host-counts').textContent = parts.join(' · ');
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateHost(data) {
|
||||||
|
var prev = hostData[data.name];
|
||||||
|
var prevSev = prev ? severity(prev) : null;
|
||||||
|
hostData[data.name] = data;
|
||||||
|
renderRow(data);
|
||||||
|
if (prevSev !== severity(data)) sortRows();
|
||||||
|
updateCounts();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- host filter -----------------------------------------------------
|
||||||
|
function applyHostFilter(row) {
|
||||||
|
var rows = row ? [row] : [...document.querySelectorAll('.hrow')];
|
||||||
|
rows.forEach(function (r) {
|
||||||
|
r.style.display = (!hostFilterRe || hostFilterRe.test(r.dataset.host)) ? '' : 'none';
|
||||||
|
});
|
||||||
|
}
|
||||||
|
document.getElementById('host-filter').addEventListener('input', function () {
|
||||||
|
var v = this.value.trim();
|
||||||
|
try {
|
||||||
|
hostFilterRe = v ? new RegExp(v, 'i') : null;
|
||||||
|
this.style.borderColor = '';
|
||||||
|
} catch (_) {
|
||||||
|
hostFilterRe = null;
|
||||||
|
this.style.borderColor = 'var(--st-crit)';
|
||||||
|
}
|
||||||
|
applyHostFilter();
|
||||||
|
});
|
||||||
|
|
||||||
|
// ---- last alert tracking ---------------------------------------------
|
||||||
|
var ALERT_LEVELS = { WARNING: 1, CRITICAL: 1, RECOVER: 1 };
|
||||||
|
|
||||||
|
function noteAlert(host, level, message, ts) {
|
||||||
|
if (!host || !ALERT_LEVELS[(level || '').toUpperCase()]) return;
|
||||||
|
var cur = lastAlert[host];
|
||||||
|
if (cur && cur.ts && ts && ts < cur.ts) return; // keep the newest
|
||||||
|
lastAlert[host] = { level: level, message: message, ts: ts };
|
||||||
|
if (hostData[host]) renderRow(hostData[host]);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function seedActiveAlerts() {
|
||||||
|
try {
|
||||||
|
var r = await fetch('/api/0/alerts');
|
||||||
|
if (!r.ok) return;
|
||||||
|
var data = await r.json();
|
||||||
|
(data.alerts || []).forEach(function (a) {
|
||||||
|
var metric = (a.metric_path.includes('.')
|
||||||
|
? a.metric_path.slice(a.metric_path.indexOf('.') + 1)
|
||||||
|
: a.metric_path).replace(/_status_code$/, '');
|
||||||
|
var msg = metric + ' = ' + a.last_value
|
||||||
|
+ (a.formatted_message ? ' ' + a.formatted_message
|
||||||
|
: (a.operator && a.threshold_value != null ? ' (' + a.operator + ' ' + a.threshold_value + ')' : ''));
|
||||||
|
noteAlert(a.hostname, a.level, msg, a.since);
|
||||||
|
});
|
||||||
|
} catch (_) { /* dashboard still works without the seed */ }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- websocket -----------------------------------------------------------
|
||||||
|
function setWsStatus(ok) {
|
||||||
|
var el = document.getElementById('ws-status');
|
||||||
|
el.classList.toggle('reconnecting', !ok);
|
||||||
|
el.textContent = ok ? 'live' : 'reconnecting…';
|
||||||
|
}
|
||||||
|
|
||||||
|
function WS_Connect() {
|
||||||
|
if (!("WebSocket" in window)) {
|
||||||
|
console.log("WebSocket NOT supported by your Browser!");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
var ws_hbd = new WebSocket("{{ heartbeat_ws_url }}");
|
||||||
|
|
||||||
|
ws_hbd.onopen = function () {
|
||||||
|
setWsStatus(true);
|
||||||
|
ws_hbd.send("heartbeat_web");
|
||||||
|
};
|
||||||
|
|
||||||
|
ws_hbd.onerror = function (event) { console.log(event); };
|
||||||
|
|
||||||
|
ws_hbd.onmessage = function (event) {
|
||||||
|
var state = JSON.parse(event.data);
|
||||||
|
if (state.type === "host") {
|
||||||
|
updateHost(state.data);
|
||||||
|
} else if (state.type === "message") {
|
||||||
|
var msg = state.data;
|
||||||
|
noteAlert(msg.host, msg.level, msg.message, msg.ts);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
ws_hbd.onclose = function () {
|
||||||
|
setWsStatus(false);
|
||||||
|
setTimeout(WS_Connect, 3000);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- boot -----------------------------------------------------------------
|
||||||
|
INITIAL_HOSTS.forEach(function (h) { hostData[h.name] = h; });
|
||||||
|
Object.values(hostData).forEach(renderRow);
|
||||||
|
sortRows();
|
||||||
|
updateCounts();
|
||||||
|
seedActiveAlerts();
|
||||||
|
WS_Connect();
|
||||||
|
|
||||||
|
// refresh relative times once a minute
|
||||||
|
setInterval(function () {
|
||||||
|
Object.values(hostData).forEach(renderRow);
|
||||||
|
}, 60000);
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -0,0 +1,228 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
{% include 'head.html' %}
|
||||||
|
<link rel="stylesheet" href="/static/hbd-ui.css">
|
||||||
|
<script src="/static/hbd-ui.js"></script>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
html, body { height: auto; overflow: visible; }
|
||||||
|
body { background: var(--st-paper); padding: 60px 0 0; }
|
||||||
|
|
||||||
|
.log-main { max-width: 1100px; margin: 0 auto; padding: 0 20px 60px; }
|
||||||
|
|
||||||
|
.st-toolbar input, .st-toolbar select {
|
||||||
|
font: 12px var(--st-mono); color: var(--st-ink);
|
||||||
|
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||||||
|
padding: 5px 12px;
|
||||||
|
}
|
||||||
|
.live-dot { display: inline-flex; align-items: center; gap: 6px; font-size: 12px; font-weight: 600; color: var(--st-ok); }
|
||||||
|
.live-dot::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: var(--st-ok); }
|
||||||
|
.live-dot.reconnecting { color: var(--st-crit); }
|
||||||
|
.live-dot.reconnecting::before { background: var(--st-crit); }
|
||||||
|
|
||||||
|
.logrow {
|
||||||
|
display: grid; grid-template-columns: 130px 74px 150px minmax(0,1fr);
|
||||||
|
gap: 4px 14px; align-items: baseline;
|
||||||
|
padding: 4px 14px; border-bottom: 1px solid var(--st-line-soft);
|
||||||
|
font-family: var(--st-mono); font-size: 11.5px;
|
||||||
|
}
|
||||||
|
.logrow:last-child { border-bottom: none; }
|
||||||
|
.logrow .t { color: var(--st-faint); font-variant-numeric: tabular-nums; white-space: nowrap; }
|
||||||
|
.logrow .h { color: var(--st-ink); font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.logrow .m { color: var(--st-muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
|
||||||
|
.empty { padding: 24px 14px; text-align: center; color: var(--st-faint); font-size: 12.5px; }
|
||||||
|
.loadmore {
|
||||||
|
display: block; margin: 14px auto 0; cursor: pointer;
|
||||||
|
font-family: var(--st-mono); font-size: 12px; color: var(--st-muted);
|
||||||
|
background: var(--st-surface); border: 1px solid var(--st-line); border-radius: 999px;
|
||||||
|
padding: 6px 16px;
|
||||||
|
}
|
||||||
|
.loadmore:hover { color: var(--st-accent); border-color: var(--st-accent); }
|
||||||
|
.loadmore:disabled { opacity: .5; cursor: default; }
|
||||||
|
|
||||||
|
@media (max-width: 760px) {
|
||||||
|
.logrow { grid-template-columns: 66px minmax(0,1fr); }
|
||||||
|
.logrow .lvlc, .logrow .h { display: none; }
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
{% include 'nav.html' %}
|
||||||
|
|
||||||
|
<div class="st-toolbar">
|
||||||
|
<span class="brand">hbd<span class="tld">·log</span></span>
|
||||||
|
<span class="hintline">connectivity and alert events — full journal history</span>
|
||||||
|
<span class="spacer"></span>
|
||||||
|
<input type="text" id="filter-host" placeholder="host">
|
||||||
|
<select id="filter-level">
|
||||||
|
<option value="">level</option>
|
||||||
|
<option value="info">INFO</option>
|
||||||
|
<option value="warning">WARNING</option>
|
||||||
|
<option value="critical">CRITICAL</option>
|
||||||
|
<option value="recover">RECOVER</option>
|
||||||
|
<option value="unknown">UNKNOWN</option>
|
||||||
|
</select>
|
||||||
|
<input type="text" id="filter-msg" placeholder="message">
|
||||||
|
<span class="live-dot reconnecting" id="ws-status">connecting…</span>
|
||||||
|
</div>
|
||||||
|
<svg class="pulse" viewBox="0 0 1200 14" preserveAspectRatio="none" aria-hidden="true">
|
||||||
|
<polyline points="0,10 340,10 352,10 358,3 364,13 370,1 378,12 384,10 560,10 572,10 578,3 584,13 590,1 598,12 604,10 1200,10"/>
|
||||||
|
</svg>
|
||||||
|
|
||||||
|
<div class="log-main">
|
||||||
|
<section class="st">
|
||||||
|
<div class="sec-head">
|
||||||
|
<h2>log<span class="colon">:</span></h2>
|
||||||
|
<span class="count" id="log-count"></span>
|
||||||
|
</div>
|
||||||
|
<div class="list" id="messages">
|
||||||
|
<div class="empty">Loading events…</div>
|
||||||
|
</div>
|
||||||
|
<button class="loadmore" id="load-older" style="display:none">load older</button>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
var PAGE_SIZE = 200;
|
||||||
|
var oldestTs = null; // ts of the oldest loaded row (pagination cursor)
|
||||||
|
var seq = 0; // guards against out-of-order fetch responses
|
||||||
|
var newestTs = null; // ts of the newest loaded row (dedup guard for the WS tail)
|
||||||
|
|
||||||
|
function escHtml(s) {
|
||||||
|
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"').replace(/'/g,''');
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtClock(ts) {
|
||||||
|
var d = new Date(ts * 1000);
|
||||||
|
function p(n) { return n < 10 ? '0' + n : '' + n; }
|
||||||
|
var t = p(d.getHours()) + ':' + p(d.getMinutes()) + ':' + p(d.getSeconds());
|
||||||
|
return (d.toDateString() === new Date().toDateString())
|
||||||
|
? t
|
||||||
|
: d.getFullYear() + '-' + p(d.getMonth() + 1) + '-' + p(d.getDate()) + ' ' + t;
|
||||||
|
}
|
||||||
|
|
||||||
|
function logRowHtml(msg) {
|
||||||
|
var lvl = (msg.level || 'INFO');
|
||||||
|
var lc = lvl.toLowerCase();
|
||||||
|
var chipCls = lc === 'critical' ? 'crit' : (lc === 'warning' ? 'warn' : (lc === 'recover' ? 'ok' : ''));
|
||||||
|
var short = lc === 'critical' ? 'CRIT' : (lc === 'warning' ? 'WARN' : escHtml(lvl));
|
||||||
|
return '<div class="logrow">'
|
||||||
|
+ '<span class="t">' + fmtClock(msg.ts) + '</span>'
|
||||||
|
+ '<span class="lvlc"><span class="chip ' + chipCls + '">' + short + '</span></span>'
|
||||||
|
+ '<span class="h">' + escHtml(msg.host || '') + (msg.service ? ' · ' + escHtml(msg.service) : '') + '</span>'
|
||||||
|
+ '<span class="m">' + escHtml(msg.message || '') + '</span>'
|
||||||
|
+ '</div>';
|
||||||
|
}
|
||||||
|
|
||||||
|
function currentFilters() {
|
||||||
|
return {
|
||||||
|
host: document.getElementById('filter-host').value.trim(),
|
||||||
|
level: document.getElementById('filter-level').value,
|
||||||
|
q: document.getElementById('filter-msg').value.trim()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchesFilters(msg) {
|
||||||
|
var f = currentFilters();
|
||||||
|
if (f.host && !(msg.host || '').toLowerCase().includes(f.host.toLowerCase())) return false;
|
||||||
|
if (f.level && (msg.level || '').toLowerCase() !== f.level) return false;
|
||||||
|
if (f.q && !(msg.message || '').toLowerCase().includes(f.q.toLowerCase())) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateCount() {
|
||||||
|
var n = document.querySelectorAll('#messages .logrow').length;
|
||||||
|
document.getElementById('log-count').textContent = n ? n + ' events loaded' : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchLog(reset) {
|
||||||
|
var mySeq = ++seq;
|
||||||
|
var box = document.getElementById('messages');
|
||||||
|
var btn = document.getElementById('load-older');
|
||||||
|
btn.disabled = true;
|
||||||
|
var f = currentFilters();
|
||||||
|
var params = new URLSearchParams();
|
||||||
|
params.set('limit', String(PAGE_SIZE));
|
||||||
|
if (f.host) params.set('host', f.host);
|
||||||
|
if (f.level) params.set('level', f.level);
|
||||||
|
if (f.q) params.set('q', f.q);
|
||||||
|
if (!reset && oldestTs != null) params.set('before', String(oldestTs));
|
||||||
|
var data;
|
||||||
|
try {
|
||||||
|
var r = await fetch('/api/0/log?' + params.toString());
|
||||||
|
if (!r.ok) throw new Error('HTTP ' + r.status);
|
||||||
|
data = await r.json();
|
||||||
|
} catch (e) {
|
||||||
|
if (mySeq === seq && reset) {
|
||||||
|
box.innerHTML = '<div class="empty">Failed to load events: ' + escHtml(e.message) + '</div>';
|
||||||
|
}
|
||||||
|
btn.disabled = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (mySeq !== seq) return; // a newer request superseded this one
|
||||||
|
if (reset) { box.innerHTML = ''; oldestTs = null; }
|
||||||
|
var html = '';
|
||||||
|
data.events.forEach(function (ev) {
|
||||||
|
html += logRowHtml(ev);
|
||||||
|
if (ev.ts != null && (oldestTs == null || ev.ts < oldestTs)) oldestTs = ev.ts;
|
||||||
|
if (ev.ts != null && (newestTs == null || ev.ts > newestTs)) newestTs = ev.ts;
|
||||||
|
});
|
||||||
|
box.insertAdjacentHTML('beforeend', html);
|
||||||
|
if (!box.children.length) box.innerHTML = '<div class="empty">No events found.</div>';
|
||||||
|
btn.style.display = data.more ? '' : 'none';
|
||||||
|
btn.disabled = false;
|
||||||
|
updateCount();
|
||||||
|
}
|
||||||
|
|
||||||
|
function onFilterChange() { fetchLog(true); }
|
||||||
|
|
||||||
|
function debounce(fn, ms) {
|
||||||
|
var t;
|
||||||
|
return function () { clearTimeout(t); t = setTimeout(fn, ms); };
|
||||||
|
}
|
||||||
|
document.getElementById('filter-host').addEventListener('input', debounce(onFilterChange, 300));
|
||||||
|
document.getElementById('filter-msg').addEventListener('input', debounce(onFilterChange, 300));
|
||||||
|
document.getElementById('filter-level').addEventListener('change', onFilterChange);
|
||||||
|
document.getElementById('load-older').addEventListener('click', function () { fetchLog(false); });
|
||||||
|
|
||||||
|
// ---- live tail over the shared websocket --------------------------------
|
||||||
|
function setWsStatus(ok) {
|
||||||
|
var el = document.getElementById('ws-status');
|
||||||
|
el.classList.toggle('reconnecting', !ok);
|
||||||
|
el.textContent = ok ? 'live' : 'reconnecting…';
|
||||||
|
}
|
||||||
|
|
||||||
|
function WS_Connect() {
|
||||||
|
if (!("WebSocket" in window)) return;
|
||||||
|
var ws_hbd = new WebSocket("{{ heartbeat_ws_url }}");
|
||||||
|
ws_hbd.onopen = function () {
|
||||||
|
setWsStatus(true);
|
||||||
|
ws_hbd.send("heartbeat_web");
|
||||||
|
};
|
||||||
|
ws_hbd.onmessage = function (event) {
|
||||||
|
var state = JSON.parse(event.data);
|
||||||
|
// history replays are covered by the API seed; only tail new events
|
||||||
|
if (state.type !== "message" || state.history) return;
|
||||||
|
var msg = state.data;
|
||||||
|
if (!matchesFilters(msg)) return;
|
||||||
|
if (msg.ts != null && newestTs != null && msg.ts <= newestTs) return; // already loaded via the API
|
||||||
|
if (msg.ts != null && (newestTs == null || msg.ts > newestTs)) newestTs = msg.ts;
|
||||||
|
var box = document.getElementById('messages');
|
||||||
|
var placeholder = box.querySelector('.empty');
|
||||||
|
if (placeholder) placeholder.remove();
|
||||||
|
box.insertAdjacentHTML('afterbegin', logRowHtml(msg));
|
||||||
|
updateCount();
|
||||||
|
};
|
||||||
|
ws_hbd.onclose = function () {
|
||||||
|
setWsStatus(false);
|
||||||
|
setTimeout(WS_Connect, 3000);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- boot ----------------------------------------------------------------
|
||||||
|
fetchLog(true);
|
||||||
|
WS_Connect();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -6,11 +6,18 @@
|
|||||||
<a href="/live"{% if active_page == "live" %} class="active"{% endif %}>Live Dashboard</a>
|
<a href="/live"{% if active_page == "live" %} class="active"{% endif %}>Live Dashboard</a>
|
||||||
<a href="/plugins"{% if active_page == "plugins" %} class="active"{% endif %}>Host Overview</a>
|
<a href="/plugins"{% if active_page == "plugins" %} class="active"{% endif %}>Host Overview</a>
|
||||||
<a href="/alerts"{% if active_page == "alerts" %} class="active"{% endif %}>Alerts</a>
|
<a href="/alerts"{% if active_page == "alerts" %} class="active"{% endif %}>Alerts</a>
|
||||||
{% if current_user and current_user.admin %}
|
<a href="/log"{% if active_page == "log" %} class="active"{% endif %}>Log</a>
|
||||||
|
{% if current_user %}
|
||||||
<a href="/settings"{% if active_page == "settings" %} class="active"{% endif %}>Settings</a>
|
<a href="/settings"{% if active_page == "settings" %} class="active"{% endif %}>Settings</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<a href="/about"{% if active_page == "about" %} class="active"{% endif %}>About</a>
|
<a href="/about"{% if active_page == "about" %} class="active"{% endif %}>About</a>
|
||||||
</div>
|
</div>
|
||||||
|
{% if current_user %}
|
||||||
|
<button id="nav-publish-btn" class="nav-publish-btn" onclick="navPublishConfig()" style="display:none" title="Publish pending config changes to .hb.yaml">⚠ Publish Config</button>
|
||||||
|
{% endif %}
|
||||||
|
<div class="nav-pie" title="Host alert status">
|
||||||
|
<canvas id="alert-pie" width="44" height="44"></canvas>
|
||||||
|
</div>
|
||||||
<div class="nav-clock" title="Click for full-screen clock">
|
<div class="nav-clock" title="Click for full-screen clock">
|
||||||
<canvas id="swiss-clock" width="44" height="44"></canvas>
|
<canvas id="swiss-clock" width="44" height="44"></canvas>
|
||||||
</div>
|
</div>
|
||||||
@@ -42,4 +49,87 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
function drawAlertPie(critical, warning, ok) {
|
||||||
|
var canvas = document.getElementById('alert-pie');
|
||||||
|
if (!canvas) return;
|
||||||
|
var ctx = canvas.getContext('2d');
|
||||||
|
var SIZE = canvas.width;
|
||||||
|
var R = SIZE / 2;
|
||||||
|
ctx.clearRect(0, 0, SIZE, SIZE);
|
||||||
|
var total = critical + warning + ok;
|
||||||
|
if (total === 0) {
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.arc(R, R, R - 1, 0, Math.PI * 2);
|
||||||
|
ctx.fillStyle = '#ccc';
|
||||||
|
ctx.fill();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
var slices = [
|
||||||
|
{ value: critical, color: '#e53935' },
|
||||||
|
{ value: warning, color: '#ffb300' },
|
||||||
|
{ value: ok, color: '#43a047' }
|
||||||
|
];
|
||||||
|
var start = -Math.PI / 2;
|
||||||
|
slices.forEach(function(s) {
|
||||||
|
if (s.value === 0) return;
|
||||||
|
var sweep = (s.value / total) * Math.PI * 2;
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.moveTo(R, R);
|
||||||
|
ctx.arc(R, R, R - 1, start, start + sweep);
|
||||||
|
ctx.closePath();
|
||||||
|
ctx.fillStyle = s.color;
|
||||||
|
ctx.fill();
|
||||||
|
start += sweep;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateAlertPie() {
|
||||||
|
fetch('/api/0/alert_summary').then(function(r) {
|
||||||
|
if (!r.ok) return;
|
||||||
|
return r.json();
|
||||||
|
}).then(function(d) {
|
||||||
|
if (d) drawAlertPie(d.critical || 0, d.warning || 0, d.ok || 0);
|
||||||
|
}).catch(function() {});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
updateAlertPie();
|
||||||
|
setInterval(updateAlertPie, 30000);
|
||||||
|
navCheckPendingConfig();
|
||||||
|
window.addEventListener('storage', navCheckPendingConfig);
|
||||||
|
});
|
||||||
|
|
||||||
|
function navCheckPendingConfig() {
|
||||||
|
var btn = document.getElementById('nav-publish-btn');
|
||||||
|
if (!btn) return;
|
||||||
|
btn.style.display = localStorage.getItem('hbd_pending_config') ? '' : 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function navPublishConfig() {
|
||||||
|
var btn = document.getElementById('nav-publish-btn');
|
||||||
|
var pending = localStorage.getItem('hbd_pending_config');
|
||||||
|
if (!pending) return;
|
||||||
|
var staged;
|
||||||
|
try { staged = JSON.parse(pending); } catch(e) { return; }
|
||||||
|
if (btn) { btn.disabled = true; btn.textContent = 'Saving…'; }
|
||||||
|
try {
|
||||||
|
var resp = await fetch('/api/0/config', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {'Content-Type': 'application/json'},
|
||||||
|
body: pending
|
||||||
|
});
|
||||||
|
if (resp.ok) {
|
||||||
|
localStorage.removeItem('hbd_pending_config');
|
||||||
|
window.location.reload();
|
||||||
|
} else {
|
||||||
|
var err = await resp.json().catch(function() { return {}; });
|
||||||
|
alert('Error: ' + (err.error || resp.statusText));
|
||||||
|
if (btn) { btn.disabled = false; btn.textContent = '⚠ Publish Config'; }
|
||||||
|
}
|
||||||
|
} catch(e) {
|
||||||
|
alert('Network error: ' + e.message);
|
||||||
|
if (btn) { btn.disabled = false; btn.textContent = '⚠ Publish Config'; }
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
+562
-262
File diff suppressed because it is too large
Load Diff
@@ -96,7 +96,7 @@
|
|||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
background: #f44336;
|
background: #f44336;
|
||||||
color: #fff;
|
color: #fff;
|
||||||
font-size: 0.85em;
|
font-size: 1.00em;
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
transition: background 0.15s;
|
transition: background 0.15s;
|
||||||
@@ -157,7 +157,7 @@
|
|||||||
gap: 6px;
|
gap: 6px;
|
||||||
padding: 4px 12px;
|
padding: 4px 12px;
|
||||||
border-radius: 16px;
|
border-radius: 16px;
|
||||||
font-size: 0.85em;
|
font-size: 1.00em;
|
||||||
font-weight: 500;
|
font-weight: 500;
|
||||||
text-decoration: none;
|
text-decoration: none;
|
||||||
}
|
}
|
||||||
@@ -204,6 +204,119 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.channel-name { color: #333; }
|
.channel-name { color: #333; }
|
||||||
|
|
||||||
|
.edit-section { margin-top: 20px; }
|
||||||
|
.edit-section h4 { font-size: .88em; font-weight: 600; color: #333; margin: 0 0 10px; text-transform: uppercase; letter-spacing: .04em; border-bottom: 1px solid #eee; padding-bottom: 6px; }
|
||||||
|
.edit-field { margin-bottom: 10px; }
|
||||||
|
.edit-field label { display: block; font-size: .82em; color: #666; margin-bottom: 3px; }
|
||||||
|
.edit-input { width: 100%; border: 1px solid #ccc; border-radius: 4px; padding: 5px 8px; font-size: .88em; box-sizing: border-box; }
|
||||||
|
.edit-input:focus { border-color: #0066cc; outline: none; }
|
||||||
|
.status-msg { font-size: .82em; margin-left: 8px; }
|
||||||
|
.save-row { display: flex; align-items: center; margin-top: 8px; }
|
||||||
|
.btn-save { background: #0066cc; color: #fff; border: none; border-radius: 4px; padding: 5px 14px; font-size: .85em; cursor: pointer; }
|
||||||
|
.btn-save:hover { background: #0055aa; }
|
||||||
|
/* ---- Channel chip picker ---- */
|
||||||
|
.ch-picker { }
|
||||||
|
.ch-picker-label { font-size: .8em; font-weight: 600; color: #888; text-transform: uppercase; letter-spacing: .04em; margin-bottom: 6px; }
|
||||||
|
.ch-chips { display: flex; flex-wrap: wrap; gap: 6px; min-height: 32px; margin-bottom: 10px; }
|
||||||
|
.ch-chip {
|
||||||
|
display: inline-flex; align-items: center; gap: 5px;
|
||||||
|
padding: 4px 10px; border-radius: 14px; font-size: .85em; font-weight: 500; cursor: pointer;
|
||||||
|
border: none; font-family: inherit;
|
||||||
|
}
|
||||||
|
.ch-chip.selected { background: #e3f2fd; color: #1565c0; }
|
||||||
|
.ch-chip.selected:hover { background: #bbdefb; }
|
||||||
|
.ch-chip.available { background: #f1f3f4; color: #555; }
|
||||||
|
.ch-chip.available:hover { background: #e8eaf6; color: #283593; }
|
||||||
|
.ch-chip-x { font-size: .9em; line-height: 1; color: inherit; opacity: .7; }
|
||||||
|
|
||||||
|
/* ---- My Channels card list ---- */
|
||||||
|
.my-ch-card {
|
||||||
|
border: 1px solid #e8eaf6; border-radius: 6px; margin-bottom: 8px; overflow: hidden;
|
||||||
|
}
|
||||||
|
.my-ch-header {
|
||||||
|
display: flex; align-items: center; gap: 8px; padding: 8px 12px;
|
||||||
|
background: #f8f9ff; border-bottom: 1px solid #e8eaf6;
|
||||||
|
}
|
||||||
|
.my-ch-name { font-weight: 600; font-size: .9em; color: #222; }
|
||||||
|
.my-ch-type { padding: 2px 7px; border-radius: 8px; font-size: .72em; font-weight: 600; background: #e8eaf6; color: #3949ab; }
|
||||||
|
.my-ch-actions { margin-left: auto; display: flex; gap: 5px; }
|
||||||
|
.btn-sm-edit { background: #888; color: #fff; border: none; border-radius: 4px; padding: 2px 8px; font-size: .78em; cursor: pointer; }
|
||||||
|
.btn-sm-edit:hover { background: #666; }
|
||||||
|
.btn-sm-del { background: transparent; color: #c62828; border: 1px solid #e0e0e0; border-radius: 4px; padding: 2px 7px; font-size: .78em; cursor: pointer; }
|
||||||
|
.btn-sm-del:hover { background: #fce4ec; }
|
||||||
|
|
||||||
|
/* ---- Theme picker ---- */
|
||||||
|
.theme-btns { display: flex; gap: 6px; }
|
||||||
|
.theme-btn {
|
||||||
|
padding: 5px 14px;
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
border-radius: 4px;
|
||||||
|
background: var(--surface-3, #f5f5f5);
|
||||||
|
color: var(--text-sec, #666);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: .88em;
|
||||||
|
font-family: inherit;
|
||||||
|
}
|
||||||
|
.theme-btn:hover { border-color: var(--link, #0066cc); color: var(--link, #0066cc); }
|
||||||
|
.theme-btn.active { background: var(--link, #0066cc); color: #fff; border-color: var(--link, #0066cc); }
|
||||||
|
|
||||||
|
/* ── Dark mode ── */
|
||||||
|
html[data-theme="dark"] h1 { color: var(--text); }
|
||||||
|
html[data-theme="dark"] .subtitle { color: var(--text-sec); }
|
||||||
|
html[data-theme="dark"] .profile-card { background: var(--surface); box-shadow: 0 1px 6px var(--shadow); }
|
||||||
|
html[data-theme="dark"] .profile-name { color: var(--text); }
|
||||||
|
html[data-theme="dark"] .profile-username { color: var(--text-sec); }
|
||||||
|
html[data-theme="dark"] .badge-admin { background: #1a3255; color: #7aa8f0; }
|
||||||
|
html[data-theme="dark"] .badge-user { background: var(--surface-3); color: var(--text-sec); }
|
||||||
|
html[data-theme="dark"] .section { background: var(--surface); box-shadow: 0 1px 6px var(--shadow); }
|
||||||
|
html[data-theme="dark"] .section h2 { color: var(--text); border-bottom-color: var(--border); }
|
||||||
|
html[data-theme="dark"] .settings-row { border-bottom-color: var(--border-4); }
|
||||||
|
html[data-theme="dark"] .settings-label { color: var(--text-sec); }
|
||||||
|
html[data-theme="dark"] .settings-value { color: var(--text); }
|
||||||
|
html[data-theme="dark"] .settings-empty { color: var(--text-dim); }
|
||||||
|
html[data-theme="dark"] .edit-section h4 { color: var(--text); border-bottom-color: var(--border); }
|
||||||
|
html[data-theme="dark"] .edit-field label { color: var(--text-sec); }
|
||||||
|
html[data-theme="dark"] .edit-input { background: var(--input-bg); border-color: var(--input-border); color: var(--text); }
|
||||||
|
html[data-theme="dark"] .channel-row { border-bottom-color: var(--border-4); }
|
||||||
|
html[data-theme="dark"] .channel-name { color: var(--text); }
|
||||||
|
html[data-theme="dark"] .ch-picker-label { color: var(--text-sec); }
|
||||||
|
html[data-theme="dark"] .ch-chip.selected { background: #1a3255; color: #60a5fa; }
|
||||||
|
html[data-theme="dark"] .ch-chip.available { background: var(--surface-3); color: var(--text-sec); }
|
||||||
|
html[data-theme="dark"] .ch-chip.available:hover { background: var(--border); color: var(--link); }
|
||||||
|
html[data-theme="dark"] .my-ch-card { border-color: var(--border); }
|
||||||
|
html[data-theme="dark"] .my-ch-header { background: var(--surface-2); border-bottom-color: var(--border); }
|
||||||
|
html[data-theme="dark"] .my-ch-name { color: var(--text); }
|
||||||
|
html[data-theme="dark"] .host-chip.owner { background: #0d2e17; color: #66bb6a; }
|
||||||
|
html[data-theme="dark"] .host-chip.manager { background: #0d1f40; color: #64b5f6; }
|
||||||
|
html[data-theme="dark"] .host-chip.monitor { background: #1e0d30; color: #ba68c8; }
|
||||||
|
html[data-theme="dark"] .no-hosts { color: var(--text-dim); }
|
||||||
|
html[data-theme="dark"] .ch-modal-box { background: var(--surface); color: var(--text); }
|
||||||
|
html[data-theme="dark"] .ch-modal-box h3 { color: var(--text); }
|
||||||
|
html[data-theme="dark"] .ch-form-row label { color: var(--text-sec); }
|
||||||
|
html[data-theme="dark"] .ch-form-divider { color: var(--text-muted); border-top-color: var(--border); }
|
||||||
|
|
||||||
|
/* ---- Channel modal (for My Channels CRUD) ---- */
|
||||||
|
.ch-modal-overlay {
|
||||||
|
position: fixed; inset: 0; background: rgba(0,0,0,.4);
|
||||||
|
display: flex; align-items: center; justify-content: center; z-index: 1001;
|
||||||
|
}
|
||||||
|
.ch-modal-box {
|
||||||
|
background: #fff; border-radius: 8px; padding: 24px;
|
||||||
|
min-width: 360px; max-width: 520px; width: 95%;
|
||||||
|
box-shadow: 0 8px 32px rgba(0,0,0,.2);
|
||||||
|
}
|
||||||
|
.ch-modal-box h3 { margin: 0 0 16px; font-size: 1em; }
|
||||||
|
.ch-form-row { margin-bottom: 12px; }
|
||||||
|
.ch-form-row label { display: block; font-size: .83em; font-weight: 600; color: #555; margin-bottom: 3px; }
|
||||||
|
.ch-form-row input[type=text], .ch-form-row input[type=password], .ch-form-row select {
|
||||||
|
width: 100%; border: 1px solid #ccc; border-radius: 4px; padding: 5px 8px;
|
||||||
|
font-size: .88em; box-sizing: border-box; font-family: inherit;
|
||||||
|
}
|
||||||
|
.ch-form-row input:focus, .ch-form-row select:focus { border-color: #0066cc; outline: none; }
|
||||||
|
.ch-form-divider { font-size: .78em; font-weight: 700; text-transform: uppercase; letter-spacing: .05em; color: #888; margin: 14px 0 8px; border-top: 1px solid #eee; padding-top: 10px; }
|
||||||
|
.ch-modal-footer { display: flex; justify-content: flex-end; gap: 8px; margin-top: 18px; }
|
||||||
|
.ch-modal-status { font-size: .83em; margin-top: 8px; }
|
||||||
</style>
|
</style>
|
||||||
|
|
||||||
<body>
|
<body>
|
||||||
@@ -266,19 +379,158 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Notification channels -->
|
{% if current_user %}
|
||||||
|
<!-- ---- Editable identity ---- -->
|
||||||
|
<div class="section edit-section">
|
||||||
|
<h4>Identity</h4>
|
||||||
|
<div class="edit-field">
|
||||||
|
<label for="profile-fullname">Display name</label>
|
||||||
|
<input id="profile-fullname" class="edit-input" type="text" value="{{ current_user.full_name | e }}" placeholder="Full name">
|
||||||
|
</div>
|
||||||
|
<div class="edit-field">
|
||||||
|
<label for="profile-avatar">Avatar URL or path</label>
|
||||||
|
<input id="profile-avatar" class="edit-input" type="text" value="{{ current_user.avatar | e }}" placeholder="/path/to/avatar.png or https://…">
|
||||||
|
</div>
|
||||||
|
<div class="save-row">
|
||||||
|
<button class="btn-save" onclick="saveIdentity()">Save</button>
|
||||||
|
<span id="identity-status" class="status-msg"></span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- ---- Change password ---- -->
|
||||||
|
<div class="section edit-section">
|
||||||
|
<h4>Change password</h4>
|
||||||
|
<div class="edit-field">
|
||||||
|
<label for="profile-current-pw">Current password</label>
|
||||||
|
<input id="profile-current-pw" class="edit-input" type="password" autocomplete="current-password">
|
||||||
|
</div>
|
||||||
|
<div class="edit-field">
|
||||||
|
<label for="profile-new-pw">New password</label>
|
||||||
|
<input id="profile-new-pw" class="edit-input" type="password" autocomplete="new-password">
|
||||||
|
</div>
|
||||||
|
<div class="save-row">
|
||||||
|
<button class="btn-save" onclick="changePassword()">Change password</button>
|
||||||
|
<span id="password-status" class="status-msg"></span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<!-- Notification channels — chip picker -->
|
||||||
<div class="section">
|
<div class="section">
|
||||||
<h2>Notification Channels</h2>
|
<h2>Notification Channels</h2>
|
||||||
{% if notification_channels %}
|
{% if current_user %}
|
||||||
{% for ch in notification_channels %}
|
<p style="font-size:.82em;color:#888;margin:0 0 12px">Click a channel to add or remove it from your alert list.</p>
|
||||||
<div class="channel-row">
|
{% if all_channels %}
|
||||||
<span class="channel-type">{{ ch.type }}</span>
|
<div class="ch-picker">
|
||||||
<span class="channel-name">{{ ch.name }}</span>
|
<div class="ch-picker-label">Selected</div>
|
||||||
|
<div id="selected-chips" class="ch-chips">
|
||||||
|
{% for ch in all_channels %}
|
||||||
|
{% if ch.name in (current_user.notification_channels or []) %}
|
||||||
|
<button class="ch-chip selected" data-ch="{{ ch.name | e }}" onclick="toggleChip(this)">
|
||||||
|
{{ ch.name | e }} <span class="ch-chip-x">×</span>
|
||||||
|
</button>
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
{% set selected_set = current_user.notification_channels or [] %}
|
||||||
|
{% set has_selected = selected_set | length > 0 %}
|
||||||
|
{% if not has_selected %}
|
||||||
|
<span style="font-size:.83em;color:#bbb;font-style:italic;align-self:center">None selected</span>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
<div class="ch-picker-label">Available</div>
|
||||||
|
<div id="available-chips" class="ch-chips">
|
||||||
|
{% for ch in all_channels %}
|
||||||
|
{% if ch.name not in (current_user.notification_channels or []) %}
|
||||||
|
<button class="ch-chip available" data-ch="{{ ch.name | e }}" onclick="toggleChip(this)">
|
||||||
|
+ {{ ch.name | e }}
|
||||||
|
</button>
|
||||||
|
{% endif %}
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{% endfor %}
|
|
||||||
{% else %}
|
{% else %}
|
||||||
<span class="no-hosts">No personal notification channels configured.</span>
|
<p style="font-size:.83em;color:#bbb;font-style:italic">No notification channels available. You can create your own below.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
<div class="save-row">
|
||||||
|
<button class="btn-save" onclick="saveChannels()">Save channels</button>
|
||||||
|
<span id="channels-status" class="status-msg"></span>
|
||||||
|
</div>
|
||||||
|
{% else %}
|
||||||
|
<span class="no-hosts">Log in to manage notification channels.</span>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- My Channels — create/edit/delete own channels -->
|
||||||
|
{% if current_user %}
|
||||||
|
<div class="section">
|
||||||
|
<h2>My Channels</h2>
|
||||||
|
<p style="font-size:.82em;color:#888;margin:0 0 12px">Channels you own are private to you. Global channels are managed by administrators.</p>
|
||||||
|
<div id="my-channels-list">
|
||||||
|
{% set my_channels = all_channels | selectattr('owner', 'equalto', current_user.username) | list %}
|
||||||
|
{% for ch in my_channels %}
|
||||||
|
<div class="my-ch-card" id="mychcard-{{ ch.name | e }}">
|
||||||
|
<div class="my-ch-header">
|
||||||
|
<span class="my-ch-name">{{ ch.name | e }}</span>
|
||||||
|
<span class="my-ch-type">{{ ch.type | e }}</span>
|
||||||
|
<span class="my-ch-actions">
|
||||||
|
<button class="btn-sm-edit" onclick="openMyChModal('{{ ch.name | e }}')">Edit</button>
|
||||||
|
<button class="btn-sm-del" onclick="deleteMyChannel('{{ ch.name | e }}')">✕</button>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
{% if not my_channels %}
|
||||||
|
<p id="my-channels-empty" style="font-size:.83em;color:#bbb;font-style:italic">No channels yet.</p>
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
<div class="save-row" style="margin-top:8px">
|
||||||
|
<button class="btn-save" onclick="openMyChModal()">+ New channel</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- My Channels modal -->
|
||||||
|
<div id="my-ch-modal" class="ch-modal-overlay" style="display:none" onclick="if(event.target===this)closeMyChModal()">
|
||||||
|
<div class="ch-modal-box">
|
||||||
|
<h3 id="my-ch-modal-title">New Channel</h3>
|
||||||
|
<div class="ch-form-row">
|
||||||
|
<label>Channel name</label>
|
||||||
|
<input type="text" id="my-ch-name" placeholder="e.g. my_pushover" autocomplete="off">
|
||||||
|
</div>
|
||||||
|
<div class="ch-form-row">
|
||||||
|
<label>Type</label>
|
||||||
|
<select id="my-ch-type" onchange="onMyChTypeChange()">
|
||||||
|
<option value="">— select —</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div id="my-ch-type-fields"></div>
|
||||||
|
<div class="ch-form-divider">Options</div>
|
||||||
|
<div class="ch-form-row">
|
||||||
|
<label>Minimum alert level</label>
|
||||||
|
<select id="my-ch-min-level">
|
||||||
|
<option value="WARNING">WARNING (and above)</option>
|
||||||
|
<option value="CRITICAL">CRITICAL only</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div id="my-ch-modal-status" class="ch-modal-status"></div>
|
||||||
|
<div class="ch-modal-footer">
|
||||||
|
<button class="btn-save" style="background:#888" onclick="closeMyChModal()">Cancel</button>
|
||||||
|
<button class="btn-save" onclick="saveMyChannel()">Save</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<!-- Appearance -->
|
||||||
|
<div class="section">
|
||||||
|
<h2>Appearance</h2>
|
||||||
|
<div class="settings-row">
|
||||||
|
<span class="settings-label">Theme</span>
|
||||||
|
<div class="theme-btns">
|
||||||
|
<button class="theme-btn" data-theme-val="auto" onclick="setTheme('auto')">Auto</button>
|
||||||
|
<button class="theme-btn" data-theme-val="light" onclick="setTheme('light')">Light</button>
|
||||||
|
<button class="theme-btn" data-theme-val="dark" onclick="setTheme('dark')">Dark</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Host access -->
|
<!-- Host access -->
|
||||||
@@ -326,5 +578,255 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
|
<script>
|
||||||
|
// ---- Theme ----
|
||||||
|
function applyTheme(pref) {
|
||||||
|
var dark = pref === 'dark' ||
|
||||||
|
(pref === 'auto' && window.matchMedia('(prefers-color-scheme: dark)').matches);
|
||||||
|
if (dark) { document.documentElement.setAttribute('data-theme', 'dark'); }
|
||||||
|
else { document.documentElement.removeAttribute('data-theme'); }
|
||||||
|
}
|
||||||
|
function setTheme(pref) {
|
||||||
|
try { localStorage.setItem('hbd_theme', pref); } catch(e) {}
|
||||||
|
applyTheme(pref);
|
||||||
|
document.querySelectorAll('.theme-btn').forEach(function(b) {
|
||||||
|
b.classList.toggle('active', b.dataset.themeVal === pref);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
(function() {
|
||||||
|
var pref = 'auto';
|
||||||
|
try { pref = localStorage.getItem('hbd_theme') || 'auto'; } catch(e) {}
|
||||||
|
document.querySelectorAll('.theme-btn').forEach(function(b) {
|
||||||
|
b.classList.toggle('active', b.dataset.themeVal === pref);
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
|
// ---- Identity ----
|
||||||
|
async function saveIdentity() {
|
||||||
|
const full_name = document.getElementById('profile-fullname').value;
|
||||||
|
const avatar = document.getElementById('profile-avatar').value;
|
||||||
|
const resp = await fetch('/api/0/users/me', {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({full_name, avatar}),
|
||||||
|
});
|
||||||
|
if (resp.ok) {
|
||||||
|
showStatus('identity-status', 'Saved', '#2e7d32');
|
||||||
|
} else {
|
||||||
|
const err = await resp.json().catch(() => ({}));
|
||||||
|
showStatus('identity-status', err.error || 'Error saving', '#c62828');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Password ----
|
||||||
|
async function changePassword() {
|
||||||
|
const current = document.getElementById('profile-current-pw').value;
|
||||||
|
const newpw = document.getElementById('profile-new-pw').value;
|
||||||
|
if (!current || !newpw) {
|
||||||
|
showStatus('password-status', 'Both fields are required', '#c62828');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const resp = await fetch('/api/0/users/me', {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({password: {current, new: newpw}}),
|
||||||
|
});
|
||||||
|
if (resp.ok) {
|
||||||
|
document.getElementById('profile-current-pw').value = '';
|
||||||
|
document.getElementById('profile-new-pw').value = '';
|
||||||
|
showStatus('password-status', 'Password changed', '#2e7d32');
|
||||||
|
} else {
|
||||||
|
const err = await resp.json().catch(() => ({}));
|
||||||
|
showStatus('password-status', err.error || 'Error', '#c62828');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Channel chip picker ----
|
||||||
|
function toggleChip(btn) {
|
||||||
|
const name = btn.dataset.ch;
|
||||||
|
const isSelected = btn.classList.contains('selected');
|
||||||
|
if (isSelected) {
|
||||||
|
// Move to available
|
||||||
|
btn.classList.remove('selected');
|
||||||
|
btn.classList.add('available');
|
||||||
|
btn.innerHTML = '+ ' + escHtml(name);
|
||||||
|
btn.onclick = function() { toggleChip(this); };
|
||||||
|
document.getElementById('available-chips').appendChild(btn);
|
||||||
|
// Remove "None selected" placeholder if it exists
|
||||||
|
} else {
|
||||||
|
// Move to selected
|
||||||
|
btn.classList.remove('available');
|
||||||
|
btn.classList.add('selected');
|
||||||
|
btn.innerHTML = escHtml(name) + ' <span class="ch-chip-x">×</span>';
|
||||||
|
btn.onclick = function() { toggleChip(this); };
|
||||||
|
document.getElementById('selected-chips').appendChild(btn);
|
||||||
|
}
|
||||||
|
// Update placeholder visibility
|
||||||
|
const sel = document.getElementById('selected-chips');
|
||||||
|
const placeholder = sel.querySelector('span[style]');
|
||||||
|
const hasChips = sel.querySelectorAll('.ch-chip.selected').length > 0;
|
||||||
|
if (placeholder) placeholder.style.display = hasChips ? 'none' : '';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveChannels() {
|
||||||
|
const notification_channels = [
|
||||||
|
...document.querySelectorAll('#selected-chips .ch-chip.selected')
|
||||||
|
].map(b => b.dataset.ch);
|
||||||
|
const resp = await fetch('/api/0/users/me', {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({notification_channels}),
|
||||||
|
});
|
||||||
|
if (resp.ok) {
|
||||||
|
showStatus('channels-status', 'Saved', '#2e7d32');
|
||||||
|
} else {
|
||||||
|
const err = await resp.json().catch(() => ({}));
|
||||||
|
showStatus('channels-status', err.error || 'Error saving', '#c62828');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- My Channels CRUD ----
|
||||||
|
let _myChSchemas = {};
|
||||||
|
let _myChEditName = null;
|
||||||
|
|
||||||
|
async function _loadMyChSchemas() {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/0/notification_channel_types');
|
||||||
|
_myChSchemas = await r.json();
|
||||||
|
const sel = document.getElementById('my-ch-type');
|
||||||
|
if (!sel) return;
|
||||||
|
Object.entries(_myChSchemas).forEach(([k, v]) => {
|
||||||
|
const opt = document.createElement('option');
|
||||||
|
opt.value = k; opt.textContent = v.label;
|
||||||
|
sel.appendChild(opt);
|
||||||
|
});
|
||||||
|
} catch(e) { console.warn('Could not load channel schemas', e); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function onMyChTypeChange() {
|
||||||
|
const type = document.getElementById('my-ch-type').value;
|
||||||
|
const container = document.getElementById('my-ch-type-fields');
|
||||||
|
container.innerHTML = '';
|
||||||
|
if (!type || !_myChSchemas[type]) return;
|
||||||
|
const divider = document.createElement('div');
|
||||||
|
divider.className = 'ch-form-divider';
|
||||||
|
divider.textContent = _myChSchemas[type].label + ' settings';
|
||||||
|
container.appendChild(divider);
|
||||||
|
(_myChSchemas[type].fields || []).forEach(sf => {
|
||||||
|
const row = document.createElement('div');
|
||||||
|
row.className = 'ch-form-row';
|
||||||
|
const lbl = document.createElement('label');
|
||||||
|
lbl.textContent = sf.label + (sf.required ? ' *' : '');
|
||||||
|
const inp = document.createElement('input');
|
||||||
|
inp.type = sf.type === 'secret' ? 'password' : 'text';
|
||||||
|
inp.id = 'mychf-' + sf.key;
|
||||||
|
inp.placeholder = sf.required ? '(required)' : '(optional)';
|
||||||
|
inp.autocomplete = 'off';
|
||||||
|
row.appendChild(lbl);
|
||||||
|
row.appendChild(inp);
|
||||||
|
container.appendChild(row);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openMyChModal(name) {
|
||||||
|
_myChEditName = name || null;
|
||||||
|
document.getElementById('my-ch-modal-status').textContent = '';
|
||||||
|
document.getElementById('my-ch-modal-title').textContent = name ? 'Edit Channel' : 'New Channel';
|
||||||
|
document.getElementById('my-ch-name').value = name || '';
|
||||||
|
document.getElementById('my-ch-name').disabled = !!name;
|
||||||
|
document.getElementById('my-ch-type').value = '';
|
||||||
|
document.getElementById('my-ch-type-fields').innerHTML = '';
|
||||||
|
document.getElementById('my-ch-min-level').value = 'WARNING';
|
||||||
|
|
||||||
|
if (name) {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/0/notification_channels');
|
||||||
|
const channels = await r.json();
|
||||||
|
const ch = channels.find(c => c.name === name);
|
||||||
|
if (ch) {
|
||||||
|
document.getElementById('my-ch-type').value = ch.type;
|
||||||
|
onMyChTypeChange();
|
||||||
|
document.getElementById('my-ch-min-level').value = ch.min_level || 'WARNING';
|
||||||
|
(ch.fields || []).forEach(f => {
|
||||||
|
const inp = document.getElementById('mychf-' + f.key);
|
||||||
|
if (inp) inp.value = f.value || '';
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch(e) { console.warn('Failed to load channel', e); }
|
||||||
|
}
|
||||||
|
document.getElementById('my-ch-modal').style.display = 'flex';
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeMyChModal() {
|
||||||
|
document.getElementById('my-ch-modal').style.display = 'none';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveMyChannel() {
|
||||||
|
const name = document.getElementById('my-ch-name').value.trim();
|
||||||
|
const type = document.getElementById('my-ch-type').value;
|
||||||
|
const minLevel = document.getElementById('my-ch-min-level').value;
|
||||||
|
const statusEl = document.getElementById('my-ch-modal-status');
|
||||||
|
statusEl.textContent = '';
|
||||||
|
|
||||||
|
if (!name) { statusEl.textContent = 'Name is required.'; statusEl.style.color = '#c62828'; return; }
|
||||||
|
if (!type) { statusEl.textContent = 'Please select a type.'; statusEl.style.color = '#c62828'; return; }
|
||||||
|
|
||||||
|
const body = { name, type, min_level: minLevel };
|
||||||
|
if (_myChSchemas[type]) {
|
||||||
|
(_myChSchemas[type].fields || []).forEach(sf => {
|
||||||
|
const inp = document.getElementById('mychf-' + sf.key);
|
||||||
|
if (inp) body[sf.key] = inp.value;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const isEdit = !!_myChEditName;
|
||||||
|
const url = isEdit
|
||||||
|
? '/api/0/notification_channels/' + encodeURIComponent(_myChEditName)
|
||||||
|
: '/api/0/notification_channels';
|
||||||
|
const method = isEdit ? 'PUT' : 'POST';
|
||||||
|
try {
|
||||||
|
const r = await fetch(url, { method, headers: {'Content-Type': 'application/json'}, body: JSON.stringify(body) });
|
||||||
|
if (r.ok) {
|
||||||
|
closeMyChModal();
|
||||||
|
window.location.reload();
|
||||||
|
} else {
|
||||||
|
const err = await r.json().catch(() => ({}));
|
||||||
|
statusEl.textContent = err.error || 'Error saving.';
|
||||||
|
statusEl.style.color = '#c62828';
|
||||||
|
}
|
||||||
|
} catch(e) {
|
||||||
|
statusEl.textContent = 'Network error: ' + e.message;
|
||||||
|
statusEl.style.color = '#c62828';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteMyChannel(name) {
|
||||||
|
if (!confirm('Delete channel "' + name + '"?')) return;
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/0/notification_channels/' + encodeURIComponent(name), { method: 'DELETE' });
|
||||||
|
if (r.ok) {
|
||||||
|
window.location.reload();
|
||||||
|
} else {
|
||||||
|
const err = await r.json().catch(() => ({}));
|
||||||
|
alert('Error: ' + (err.error || 'Could not delete.'));
|
||||||
|
}
|
||||||
|
} catch(e) { alert('Network error: ' + e.message); }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Utilities ----
|
||||||
|
function showStatus(id, msg, color) {
|
||||||
|
const el = document.getElementById(id);
|
||||||
|
if (!el) return;
|
||||||
|
el.textContent = msg;
|
||||||
|
el.style.color = color;
|
||||||
|
setTimeout(() => { el.textContent = ''; }, 3000);
|
||||||
|
}
|
||||||
|
|
||||||
|
function escHtml(s) {
|
||||||
|
return String(s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', _loadMyChSchemas);
|
||||||
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
+1271
-439
File diff suppressed because it is too large
Load Diff
+492
-120
@@ -9,10 +9,11 @@ This module provides a flexible threshold checking system that:
|
|||||||
- Supports multiple comparison operators
|
- Supports multiple comparison operators
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Dict, Any, Optional, Tuple, Callable
|
from typing import Dict, List, Any, Optional, Tuple, Callable
|
||||||
from . import notify as notify_mod
|
from . import notify as notify_mod
|
||||||
from .config import THRESHOLD_DEFAULTS
|
from .config import THRESHOLD_DEFAULTS
|
||||||
|
|
||||||
@@ -29,12 +30,13 @@ class AlertLevel(Enum):
|
|||||||
|
|
||||||
class ComparisonOperator(Enum):
|
class ComparisonOperator(Enum):
|
||||||
"""Supported comparison operators for threshold checks."""
|
"""Supported comparison operators for threshold checks."""
|
||||||
GT = ">" # Greater than
|
GT = ">" # Greater than
|
||||||
GTE = ">=" # Greater than or equal
|
GTE = ">=" # Greater than or equal
|
||||||
LT = "<" # Less than
|
LT = "<" # Less than
|
||||||
LTE = "<=" # Less than or equal
|
LTE = "<=" # Less than or equal
|
||||||
EQ = "==" # Equal to
|
EQ = "==" # Equal to
|
||||||
NEQ = "!=" # Not equal to
|
NEQ = "!=" # Not equal to
|
||||||
|
NAGIOS = "nagios" # Nagios exit-code semantics: 0=OK 1=WARNING 2=CRITICAL 3=UNKNOWN
|
||||||
|
|
||||||
|
|
||||||
class AlertState:
|
class AlertState:
|
||||||
@@ -56,6 +58,7 @@ class AlertState:
|
|||||||
self.last_notification = None
|
self.last_notification = None
|
||||||
self.threshold_value = None # The threshold value that triggered alert
|
self.threshold_value = None # The threshold value that triggered alert
|
||||||
self.operator = None # The comparison operator (>, <, >=, etc.)
|
self.operator = None # The comparison operator (>, <, >=, etc.)
|
||||||
|
self.hysteresis: Optional[float] = None # Hysteresis fraction used for recovery
|
||||||
self.formatted_message = None # Formatted display message for UI
|
self.formatted_message = None # Formatted display message for UI
|
||||||
self.acknowledged = False # Whether alert has been acknowledged
|
self.acknowledged = False # Whether alert has been acknowledged
|
||||||
self.acknowledged_at = None # Timestamp when acknowledged
|
self.acknowledged_at = None # Timestamp when acknowledged
|
||||||
@@ -151,6 +154,15 @@ class AlertState:
|
|||||||
if self.formatted_message is not None:
|
if self.formatted_message is not None:
|
||||||
result["formatted_message"] = self.formatted_message
|
result["formatted_message"] = self.formatted_message
|
||||||
|
|
||||||
|
# Compute and expose the recovery threshold so the UI can display it
|
||||||
|
if (self.hysteresis and self.threshold_value is not None
|
||||||
|
and self.operator is not None):
|
||||||
|
ha = abs(self.threshold_value * self.hysteresis)
|
||||||
|
if self.operator in ('>', '>='):
|
||||||
|
result["recovery_threshold"] = round(self.threshold_value - ha, 4)
|
||||||
|
elif self.operator in ('<', '<='):
|
||||||
|
result["recovery_threshold"] = round(self.threshold_value + ha, 4)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def __setstate__(self, state):
|
def __setstate__(self, state):
|
||||||
@@ -158,6 +170,8 @@ class AlertState:
|
|||||||
self.__dict__.update(state)
|
self.__dict__.update(state)
|
||||||
if not hasattr(self, 'consecutive_count'):
|
if not hasattr(self, 'consecutive_count'):
|
||||||
self.consecutive_count = 0
|
self.consecutive_count = 0
|
||||||
|
if not hasattr(self, 'hysteresis'):
|
||||||
|
self.hysteresis = None
|
||||||
|
|
||||||
def acknowledge(self):
|
def acknowledge(self):
|
||||||
"""Acknowledge this alert to stop reminder notifications."""
|
"""Acknowledge this alert to stop reminder notifications."""
|
||||||
@@ -181,6 +195,7 @@ class ThresholdConfig:
|
|||||||
hysteresis: float = 0.0,
|
hysteresis: float = 0.0,
|
||||||
enabled: bool = True,
|
enabled: bool = True,
|
||||||
count: int = 1,
|
count: int = 1,
|
||||||
|
grace: Optional[float] = None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Initialize threshold configuration.
|
Initialize threshold configuration.
|
||||||
@@ -193,6 +208,7 @@ class ThresholdConfig:
|
|||||||
hysteresis: Hysteresis percentage to prevent flapping (0.0-1.0)
|
hysteresis: Hysteresis percentage to prevent flapping (0.0-1.0)
|
||||||
enabled: Whether this threshold is enabled
|
enabled: Whether this threshold is enabled
|
||||||
count: Number of consecutive exceedances required before alerting (default 1)
|
count: Number of consecutive exceedances required before alerting (default 1)
|
||||||
|
grace: Per-metric grace period in seconds; overrides global grace when set
|
||||||
"""
|
"""
|
||||||
self.metric_path = metric_path
|
self.metric_path = metric_path
|
||||||
self.warning = warning
|
self.warning = warning
|
||||||
@@ -201,6 +217,7 @@ class ThresholdConfig:
|
|||||||
self.hysteresis = hysteresis
|
self.hysteresis = hysteresis
|
||||||
self.display = display
|
self.display = display
|
||||||
self.count = max(1, int(count))
|
self.count = max(1, int(count))
|
||||||
|
self.grace = float(grace) if grace is not None else None
|
||||||
|
|
||||||
# Parse operator
|
# Parse operator
|
||||||
try:
|
try:
|
||||||
@@ -226,6 +243,16 @@ class ThresholdConfig:
|
|||||||
if not self.enabled:
|
if not self.enabled:
|
||||||
return AlertLevel.OK
|
return AlertLevel.OK
|
||||||
|
|
||||||
|
# Nagios exit-code semantics: value IS the severity
|
||||||
|
if self.operator == ComparisonOperator.NAGIOS:
|
||||||
|
try:
|
||||||
|
code = int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return AlertLevel.UNKNOWN
|
||||||
|
return {0: AlertLevel.OK, 1: AlertLevel.WARNING, 2: AlertLevel.CRITICAL}.get(
|
||||||
|
code, AlertLevel.UNKNOWN
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Convert value to float for comparison
|
# Convert value to float for comparison
|
||||||
value = float(value)
|
value = float(value)
|
||||||
@@ -262,6 +289,10 @@ class ThresholdConfig:
|
|||||||
"""
|
"""
|
||||||
new_level = self.evaluate(value)
|
new_level = self.evaluate(value)
|
||||||
|
|
||||||
|
# Nagios exit codes are discrete integers — hysteresis doesn't apply
|
||||||
|
if self.operator == ComparisonOperator.NAGIOS:
|
||||||
|
return new_level
|
||||||
|
|
||||||
# If no hysteresis, return new level
|
# If no hysteresis, return new level
|
||||||
if self.hysteresis == 0.0:
|
if self.hysteresis == 0.0:
|
||||||
return new_level
|
return new_level
|
||||||
@@ -328,14 +359,17 @@ class ThresholdChecker:
|
|||||||
renotify_interval: Seconds between repeat notifications (default: 1 hour)
|
renotify_interval: Seconds between repeat notifications (default: 1 hour)
|
||||||
journal: Optional MessageJournal instance for logging threshold events
|
journal: Optional MessageJournal instance for logging threshold events
|
||||||
"""
|
"""
|
||||||
# Named threshold configurations: {config_name: {metric_path: ThresholdConfig}}
|
# Named threshold configurations (pre-merged: defaults + overrides): {config_name: {metric_path: ThresholdConfig}}
|
||||||
self.threshold_configs = {}
|
self.threshold_configs = {}
|
||||||
|
|
||||||
|
# Raw overrides only for each named config (no defaults baked in): {config_name: {metric_path: ThresholdConfig}}
|
||||||
|
self.threshold_raw_configs: Dict[str, Dict[str, ThresholdConfig]] = {}
|
||||||
|
|
||||||
# Single threshold set for backward compatibility: {metric_path: ThresholdConfig}
|
# Single threshold set for backward compatibility: {metric_path: ThresholdConfig}
|
||||||
self.thresholds = {}
|
self.thresholds = {}
|
||||||
|
|
||||||
# Host to config name mapping: {host_name: config_name}
|
# Host to ordered list of config names: {host_name: [config_name, ...]}
|
||||||
self.host_config_mapping = {}
|
self.host_config_mapping: Dict[str, List[str]] = {}
|
||||||
|
|
||||||
# Default config name to use when no mapping exists
|
# Default config name to use when no mapping exists
|
||||||
self.default_config = "default"
|
self.default_config = "default"
|
||||||
@@ -372,6 +406,7 @@ class ThresholdChecker:
|
|||||||
|
|
||||||
# Clear old configuration
|
# Clear old configuration
|
||||||
self.threshold_configs.clear()
|
self.threshold_configs.clear()
|
||||||
|
self.threshold_raw_configs.clear()
|
||||||
self.thresholds.clear()
|
self.thresholds.clear()
|
||||||
self.host_config_mapping.clear()
|
self.host_config_mapping.clear()
|
||||||
self.grace_seconds = float(config.get("grace", 2))
|
self.grace_seconds = float(config.get("grace", 2))
|
||||||
@@ -391,10 +426,24 @@ class ThresholdChecker:
|
|||||||
Supports two formats:
|
Supports two formats:
|
||||||
1. Legacy format with direct 'thresholds' section
|
1. Legacy format with direct 'thresholds' section
|
||||||
2. New format with 'threshold_configs' and 'host_threshold_mapping'
|
2. New format with 'threshold_configs' and 'host_threshold_mapping'
|
||||||
|
|
||||||
|
In all cases, THRESHOLD_DEFAULTS are seeded into threshold_configs["default"]
|
||||||
|
so the Settings page always shows the built-in defaults.
|
||||||
|
_parse_multi_config() overwrites this with the fully-merged effective defaults.
|
||||||
"""
|
"""
|
||||||
|
# Always expose built-in defaults through threshold_configs["default"] so
|
||||||
|
# the Settings page has something to display even in legacy/no-config mode.
|
||||||
|
seed: Dict[str, ThresholdConfig] = {}
|
||||||
|
for plugin_name, plugin_thresholds in THRESHOLD_DEFAULTS.get("thresholds", {}).items():
|
||||||
|
if isinstance(plugin_thresholds, dict):
|
||||||
|
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=seed)
|
||||||
|
if seed:
|
||||||
|
self.threshold_configs["default"] = seed
|
||||||
|
self.threshold_raw_configs["default"] = {}
|
||||||
|
|
||||||
# Check for new multi-config format
|
# Check for new multi-config format
|
||||||
if "threshold_configs" in config:
|
if "threshold_configs" in config:
|
||||||
self._parse_multi_config(config)
|
self._parse_multi_config(config) # overwrites threshold_configs["default"]
|
||||||
elif "thresholds" in config:
|
elif "thresholds" in config:
|
||||||
# Legacy single threshold configuration
|
# Legacy single threshold configuration
|
||||||
self._parse_legacy_config(config)
|
self._parse_legacy_config(config)
|
||||||
@@ -424,9 +473,10 @@ class ThresholdChecker:
|
|||||||
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=effective_defaults)
|
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=effective_defaults)
|
||||||
|
|
||||||
self.threshold_configs["default"] = dict(effective_defaults)
|
self.threshold_configs["default"] = dict(effective_defaults)
|
||||||
|
self.threshold_raw_configs["default"] = {}
|
||||||
logger.info("Registered 'default' threshold config with %d metrics", len(effective_defaults))
|
logger.info("Registered 'default' threshold config with %d metrics", len(effective_defaults))
|
||||||
|
|
||||||
# Parse each named configuration, seeding it with effective_defaults first
|
# Parse each named configuration
|
||||||
for config_name, config_data in threshold_configs.items():
|
for config_name, config_data in threshold_configs.items():
|
||||||
if config_name == "default":
|
if config_name == "default":
|
||||||
continue # already handled above
|
continue # already handled above
|
||||||
@@ -440,33 +490,61 @@ class ThresholdChecker:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
logger.info("Parsing threshold configuration: %s", config_name)
|
logger.info("Parsing threshold configuration: %s", config_name)
|
||||||
self.threshold_configs[config_name] = dict(effective_defaults)
|
|
||||||
|
|
||||||
|
# Raw overrides only (used for multi-config layering)
|
||||||
|
raw_overrides: Dict[str, ThresholdConfig] = {}
|
||||||
thresholds_config = config_data["thresholds"]
|
thresholds_config = config_data["thresholds"]
|
||||||
for plugin_name, plugin_thresholds in thresholds_config.items():
|
for plugin_name, plugin_thresholds in thresholds_config.items():
|
||||||
if not isinstance(plugin_thresholds, dict):
|
if not isinstance(plugin_thresholds, dict):
|
||||||
continue
|
continue
|
||||||
|
plugin_enabled = plugin_thresholds.get('enabled', plugin_thresholds.get('enable', True))
|
||||||
|
if not plugin_enabled:
|
||||||
|
# raw_overrides is empty at this point so there's nothing to delete.
|
||||||
|
# Instead, inject disabled stubs for every matching effective_default so
|
||||||
|
# the merge step overwrites the inherited defaults.
|
||||||
|
for key, tc in effective_defaults.items():
|
||||||
|
if key.startswith(f"{plugin_name}."):
|
||||||
|
raw_overrides[key] = ThresholdConfig(
|
||||||
|
metric_path=key,
|
||||||
|
warning=tc.warning,
|
||||||
|
critical=tc.critical,
|
||||||
|
operator=tc.operator.value,
|
||||||
|
enabled=False,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
"Plugin-level disable in config '%s': disabled all thresholds for %s",
|
||||||
|
config_name, plugin_name,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self._parse_plugin_thresholds(plugin_name, plugin_thresholds, target_dict=raw_overrides)
|
||||||
|
self.threshold_raw_configs[config_name] = raw_overrides
|
||||||
|
|
||||||
self._parse_plugin_thresholds(
|
# Pre-merged version (defaults + overrides) for single-config fast path
|
||||||
plugin_name,
|
self.threshold_configs[config_name] = dict(effective_defaults)
|
||||||
plugin_thresholds,
|
self.threshold_configs[config_name].update(raw_overrides)
|
||||||
target_dict=self.threshold_configs[config_name]
|
|
||||||
)
|
|
||||||
|
|
||||||
# Parse host to config mapping from two possible sources
|
# Parse host → config list mapping from two possible sources
|
||||||
# 1. New format: hosts section with threshold_config attribute
|
|
||||||
|
def _normalise(value) -> List[str]:
|
||||||
|
"""Accept a string or list; always return a list."""
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [str(v) for v in value]
|
||||||
|
return [str(value)]
|
||||||
|
|
||||||
|
# 1. hosts section with threshold_config attribute (string or list)
|
||||||
if "hosts" in config:
|
if "hosts" in config:
|
||||||
hosts_config = config["hosts"]
|
hosts_config = config["hosts"]
|
||||||
if isinstance(hosts_config, dict):
|
if isinstance(hosts_config, dict):
|
||||||
for host_name, host_attrs in hosts_config.items():
|
for host_name, host_attrs in hosts_config.items():
|
||||||
if isinstance(host_attrs, dict) and "threshold_config" in host_attrs:
|
if isinstance(host_attrs, dict) and "threshold_config" in host_attrs:
|
||||||
self.host_config_mapping[host_name] = host_attrs["threshold_config"]
|
self.host_config_mapping[host_name] = _normalise(host_attrs["threshold_config"])
|
||||||
|
|
||||||
# 2. Legacy format: host_threshold_mapping section (for backward compatibility)
|
# 2. Legacy host_threshold_mapping section (string values only)
|
||||||
if "host_threshold_mapping" in config:
|
if "host_threshold_mapping" in config:
|
||||||
legacy_mapping = config.get("host_threshold_mapping", {})
|
legacy_mapping = config.get("host_threshold_mapping", {})
|
||||||
if isinstance(legacy_mapping, dict):
|
if isinstance(legacy_mapping, dict):
|
||||||
self.host_config_mapping.update(legacy_mapping)
|
for host_name, value in legacy_mapping.items():
|
||||||
|
self.host_config_mapping[host_name] = _normalise(value)
|
||||||
|
|
||||||
# Set default config (first one alphabetically or explicitly set)
|
# Set default config (first one alphabetically or explicitly set)
|
||||||
self.default_config = config.get("default_threshold_config", "default")
|
self.default_config = config.get("default_threshold_config", "default")
|
||||||
@@ -516,14 +594,26 @@ class ThresholdChecker:
|
|||||||
self._parse_rtt_thresholds(thresholds, target_dict)
|
self._parse_rtt_thresholds(thresholds, target_dict)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Plugin-level enabled: false (also accept 'enable' as a common typo) removes all
|
||||||
|
# thresholds for this plugin — e.g. memory_monitor: {enabled: false}.
|
||||||
|
plugin_enabled = thresholds.get('enabled', thresholds.get('enable', True))
|
||||||
|
if not plugin_enabled:
|
||||||
|
for key in [k for k in target_dict if k.startswith(f"{plugin_name}.")]:
|
||||||
|
del target_dict[key]
|
||||||
|
logger.info("Plugin-level disable: removed all thresholds for %s", plugin_name)
|
||||||
|
return
|
||||||
|
|
||||||
for metric_name, threshold_config in thresholds.items():
|
for metric_name, threshold_config in thresholds.items():
|
||||||
if not isinstance(threshold_config, dict):
|
if not isinstance(threshold_config, dict):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Handle nested metrics (e.g., partitions./.percent)
|
# Handle nested metrics (e.g., partitions./.percent or pools.*.status)
|
||||||
if metric_name == "partitions":
|
if metric_name == "partitions":
|
||||||
self._parse_partition_thresholds(plugin_name, threshold_config, target_dict)
|
self._parse_partition_thresholds(plugin_name, threshold_config, target_dict)
|
||||||
continue
|
continue
|
||||||
|
if metric_name == "pools":
|
||||||
|
self._parse_pool_thresholds(plugin_name, threshold_config, target_dict)
|
||||||
|
continue
|
||||||
|
|
||||||
metric_path = f"{plugin_name}.{metric_name}"
|
metric_path = f"{plugin_name}.{metric_name}"
|
||||||
|
|
||||||
@@ -531,11 +621,15 @@ class ThresholdChecker:
|
|||||||
warning = threshold_config.get("warning")
|
warning = threshold_config.get("warning")
|
||||||
critical = threshold_config.get("critical")
|
critical = threshold_config.get("critical")
|
||||||
operator = threshold_config.get("operator", ">")
|
operator = threshold_config.get("operator", ">")
|
||||||
display = threshold_config.get("display", "(threshold: {op_symbol} {threshold_value})")
|
# Nagios operator maps exit codes directly; no numeric thresholds needed
|
||||||
hysteresis = threshold_config.get("hysteresis", 0.1) # 10% default
|
is_nagios_op = (operator == "nagios")
|
||||||
|
default_display = "{check_name}: {output}" if is_nagios_op else "(threshold: {op_symbol} {threshold_value})"
|
||||||
|
display = threshold_config.get("display", default_display)
|
||||||
|
hysteresis = threshold_config.get("hysteresis", 0.0 if is_nagios_op else 0.02)
|
||||||
enabled = threshold_config.get("enabled", True)
|
enabled = threshold_config.get("enabled", True)
|
||||||
|
grace = threshold_config.get("grace", None)
|
||||||
|
|
||||||
if warning is None and critical is None:
|
if warning is None and critical is None and not is_nagios_op:
|
||||||
logger.warning("No thresholds defined for %s, skipping", metric_path)
|
logger.warning("No thresholds defined for %s, skipping", metric_path)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -546,7 +640,8 @@ class ThresholdChecker:
|
|||||||
operator=operator,
|
operator=operator,
|
||||||
hysteresis=hysteresis,
|
hysteresis=hysteresis,
|
||||||
enabled=enabled,
|
enabled=enabled,
|
||||||
display=display
|
display=display,
|
||||||
|
grace=grace,
|
||||||
)
|
)
|
||||||
|
|
||||||
target_dict[metric_path] = threshold
|
target_dict[metric_path] = threshold
|
||||||
@@ -591,6 +686,7 @@ class ThresholdChecker:
|
|||||||
hysteresis = threshold_config.get("hysteresis", 0.1)
|
hysteresis = threshold_config.get("hysteresis", 0.1)
|
||||||
enabled = threshold_config.get("enabled", True)
|
enabled = threshold_config.get("enabled", True)
|
||||||
display = threshold_config.get("display")
|
display = threshold_config.get("display")
|
||||||
|
grace = threshold_config.get("grace", None)
|
||||||
if warning is None and critical is None:
|
if warning is None and critical is None:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -601,11 +697,64 @@ class ThresholdChecker:
|
|||||||
operator=operator,
|
operator=operator,
|
||||||
hysteresis=hysteresis,
|
hysteresis=hysteresis,
|
||||||
enabled=enabled,
|
enabled=enabled,
|
||||||
display=display
|
display=display,
|
||||||
|
grace=grace,
|
||||||
)
|
)
|
||||||
|
|
||||||
target_dict[metric_path] = threshold
|
target_dict[metric_path] = threshold
|
||||||
|
|
||||||
|
def _parse_pool_thresholds(
|
||||||
|
self,
|
||||||
|
plugin_name: str,
|
||||||
|
pools: Dict[str, Any],
|
||||||
|
target_dict: Optional[Dict[str, ThresholdConfig]] = None,
|
||||||
|
):
|
||||||
|
"""Parse ZFS pool thresholds. Pool names may be literal or '*' (all pools).
|
||||||
|
|
||||||
|
Config shape::
|
||||||
|
|
||||||
|
zfs_monitor:
|
||||||
|
pools:
|
||||||
|
'*':
|
||||||
|
status:
|
||||||
|
warning: 1
|
||||||
|
critical: 2
|
||||||
|
operator: '>'
|
||||||
|
tank:
|
||||||
|
capacity:
|
||||||
|
warning: 80
|
||||||
|
critical: 90
|
||||||
|
"""
|
||||||
|
if target_dict is None:
|
||||||
|
target_dict = self.thresholds
|
||||||
|
|
||||||
|
for pool_name, metrics in pools.items():
|
||||||
|
if not isinstance(metrics, dict):
|
||||||
|
continue
|
||||||
|
for metric_name, threshold_config in metrics.items():
|
||||||
|
if not isinstance(threshold_config, dict):
|
||||||
|
continue
|
||||||
|
metric_path = f"{plugin_name}.{pool_name}.{metric_name}"
|
||||||
|
warning = threshold_config.get("warning")
|
||||||
|
critical = threshold_config.get("critical")
|
||||||
|
operator = threshold_config.get("operator", ">")
|
||||||
|
hysteresis = threshold_config.get("hysteresis", 0.02)
|
||||||
|
enabled = threshold_config.get("enabled", True)
|
||||||
|
display = threshold_config.get("display")
|
||||||
|
grace = threshold_config.get("grace", None)
|
||||||
|
if warning is None and critical is None:
|
||||||
|
continue
|
||||||
|
target_dict[metric_path] = ThresholdConfig(
|
||||||
|
metric_path=metric_path,
|
||||||
|
warning=warning,
|
||||||
|
critical=critical,
|
||||||
|
operator=operator,
|
||||||
|
hysteresis=hysteresis,
|
||||||
|
enabled=enabled,
|
||||||
|
display=display,
|
||||||
|
grace=grace,
|
||||||
|
)
|
||||||
|
|
||||||
def _parse_rtt_thresholds(
|
def _parse_rtt_thresholds(
|
||||||
self,
|
self,
|
||||||
rtt_thresholds: Dict[str, Any],
|
rtt_thresholds: Dict[str, Any],
|
||||||
@@ -635,10 +784,11 @@ class ThresholdChecker:
|
|||||||
warning = rtt_thresholds.get("warning")
|
warning = rtt_thresholds.get("warning")
|
||||||
critical = rtt_thresholds.get("critical")
|
critical = rtt_thresholds.get("critical")
|
||||||
operator = rtt_thresholds.get("operator", ">")
|
operator = rtt_thresholds.get("operator", ">")
|
||||||
hysteresis = rtt_thresholds.get("hysteresis", 0.1) # 10% default
|
hysteresis = rtt_thresholds.get("hysteresis", 0.02) # 2% default
|
||||||
enabled = rtt_thresholds.get("enabled", True)
|
enabled = rtt_thresholds.get("enabled", True)
|
||||||
display = rtt_thresholds.get("display")
|
display = rtt_thresholds.get("display")
|
||||||
count = rtt_thresholds.get("count", 1)
|
count = rtt_thresholds.get("count", 1)
|
||||||
|
grace = rtt_thresholds.get("grace", None)
|
||||||
|
|
||||||
if warning is None and critical is None:
|
if warning is None and critical is None:
|
||||||
logger.warning("No RTT thresholds defined, skipping")
|
logger.warning("No RTT thresholds defined, skipping")
|
||||||
@@ -653,6 +803,7 @@ class ThresholdChecker:
|
|||||||
enabled=enabled,
|
enabled=enabled,
|
||||||
display=display,
|
display=display,
|
||||||
count=count,
|
count=count,
|
||||||
|
grace=grace,
|
||||||
)
|
)
|
||||||
|
|
||||||
target_dict[metric_path] = threshold
|
target_dict[metric_path] = threshold
|
||||||
@@ -664,7 +815,10 @@ class ThresholdChecker:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def get_thresholds_for_host(self, host_name: str) -> Dict[str, ThresholdConfig]:
|
def get_thresholds_for_host(self, host_name: str) -> Dict[str, ThresholdConfig]:
|
||||||
"""Get the appropriate threshold configuration for a host.
|
"""Get the effective threshold configuration for a host.
|
||||||
|
|
||||||
|
When threshold_config is a list, configs are applied left-to-right on top
|
||||||
|
of the default thresholds so earlier entries can be overridden by later ones.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
host_name: Name of the host
|
host_name: Name of the host
|
||||||
@@ -676,23 +830,40 @@ class ThresholdChecker:
|
|||||||
if self.thresholds and not self.threshold_configs:
|
if self.thresholds and not self.threshold_configs:
|
||||||
return self.thresholds
|
return self.thresholds
|
||||||
|
|
||||||
# Multi-config mode: look up host-specific configuration
|
if not self.threshold_configs:
|
||||||
if self.threshold_configs:
|
return {}
|
||||||
config_name = self.host_config_mapping.get(host_name, self.default_config)
|
|
||||||
|
|
||||||
if config_name in self.threshold_configs:
|
config_names = self.host_config_mapping.get(host_name)
|
||||||
return self.threshold_configs[config_name]
|
|
||||||
else:
|
# No host-specific mapping → return pre-merged default
|
||||||
|
if not config_names:
|
||||||
|
return self.threshold_configs.get(self.default_config, {})
|
||||||
|
|
||||||
|
# Single config → fast path using pre-merged copy
|
||||||
|
if len(config_names) == 1:
|
||||||
|
name = config_names[0]
|
||||||
|
if name in self.threshold_configs:
|
||||||
|
return self.threshold_configs[name]
|
||||||
|
logger.warning(
|
||||||
|
"Threshold config '%s' not found for host '%s', using default '%s'",
|
||||||
|
name, host_name, self.default_config,
|
||||||
|
)
|
||||||
|
return self.threshold_configs.get(self.default_config, {})
|
||||||
|
|
||||||
|
# Multiple configs → start from defaults, layer raw overrides in order
|
||||||
|
result = dict(self.threshold_configs.get(self.default_config, {}))
|
||||||
|
for name in config_names:
|
||||||
|
if name == self.default_config:
|
||||||
|
continue # defaults already the base
|
||||||
|
raw = self.threshold_raw_configs.get(name)
|
||||||
|
if raw is None:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Threshold config '%s' not found for host '%s', using default '%s'",
|
"Threshold config '%s' not found for host '%s', skipping",
|
||||||
config_name,
|
name, host_name,
|
||||||
host_name,
|
|
||||||
self.default_config
|
|
||||||
)
|
)
|
||||||
return self.threshold_configs.get(self.default_config, {})
|
else:
|
||||||
|
result.update(raw)
|
||||||
# No thresholds configured
|
return result
|
||||||
return {}
|
|
||||||
|
|
||||||
def check_value(
|
def check_value(
|
||||||
self,
|
self,
|
||||||
@@ -760,6 +931,12 @@ class ThresholdChecker:
|
|||||||
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||||
threshold_value = threshold.warning
|
threshold_value = threshold.warning
|
||||||
|
|
||||||
|
# Keep hysteresis on the state so the UI can show the recovery threshold
|
||||||
|
if new_level != AlertLevel.OK:
|
||||||
|
alert_state.hysteresis = threshold.hysteresis
|
||||||
|
else:
|
||||||
|
alert_state.hysteresis = None
|
||||||
|
|
||||||
# Update state and check for changes
|
# Update state and check for changes
|
||||||
old_level = alert_state.level
|
old_level = alert_state.level
|
||||||
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
||||||
@@ -769,6 +946,36 @@ class ThresholdChecker:
|
|||||||
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, None)
|
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, None)
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
def _find_threshold(
|
||||||
|
self, thresholds: Dict[str, "ThresholdConfig"], metric_path: str
|
||||||
|
) -> Tuple[Optional["ThresholdConfig"], Optional[str]]:
|
||||||
|
"""Return (threshold, check_name) for *metric_path*, falling back to suffix matches.
|
||||||
|
|
||||||
|
Allows generic thresholds like ``nagios_runner.status_code`` to match
|
||||||
|
fully-qualified paths like ``nagios_runner.check_disk_root_status_code``.
|
||||||
|
The exact match is always tried first; then successive leading
|
||||||
|
underscore-delimited segments are stripped from the field name until
|
||||||
|
a match is found or no segments remain.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(ThresholdConfig, None) for an exact match.
|
||||||
|
(ThresholdConfig, "check_disk_root") for a suffix match — the second
|
||||||
|
element is the stripped prefix, available as ``{check_name}`` in
|
||||||
|
display format templates.
|
||||||
|
(None, None) when no threshold is found.
|
||||||
|
"""
|
||||||
|
if metric_path in thresholds:
|
||||||
|
return thresholds[metric_path], None
|
||||||
|
plugin, sep, field = metric_path.partition(".")
|
||||||
|
if not sep:
|
||||||
|
return None, None
|
||||||
|
parts = field.split("_")
|
||||||
|
for i in range(1, len(parts)):
|
||||||
|
candidate = plugin + "." + "_".join(parts[i:])
|
||||||
|
if candidate in thresholds:
|
||||||
|
return thresholds[candidate], "_".join(parts[:i])
|
||||||
|
return None, None
|
||||||
|
|
||||||
def check_plugin_data(
|
def check_plugin_data(
|
||||||
self,
|
self,
|
||||||
host_name: str,
|
host_name: str,
|
||||||
@@ -797,11 +1004,10 @@ class ThresholdChecker:
|
|||||||
for metric_name, value in data.items():
|
for metric_name, value in data.items():
|
||||||
metric_path = f"{plugin_name}.{metric_name}"
|
metric_path = f"{plugin_name}.{metric_name}"
|
||||||
|
|
||||||
if metric_path not in thresholds:
|
threshold, check_name = self._find_threshold(thresholds, metric_path)
|
||||||
|
if threshold is None:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
threshold = thresholds[metric_path]
|
|
||||||
|
|
||||||
# Get or create alert state
|
# Get or create alert state
|
||||||
if metric_path not in alert_states:
|
if metric_path not in alert_states:
|
||||||
alert_states[metric_path] = AlertState(metric_path)
|
alert_states[metric_path] = AlertState(metric_path)
|
||||||
@@ -821,13 +1027,15 @@ class ThresholdChecker:
|
|||||||
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||||
threshold_value = threshold.warning
|
threshold_value = threshold.warning
|
||||||
|
|
||||||
|
alert_state.hysteresis = threshold.hysteresis if new_level != AlertLevel.OK else None
|
||||||
|
|
||||||
# Update state and check for changes
|
# Update state and check for changes
|
||||||
old_level = alert_state.level
|
old_level = alert_state.level
|
||||||
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
||||||
state_changes.append((metric_path, old_level, new_level, value))
|
state_changes.append((metric_path, old_level, new_level, value))
|
||||||
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, data)
|
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, data, check_name=check_name, metric_name=metric_name)
|
||||||
elif new_level != AlertLevel.OK:
|
elif new_level != AlertLevel.OK:
|
||||||
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, data)
|
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, data, check_name=check_name, metric_name=metric_name)
|
||||||
|
|
||||||
# Check nested metrics (e.g., partition data in disk_monitor)
|
# Check nested metrics (e.g., partition data in disk_monitor)
|
||||||
self._check_nested_metrics(
|
self._check_nested_metrics(
|
||||||
@@ -852,6 +1060,44 @@ class ThresholdChecker:
|
|||||||
# Get host-specific thresholds
|
# Get host-specific thresholds
|
||||||
thresholds = self.get_thresholds_for_host(host_name)
|
thresholds = self.get_thresholds_for_host(host_name)
|
||||||
|
|
||||||
|
# ZFS pool health checks
|
||||||
|
if plugin_name == "zfs_monitor" and "pools" in data:
|
||||||
|
pools = data["pools"]
|
||||||
|
if isinstance(pools, dict):
|
||||||
|
for pool_name, pool_metrics in pools.items():
|
||||||
|
if not isinstance(pool_metrics, dict):
|
||||||
|
continue
|
||||||
|
# Synthesize status from health string for older clients
|
||||||
|
# that predate the status field.
|
||||||
|
pool_metrics_effective = dict(pool_metrics)
|
||||||
|
if "health" in pool_metrics and "status" not in pool_metrics:
|
||||||
|
pool_metrics_effective["status"] = 0 if pool_metrics["health"] == "ONLINE" else 1
|
||||||
|
for metric_name, value in pool_metrics_effective.items():
|
||||||
|
# Try specific pool name first, then wildcard '*'
|
||||||
|
metric_path = f"{plugin_name}.{pool_name}.{metric_name}"
|
||||||
|
wildcard_path = f"{plugin_name}.*.{metric_name}"
|
||||||
|
threshold = thresholds.get(metric_path) or thresholds.get(wildcard_path)
|
||||||
|
if threshold is None:
|
||||||
|
continue
|
||||||
|
if metric_path not in alert_states:
|
||||||
|
alert_states[metric_path] = AlertState(metric_path)
|
||||||
|
alert_state = alert_states[metric_path]
|
||||||
|
new_level = threshold.evaluate_with_hysteresis(value, alert_state.level)
|
||||||
|
threshold_value = None
|
||||||
|
if new_level == AlertLevel.CRITICAL and threshold.critical is not None:
|
||||||
|
threshold_value = threshold.critical
|
||||||
|
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||||
|
threshold_value = threshold.warning
|
||||||
|
alert_state.hysteresis = threshold.hysteresis if new_level != AlertLevel.OK else None
|
||||||
|
pool_context = dict(pool_metrics_effective)
|
||||||
|
pool_context["pool_name"] = pool_name
|
||||||
|
old_level = alert_state.level
|
||||||
|
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
||||||
|
state_changes.append((metric_path, old_level, new_level, value))
|
||||||
|
self._apply_grace(host_name, alert_state, metric_path, old_level, new_level, value, threshold, pool_context, metric_name=pool_name)
|
||||||
|
elif new_level != AlertLevel.OK:
|
||||||
|
self._check_pending_or_renotify(host_name, alert_state, metric_path, value, threshold, pool_context, metric_name=pool_name)
|
||||||
|
|
||||||
# Look for partition data in disk_monitor
|
# Look for partition data in disk_monitor
|
||||||
if plugin_name == "disk_monitor" and "partitions" in data:
|
if plugin_name == "disk_monitor" and "partitions" in data:
|
||||||
partitions = data["partitions"]
|
partitions = data["partitions"]
|
||||||
@@ -887,6 +1133,8 @@ class ThresholdChecker:
|
|||||||
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
elif new_level == AlertLevel.WARNING and threshold.warning is not None:
|
||||||
threshold_value = threshold.warning
|
threshold_value = threshold.warning
|
||||||
|
|
||||||
|
alert_state.hysteresis = threshold.hysteresis if new_level != AlertLevel.OK else None
|
||||||
|
|
||||||
old_level = alert_state.level
|
old_level = alert_state.level
|
||||||
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
if alert_state.update(new_level, value, threshold_value, threshold.operator.value):
|
||||||
state_changes.append((metric_path, old_level, new_level, value))
|
state_changes.append((metric_path, old_level, new_level, value))
|
||||||
@@ -903,6 +1151,8 @@ class ThresholdChecker:
|
|||||||
value: Any,
|
value: Any,
|
||||||
threshold: ThresholdConfig,
|
threshold: ThresholdConfig,
|
||||||
plugin_data: Optional[Dict[str, Any]] = None,
|
plugin_data: Optional[Dict[str, Any]] = None,
|
||||||
|
check_name: Optional[str] = None,
|
||||||
|
metric_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
"""Trigger a notification for an alert state change.
|
"""Trigger a notification for an alert state change.
|
||||||
|
|
||||||
@@ -925,55 +1175,53 @@ class ThresholdChecker:
|
|||||||
# Format operator symbol
|
# Format operator symbol
|
||||||
op_symbol = threshold.operator.value
|
op_symbol = threshold.operator.value
|
||||||
|
|
||||||
|
# Short metric label: strip the plugin-name prefix and _status_code suffix
|
||||||
|
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
|
||||||
|
|
||||||
# Use a display-friendly value (inf is the sentinel for "overdue")
|
# Use a display-friendly value (inf is the sentinel for "overdue")
|
||||||
import math
|
import math
|
||||||
display_value = "overdue" if isinstance(value, float) and math.isinf(value) else value
|
display_value = "overdue" if isinstance(value, float) and math.isinf(value) else value
|
||||||
|
|
||||||
# Format message
|
# Format message — for the nagios operator there is no numeric threshold_value;
|
||||||
if new_level == AlertLevel.OK:
|
# render the display template whenever one is available.
|
||||||
lvl = "RECOVER"
|
has_display = threshold_value is not None or threshold.operator == ComparisonOperator.NAGIOS
|
||||||
message = f"{metric_path} = {display_value} ({old_level.name} -> OK)"
|
|
||||||
elif new_level == AlertLevel.WARNING:
|
|
||||||
lvl = "WARNING"
|
|
||||||
if threshold_value is not None:
|
|
||||||
threshold_info = self._format_display(
|
|
||||||
threshold.display,
|
|
||||||
value=display_value,
|
|
||||||
threshold_value=threshold_value,
|
|
||||||
op_symbol=op_symbol,
|
|
||||||
plugin_data=plugin_data
|
|
||||||
)
|
|
||||||
message = f"{metric_path} = {display_value} {threshold_info}"
|
|
||||||
else:
|
|
||||||
message = f"{metric_path} = {display_value}"
|
|
||||||
elif new_level == AlertLevel.CRITICAL:
|
|
||||||
lvl = "CRITICAL"
|
|
||||||
if threshold_value is not None:
|
|
||||||
threshold_info = self._format_display(
|
|
||||||
threshold.display,
|
|
||||||
value=display_value,
|
|
||||||
threshold_value=threshold_value,
|
|
||||||
op_symbol=op_symbol,
|
|
||||||
plugin_data=plugin_data
|
|
||||||
)
|
|
||||||
message = f"{metric_path} = {display_value} {threshold_info}"
|
|
||||||
else:
|
|
||||||
message = f"{metric_path} = {display_value}"
|
|
||||||
else:
|
|
||||||
lvl = "UNKNOWN"
|
|
||||||
message = f"{metric_path} = {display_value}"
|
|
||||||
|
|
||||||
# Return the formatted threshold info for storing in AlertState
|
def _fmt():
|
||||||
formatted_threshold_msg = None
|
return self._format_display(
|
||||||
if threshold_value is not None and new_level != AlertLevel.OK:
|
|
||||||
formatted_threshold_msg = self._format_display(
|
|
||||||
threshold.display,
|
threshold.display,
|
||||||
value=display_value,
|
value=display_value,
|
||||||
threshold_value=threshold_value,
|
threshold_value=threshold_value,
|
||||||
op_symbol=op_symbol,
|
op_symbol=op_symbol,
|
||||||
plugin_data=plugin_data
|
plugin_data=plugin_data,
|
||||||
|
check_name=check_name,
|
||||||
|
metric_name=metric_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if new_level == AlertLevel.OK:
|
||||||
|
lvl = "RECOVER"
|
||||||
|
message = f"{short_path} = {display_value} ({old_level.name} -> OK)"
|
||||||
|
elif new_level == AlertLevel.WARNING:
|
||||||
|
lvl = "WARNING"
|
||||||
|
if has_display:
|
||||||
|
message = f"{short_path} = {display_value} {_fmt()}"
|
||||||
|
else:
|
||||||
|
message = f"{short_path} = {display_value}"
|
||||||
|
elif new_level == AlertLevel.CRITICAL:
|
||||||
|
lvl = "CRITICAL"
|
||||||
|
if has_display:
|
||||||
|
message = f"{short_path} = {display_value} {_fmt()}"
|
||||||
|
else:
|
||||||
|
message = f"{short_path} = {display_value}"
|
||||||
|
else:
|
||||||
|
lvl = "UNKNOWN"
|
||||||
|
if has_display:
|
||||||
|
message = f"{short_path} = {display_value} {_fmt()}"
|
||||||
|
else:
|
||||||
|
message = f"{short_path} = {display_value}"
|
||||||
|
|
||||||
|
# Formatted threshold info stored on AlertState for the UI
|
||||||
|
formatted_threshold_msg = _fmt() if has_display and new_level != AlertLevel.OK else None
|
||||||
|
|
||||||
return lvl, message, formatted_threshold_msg
|
return lvl, message, formatted_threshold_msg
|
||||||
|
|
||||||
def _send_notification(
|
def _send_notification(
|
||||||
@@ -987,19 +1235,29 @@ class ThresholdChecker:
|
|||||||
value: Any,
|
value: Any,
|
||||||
):
|
):
|
||||||
"""Send notification and log to journal/eventlog."""
|
"""Send notification and log to journal/eventlog."""
|
||||||
|
from . import hbdclass
|
||||||
|
host = hbdclass.Host.hosts.get(host_name)
|
||||||
|
if host is not None and not host.watched:
|
||||||
|
eventlog(host_name, lvl, message, service="threshold")
|
||||||
|
return
|
||||||
|
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
|
||||||
|
title = f"[{lvl}] {host_name} {short_path}"
|
||||||
|
# Strip the "metric = " prefix from message so body is just the value/detail
|
||||||
|
prefix = short_path + " = "
|
||||||
|
body = message[len(prefix):] if message.startswith(prefix) else message
|
||||||
asyncio.get_event_loop().create_task(notify_mod.send_notification(
|
asyncio.get_event_loop().create_task(notify_mod.send_notification(
|
||||||
host_name,
|
host_name,
|
||||||
notify_mod.Notification(
|
notify_mod.Notification(
|
||||||
title=f"[{lvl}] {host_name}",
|
title=title,
|
||||||
body=message,
|
body=body,
|
||||||
level=lvl,
|
level=lvl,
|
||||||
|
service=short_path,
|
||||||
),
|
),
|
||||||
))
|
))
|
||||||
|
|
||||||
# Log to journal
|
# Log to journal
|
||||||
if self.journal is not None:
|
if self.journal is not None:
|
||||||
try:
|
try:
|
||||||
import asyncio
|
|
||||||
loop = asyncio.get_event_loop()
|
loop = asyncio.get_event_loop()
|
||||||
loop.create_task(self.journal.log_threshold_event(
|
loop.create_task(self.journal.log_threshold_event(
|
||||||
host_name=host_name,
|
host_name=host_name,
|
||||||
@@ -1017,33 +1275,62 @@ class ThresholdChecker:
|
|||||||
self,
|
self,
|
||||||
display_format: str,
|
display_format: str,
|
||||||
value: Any,
|
value: Any,
|
||||||
threshold_value: float,
|
threshold_value: Optional[float],
|
||||||
op_symbol: str,
|
op_symbol: str,
|
||||||
plugin_data: Optional[Dict[str, Any]] = None,
|
plugin_data: Optional[Dict[str, Any]] = None,
|
||||||
|
check_name: Optional[str] = None,
|
||||||
|
metric_name: Optional[str] = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Format the display string using available data.
|
"""Format the display string using available data.
|
||||||
|
|
||||||
Args:
|
Available template variables:
|
||||||
display_format: Format string from threshold config
|
{value} - current metric value
|
||||||
value: Current metric value
|
{threshold_value} - threshold that was exceeded
|
||||||
threshold_value: Threshold value that was exceeded
|
{op_symbol} - comparison operator (>, <, >=, <=, ==, !=)
|
||||||
op_symbol: Comparison operator symbol
|
{check_name} - prefix stripped for generic threshold match
|
||||||
plugin_data: Optional dictionary of plugin data fields
|
(e.g. "check_disk_root" when metric
|
||||||
|
"check_disk_root_status_code" matched generic
|
||||||
|
threshold "status_code")
|
||||||
|
{metric_name} - field name within the plugin data dict
|
||||||
|
Any key from plugin_data is also available.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Formatted display string
|
Formatted display string
|
||||||
"""
|
"""
|
||||||
|
if not display_format:
|
||||||
|
display_format = "(threshold: {op_symbol} {threshold_value})" if threshold_value is not None else ""
|
||||||
|
|
||||||
# Build format context with standard variables
|
# Build format context with standard variables
|
||||||
format_context = {
|
format_context = {
|
||||||
'value': value,
|
'value': value,
|
||||||
'threshold_value': threshold_value,
|
|
||||||
'op_symbol': op_symbol,
|
'op_symbol': op_symbol,
|
||||||
}
|
}
|
||||||
|
if threshold_value is not None:
|
||||||
|
format_context['threshold_value'] = threshold_value
|
||||||
|
|
||||||
|
# Add generic-match context variables when available
|
||||||
|
if check_name is not None:
|
||||||
|
format_context['check_name'] = check_name
|
||||||
|
if metric_name is not None:
|
||||||
|
format_context['metric_name'] = metric_name
|
||||||
|
|
||||||
# Add all plugin data fields if available
|
# Add all plugin data fields if available
|
||||||
if plugin_data:
|
if plugin_data:
|
||||||
format_context.update(plugin_data)
|
format_context.update(plugin_data)
|
||||||
|
|
||||||
|
# For nagios_runner generic matches, expose the matched check's output
|
||||||
|
# and status as short aliases {output} and {status} so display templates
|
||||||
|
# don't need to use the full {check_disk_root_output} form.
|
||||||
|
if check_name and plugin_data:
|
||||||
|
if 'output' not in format_context:
|
||||||
|
output = plugin_data.get(f"{check_name}_output")
|
||||||
|
if output is not None:
|
||||||
|
format_context['output'] = output
|
||||||
|
if 'status' not in format_context:
|
||||||
|
status = plugin_data.get(f"{check_name}_status")
|
||||||
|
if status is not None:
|
||||||
|
format_context['status'] = status
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Format the display string
|
# Format the display string
|
||||||
return display_format.format(**format_context)
|
return display_format.format(**format_context)
|
||||||
@@ -1073,17 +1360,26 @@ class ThresholdChecker:
|
|||||||
value: Any,
|
value: Any,
|
||||||
threshold: ThresholdConfig,
|
threshold: ThresholdConfig,
|
||||||
plugin_data: Optional[Dict[str, Any]],
|
plugin_data: Optional[Dict[str, Any]],
|
||||||
|
check_name: Optional[str] = None,
|
||||||
|
metric_name: Optional[str] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Handle a state-change transition with grace-period logic.
|
"""Handle a state-change transition with grace-period logic.
|
||||||
|
|
||||||
Transitioning INTO alert: defers the notification for grace_seconds.
|
Transitioning INTO alert (worsening): defers the notification for the effective
|
||||||
|
grace period (threshold.grace if set, else self.grace_seconds). Grace of 0 fires
|
||||||
|
the notification immediately with no deferral.
|
||||||
|
De-escalation within alert states (e.g. CRITICAL→WARNING): no new notification;
|
||||||
|
the metric is still alerting so no RECOVER was sent.
|
||||||
Transitioning TO OK:
|
Transitioning TO OK:
|
||||||
- Still in grace window (pending_since set): suppresses both the alert
|
- Still in grace window (pending_since set): suppresses both the alert
|
||||||
and the recovery — the spike never warranted a page.
|
and the recovery — the spike never warranted a page.
|
||||||
- Past grace: fires the RECOVER notification normally.
|
- Past grace: fires the RECOVER notification normally.
|
||||||
"""
|
"""
|
||||||
|
effective_grace = threshold.grace if threshold.grace is not None else self.grace_seconds
|
||||||
|
|
||||||
lvl, message, formatted_msg = self._trigger_notification(
|
lvl, message, formatted_msg = self._trigger_notification(
|
||||||
host_name, metric_path, old_level, new_level, value, threshold, plugin_data
|
host_name, metric_path, old_level, new_level, value, threshold, plugin_data,
|
||||||
|
check_name=check_name, metric_name=metric_name,
|
||||||
)
|
)
|
||||||
alert_state.formatted_message = formatted_msg
|
alert_state.formatted_message = formatted_msg
|
||||||
|
|
||||||
@@ -1091,16 +1387,31 @@ class ThresholdChecker:
|
|||||||
if alert_state.pending_since is not None:
|
if alert_state.pending_since is not None:
|
||||||
logger.info(
|
logger.info(
|
||||||
"Alert suppressed (recovered within %.0fs grace): %s on %s",
|
"Alert suppressed (recovered within %.0fs grace): %s on %s",
|
||||||
self.grace_seconds, metric_path, host_name,
|
effective_grace, metric_path, host_name,
|
||||||
)
|
)
|
||||||
alert_state.pending_since = None
|
alert_state.pending_since = None
|
||||||
else:
|
else:
|
||||||
self._send_notification(host_name, lvl, message, metric_path, old_level, new_level, value)
|
self._send_notification(host_name, lvl, message, metric_path, old_level, new_level, value)
|
||||||
|
elif new_level.value > old_level.value:
|
||||||
|
# Worsening (OK→WARNING, OK→CRITICAL, WARNING→CRITICAL).
|
||||||
|
if effective_grace <= 0:
|
||||||
|
# No grace period — fire immediately.
|
||||||
|
self._send_notification(host_name, lvl, message, metric_path, old_level, new_level, value)
|
||||||
|
now = time.time()
|
||||||
|
alert_state.last_notification = now
|
||||||
|
alert_state.notification_count = 1
|
||||||
|
else:
|
||||||
|
alert_state.pending_since = time.time()
|
||||||
|
logger.debug(
|
||||||
|
"Alert deferred (%.0fs grace): %s on %s = %s",
|
||||||
|
effective_grace, metric_path, host_name, value,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
alert_state.pending_since = time.time()
|
# De-escalation within alert states (e.g. CRITICAL→WARNING): metric is still
|
||||||
|
# alerting but did not recover, so no new notification.
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Alert deferred (%.0fs grace): %s on %s = %s",
|
"De-escalation %s→%s for %s on %s, no notification",
|
||||||
self.grace_seconds, metric_path, host_name, value,
|
old_level.name, new_level.name, metric_path, host_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _check_pending_or_renotify(
|
def _check_pending_or_renotify(
|
||||||
@@ -1111,25 +1422,43 @@ class ThresholdChecker:
|
|||||||
value: Any,
|
value: Any,
|
||||||
threshold: ThresholdConfig,
|
threshold: ThresholdConfig,
|
||||||
plugin_data: Optional[Dict[str, Any]],
|
plugin_data: Optional[Dict[str, Any]],
|
||||||
|
check_name: Optional[str] = None,
|
||||||
|
metric_name: Optional[str] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Called when alert level is unchanged and non-OK.
|
"""Called when alert level is unchanged and non-OK.
|
||||||
|
|
||||||
If a deferred notification is pending and grace_seconds have elapsed,
|
If a deferred notification is pending and grace_seconds have elapsed,
|
||||||
fires it now. Otherwise falls through to normal reminder logic.
|
fires it now. Otherwise falls through to normal reminder logic.
|
||||||
"""
|
"""
|
||||||
|
effective_grace = threshold.grace if threshold.grace is not None else self.grace_seconds
|
||||||
if alert_state.pending_since is not None:
|
if alert_state.pending_since is not None:
|
||||||
if time.time() - alert_state.pending_since >= self.grace_seconds:
|
if time.time() - alert_state.pending_since >= effective_grace:
|
||||||
lvl, message, formatted_msg = self._trigger_notification(
|
lvl, message, formatted_msg = self._trigger_notification(
|
||||||
host_name, metric_path, AlertLevel.OK, alert_state.level, value, threshold, plugin_data
|
host_name, metric_path, AlertLevel.OK, alert_state.level, value, threshold, plugin_data,
|
||||||
|
check_name=check_name, metric_name=metric_name,
|
||||||
)
|
)
|
||||||
alert_state.formatted_message = formatted_msg
|
alert_state.formatted_message = formatted_msg
|
||||||
self._send_notification(
|
self._send_notification(
|
||||||
host_name, lvl, message, metric_path, AlertLevel.OK, alert_state.level, value
|
host_name, lvl, message, metric_path, AlertLevel.OK, alert_state.level, value
|
||||||
)
|
)
|
||||||
alert_state.pending_since = None
|
alert_state.pending_since = None
|
||||||
|
now = time.time()
|
||||||
|
alert_state.last_notification = now
|
||||||
|
alert_state.notification_count = 1
|
||||||
# else: still within grace window, do nothing
|
# else: still within grace window, do nothing
|
||||||
else:
|
else:
|
||||||
self._check_renotify(host_name, alert_state, metric_path, value, threshold, plugin_data)
|
self._check_renotify(host_name, alert_state, metric_path, value, threshold, plugin_data, check_name=check_name, metric_name=metric_name)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _human_duration(seconds: float) -> str:
|
||||||
|
s = int(seconds)
|
||||||
|
if s < 120:
|
||||||
|
return f"{s}s"
|
||||||
|
if s < 3600:
|
||||||
|
return f"{s // 60}m {s % 60}s"
|
||||||
|
h, rem = divmod(s, 3600)
|
||||||
|
m = rem // 60
|
||||||
|
return f"{h}h {m}m" if m else f"{h}h"
|
||||||
|
|
||||||
def _check_renotify(
|
def _check_renotify(
|
||||||
self,
|
self,
|
||||||
@@ -1139,6 +1468,8 @@ class ThresholdChecker:
|
|||||||
value: Any,
|
value: Any,
|
||||||
threshold: ThresholdConfig,
|
threshold: ThresholdConfig,
|
||||||
plugin_data: Optional[Dict[str, Any]] = None,
|
plugin_data: Optional[Dict[str, Any]] = None,
|
||||||
|
check_name: Optional[str] = None,
|
||||||
|
metric_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
"""Check if we should send a repeat notification.
|
"""Check if we should send a repeat notification.
|
||||||
|
|
||||||
@@ -1176,6 +1507,7 @@ class ThresholdChecker:
|
|||||||
|
|
||||||
# Format operator symbol
|
# Format operator symbol
|
||||||
op_symbol = threshold.operator.value
|
op_symbol = threshold.operator.value
|
||||||
|
short_path = (metric_path.partition(".")[2] or metric_path).removesuffix("_status_code")
|
||||||
|
|
||||||
# Time to re-notify
|
# Time to re-notify
|
||||||
if threshold_value is not None:
|
if threshold_value is not None:
|
||||||
@@ -1185,23 +1517,63 @@ class ThresholdChecker:
|
|||||||
value=value,
|
value=value,
|
||||||
threshold_value=threshold_value,
|
threshold_value=threshold_value,
|
||||||
op_symbol=op_symbol,
|
op_symbol=op_symbol,
|
||||||
plugin_data=plugin_data
|
plugin_data=plugin_data,
|
||||||
|
check_name=check_name,
|
||||||
|
metric_name=metric_name,
|
||||||
)
|
)
|
||||||
message = f"REMINDER ({alert_state.level.name}): {host_name} - {metric_path} = {value} {threshold_info}, ongoing for {int(now - alert_state.since)}s"
|
body = f"{value} {threshold_info}, ongoing for {self._human_duration(now - alert_state.since)}"
|
||||||
else:
|
else:
|
||||||
message = f"REMINDER ({alert_state.level.name}): {host_name} - {metric_path} = {value} (ongoing for {int(now - alert_state.since)}s)"
|
body = f"{value} (ongoing for {self._human_duration(now - alert_state.since)})"
|
||||||
|
message = f"REMINDER ({alert_state.level.name}): {host_name} - {short_path} = {body}"
|
||||||
|
|
||||||
asyncio.get_event_loop().create_task(notify_mod.send_notification(
|
from . import hbdclass
|
||||||
host_name,
|
host = hbdclass.Host.hosts.get(host_name)
|
||||||
notify_mod.Notification(
|
if host is None or host.watched:
|
||||||
title=f"[REMINDER/{alert_state.level.name}] {host_name}",
|
asyncio.get_event_loop().create_task(notify_mod.send_notification(
|
||||||
body=message,
|
host_name,
|
||||||
level=alert_state.level.name,
|
notify_mod.Notification(
|
||||||
),
|
title=f"[REMINDER/{alert_state.level.name}] {host_name} {short_path}",
|
||||||
))
|
body=body,
|
||||||
|
level=alert_state.level.name,
|
||||||
|
service=short_path,
|
||||||
|
),
|
||||||
|
))
|
||||||
|
logger.info("Re-notification sent: %s", message)
|
||||||
alert_state.last_notification = now
|
alert_state.last_notification = now
|
||||||
alert_state.notification_count += 1
|
alert_state.notification_count += 1
|
||||||
logger.info("Re-notification sent: %s", message)
|
|
||||||
|
def purge_stale_alerts(self, hbdclass) -> None:
|
||||||
|
"""Remove alert states that have no matching threshold configuration.
|
||||||
|
|
||||||
|
Called after startup (pickle restore) and after each config reload so
|
||||||
|
that alerts orphaned by configuration changes do not linger forever.
|
||||||
|
Alerts whose metric_path is not present in the current threshold config
|
||||||
|
for that host are silently dropped.
|
||||||
|
"""
|
||||||
|
for hostname, host in hbdclass.Host.hosts.items():
|
||||||
|
if not host.alert_states:
|
||||||
|
continue
|
||||||
|
configured = self.get_thresholds_for_host(hostname)
|
||||||
|
stale = []
|
||||||
|
for mp in host.alert_states:
|
||||||
|
# connectivity.* and rtt are managed by the connection state
|
||||||
|
# machine, not by threshold config — never purge them.
|
||||||
|
if mp == "rtt" or mp.startswith("connectivity"):
|
||||||
|
continue
|
||||||
|
if self._find_threshold(configured, mp)[0] is not None:
|
||||||
|
continue
|
||||||
|
# Also match wildcard pool/partition thresholds (e.g. "zfs_monitor.*.status"
|
||||||
|
# covers alert state "zfs_monitor.tank.status").
|
||||||
|
parts = mp.split(".")
|
||||||
|
if len(parts) == 3 and f"{parts[0]}.*.{parts[2]}" in configured:
|
||||||
|
continue
|
||||||
|
stale.append(mp)
|
||||||
|
for mp in stale:
|
||||||
|
logger.info(
|
||||||
|
"Purging stale alert state for %s / %s (no threshold configured)",
|
||||||
|
hostname, mp,
|
||||||
|
)
|
||||||
|
del host.alert_states[mp]
|
||||||
|
|
||||||
def get_active_alerts(self, alert_states: Dict[str, AlertState]) -> list:
|
def get_active_alerts(self, alert_states: Dict[str, AlertState]) -> list:
|
||||||
"""
|
"""
|
||||||
|
|||||||
+125
-26
@@ -16,6 +16,11 @@ from . import notify as notify_mod
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
eventlog = notify_mod.eventlog
|
eventlog = notify_mod.eventlog
|
||||||
|
|
||||||
|
# Plugin data is declared stale after this many collection intervals without a
|
||||||
|
# fresh sample. >= 2 so a single missed sample never purges live data (the user
|
||||||
|
# directive: wait at least two full intervals after recovery before going stale).
|
||||||
|
_STALE_INTERVAL_MULTIPLIER = 3
|
||||||
|
|
||||||
# SO_TIMESTAMP: kernel attaches a struct timeval to each received datagram.
|
# SO_TIMESTAMP: kernel attaches a struct timeval to each received datagram.
|
||||||
# Supported on Linux, FreeBSD, and macOS. The constant is not exposed by
|
# Supported on Linux, FreeBSD, and macOS. The constant is not exposed by
|
||||||
# Python's socket module on all platforms
|
# Python's socket module on all platforms
|
||||||
@@ -211,10 +216,11 @@ def _make_timer_callbacks(uname, host, ctx):
|
|||||||
connection.newstate(connection.__class__.OVERDUE, now, cfg.get("grace", 2))
|
connection.newstate(connection.__class__.OVERDUE, now, cfg.get("grace", 2))
|
||||||
msg = f"{connection.afam} overdue"
|
msg = f"{connection.afam} overdue"
|
||||||
eventlog(uname, "CRITICAL", msg)
|
eventlog(uname, "CRITICAL", msg)
|
||||||
asyncio.create_task(notify_mod.send_notification(
|
if host.watched:
|
||||||
uname,
|
asyncio.create_task(notify_mod.send_notification(
|
||||||
notify_mod.Notification(title=f"[CRITICAL] {uname}", body=msg, level="CRITICAL"),
|
uname,
|
||||||
))
|
notify_mod.Notification(title=f"[CRITICAL] {uname}", body=msg, level="CRITICAL"),
|
||||||
|
))
|
||||||
# Track in alert_states so the Alerts Dashboard shows this
|
# Track in alert_states so the Alerts Dashboard shows this
|
||||||
_set_connectivity_alert(host, connection.afam, "CRITICAL")
|
_set_connectivity_alert(host, connection.afam, "CRITICAL")
|
||||||
if threshold_checker:
|
if threshold_checker:
|
||||||
@@ -231,6 +237,23 @@ def _make_timer_callbacks(uname, host, ctx):
|
|||||||
return on_overdue, on_unknown
|
return on_overdue, on_unknown
|
||||||
|
|
||||||
|
|
||||||
|
def _make_plugin_stale_callback(uname, ctx):
|
||||||
|
"""Return an async callback that clears stale plugin data and its alerts."""
|
||||||
|
msg_to_websockets = ctx.get("msg_to_websockets")
|
||||||
|
|
||||||
|
async def on_plugin_stale(host, plugin_name):
|
||||||
|
host.plugin_data.pop(plugin_name, None)
|
||||||
|
stale_keys = [k for k in host.alert_states if k.startswith(f"{plugin_name}.")]
|
||||||
|
for k in stale_keys:
|
||||||
|
del host.alert_states[k]
|
||||||
|
eventlog(uname, "INFO", f"plugin data stale: {plugin_name}")
|
||||||
|
if msg_to_websockets:
|
||||||
|
msg_to_websockets("plugin_stale", {"host": uname, "plugin": plugin_name})
|
||||||
|
msg_to_websockets("host", host.stateinfo())
|
||||||
|
|
||||||
|
return on_plugin_stale
|
||||||
|
|
||||||
|
|
||||||
def restore_connection_timers(hbdclass, ctx):
|
def restore_connection_timers(hbdclass, ctx):
|
||||||
"""Restore overdue timers for all loaded connections after a pickle restore.
|
"""Restore overdue timers for all loaded connections after a pickle restore.
|
||||||
|
|
||||||
@@ -248,10 +271,15 @@ def restore_connection_timers(hbdclass, ctx):
|
|||||||
for afam, conn in list(host.connections.items()):
|
for afam, conn in list(host.connections.items()):
|
||||||
state = conn.getstate()
|
state = conn.getstate()
|
||||||
if state == hbdclass.Connection.DOWN:
|
if state == hbdclass.Connection.DOWN:
|
||||||
|
_set_connectivity_alert(host, afam, "CRITICAL")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
on_overdue, on_unknown = _make_timer_callbacks(uname, host, ctx)
|
on_overdue, on_unknown = _make_timer_callbacks(uname, host, ctx)
|
||||||
|
|
||||||
|
if state == hbdclass.Connection.UNKNOWN:
|
||||||
|
_set_connectivity_alert(host, afam, "CRITICAL")
|
||||||
|
continue
|
||||||
|
|
||||||
if state == hbdclass.Connection.UP and interval > 0:
|
if state == hbdclass.Connection.UP and interval > 0:
|
||||||
elapsed = now - conn.lastbeat
|
elapsed = now - conn.lastbeat
|
||||||
# Give hosts one full (interval + grace) of extra time on startup
|
# Give hosts one full (interval + grace) of extra time on startup
|
||||||
@@ -282,6 +310,10 @@ def restore_connection_timers(hbdclass, ctx):
|
|||||||
"Restored OVERDUE timer %s/%s: %.0fs remaining",
|
"Restored OVERDUE timer %s/%s: %.0fs remaining",
|
||||||
uname, afam, remaining,
|
uname, afam, remaining,
|
||||||
)
|
)
|
||||||
|
# Ensure the connectivity alert is set — it may be missing if
|
||||||
|
# hbd was shut down before the on_overdue callback had a chance
|
||||||
|
# to record it.
|
||||||
|
_set_connectivity_alert(host, afam, "CRITICAL")
|
||||||
restored += 1
|
restored += 1
|
||||||
|
|
||||||
logger.info("Restored timers for %d connection(s)", restored)
|
logger.info("Restored timers for %d connection(s)", restored)
|
||||||
@@ -332,11 +364,12 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
# Use new config function to check dyndns
|
# Use new config function to check dyndns
|
||||||
dyndnshosts = config_mod.get_dyndnshosts(cfg)
|
dyndnshosts = config_mod.get_dyndnshosts(cfg)
|
||||||
host.dyn = uname in dyndnshosts
|
host.dyn = uname in dyndnshosts
|
||||||
|
watchhosts = config_mod.get_watchhosts(cfg)
|
||||||
|
host.watched = uname in watchhosts
|
||||||
# Apply user-access settings from config
|
# Apply user-access settings from config
|
||||||
access = config_mod.get_host_access(cfg, uname)
|
access = config_mod.get_host_access(cfg, uname)
|
||||||
host.apply_access(access["owner"], access["managers"], access["monitors"])
|
host.apply_access(access["owner"], access["managers"], access["monitors"])
|
||||||
if verbose:
|
logger.info("New host signed on: %s (dyn=%s, access=%s)", uname, host.dyn, access)
|
||||||
print(("XX: New host, num now %s" % (len(hbdcls.Host.hosts))))
|
|
||||||
newh = True
|
newh = True
|
||||||
else:
|
else:
|
||||||
host = hbdcls.Host.hosts[uname]
|
host = hbdcls.Host.hosts[uname]
|
||||||
@@ -350,8 +383,10 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
|
|
||||||
if msg.get("ID") == "HTB":
|
if msg.get("ID") == "HTB":
|
||||||
host.doesack = msg.get("acks", -1)
|
host.doesack = msg.get("acks", -1)
|
||||||
# send ACK back
|
# send ACK back; ask client to resend plugin info when we have none yet
|
||||||
rmsg = {"time": time.time()}
|
rmsg = {"time": time.time()}
|
||||||
|
if not host.plugin_data:
|
||||||
|
rmsg["request_update"] = 1
|
||||||
opkt = dicttos("ACK", rmsg)
|
opkt = dicttos("ACK", rmsg)
|
||||||
try:
|
try:
|
||||||
transport.sendto(opkt, addr)
|
transport.sendto(opkt, addr)
|
||||||
@@ -364,10 +399,52 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
plugin_name = msg.get("plugin")
|
plugin_name = msg.get("plugin")
|
||||||
if plugin_name:
|
if plugin_name:
|
||||||
# Extract plugin fields, dropping protocol metadata fields
|
# Extract plugin fields, dropping protocol metadata fields
|
||||||
|
# (_interval is the client-declared collection interval, not a metric).
|
||||||
plugin_data = {k: v for k, v in msg.items()
|
plugin_data = {k: v for k, v in msg.items()
|
||||||
if k not in ("ID", "plugin", "id", "name")}
|
if k not in ("ID", "plugin", "id", "name", "_interval")}
|
||||||
# Store plugin data with timestamp
|
# Store plugin data with timestamp
|
||||||
host.add_plugin_data(plugin_name, plugin_data, timestamp=now)
|
host.add_plugin_data(plugin_name, plugin_data, timestamp=now)
|
||||||
|
|
||||||
|
# Set the stale timer. Prefer the interval the client declares: it is
|
||||||
|
# the true collection cadence and is correct from the very first
|
||||||
|
# post-recovery sample, so we never purge data too early after an
|
||||||
|
# outage. interval == 0 means a collect-once InfoPlugin that never
|
||||||
|
# goes stale. Legacy clients omit _interval, so fall back to inferring
|
||||||
|
# the cadence from the gap between the last two received samples.
|
||||||
|
history = host.plugin_data.get(plugin_name, [])
|
||||||
|
declared = msg.get("_interval")
|
||||||
|
if declared is not None:
|
||||||
|
if declared > 0:
|
||||||
|
host.reset_plugin_timer(plugin_name, declared * _STALE_INTERVAL_MULTIPLIER,
|
||||||
|
_make_plugin_stale_callback(uname, ctx))
|
||||||
|
else:
|
||||||
|
host.cancel_plugin_timer(plugin_name)
|
||||||
|
elif len(history) >= 2:
|
||||||
|
plugin_interval = max(history[-1][0] - history[-2][0], 1)
|
||||||
|
host.reset_plugin_timer(plugin_name, plugin_interval * _STALE_INTERVAL_MULTIPLIER,
|
||||||
|
_make_plugin_stale_callback(uname, ctx))
|
||||||
|
else:
|
||||||
|
host.cancel_plugin_timer(plugin_name)
|
||||||
|
|
||||||
|
# Remove alert states for metrics present in the previous sample but
|
||||||
|
# absent now (e.g. a nagios check removed from configuration).
|
||||||
|
if len(history) >= 2:
|
||||||
|
prev_keys = set(history[-2][1].keys())
|
||||||
|
curr_keys = set(plugin_data.keys())
|
||||||
|
for metric_name in prev_keys - curr_keys:
|
||||||
|
metric_path = f"{plugin_name}.{metric_name}"
|
||||||
|
if host.alert_states.pop(metric_path, None) is not None:
|
||||||
|
eventlog(uname, "INFO", f"stale check removed: {metric_path}")
|
||||||
|
if (prev_keys - curr_keys) and msg_to_websockets:
|
||||||
|
msg_to_websockets("host", host.stateinfo())
|
||||||
|
|
||||||
|
# If os_info reports an owner and none is configured server-side, apply it
|
||||||
|
if plugin_name == "os_info":
|
||||||
|
config_owner = config_mod.get_host_access(cfg, uname).get("owner")
|
||||||
|
default_owner = config_mod.get_default_owner(cfg)
|
||||||
|
inferred_owner = config_owner or plugin_data.get("owner") or default_owner
|
||||||
|
host.owner = inferred_owner
|
||||||
|
logger.info(f"owner for {uname} is {host.owner}")
|
||||||
if DEBUG > 1:
|
if DEBUG > 1:
|
||||||
print(f"Stored plugin data for {uname}: {plugin_name}")
|
print(f"Stored plugin data for {uname}: {plugin_name}")
|
||||||
|
|
||||||
@@ -407,10 +484,11 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
|
|
||||||
if res:
|
if res:
|
||||||
eventlog(uname, "WARNING", res)
|
eventlog(uname, "WARNING", res)
|
||||||
asyncio.create_task(notify_mod.send_notification(
|
if host.watched:
|
||||||
uname,
|
asyncio.create_task(notify_mod.send_notification(
|
||||||
notify_mod.Notification(title=f"[WARNING] {uname}", body=res, level="WARNING"),
|
uname,
|
||||||
))
|
notify_mod.Notification(title=f"[WARNING] {uname}", body=res, level="WARNING"),
|
||||||
|
))
|
||||||
|
|
||||||
interval = int(msg.get("interval", 0) or 0)
|
interval = int(msg.get("interval", 0) or 0)
|
||||||
shutdown = msg.get("shutdown", 0)
|
shutdown = msg.get("shutdown", 0)
|
||||||
@@ -419,17 +497,32 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
boot = msg.get("boot", 0)
|
boot = msg.get("boot", 0)
|
||||||
|
|
||||||
if boot:
|
if boot:
|
||||||
|
# hbc was stared with a -b flag
|
||||||
eventlog(uname, "INFO", "booted")
|
eventlog(uname, "INFO", "booted")
|
||||||
asyncio.create_task(notify_mod.send_notification(
|
if host.watched:
|
||||||
uname,
|
asyncio.create_task(notify_mod.send_notification(
|
||||||
notify_mod.Notification(title=f"[INFO] {uname}", body=f"{host.name} booted", level="INFO"),
|
uname,
|
||||||
))
|
notify_mod.Notification(title=f"[INFO] {uname}", body=f"{host.name} booted", level="INFO"),
|
||||||
|
))
|
||||||
if message:
|
if message:
|
||||||
eventlog(uname, "INFO", "msg: %s" % message, service=service)
|
eventlog(uname, "INFO", message, service=service)
|
||||||
|
|
||||||
if conn.getstate() != hbdcls.Connection.UP:
|
if conn.getstate() != hbdcls.Connection.UP:
|
||||||
|
# Transition to UP and log/notify if appropriate
|
||||||
lasts = conn.state
|
lasts = conn.state
|
||||||
d = conn.newstate(hbdcls.Connection.UP, now)
|
d = conn.newstate(hbdcls.Connection.UP, now)
|
||||||
|
# On reboot, pre-boot plugin data and derived alerts are stale.
|
||||||
|
# Cancel all plugin timers and wipe plugin state so timers restart
|
||||||
|
# cleanly from the first two post-boot samples.
|
||||||
|
for pname in list(host.plugin_timers):
|
||||||
|
host.cancel_plugin_timer(pname)
|
||||||
|
host.plugin_data.clear()
|
||||||
|
stale_plugin_keys = [
|
||||||
|
k for k in host.alert_states
|
||||||
|
if k not in ("rtt",) and not k.startswith("connectivity.")
|
||||||
|
]
|
||||||
|
for k in stale_plugin_keys:
|
||||||
|
del host.alert_states[k]
|
||||||
# Clear connectivity alert now that the host is back up
|
# Clear connectivity alert now that the host is back up
|
||||||
_set_connectivity_alert(host, conn.afam, "OK")
|
_set_connectivity_alert(host, conn.afam, "OK")
|
||||||
# Don't log/notify RECOVER for a brand-new host seen for the first time —
|
# Don't log/notify RECOVER for a brand-new host seen for the first time —
|
||||||
@@ -437,13 +530,18 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
if not newh:
|
if not newh:
|
||||||
if d == 0 or lasts == "unknown":
|
if d == 0 or lasts == "unknown":
|
||||||
m = "%s is up" % (conn.afam)
|
m = "%s is up" % (conn.afam)
|
||||||
|
elif d < 4:
|
||||||
|
# Transient blip (likely client restart) — skip log and notification
|
||||||
|
m = None
|
||||||
else:
|
else:
|
||||||
m = "%s back after being %s for %s" % (conn.afam, lasts, dur(d))
|
m = "%s back after being %s for %s" % (conn.afam, lasts, dur(d))
|
||||||
eventlog(uname, "RECOVER", m)
|
if m:
|
||||||
asyncio.create_task(notify_mod.send_notification(
|
eventlog(uname, "RECOVER", m)
|
||||||
uname,
|
if host.watched:
|
||||||
notify_mod.Notification(title=f"[RECOVER] {uname}", body=m, level="RECOVER"),
|
asyncio.create_task(notify_mod.send_notification(
|
||||||
))
|
uname,
|
||||||
|
notify_mod.Notification(title=f"[RECOVER] {uname}", body=m, level="RECOVER"),
|
||||||
|
))
|
||||||
|
|
||||||
if boot or newh:
|
if boot or newh:
|
||||||
host.upcount = host.doesack
|
host.upcount = host.doesack
|
||||||
@@ -453,10 +551,11 @@ def handle_datagram(msg: dict, addr, transport, ctx: dict):
|
|||||||
if shutdown:
|
if shutdown:
|
||||||
m = "%s shutdown" % conn.afam
|
m = "%s shutdown" % conn.afam
|
||||||
eventlog(uname, "INFO", m)
|
eventlog(uname, "INFO", m)
|
||||||
asyncio.create_task(notify_mod.send_notification(
|
if host.watched:
|
||||||
uname,
|
asyncio.create_task(notify_mod.send_notification(
|
||||||
notify_mod.Notification(title=f"[INFO] {uname}", body=m, level="INFO"),
|
uname,
|
||||||
))
|
notify_mod.Notification(title=f"[INFO] {uname}", body=m, level="INFO"),
|
||||||
|
))
|
||||||
conn.newstate(hbdcls.Connection.DOWN, now)
|
conn.newstate(hbdcls.Connection.DOWN, now)
|
||||||
_set_connectivity_alert(host, conn.afam, "CRITICAL")
|
_set_connectivity_alert(host, conn.afam, "CRITICAL")
|
||||||
|
|
||||||
|
|||||||
@@ -146,9 +146,14 @@ def load_users(config: dict) -> dict:
|
|||||||
Returns the new ``users`` dict.
|
Returns the new ``users`` dict.
|
||||||
"""
|
"""
|
||||||
global users
|
global users
|
||||||
|
old_users = dict(users) # snapshot before rebuild
|
||||||
users_cfg = config.get("users", {})
|
users_cfg = config.get("users", {})
|
||||||
if not isinstance(users_cfg, dict):
|
if not isinstance(users_cfg, dict):
|
||||||
users = {}
|
users = {}
|
||||||
|
# Preserve OAuth-provisioned users (password_hash == "") that aren't in config.
|
||||||
|
for username, existing_user in old_users.items():
|
||||||
|
if not existing_user.password_hash and username not in users:
|
||||||
|
users[username] = existing_user
|
||||||
return users
|
return users
|
||||||
|
|
||||||
result: dict = {}
|
result: dict = {}
|
||||||
@@ -166,6 +171,10 @@ def load_users(config: dict) -> dict:
|
|||||||
)
|
)
|
||||||
|
|
||||||
users = result
|
users = result
|
||||||
|
# Preserve OAuth-provisioned users (password_hash == "") that aren't in config.
|
||||||
|
for username, existing_user in old_users.items():
|
||||||
|
if not existing_user.password_hash and username not in users:
|
||||||
|
users[username] = existing_user
|
||||||
logger.info("Loaded %d user(s) from config", len(users))
|
logger.info("Loaded %d user(s) from config", len(users))
|
||||||
return users
|
return users
|
||||||
|
|
||||||
@@ -187,6 +196,26 @@ def authenticate(username: str, password: str) -> "User | None":
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def provision_oauth_user(username: str, full_name: str, avatar: str) -> "User":
|
||||||
|
"""Create or update a user sourced from an OAuth2 provider.
|
||||||
|
|
||||||
|
New users are inserted with no password_hash — they can only authenticate
|
||||||
|
via OAuth. Existing users (e.g. defined in config with a password) have
|
||||||
|
their display name and avatar refreshed; all other attributes are preserved.
|
||||||
|
"""
|
||||||
|
user = users.get(username)
|
||||||
|
if user is None:
|
||||||
|
user = User(username=username, full_name=full_name, avatar=avatar)
|
||||||
|
users[username] = user
|
||||||
|
logger.info("Provisioned OAuth user %r", username)
|
||||||
|
else:
|
||||||
|
if full_name:
|
||||||
|
user.full_name = full_name
|
||||||
|
if avatar:
|
||||||
|
user.avatar = avatar
|
||||||
|
return user
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Session management
|
# Session management
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
+62
-13
@@ -13,7 +13,8 @@ from . import data
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
_connections: set = set()
|
# Map of WebSocket → User object (or None when auth is disabled)
|
||||||
|
_connections: dict = {}
|
||||||
_loop: Optional[asyncio.AbstractEventLoop] = None
|
_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||||
_get_hosts: Optional[Callable[[], Iterable]] = None
|
_get_hosts: Optional[Callable[[], Iterable]] = None
|
||||||
_verbose: bool = False
|
_verbose: bool = False
|
||||||
@@ -34,31 +35,65 @@ def setup(
|
|||||||
_verbose = verbose
|
_verbose = verbose
|
||||||
|
|
||||||
|
|
||||||
|
def _user_can_see_host(user, host_name: str) -> bool:
|
||||||
|
"""Return True if *user* may see updates for *host_name* (manager or higher)."""
|
||||||
|
from . import hbdclass, users as users_mod
|
||||||
|
if user is None or not users_mod.users_enabled():
|
||||||
|
return True
|
||||||
|
if user.admin:
|
||||||
|
return True
|
||||||
|
host = hbdclass.Host.hosts.get(host_name)
|
||||||
|
if host is None:
|
||||||
|
return False
|
||||||
|
return host.is_manager(user.username)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_token(request) -> str:
|
||||||
|
"""Extract session token from request (mirrors logic in http.py)."""
|
||||||
|
auth = request.headers.get("Authorization", "")
|
||||||
|
if auth.startswith("Bearer "):
|
||||||
|
return auth[7:].strip()
|
||||||
|
token = request.headers.get("X-Auth-Token", "")
|
||||||
|
if token:
|
||||||
|
return token
|
||||||
|
return request.cookies.get("hbd_session", "")
|
||||||
|
|
||||||
|
|
||||||
async def handler(request):
|
async def handler(request):
|
||||||
"""aiohttp WebSocket upgrade handler — register as GET /ws."""
|
"""aiohttp WebSocket upgrade handler — register as GET /ws."""
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
from . import users as users_mod
|
||||||
|
|
||||||
ws = web.WebSocketResponse()
|
ws = web.WebSocketResponse()
|
||||||
await ws.prepare(request)
|
await ws.prepare(request)
|
||||||
|
|
||||||
_connections.add(ws)
|
token = _get_token(request)
|
||||||
|
user = users_mod.get_session_user(token) if token else None
|
||||||
|
|
||||||
|
_connections[ws] = user
|
||||||
remote = request.remote
|
remote = request.remote
|
||||||
logger.info("WebSocket connected from %s", remote)
|
logger.info("WebSocket connected from %s", remote)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Send current host state to the new client
|
# Send current host state, filtered to hosts this user may see
|
||||||
if _get_hosts:
|
if _get_hosts:
|
||||||
try:
|
try:
|
||||||
for h in list(_get_hosts()):
|
for h in list(_get_hosts()):
|
||||||
await ws.send_str(json.dumps({"type": "host", "data": h}))
|
host_name = h.get("raw_name") or h.get("name", "")
|
||||||
|
if _user_can_see_host(user, host_name):
|
||||||
|
await ws.send_str(json.dumps({"type": "host", "data": h}))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Error sending initial hosts: %s", e)
|
logger.error("Error sending initial hosts: %s", e)
|
||||||
|
|
||||||
# Send recent messages
|
# Send recent messages newest-first so the client can append them in
|
||||||
|
# display order without reordering on arrival (tagged history=True so
|
||||||
|
# the client knows to append rather than prepend).
|
||||||
if data.msgs:
|
if data.msgs:
|
||||||
try:
|
try:
|
||||||
for m in data.msgs:
|
for m in reversed(data.msgs[-30:]):
|
||||||
await ws.send_str(json.dumps({"type": "message", "data": m}))
|
host_name = m.get("host") if isinstance(m, dict) else None
|
||||||
|
if not host_name or _user_can_see_host(user, host_name):
|
||||||
|
await ws.send_str(json.dumps({"type": "message", "data": m, "history": True}))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error("Error sending initial messages: %s", e)
|
logger.error("Error sending initial messages: %s", e)
|
||||||
|
|
||||||
@@ -74,7 +109,7 @@ async def handler(request):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("WebSocket handler error from %s: %s", remote, e)
|
logger.exception("WebSocket handler error from %s: %s", remote, e)
|
||||||
finally:
|
finally:
|
||||||
_connections.discard(ws)
|
_connections.pop(ws, None)
|
||||||
logger.info("WebSocket disconnected from %s", remote)
|
logger.info("WebSocket disconnected from %s", remote)
|
||||||
|
|
||||||
return ws
|
return ws
|
||||||
@@ -83,25 +118,39 @@ async def handler(request):
|
|||||||
def broadcast(typ: str, payload) -> bool:
|
def broadcast(typ: str, payload) -> bool:
|
||||||
"""Thread-safe broadcast to all connected WebSocket clients.
|
"""Thread-safe broadcast to all connected WebSocket clients.
|
||||||
|
|
||||||
|
For host and plugin updates, only sends to clients whose user has
|
||||||
|
manager-or-higher access to that host. Other message types are
|
||||||
|
broadcast to all clients.
|
||||||
|
|
||||||
Can be called from any thread; schedules sends on the event loop.
|
Can be called from any thread; schedules sends on the event loop.
|
||||||
Returns False if the loop is not running yet.
|
Returns False if the loop is not running yet.
|
||||||
"""
|
"""
|
||||||
if not _loop:
|
if not _loop:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
# Determine the host name for access-filtered message types
|
||||||
|
host_name: Optional[str] = None
|
||||||
|
if typ in ("host", "plugin"):
|
||||||
|
host_name = payload.get("raw_name") or payload.get("host") or payload.get("name")
|
||||||
|
elif typ == "message" and isinstance(payload, dict):
|
||||||
|
host_name = payload.get("host")
|
||||||
|
|
||||||
jmsg = json.dumps({"type": typ, "data": payload})
|
jmsg = json.dumps({"type": typ, "data": payload})
|
||||||
|
|
||||||
async def _send_all():
|
async def _send_all():
|
||||||
dead = set()
|
dead = set()
|
||||||
for ws in list(_connections):
|
for ws, user in list(_connections.items()):
|
||||||
try:
|
try:
|
||||||
if not ws.closed:
|
if ws.closed:
|
||||||
await ws.send_str(jmsg)
|
|
||||||
else:
|
|
||||||
dead.add(ws)
|
dead.add(ws)
|
||||||
|
continue
|
||||||
|
if host_name is not None and not _user_can_see_host(user, host_name):
|
||||||
|
continue
|
||||||
|
await ws.send_str(jmsg)
|
||||||
except Exception:
|
except Exception:
|
||||||
dead.add(ws)
|
dead.add(ws)
|
||||||
for ws in dead:
|
for ws in dead:
|
||||||
_connections.discard(ws)
|
_connections.pop(ws, None)
|
||||||
|
|
||||||
asyncio.run_coroutine_threadsafe(_send_all(), _loop)
|
asyncio.run_coroutine_threadsafe(_send_all(), _loop)
|
||||||
return True
|
return True
|
||||||
|
|||||||
+21
-8
@@ -4,20 +4,32 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "hbd"
|
name = "hbd"
|
||||||
version = "5.1.9"
|
version = "5.4.0"
|
||||||
description = "Heartbeat monitoring system — client (hbc) and server (hbd)"
|
description = "Heartbeat monitoring system — client (hbc) and server (hbd)"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
license = "MIT"
|
|
||||||
keywords = ["heartbeat", "monitoring", "dns", "websocket", "system-monitoring"]
|
|
||||||
authors = [
|
|
||||||
{ name = "heartbeat contributors" }
|
|
||||||
]
|
|
||||||
|
|
||||||
# Core dependencies (required for both client and server)
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"PyYAML>=6.0",
|
"PyYAML>=6.0",
|
||||||
]
|
]
|
||||||
|
license = "MIT"
|
||||||
|
license-files = ["LICENSE.md"]
|
||||||
|
keywords = ["heartbeat", "monitoring", "dns", "websocket", "system-monitoring"]
|
||||||
|
authors = [
|
||||||
|
{ name = "Andreas Wrede" }
|
||||||
|
]
|
||||||
|
classifiers = [
|
||||||
|
"Programming Language :: Python :: 3",
|
||||||
|
"Programming Language :: Python :: 3.11",
|
||||||
|
"Programming Language :: Python :: 3.12",
|
||||||
|
"Programming Language :: Python :: 3.13",
|
||||||
|
"Operating System :: POSIX :: Linux",
|
||||||
|
"Operating System :: POSIX :: BSD",
|
||||||
|
"Topic :: System :: Monitoring",
|
||||||
|
"Topic :: System :: Networking :: Monitoring",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.urls]
|
||||||
|
Repository = "https://git.wrede.ca/andreas/heartbeat"
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
# Client-only dependencies (hbc - system monitoring client)
|
# Client-only dependencies (hbc - system monitoring client)
|
||||||
@@ -32,6 +44,7 @@ server = [
|
|||||||
"aiohttp>=3.11",
|
"aiohttp>=3.11",
|
||||||
"Jinja2>=3.1.6",
|
"Jinja2>=3.1.6",
|
||||||
"matrix-nio>=0.24",
|
"matrix-nio>=0.24",
|
||||||
|
"ruamel.yaml>=0.18",
|
||||||
]
|
]
|
||||||
|
|
||||||
# Minimal client — hbc_mini only, no external dependencies
|
# Minimal client — hbc_mini only, no external dependencies
|
||||||
|
|||||||
@@ -1,4 +0,0 @@
|
|||||||
key "rndc-key" {
|
|
||||||
algorithm hmac-md5;
|
|
||||||
secret "qlGa+AYKtyOgWNuozqECMw==";
|
|
||||||
};
|
|
||||||
+18
-1
@@ -4,12 +4,29 @@ set -e
|
|||||||
uv version --bump patch
|
uv version --bump patch
|
||||||
VER=$(uv version --short)
|
VER=$(uv version --short)
|
||||||
sed -i".bak" "s/__version__ = \"[0-9.]*\"\(.*\)$/__version__ = \"$VER\"\1/" hbd/__init__.py
|
sed -i".bak" "s/__version__ = \"[0-9.]*\"\(.*\)$/__version__ = \"$VER\"\1/" hbd/__init__.py
|
||||||
|
sed -i".bak" "s/__version__ = \"[0-9.]*\"\(.*\)$/__version__ = \"$VER\"\1/" scripts/hbc_mini.py
|
||||||
|
sed -i".bak" "s/\*\*Package:\*\* \`hbd\` v[0-9.]*/\*\*Package:\*\* \`hbd\` v$VER/" README.md
|
||||||
|
|
||||||
|
# Update CHANGELOG.md with commits since last tag
|
||||||
|
LASTTAG=$(git describe --tags --abbrev=0 2>/dev/null || true)
|
||||||
|
ADDED=$(git log "${LASTTAG:+$LASTTAG..}HEAD" --pretty="%s" | grep "^feat:" | sed 's/^feat: /- /')
|
||||||
|
FIXED=$(git log "${LASTTAG:+$LASTTAG..}HEAD" --pretty="%s" | grep "^fix:" | sed 's/^fix: /- /')
|
||||||
|
{
|
||||||
|
printf "## [%s]\n" "$VER"
|
||||||
|
[ -n "$ADDED" ] && printf "\n### Added\n%s\n" "$ADDED"
|
||||||
|
[ -n "$FIXED" ] && printf "\n### Fixed\n%s\n" "$FIXED"
|
||||||
|
printf "\n---\n\n"
|
||||||
|
} > /tmp/changelog_entry.txt
|
||||||
|
sed -i".bak" "4r /tmp/changelog_entry.txt" CHANGELOG.md
|
||||||
|
rm /tmp/changelog_entry.txt CHANGELOG.md.bak
|
||||||
|
|
||||||
# commit pyproject.toml
|
# commit pyproject.toml
|
||||||
git commit -m "version $VER" pyproject.toml hbd/__init__.py
|
git commit -m "version $VER" pyproject.toml hbd/__init__.py scripts/hbc_mini.py README.md CHANGELOG.md
|
||||||
git push
|
git push
|
||||||
# tag version
|
# tag version
|
||||||
git tag -a v$VER -m "Version $VER"
|
git tag -a v$VER -m "Version $VER"
|
||||||
git push --tags
|
git push --tags
|
||||||
|
|
||||||
rm hbd/__init__.py.bak
|
rm hbd/__init__.py.bak
|
||||||
|
rm scripts/hbc_mini.py.bak
|
||||||
|
rm README.md.bak
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
hbc_mini
|
||||||
|
hbc_mini_dbg
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
CC ?= cc
|
||||||
|
CFLAGS = -O2 -Wall -Wextra -std=c11
|
||||||
|
LDFLAGS = -lz -lpthread -lm
|
||||||
|
TARGET = hbc_mini
|
||||||
|
SRC = hbc_mini.c
|
||||||
|
|
||||||
|
# FreeBSD/NetBSD keep zlib in base; no extra flags needed.
|
||||||
|
# On some NetBSD installs pthreads may need -lpthread from pkgsrc.
|
||||||
|
|
||||||
|
.PHONY: all clean debug
|
||||||
|
|
||||||
|
all: $(TARGET)
|
||||||
|
|
||||||
|
$(TARGET): $(SRC)
|
||||||
|
$(CC) $(CFLAGS) -o $@ $< $(LDFLAGS)
|
||||||
|
|
||||||
|
debug: $(SRC)
|
||||||
|
$(CC) -g -fsanitize=address,undefined -o $(TARGET)_dbg $< $(LDFLAGS)
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -f $(TARGET) $(TARGET)_dbg
|
||||||
File diff suppressed because it is too large
Load Diff
+37
-46
@@ -17,9 +17,9 @@ venv=""
|
|||||||
[ "$2" = "HA" ] && on_ha=1
|
[ "$2" = "HA" ] && on_ha=1
|
||||||
[ -z "$what" ] && what="client"
|
[ -z "$what" ] && what="client"
|
||||||
|
|
||||||
if [ -d /homeassistant ]; then
|
if [ -d /homeassistant ]; then # if running from HA command line
|
||||||
echo "HA, running \"docker exec homeassistant $0 $@\""
|
echo "HA, running \"docker exec homeassistant /config/bin/hb_install.sh $@\""
|
||||||
docker exec homeassistant $0 $@ HA
|
docker exec homeassistant /config/bin/hb_install.sh $@ HA
|
||||||
rc=$?
|
rc=$?
|
||||||
if [ $rc -ne 0 ]; then
|
if [ $rc -ne 0 ]; then
|
||||||
echo "Failed to install heartbeat in HA, please check the logs for more details"
|
echo "Failed to install heartbeat in HA, please check the logs for more details"
|
||||||
@@ -28,8 +28,9 @@ if [ -d /homeassistant ]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ $on_ha -eq 1 ]; then
|
if [ $on_ha -eq 1 ] || [ -r /.dockerenv ] && [ -d /config/bin ]; then
|
||||||
echo "Installing under docker on Home Assistant OS, using /config/bin for executables and /config/venvs for virtual environments "
|
# Installing under docker on Home Assistant OS, using /config/bin for executables and /config/venvs for virtual environments
|
||||||
|
echo "Home Assistant OS detected, installing under docker"
|
||||||
where="/config/bin"
|
where="/config/bin"
|
||||||
venv="/config/venvs"
|
venv="/config/venvs"
|
||||||
else
|
else
|
||||||
@@ -52,23 +53,26 @@ else
|
|||||||
venv="$HOME/venvs"
|
venv="$HOME/venvs"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
echo "Installing $what to $where"
|
||||||
echo "Installing heartbeat $what"
|
if [ ! -z "$venv" ]; then
|
||||||
|
echo "Using virtual environment at $venv/hbd"
|
||||||
|
fi
|
||||||
|
|
||||||
if [ "$venv" != "" ] && [ ! -d $venv/hbd ]; then
|
if [ "$venv" != "" ] && [ ! -d $venv/hbd ]; then
|
||||||
set +e
|
|
||||||
python3 -m pip --version > /dev/null 2>&1
|
|
||||||
rc=$?
|
|
||||||
set -e
|
|
||||||
arg=""
|
arg=""
|
||||||
if [ $rc -ne 0 ]; then
|
have_pip=$(python3 -c "import pip" 2>/dev/null &> /dev/null && echo "Installed" || echo "Not Installed")
|
||||||
|
if [ "$have_pip" = "Not Installed" ]; then
|
||||||
# some systems do not have pip installed by default, so we need to fetch get-pip.py and install pip
|
# some systems do not have pip installed by default, so we need to fetch get-pip.py and install pip
|
||||||
echo "pip is not installed, fetching get-pip.py and installing pip"
|
echo "pip is not installed, fetching get-pip.py and installing pip"
|
||||||
arg="--without-pip"
|
arg="--without-pip"
|
||||||
fi
|
fi
|
||||||
mkdir -p $venv
|
mkdir -p $venv
|
||||||
have_venv=$(python3 -c "import venv" &> /dev/null && echo "Installed" || echo "Not Installed")
|
have_venv=$(python3 -c "import venv" 2>/dev/null &> /dev/null && echo "Installed" || echo "Not Installed")
|
||||||
if [ "$have_venv" = "Not Installed" ]; then
|
if [ "$have_venv" = "Not Installed" ]; then
|
||||||
|
if [ "$have_pip" = "Not Installed" ]; then
|
||||||
|
echo "python has no venv, and no pip to install virtualenv, cannot continue"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "python venv module not found, installing virtualenv"
|
echo "python venv module not found, installing virtualenv"
|
||||||
python3 -m pip install --user virtualenv
|
python3 -m pip install --user virtualenv
|
||||||
python3 -m virtualenv $venv/hbd --system-site-packages $arg
|
python3 -m virtualenv $venv/hbd --system-site-packages $arg
|
||||||
@@ -82,43 +86,30 @@ if [ "$venv" != "" ] && [ ! -d $venv/hbd ]; then
|
|||||||
deactivate
|
deactivate
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -z "$venv" ]; then
|
if [ ! -z "$venv" ]; then
|
||||||
echo "Installing heartbeat $what globally"
|
|
||||||
else
|
|
||||||
echo "Installing heartbeat $what in virtual environment $venv/hbd"
|
|
||||||
. $venv/hbd/bin/activate
|
. $venv/hbd/bin/activate
|
||||||
fi
|
fi
|
||||||
python3 -mpip install --upgrade --index-url https://git.wrede.ca/api/packages/andreas/pypi/simple/ --extra-index-url https://pypi.org/simple hbd[$what]
|
if [ "$what" = "mini" ]; then
|
||||||
|
curl -s -o $where/hbc_mini https://git.wrede.ca/andreas/heartbeat/raw/branch/master/scripts/hbc_mini.py
|
||||||
|
chmod +x $where/hbc_mini
|
||||||
|
else
|
||||||
|
python3 -mpip install --upgrade --index-url https://git.wrede.ca/api/packages/andreas/pypi/simple/ --extra-index-url https://pypi.org/simple hbd[$what]
|
||||||
|
fi
|
||||||
|
|
||||||
if [ "$what" = "server" ]; then
|
if [ ! -z "$venv" ]; then
|
||||||
rm -f $where/hbd
|
echo "linking executables to $where"
|
||||||
ln -sf $(which hbd) $where/hbd
|
if [ "$what" = "server" ]; then
|
||||||
echo "hbd installed, you can run it with \"$where/hbd\" or \"hbd\" if $where is in your PATH"
|
rm -f $where/hbd
|
||||||
elif [ "$what" = "client" ]; then
|
ln -sf $(which hbd) $where/hbd
|
||||||
hbc_path=$(which hbc)
|
elif [ "$what" = "client" ]; then
|
||||||
if [ -z "$hbc_path" ]; then
|
|
||||||
echo "hbc not found in PATH, installation failed"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ "$hbc_path" != "$where/hbc" ]; then
|
|
||||||
rm -f $where/hbc
|
rm -f $where/hbc
|
||||||
ln -sf $(which hbc) $where/hbc
|
ln -sf $(which hbc) $where/hbc
|
||||||
fi
|
fi
|
||||||
if [ "$0" != "$where/hb_install.sh" ]; then
|
rm -f $where/hb_install.sh
|
||||||
cp "$0" $where/hb_install.sh
|
ln -sf $(which hb_install.sh) $where/hb_install.sh
|
||||||
chmod +x $where/hb_install.sh
|
|
||||||
fi
|
|
||||||
if [ $on_ha -eq 1 ]; then
|
|
||||||
echo "restarting hbc "
|
|
||||||
job=$(grep run_hbc configuration.yaml | sed 's/run_hbc://')
|
|
||||||
$job
|
|
||||||
else
|
|
||||||
echo "hbc installed, you can run it with \"$where/hbc\" or \"hbc\" if $where is in your PATH"
|
|
||||||
fi
|
|
||||||
elif [ "$what" = "mini" ]; then
|
|
||||||
hbc_path=$(which hbc_mini)
|
|
||||||
if [ "$hbc_path" != "$where/hbc_mini" ]; then
|
|
||||||
ln -sf $(which hbc) $where/hbc_mini
|
|
||||||
fi
|
|
||||||
echo "hbc mini installed, you can run it with \"$where/hbc_mini\" or \"hbc_mini\" if $where is in your PATH"
|
|
||||||
fi
|
fi
|
||||||
|
echo "Installation complete. To upgrade, run the following:"
|
||||||
|
echo " $where/hb_install.sh $what"
|
||||||
|
echo "To install on another machine, run the following obtain the install script and run it:"
|
||||||
|
echo "from https://git.wrede.ca/andreas/heartbeat/raw/branch/master/scripts/hb_install.sh"
|
||||||
|
echo "and then run sh hb_install.sh [mini|client]"
|
||||||
+111
-38
@@ -40,6 +40,9 @@ from logging.handlers import SysLogHandler
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
# updated by scripts/bumpminor.sh
|
||||||
|
__version__ = "5.4.0"
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Protocol (mirrors hbd/common/proto.py)
|
# Protocol (mirrors hbd/common/proto.py)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -111,9 +114,15 @@ def _stodict(data: bytes) -> Dict[str, Any]:
|
|||||||
_DEFAULTS: Dict[str, Any] = {
|
_DEFAULTS: Dict[str, Any] = {
|
||||||
"hb_port": 50003,
|
"hb_port": 50003,
|
||||||
"interval": 10,
|
"interval": 10,
|
||||||
|
"owner": None,
|
||||||
|
"allow_remote_command": False, # Execute shell commands received in CMD packets
|
||||||
"plugins": {},
|
"plugins": {},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Set from config in main(). Off by default: a CMD packet is an unauthenticated
|
||||||
|
# UDP datagram, so executing one must be opted into per host.
|
||||||
|
_allow_remote_command = False
|
||||||
|
|
||||||
|
|
||||||
def _load_config(path: Optional[str] = None) -> Dict[str, Any]:
|
def _load_config(path: Optional[str] = None) -> Dict[str, Any]:
|
||||||
cfg = dict(_DEFAULTS)
|
cfg = dict(_DEFAULTS)
|
||||||
@@ -233,9 +242,11 @@ class OSInfoPlugin(InfoPlugin):
|
|||||||
"machine": platform.machine(),
|
"machine": platform.machine(),
|
||||||
"architecture": platform.architecture()[0],
|
"architecture": platform.architecture()[0],
|
||||||
"python_version": platform.python_version(),
|
"python_version": platform.python_version(),
|
||||||
"hbc_version": "5.1.8",
|
"hbc_version": __version__,
|
||||||
"hbc_type": "mini",
|
"hbc_type": "mini",
|
||||||
}
|
}
|
||||||
|
if self.config.get("owner"):
|
||||||
|
data["owner"] = self.config["owner"]
|
||||||
if platform.system() == "Linux":
|
if platform.system() == "Linux":
|
||||||
data.update(_linux_distro())
|
data.update(_linux_distro())
|
||||||
elif platform.system() == "Darwin":
|
elif platform.system() == "Darwin":
|
||||||
@@ -385,7 +396,6 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
|
|
||||||
async def _collect_metrics(self) -> Dict[str, Any]:
|
async def _collect_metrics(self) -> Dict[str, Any]:
|
||||||
results: Dict[str, Any] = {}
|
results: Dict[str, Any] = {}
|
||||||
worst = 0
|
|
||||||
for cmd_cfg in self.commands:
|
for cmd_cfg in self.commands:
|
||||||
name = cmd_cfg.get("name")
|
name = cmd_cfg.get("name")
|
||||||
command = cmd_cfg.get("command")
|
command = cmd_cfg.get("command")
|
||||||
@@ -396,10 +406,6 @@ class NagiosRunnerPlugin(MonitorPlugin):
|
|||||||
results[f"{name}_status_code"] = rc
|
results[f"{name}_status_code"] = rc
|
||||||
results[f"{name}_output"] = msg
|
results[f"{name}_output"] = msg
|
||||||
results.update({f"{name}_{k}": v for k, v in perf.items()})
|
results.update({f"{name}_{k}": v for k, v in perf.items()})
|
||||||
worst = max(worst, rc)
|
|
||||||
results["overall_status"] = _NAGIOS_STATUS.get(worst, "UNKNOWN")
|
|
||||||
results["overall_status_code"] = worst
|
|
||||||
results["plugin_count"] = len(self.commands)
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
@@ -484,6 +490,12 @@ class CPUMonitorPlugin(MonitorPlugin):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open("/proc/uptime") as fh:
|
||||||
|
data["uptime_seconds"] = int(float(fh.read().split()[0]))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
@@ -532,6 +544,20 @@ class MemoryMonitorPlugin(MonitorPlugin):
|
|||||||
total = mi.get("MemTotal", 0)
|
total = mi.get("MemTotal", 0)
|
||||||
avail = mi.get("MemAvailable", mi.get("MemFree", 0))
|
avail = mi.get("MemAvailable", mi.get("MemFree", 0))
|
||||||
free = mi.get("MemFree", 0)
|
free = mi.get("MemFree", 0)
|
||||||
|
|
||||||
|
# ZFS ARC is reclaimable but not included in MemAvailable; add it.
|
||||||
|
arc_kb = 0
|
||||||
|
try:
|
||||||
|
with open("/proc/spl/kstat/zfs/arcstats") as _f:
|
||||||
|
for _line in _f:
|
||||||
|
_p = _line.split()
|
||||||
|
if len(_p) >= 3 and _p[0] == "size":
|
||||||
|
arc_kb = int(_p[2]) // 1024
|
||||||
|
break
|
||||||
|
except (OSError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
avail = min(avail + arc_kb, total)
|
||||||
used = total - avail
|
used = total - avail
|
||||||
data: Dict[str, Any] = {
|
data: Dict[str, Any] = {
|
||||||
"memory_total": total * 1024,
|
"memory_total": total * 1024,
|
||||||
@@ -698,7 +724,9 @@ async def _load_plugins(cfg: Dict[str, Any]) -> List[Plugin]:
|
|||||||
plugins_cfg: Dict[str, Any] = cfg.get("plugins", {})
|
plugins_cfg: Dict[str, Any] = cfg.get("plugins", {})
|
||||||
loaded: List[Plugin] = []
|
loaded: List[Plugin] = []
|
||||||
for cls in _ALL_PLUGIN_CLASSES:
|
for cls in _ALL_PLUGIN_CLASSES:
|
||||||
plugin_cfg = plugins_cfg.get(cls.name) or cfg.get(cls.name, {})
|
plugin_cfg = dict(plugins_cfg.get(cls.name) or cfg.get(cls.name) or {})
|
||||||
|
if "owner" in cfg and "owner" not in plugin_cfg:
|
||||||
|
plugin_cfg["owner"] = cfg["owner"]
|
||||||
plugin: Plugin = cls(config=plugin_cfg)
|
plugin: Plugin = cls(config=plugin_cfg)
|
||||||
try:
|
try:
|
||||||
ok = await plugin.initialize()
|
ok = await plugin.initialize()
|
||||||
@@ -768,7 +796,7 @@ class _HeartbeatProtocol(asyncio.DatagramProtocol):
|
|||||||
msg_id = msg.get("ID")
|
msg_id = msg.get("ID")
|
||||||
now = time.time()
|
now = time.time()
|
||||||
if msg_id == "ACK":
|
if msg_id == "ACK":
|
||||||
self._conn._handle_ack(now)
|
self._conn._handle_ack(msg, now)
|
||||||
elif msg_id == "CMD":
|
elif msg_id == "CMD":
|
||||||
asyncio.create_task(_handle_command(self._conn, msg))
|
asyncio.create_task(_handle_command(self._conn, msg))
|
||||||
elif msg_id == "UPD":
|
elif msg_id == "UPD":
|
||||||
@@ -779,8 +807,7 @@ class _HeartbeatProtocol(asyncio.DatagramProtocol):
|
|||||||
self._log.error("datagram error: %s", e)
|
self._log.error("datagram error: %s", e)
|
||||||
|
|
||||||
def error_received(self, exc):
|
def error_received(self, exc):
|
||||||
self._log.warning("protocol error on %s: %s — dropping connection", self._conn.addr, exc)
|
self._log.warning("protocol error on %s: %s — will retry", self._conn.addr, exc)
|
||||||
self._conn._dead = True
|
|
||||||
self._conn.close()
|
self._conn.close()
|
||||||
|
|
||||||
|
|
||||||
@@ -796,6 +823,7 @@ class AsyncConnection:
|
|||||||
self.rtts: List[float] = [0.0]
|
self.rtts: List[float] = [0.0]
|
||||||
self._transport: Optional[asyncio.DatagramTransport] = None
|
self._transport: Optional[asyncio.DatagramTransport] = None
|
||||||
self._dead = False
|
self._dead = False
|
||||||
|
self._request_info: asyncio.Event = asyncio.Event()
|
||||||
self._log = logging.getLogger(f"hbc.conn.{addr}")
|
self._log = logging.getLogger(f"hbc.conn.{addr}")
|
||||||
|
|
||||||
async def open(self) -> bool:
|
async def open(self) -> bool:
|
||||||
@@ -814,12 +842,14 @@ class AsyncConnection:
|
|||||||
self._transport.close()
|
self._transport.close()
|
||||||
self._transport = None
|
self._transport = None
|
||||||
|
|
||||||
def _handle_ack(self, now: float):
|
def _handle_ack(self, msg: Dict[str, Any], now: float):
|
||||||
rtt = (now - self.lastsend) * 1000.0
|
rtt = (now - self.lastsend) * 1000.0
|
||||||
self.rtts.append(rtt)
|
self.rtts.append(rtt)
|
||||||
if len(self.rtts) > 10:
|
if len(self.rtts) > 10:
|
||||||
self.rtts.pop(0)
|
self.rtts.pop(0)
|
||||||
self.ackcount += 1
|
self.ackcount += 1
|
||||||
|
if msg.get("request_update"):
|
||||||
|
self._request_info.set()
|
||||||
|
|
||||||
async def sendto(self, msg: Dict[str, Any], msg_id: str = "HTB"):
|
async def sendto(self, msg: Dict[str, Any], msg_id: str = "HTB"):
|
||||||
if self._dead:
|
if self._dead:
|
||||||
@@ -845,6 +875,10 @@ async def _handle_command(conn: AsyncConnection, msg: Dict[str, Any]):
|
|||||||
if not cmd:
|
if not cmd:
|
||||||
return
|
return
|
||||||
log = logging.getLogger("hbc.cmd")
|
log = logging.getLogger("hbc.cmd")
|
||||||
|
if not _allow_remote_command:
|
||||||
|
log.warning("refused command (allow_remote_command is false): %s", cmd)
|
||||||
|
await conn.sendto({"service": "command", "msg": "Refused: allow_remote_command is false"})
|
||||||
|
return
|
||||||
log.info("exec: %s", cmd)
|
log.info("exec: %s", cmd)
|
||||||
try:
|
try:
|
||||||
out = subprocess.check_output(
|
out = subprocess.check_output(
|
||||||
@@ -875,7 +909,7 @@ async def _handle_update(conn: AsyncConnection):
|
|||||||
log.info("running installer: %s", installer)
|
log.info("running installer: %s", installer)
|
||||||
try:
|
try:
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
installer, "miniclient",
|
installer, "mini",
|
||||||
stdout=asyncio.subprocess.PIPE,
|
stdout=asyncio.subprocess.PIPE,
|
||||||
stderr=asyncio.subprocess.STDOUT,
|
stderr=asyncio.subprocess.STDOUT,
|
||||||
)
|
)
|
||||||
@@ -930,7 +964,7 @@ async def _run_info_plugins(conn: AsyncConnection, plugins: List[Plugin]):
|
|||||||
try:
|
try:
|
||||||
data = await plugin.collect()
|
data = await plugin.collect()
|
||||||
if data:
|
if data:
|
||||||
await conn.sendto({"plugin": plugin.name, **data}, "PLG")
|
await conn.sendto({"plugin": plugin.name, **data, "_interval": plugin.interval}, "PLG")
|
||||||
log.info("sent %s", plugin.name)
|
log.info("sent %s", plugin.name)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.error("%s collect: %s", plugin.name, e)
|
log.error("%s collect: %s", plugin.name, e)
|
||||||
@@ -943,7 +977,7 @@ async def _run_monitor_group(conn: AsyncConnection, plugins: List[Plugin], inter
|
|||||||
try:
|
try:
|
||||||
data = await plugin.collect()
|
data = await plugin.collect()
|
||||||
if data:
|
if data:
|
||||||
await conn.sendto({"plugin": plugin.name, **data}, "PLG")
|
await conn.sendto({"plugin": plugin.name, **data, "_interval": plugin.interval}, "PLG")
|
||||||
log.debug("sent %s", plugin.name)
|
log.debug("sent %s", plugin.name)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
raise
|
raise
|
||||||
@@ -952,6 +986,19 @@ async def _run_monitor_group(conn: AsyncConnection, plugins: List[Plugin], inter
|
|||||||
await _sleep(interval)
|
await _sleep(interval)
|
||||||
|
|
||||||
|
|
||||||
|
async def _info_refresh_loop(conn: AsyncConnection, info: List[Plugin]):
|
||||||
|
log = logging.getLogger("hbc.plugins")
|
||||||
|
while _running:
|
||||||
|
await conn._request_info.wait()
|
||||||
|
if not _running:
|
||||||
|
break
|
||||||
|
conn._request_info.clear()
|
||||||
|
log.info("refreshing InfoPlugins on server request")
|
||||||
|
for plugin in info:
|
||||||
|
plugin._cache = None
|
||||||
|
await _run_info_plugins(conn, info)
|
||||||
|
|
||||||
|
|
||||||
async def _plugin_collector(conn: AsyncConnection, plugins: List[Plugin]):
|
async def _plugin_collector(conn: AsyncConnection, plugins: List[Plugin]):
|
||||||
info = [p for p in plugins if isinstance(p, InfoPlugin)]
|
info = [p for p in plugins if isinstance(p, InfoPlugin)]
|
||||||
monitor = [p for p in plugins if isinstance(p, MonitorPlugin)]
|
monitor = [p for p in plugins if isinstance(p, MonitorPlugin)]
|
||||||
@@ -962,12 +1009,10 @@ async def _plugin_collector(conn: AsyncConnection, plugins: List[Plugin]):
|
|||||||
for p in monitor:
|
for p in monitor:
|
||||||
by_interval[p.interval].append(p)
|
by_interval[p.interval].append(p)
|
||||||
|
|
||||||
if by_interval:
|
tasks = [asyncio.create_task(_info_refresh_loop(conn, info))]
|
||||||
await asyncio.gather(
|
tasks += [asyncio.create_task(_run_monitor_group(conn, grp, iv))
|
||||||
*[asyncio.create_task(_run_monitor_group(conn, grp, iv))
|
for iv, grp in by_interval.items()]
|
||||||
for iv, grp in by_interval.items()],
|
await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
return_exceptions=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -1011,7 +1056,7 @@ def _reconfigure_syslog(level: int):
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async def _async_main(args, cfg: Dict[str, Any]) -> int:
|
async def _async_main(args, cfg: Dict[str, Any]) -> int:
|
||||||
global _running, _shutdown_event, _active_tasks
|
global _running, _shutdown_event, _active_tasks, send_shutdown
|
||||||
_running = True
|
_running = True
|
||||||
_shutdown_event = asyncio.Event()
|
_shutdown_event = asyncio.Event()
|
||||||
_active_tasks = []
|
_active_tasks = []
|
||||||
@@ -1021,36 +1066,47 @@ async def _async_main(args, cfg: Dict[str, Any]) -> int:
|
|||||||
port = cfg.get("hb_port", PORT)
|
port = cfg.get("hb_port", PORT)
|
||||||
interval = cfg.get("interval", INTERVAL)
|
interval = cfg.get("interval", INTERVAL)
|
||||||
|
|
||||||
log.info("starting: %s -> %s port=%d interval=%ds", iam, args.hosts, port, interval)
|
log.info("hbc_mini %s on %s -> %s port=%d interval=%ds",__version__, iam, args.hosts, port, interval)
|
||||||
|
|
||||||
|
af_filter = (socket.AF_INET if getattr(args, "ipv4_only", False)
|
||||||
|
else socket.AF_INET6 if getattr(args, "ipv6_only", False)
|
||||||
|
else 0)
|
||||||
|
|
||||||
connections: List[AsyncConnection] = []
|
connections: List[AsyncConnection] = []
|
||||||
conn_id = 1
|
conn_id = 1
|
||||||
for host in args.hosts:
|
_retry_delay = 5
|
||||||
try:
|
while _running and not connections:
|
||||||
addrs = socket.getaddrinfo(host, port, 0, 0, socket.SOL_UDP)
|
for host in args.hosts:
|
||||||
except socket.gaierror as e:
|
try:
|
||||||
log.error("cannot resolve %s: %s", host, e)
|
addrs = socket.getaddrinfo(host, port, af_filter, 0, socket.SOL_UDP)
|
||||||
continue
|
except socket.gaierror as e:
|
||||||
for ai in addrs:
|
log.warning("cannot resolve %s: %s — retrying in %ds", host, e, _retry_delay)
|
||||||
conn = AsyncConnection(conn_id, ai[4][0], port, ai[0], iam)
|
continue
|
||||||
if await conn.open():
|
for ai in addrs:
|
||||||
connections.append(conn)
|
conn = AsyncConnection(conn_id, ai[4][0], port, ai[0], iam)
|
||||||
conn_id += 1
|
if await conn.open():
|
||||||
|
connections.append(conn)
|
||||||
|
conn_id += 1
|
||||||
|
if not connections:
|
||||||
|
await _sleep(_retry_delay)
|
||||||
|
_retry_delay = min(_retry_delay * 2, 60)
|
||||||
|
|
||||||
if not connections:
|
if not connections:
|
||||||
log.error("no connections established")
|
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
# Boot / one-shot message
|
# Boot / one-shot message
|
||||||
|
send_shutdown = False
|
||||||
if args.boot or args.message:
|
if args.boot or args.message:
|
||||||
bmsg: Dict[str, Any] = {"acks": 0}
|
bmsg: Dict[str, Any] = {"acks": 0}
|
||||||
if args.boot:
|
if args.boot:
|
||||||
bmsg["boot"] = 1
|
bmsg["boot"] = 1
|
||||||
|
args.boot = False # don't repeat on restart
|
||||||
|
send_shutdown = True
|
||||||
if args.message:
|
if args.message:
|
||||||
bmsg["service"] = "service"
|
bmsg["service"] = "service"
|
||||||
bmsg["msg"] = args.message
|
bmsg["msg"] = args.message
|
||||||
for c in connections:
|
target = next((c for c in connections if c._transport), connections[0])
|
||||||
await c.sendto(bmsg)
|
await target.sendto(bmsg)
|
||||||
if args.message and not args.daemon:
|
if args.message and not args.daemon:
|
||||||
await asyncio.sleep(0.3)
|
await asyncio.sleep(0.3)
|
||||||
for c in connections:
|
for c in connections:
|
||||||
@@ -1063,6 +1119,13 @@ async def _async_main(args, cfg: Dict[str, Any]) -> int:
|
|||||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||||
loop.add_signal_handler(sig, _stop)
|
loop.add_signal_handler(sig, _stop)
|
||||||
|
|
||||||
|
def _sighup():
|
||||||
|
global _dorestart
|
||||||
|
_dorestart = True
|
||||||
|
_stop()
|
||||||
|
|
||||||
|
loop.add_signal_handler(signal.SIGHUP, _sighup)
|
||||||
|
|
||||||
for conn in connections:
|
for conn in connections:
|
||||||
_active_tasks.append(asyncio.create_task(_heartbeat_sender(conn, interval)))
|
_active_tasks.append(asyncio.create_task(_heartbeat_sender(conn, interval)))
|
||||||
|
|
||||||
@@ -1075,11 +1138,13 @@ async def _async_main(args, cfg: Dict[str, Any]) -> int:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
log.info("shutting down")
|
log.info("shutting down")
|
||||||
for conn in connections:
|
target = next((c for c in connections if c._transport), connections[0] if connections else None)
|
||||||
|
if target and send_shutdown:
|
||||||
try:
|
try:
|
||||||
await conn.sendto({"shutdown": 1, "acks": conn.ackcount})
|
await target.sendto({"shutdown": 1, "acks": target.ackcount})
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
for conn in connections:
|
||||||
conn.close()
|
conn.close()
|
||||||
await asyncio.sleep(0.3)
|
await asyncio.sleep(0.3)
|
||||||
for plugin in plugins:
|
for plugin in plugins:
|
||||||
@@ -1105,6 +1170,9 @@ def main(argv=None):
|
|||||||
parser.add_argument("-d", "--daemon", action="store_true", help="Run as daemon")
|
parser.add_argument("-d", "--daemon", action="store_true", help="Run as daemon")
|
||||||
parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
|
parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
|
||||||
parser.add_argument("-x", "--debug", action="count", default=0, help="Debug level")
|
parser.add_argument("-x", "--debug", action="count", default=0, help="Debug level")
|
||||||
|
af_group = parser.add_mutually_exclusive_group()
|
||||||
|
af_group.add_argument("-4", dest="ipv4_only", action="store_true", help="Use IPv4 only")
|
||||||
|
af_group.add_argument("-6", dest="ipv6_only", action="store_true", help="Use IPv6 only")
|
||||||
parser.add_argument("hosts", nargs="+", help="HBD server(s)")
|
parser.add_argument("hosts", nargs="+", help="HBD server(s)")
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
@@ -1121,6 +1189,11 @@ def main(argv=None):
|
|||||||
|
|
||||||
cfg = _load_config(args.configfile)
|
cfg = _load_config(args.configfile)
|
||||||
|
|
||||||
|
global _allow_remote_command
|
||||||
|
_allow_remote_command = bool(cfg.get("allow_remote_command", False))
|
||||||
|
if _allow_remote_command:
|
||||||
|
logging.warning("allow_remote_command is true — CMD packets from the server will be executed")
|
||||||
|
|
||||||
if args.daemon:
|
if args.daemon:
|
||||||
_daemonize()
|
_daemonize()
|
||||||
_reconfigure_syslog(level)
|
_reconfigure_syslog(level)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,49 @@
|
|||||||
|
# PyInstaller spec for hbc_windows.exe
|
||||||
|
# Build with: pyinstaller hbc_windows.spec
|
||||||
|
#
|
||||||
|
# Requirements (on Windows):
|
||||||
|
# pip install pyinstaller
|
||||||
|
|
||||||
|
block_cipher = None
|
||||||
|
|
||||||
|
a = Analysis(
|
||||||
|
['hbc_windows.py'],
|
||||||
|
pathex=[],
|
||||||
|
binaries=[],
|
||||||
|
datas=[],
|
||||||
|
hiddenimports=[],
|
||||||
|
hookspath=[],
|
||||||
|
hooksconfig={},
|
||||||
|
runtime_hooks=[],
|
||||||
|
excludes=['tkinter', 'unittest', 'email', 'html', 'http', 'urllib', 'xml'],
|
||||||
|
win_no_prefer_redirects=False,
|
||||||
|
win_private_assemblies=False,
|
||||||
|
cipher=block_cipher,
|
||||||
|
noarchive=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
pyz = PYZ(a.pure, a.zlib_archive, cipher=block_cipher)
|
||||||
|
|
||||||
|
exe = EXE(
|
||||||
|
pyz,
|
||||||
|
a.scripts,
|
||||||
|
a.binaries,
|
||||||
|
a.zipfiles,
|
||||||
|
a.datas,
|
||||||
|
[],
|
||||||
|
name='hbc_windows',
|
||||||
|
debug=False,
|
||||||
|
bootloader_ignore_signals=False,
|
||||||
|
strip=False,
|
||||||
|
upx=False,
|
||||||
|
upx_exclude=[],
|
||||||
|
runtime_tmpdir=None,
|
||||||
|
console=True,
|
||||||
|
disable_windowed_traceback=False,
|
||||||
|
argv_emulation=False,
|
||||||
|
target_arch=None,
|
||||||
|
codesign_identity=None,
|
||||||
|
entitlements_file=None,
|
||||||
|
icon=None,
|
||||||
|
version=None,
|
||||||
|
)
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
#Requires -RunAsAdministrator
|
||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Install hbc_windows.exe as a Windows Service using NSSM.
|
||||||
|
|
||||||
|
.DESCRIPTION
|
||||||
|
Installs the HeartBeat Client as a Windows Service that starts automatically.
|
||||||
|
Requires NSSM (Non-Sucking Service Manager) in PATH or alongside this script.
|
||||||
|
Requires hbc_windows.exe built via: pyinstaller hbc_windows.spec
|
||||||
|
|
||||||
|
.PARAMETER Server
|
||||||
|
HBD server hostname or IP address (required).
|
||||||
|
|
||||||
|
.PARAMETER ExePath
|
||||||
|
Path to hbc_windows.exe. Defaults to the directory containing this script.
|
||||||
|
|
||||||
|
.PARAMETER ServiceName
|
||||||
|
Windows service name. Default: heartbeat-client
|
||||||
|
|
||||||
|
.PARAMETER ConfigFile
|
||||||
|
Path to hbc.json config file. Optional.
|
||||||
|
|
||||||
|
.PARAMETER LogFile
|
||||||
|
Path to log file. Default: C:\ProgramData\heartbeat\hbc.log
|
||||||
|
|
||||||
|
.PARAMETER Interval
|
||||||
|
Heartbeat interval in seconds. Default: 10
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
.\install_hbc_windows.ps1 -Server hbd.example.com
|
||||||
|
.\install_hbc_windows.ps1 -Server hbd.example.com -ConfigFile C:\ProgramData\heartbeat\hbc.json
|
||||||
|
#>
|
||||||
|
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[string]$Server,
|
||||||
|
|
||||||
|
[string]$ExePath = "",
|
||||||
|
[string]$ServiceName = "heartbeat-client",
|
||||||
|
[string]$ConfigFile = "",
|
||||||
|
[string]$LogFile = "C:\ProgramData\heartbeat\hbc.log",
|
||||||
|
[int]$Interval = 10
|
||||||
|
)
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
|
||||||
|
# Locate hbc_windows.exe
|
||||||
|
if ($ExePath -eq "") {
|
||||||
|
$ExePath = Join-Path $PSScriptRoot "hbc_windows.exe"
|
||||||
|
}
|
||||||
|
if (-not (Test-Path $ExePath)) {
|
||||||
|
Write-Error "hbc_windows.exe not found at: $ExePath`nBuild it first with: pyinstaller hbc_windows.spec"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Locate NSSM
|
||||||
|
$nssm = Get-Command nssm -ErrorAction SilentlyContinue
|
||||||
|
if (-not $nssm) {
|
||||||
|
$nssmLocal = Join-Path $PSScriptRoot "nssm.exe"
|
||||||
|
if (Test-Path $nssmLocal) {
|
||||||
|
$nssm = $nssmLocal
|
||||||
|
} else {
|
||||||
|
Write-Error "nssm.exe not found in PATH or alongside this script.`nDownload from https://nssm.cc/download"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$nssm = $nssm.Source
|
||||||
|
}
|
||||||
|
|
||||||
|
# Build argument list
|
||||||
|
$args_list = "--daemon $Server"
|
||||||
|
if ($ConfigFile -ne "") {
|
||||||
|
$args_list = "--daemon -c `"$ConfigFile`" $Server"
|
||||||
|
}
|
||||||
|
if ($LogFile -ne "") {
|
||||||
|
$args_list = "$args_list --log-file `"$LogFile`""
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create data directory
|
||||||
|
$dataDir = "C:\ProgramData\heartbeat"
|
||||||
|
if (-not (Test-Path $dataDir)) {
|
||||||
|
New-Item -ItemType Directory -Path $dataDir | Out-Null
|
||||||
|
Write-Host "Created $dataDir"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Remove existing service if present
|
||||||
|
$existing = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
|
||||||
|
if ($existing) {
|
||||||
|
Write-Host "Removing existing service '$ServiceName'..."
|
||||||
|
& $nssm stop $ServiceName 2>$null
|
||||||
|
& $nssm remove $ServiceName confirm
|
||||||
|
}
|
||||||
|
|
||||||
|
# Install service
|
||||||
|
Write-Host "Installing service '$ServiceName'..."
|
||||||
|
& $nssm install $ServiceName $ExePath $args_list
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
Write-Error "nssm install failed (exit $LASTEXITCODE)"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# Configure service
|
||||||
|
& $nssm set $ServiceName DisplayName "HeartBeat Client"
|
||||||
|
& $nssm set $ServiceName Description "Sends heartbeat and plugin metrics to the HBD monitoring server."
|
||||||
|
& $nssm set $ServiceName Start SERVICE_AUTO_START
|
||||||
|
& $nssm set $ServiceName AppStdout (Join-Path $dataDir "nssm_stdout.log")
|
||||||
|
& $nssm set $ServiceName AppStderr (Join-Path $dataDir "nssm_stderr.log")
|
||||||
|
& $nssm set $ServiceName AppRotateFiles 1
|
||||||
|
& $nssm set $ServiceName AppRotateBytes 5242880
|
||||||
|
|
||||||
|
# Start service
|
||||||
|
Write-Host "Starting service '$ServiceName'..."
|
||||||
|
& $nssm start $ServiceName
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
Write-Warning "Service installed but failed to start — check logs in $dataDir"
|
||||||
|
} else {
|
||||||
|
Write-Host "Service '$ServiceName' started successfully."
|
||||||
|
Write-Host "Log file: $LogFile"
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host "Useful commands:"
|
||||||
|
Write-Host " nssm status $ServiceName"
|
||||||
|
Write-Host " nssm stop $ServiceName"
|
||||||
|
Write-Host " nssm restart $ServiceName"
|
||||||
|
Write-Host " nssm remove $ServiceName confirm"
|
||||||
|
}
|
||||||
@@ -68,8 +68,7 @@ async def test_nagios_runner():
|
|||||||
print(f" ✓ Collected {len(data)} data points")
|
print(f" ✓ Collected {len(data)} data points")
|
||||||
|
|
||||||
print(f"\n4. Results:")
|
print(f"\n4. Results:")
|
||||||
print(f" Overall Status: {data.get('overall_status')} (code: {data.get('overall_status_code')})")
|
print(f" Data points collected: {len(data)}")
|
||||||
print(f" Plugins Executed: {data.get('plugin_count')}")
|
|
||||||
|
|
||||||
# Show individual plugin results
|
# Show individual plugin results
|
||||||
print(f"\n5. Individual Plugin Results:")
|
print(f"\n5. Individual Plugin Results:")
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
"""Tests for the allow_remote_command gate on CMD packets in hbc."""
|
||||||
|
import asyncio
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from hbd.client import main as client_main
|
||||||
|
from hbd.client.config import CLIENT_DEFAULTS, load_config
|
||||||
|
|
||||||
|
|
||||||
|
class FakeConn:
|
||||||
|
"""Stand-in for AsyncConnection that records what the client sends back."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.sent = []
|
||||||
|
|
||||||
|
async def sendto(self, msg, msg_id="HTB"):
|
||||||
|
self.sent.append(msg)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def conn(monkeypatch):
|
||||||
|
"""A fake connection with remote commands disabled and subprocess booby-trapped."""
|
||||||
|
monkeypatch.setattr(client_main, "allow_remote_command", False)
|
||||||
|
|
||||||
|
def explode(*a, **kw):
|
||||||
|
raise AssertionError("subprocess must not run when the command is refused")
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "check_output", explode)
|
||||||
|
return FakeConn()
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_is_false():
|
||||||
|
assert CLIENT_DEFAULTS["allow_remote_command"] is False
|
||||||
|
assert load_config("/nonexistent/hbc.yaml")["allow_remote_command"] is False
|
||||||
|
assert client_main.allow_remote_command is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_refused_when_disabled(conn):
|
||||||
|
asyncio.run(client_main.handle_command(conn, {"cmd": "id"}))
|
||||||
|
assert conn.sent == [{
|
||||||
|
"service": "command",
|
||||||
|
"msg": "Refused: allow_remote_command is false",
|
||||||
|
}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_refusal_reported_for_every_attempt(conn):
|
||||||
|
for _ in range(3):
|
||||||
|
asyncio.run(client_main.handle_command(conn, {"cmd": "rm -rf /"}))
|
||||||
|
assert len(conn.sent) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_command_is_ignored_either_way(conn, monkeypatch):
|
||||||
|
asyncio.run(client_main.handle_command(conn, {}))
|
||||||
|
monkeypatch.setattr(client_main, "allow_remote_command", True)
|
||||||
|
asyncio.run(client_main.handle_command(conn, {"cmd": ""}))
|
||||||
|
assert conn.sent == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_executed_when_enabled(conn, monkeypatch):
|
||||||
|
monkeypatch.setattr(client_main, "allow_remote_command", True)
|
||||||
|
monkeypatch.setattr(subprocess, "check_output", lambda *a, **kw: b"uid=0(root)")
|
||||||
|
asyncio.run(client_main.handle_command(conn, {"cmd": "id"}))
|
||||||
|
assert conn.sent == [{"service": "command", "msg": "OK uid=0(root)"}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_failure_reported_when_enabled(conn, monkeypatch):
|
||||||
|
monkeypatch.setattr(client_main, "allow_remote_command", True)
|
||||||
|
|
||||||
|
def fail(*a, **kw):
|
||||||
|
raise subprocess.CalledProcessError(1, "false")
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "check_output", fail)
|
||||||
|
asyncio.run(client_main.handle_command(conn, {"cmd": "false"}))
|
||||||
|
assert len(conn.sent) == 1
|
||||||
|
assert conn.sent[0]["msg"].startswith("CalledProcessError ")
|
||||||
|
|
||||||
|
|
||||||
|
def test_config_file_can_opt_in(tmp_path):
|
||||||
|
cfg_file = tmp_path / "hbc.yaml"
|
||||||
|
cfg_file.write_text("allow_remote_command: true\n", encoding="utf-8")
|
||||||
|
assert load_config(str(cfg_file))["allow_remote_command"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("configured, expected", [(True, True), (False, False), (None, False)])
|
||||||
|
def test_async_main_sets_the_flag_from_config(monkeypatch, configured, expected):
|
||||||
|
"""async_main must apply the config value before any CMD packet can arrive."""
|
||||||
|
monkeypatch.setattr(client_main, "allow_remote_command", not expected)
|
||||||
|
|
||||||
|
class Bail(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# gethostname() is the first call after the flag is assigned — bail there so the
|
||||||
|
# test never opens a socket.
|
||||||
|
def bail():
|
||||||
|
raise Bail
|
||||||
|
|
||||||
|
monkeypatch.setattr(client_main.socket, "gethostname", bail)
|
||||||
|
|
||||||
|
cfg = {} if configured is None else {"allow_remote_command": configured}
|
||||||
|
with pytest.raises(Bail):
|
||||||
|
asyncio.run(client_main.async_main(object(), cfg))
|
||||||
|
assert client_main.allow_remote_command is expected
|
||||||
@@ -0,0 +1,247 @@
|
|||||||
|
"""Tests for ownership rules and scoped config merges (hbd.server.config_access)."""
|
||||||
|
import pytest
|
||||||
|
from hbd.server import config_access as ca
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Visibility helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_is_global_when_no_owner():
|
||||||
|
assert ca.is_global({"type": "pushover"})
|
||||||
|
assert ca.is_global({"type": "email", "owner": ""})
|
||||||
|
assert ca.is_global(None) # non-dict is treated as global
|
||||||
|
assert not ca.is_global({"type": "email", "owner": "alice"})
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_can_use_global_or_own():
|
||||||
|
assert ca.user_can_use({"type": "pushover"}, "alice")
|
||||||
|
assert ca.user_can_use({"owner": "alice"}, "alice")
|
||||||
|
assert not ca.user_can_use({"owner": "bob"}, "alice")
|
||||||
|
|
||||||
|
|
||||||
|
HOSTS = {
|
||||||
|
"web1": {"owner": "alice", "watch": True},
|
||||||
|
"web2": {"owner": "bob", "managers": ["alice"]},
|
||||||
|
"web3": {"owner": "bob", "managers": "carol"}, # string manager form
|
||||||
|
"web4": {"watch": True}, # unowned
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_hosts_owner_and_manager():
|
||||||
|
assert set(ca.user_hosts(HOSTS, "alice")) == {"web1", "web2"}
|
||||||
|
assert set(ca.user_hosts(HOSTS, "carol")) == {"web3"}
|
||||||
|
assert ca.user_hosts(HOSTS, "dave") == {}
|
||||||
|
assert ca.user_hosts(None, "alice") == {}
|
||||||
|
|
||||||
|
|
||||||
|
CHANNELS = {
|
||||||
|
"global_ch": {"type": "pushover"},
|
||||||
|
"alice_ch": {"type": "email", "owner": "alice"},
|
||||||
|
"bob_ch": {"type": "signal", "owner": "bob"},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_channels_global_plus_own():
|
||||||
|
assert set(ca.user_channels(CHANNELS, "alice")) == {"global_ch", "alice_ch"}
|
||||||
|
assert set(ca.user_channels(CHANNELS, "dave")) == {"global_ch"}
|
||||||
|
|
||||||
|
|
||||||
|
TCS = {
|
||||||
|
"default": {"thresholds": {}},
|
||||||
|
"alice_tc": {"owner": "alice", "thresholds": {}},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_threshold_configs_global_plus_own():
|
||||||
|
assert set(ca.user_threshold_configs(TCS, "alice")) == {"default", "alice_tc"}
|
||||||
|
assert set(ca.user_threshold_configs(TCS, "bob")) == {"default"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# merge_hosts_scoped
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
MERGE_HOSTS = {
|
||||||
|
"mine": {"owner": "alice", "watch": True, "notification_channels": ["global_ch"]},
|
||||||
|
"managed": {"owner": "bob", "managers": ["alice"], "watch": True,
|
||||||
|
"notification_channels": ["bob_ch"]},
|
||||||
|
"foreign": {"owner": "bob", "watch": False},
|
||||||
|
}
|
||||||
|
MERGE_CHANNELS = {
|
||||||
|
"global_ch": {"type": "pushover"},
|
||||||
|
"alice_ch": {"type": "email", "owner": "alice"},
|
||||||
|
"bob_ch": {"type": "signal", "owner": "bob"},
|
||||||
|
}
|
||||||
|
MERGE_TCS = {"alice_tc": {"owner": "alice"}, "bob_tc": {"owner": "bob"}}
|
||||||
|
|
||||||
|
|
||||||
|
def _merge_hosts(payload, existing=None):
|
||||||
|
return ca.merge_hosts_scoped(
|
||||||
|
existing if existing is not None else dict(MERGE_HOSTS),
|
||||||
|
payload, "alice", MERGE_CHANNELS, MERGE_TCS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _full_payload(**overrides):
|
||||||
|
"""Payload covering alice's full visible subset, with per-host overrides."""
|
||||||
|
p = {
|
||||||
|
"mine": {"owner": "alice", "watch": True, "notification_channels": ["global_ch"]},
|
||||||
|
"managed": {"watch": True, "notification_channels": ["bob_ch"]},
|
||||||
|
}
|
||||||
|
p.update(overrides)
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_preserves_foreign_hosts():
|
||||||
|
result = _merge_hosts(_full_payload())
|
||||||
|
assert result["foreign"] == {"owner": "bob", "watch": False}
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_manager_edits_settings():
|
||||||
|
result = _merge_hosts(_full_payload(managed={"watch": False, "dyndns": True,
|
||||||
|
"notification_channels": ["bob_ch"]}))
|
||||||
|
assert result["managed"]["watch"] is False
|
||||||
|
assert result["managed"]["dyndns"] is True
|
||||||
|
# access fields carried over untouched
|
||||||
|
assert result["managed"]["owner"] == "bob"
|
||||||
|
assert result["managed"]["managers"] == ["alice"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_manager_cannot_change_owner():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="managed"):
|
||||||
|
_merge_hosts(_full_payload(managed={"watch": True, "owner": "alice"}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_manager_cannot_change_managers():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="managers"):
|
||||||
|
_merge_hosts(_full_payload(managed={"watch": True, "managers": ["alice", "carol"]}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_manager_same_access_values_ok():
|
||||||
|
# Submitting unchanged access fields is not a violation.
|
||||||
|
result = _merge_hosts(_full_payload(managed={"watch": True, "owner": "bob",
|
||||||
|
"managers": ["alice"],
|
||||||
|
"notification_channels": ["bob_ch"]}))
|
||||||
|
assert result["managed"]["owner"] == "bob"
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_manager_cannot_delete():
|
||||||
|
payload = {"mine": {"owner": "alice", "watch": True}} # 'managed' missing
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="managed"):
|
||||||
|
_merge_hosts(payload)
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_owner_can_delete():
|
||||||
|
payload = _full_payload()
|
||||||
|
del payload["mine"]
|
||||||
|
result = _merge_hosts(payload)
|
||||||
|
assert "mine" not in result
|
||||||
|
assert "managed" in result and "foreign" in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_owner_can_transfer_ownership():
|
||||||
|
result = _merge_hosts(_full_payload(mine={"owner": "bob", "watch": True}))
|
||||||
|
assert result["mine"]["owner"] == "bob"
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_new_host_owner_forced():
|
||||||
|
result = _merge_hosts(_full_payload(newhost={"watch": True, "owner": "bob"}))
|
||||||
|
assert result["newhost"]["owner"] == "alice"
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_cannot_touch_foreign_host():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="foreign"):
|
||||||
|
_merge_hosts(_full_payload(foreign={"watch": True}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_cannot_add_foreign_private_channel():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="bob_ch"):
|
||||||
|
_merge_hosts(_full_payload(mine={"owner": "alice", "watch": True,
|
||||||
|
"notification_channels": ["bob_ch"]}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_keeps_preexisting_foreign_assignment():
|
||||||
|
# 'managed' already has bob_ch; keeping it is fine.
|
||||||
|
result = _merge_hosts(_full_payload())
|
||||||
|
assert result["managed"]["notification_channels"] == ["bob_ch"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_can_add_global_and_own_channel():
|
||||||
|
result = _merge_hosts(_full_payload(mine={"owner": "alice", "watch": True,
|
||||||
|
"notification_channels": ["global_ch", "alice_ch"]}))
|
||||||
|
assert result["mine"]["notification_channels"] == ["global_ch", "alice_ch"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_threshold_assignment_validated():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="bob_tc"):
|
||||||
|
_merge_hosts(_full_payload(mine={"owner": "alice", "watch": True,
|
||||||
|
"threshold_config": ["bob_tc"]}))
|
||||||
|
result = _merge_hosts(_full_payload(mine={"owner": "alice", "watch": True,
|
||||||
|
"threshold_config": ["default", "alice_tc"]}))
|
||||||
|
assert result["mine"]["threshold_config"] == ["default", "alice_tc"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_rejects_unknown_fields():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="sneaky"):
|
||||||
|
_merge_hosts(_full_payload(mine={"owner": "alice", "watch": True, "sneaky": 1}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_hosts_owner_clearing_list_removes_key():
|
||||||
|
result = _merge_hosts(_full_payload(mine={"owner": "alice", "watch": True}))
|
||||||
|
assert "notification_channels" not in result["mine"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# merge_threshold_configs_scoped
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
MERGE_EXISTING_TCS = {
|
||||||
|
"default": {"thresholds": {"cpu": {"load": {"warning": 2}}}},
|
||||||
|
"alice_tc": {"owner": "alice", "thresholds": {"cpu": {"load": {"warning": 3}}}},
|
||||||
|
"bob_tc": {"owner": "bob", "thresholds": {}},
|
||||||
|
"global_tc": {"thresholds": {}},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _merge_tcs(payload):
|
||||||
|
return ca.merge_threshold_configs_scoped(dict(MERGE_EXISTING_TCS), payload, "alice")
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tcs_new_config_owner_forced():
|
||||||
|
result = _merge_tcs({"alice_tc": {"thresholds": {}},
|
||||||
|
"new_tc": {"thresholds": {}, "owner": "bob"}})
|
||||||
|
assert result["new_tc"]["owner"] == "alice"
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tcs_edit_own():
|
||||||
|
result = _merge_tcs({"alice_tc": {"thresholds": {"mem": {"used": {"warning": 90}}}}})
|
||||||
|
assert result["alice_tc"]["thresholds"] == {"mem": {"used": {"warning": 90}}}
|
||||||
|
assert result["alice_tc"]["owner"] == "alice"
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tcs_delete_own_when_missing():
|
||||||
|
result = _merge_tcs({})
|
||||||
|
assert "alice_tc" not in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tcs_preserves_global_and_foreign():
|
||||||
|
result = _merge_tcs({"alice_tc": {"thresholds": {}}})
|
||||||
|
assert result["default"] == MERGE_EXISTING_TCS["default"]
|
||||||
|
assert result["bob_tc"] == MERGE_EXISTING_TCS["bob_tc"]
|
||||||
|
assert result["global_tc"] == MERGE_EXISTING_TCS["global_tc"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tcs_rejects_default():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="default"):
|
||||||
|
_merge_tcs({"alice_tc": {"thresholds": {}}, "default": {"thresholds": {}}})
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tcs_rejects_global_name_collision():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="global_tc"):
|
||||||
|
_merge_tcs({"alice_tc": {"thresholds": {}}, "global_tc": {"thresholds": {}}})
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_tcs_rejects_foreign_owned():
|
||||||
|
with pytest.raises(ca.ScopedMergeError, match="bob_tc"):
|
||||||
|
_merge_tcs({"alice_tc": {"thresholds": {}}, "bob_tc": {"thresholds": {}}})
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
import glob
|
||||||
|
import os
|
||||||
|
import pytest
|
||||||
|
from hbd.server import configio
|
||||||
|
|
||||||
|
SAMPLE_YAML = """\
|
||||||
|
# Server configuration
|
||||||
|
hbd_port: 50004 # HTTP API port
|
||||||
|
interval: 20
|
||||||
|
users:
|
||||||
|
alice:
|
||||||
|
full_name: Alice Smith
|
||||||
|
admin: true
|
||||||
|
notification_channels:
|
||||||
|
pushover_ops:
|
||||||
|
type: pushover
|
||||||
|
token: abc123
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_roundtrip_loads_values(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
assert data["hbd_port"] == 50004
|
||||||
|
assert data["interval"] == 20
|
||||||
|
assert data["users"]["alice"]["full_name"] == "Alice Smith"
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_config_creates_backup(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
data["interval"] = 30
|
||||||
|
configio.write_config(str(f), data)
|
||||||
|
backups = configio.list_backups(str(f))
|
||||||
|
assert len(backups) == 1
|
||||||
|
assert ".bak." in backups[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_config_preserves_comments(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
data["interval"] = 30
|
||||||
|
configio.write_config(str(f), data)
|
||||||
|
content = f.read_text()
|
||||||
|
assert "# Server configuration" in content
|
||||||
|
assert "# HTTP API port" in content
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_config_atomically_replaces_file(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
data["interval"] = 99
|
||||||
|
configio.write_config(str(f), data)
|
||||||
|
assert not (tmp_path / ".hb.yaml.tmp").exists()
|
||||||
|
data2 = configio.read_roundtrip(str(f))
|
||||||
|
assert data2["interval"] == 99
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_config_backup_rotation(tmp_path):
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text(SAMPLE_YAML)
|
||||||
|
# Pre-create 10 existing backups with old timestamps
|
||||||
|
for i in range(10):
|
||||||
|
(tmp_path / f".hb.yaml.bak.20260101-{i:06d}").write_text("old")
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
backups = configio.list_backups(str(cfg))
|
||||||
|
assert len(backups) == 10
|
||||||
|
assert not (tmp_path / ".hb.yaml.bak.20260101-000000").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_backups_newest_first(tmp_path):
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text(SAMPLE_YAML)
|
||||||
|
for i in range(3):
|
||||||
|
(tmp_path / f".hb.yaml.bak.20260101-{i:02d}0000").write_text("b")
|
||||||
|
backups = configio.list_backups(str(cfg))
|
||||||
|
assert len(backups) == 3
|
||||||
|
assert backups == sorted(backups, reverse=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_structured_section_server_updates_keys(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.apply_structured_section(data, "server", {"interval": 60, "hbd_port": 8080})
|
||||||
|
assert data["interval"] == 60
|
||||||
|
assert data["hbd_port"] == 8080
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_structured_section_server_ignores_unknown_keys(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.apply_structured_section(data, "server", {"interval": 60, "not_a_key": "x"})
|
||||||
|
assert "not_a_key" not in data
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_structured_section_users_replaces_dict(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
new_users = {"bob": {"full_name": "Bob Jones", "admin": False}}
|
||||||
|
configio.apply_structured_section(data, "users", new_users)
|
||||||
|
assert "alice" not in data["users"]
|
||||||
|
assert data["users"]["bob"]["full_name"] == "Bob Jones"
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_yaml_section_notification_channels(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
new_yaml = "email_ops:\n type: email\n recipients: [ops@example.com]\n"
|
||||||
|
configio.apply_yaml_section(data, "notification_channels", new_yaml)
|
||||||
|
assert "email_ops" in data["notification_channels"]
|
||||||
|
assert "pushover_ops" not in data["notification_channels"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_yaml_section_thresholds_maps_to_threshold_configs(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.apply_yaml_section(data, "thresholds", "default:\n cpu: 80\n")
|
||||||
|
assert "threshold_configs" in data
|
||||||
|
assert data["threshold_configs"]["default"]["cpu"] == 80
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_yaml_section_dns_replaces_each_key(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.apply_yaml_section(
|
||||||
|
data, "dns",
|
||||||
|
"nsupdate_bin: /usr/bin/nsupdate\ndyndomains: [dyn.example.com]\n"
|
||||||
|
)
|
||||||
|
assert data["nsupdate_bin"] == "/usr/bin/nsupdate"
|
||||||
|
assert data["dyndomains"] == ["dyn.example.com"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_yaml_section_unknown_raises(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
with pytest.raises(ValueError, match="Unknown YAML section"):
|
||||||
|
configio.apply_yaml_section(data, "nope", "x: 1\n")
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_structured_section_unknown_raises(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
with pytest.raises(ValueError, match="Unknown structured section"):
|
||||||
|
configio.apply_structured_section(data, "nope", {"x": 1})
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_roundtrip_missing_file_raises(tmp_path):
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
configio.read_roundtrip(str(tmp_path / "nonexistent.yaml"))
|
||||||
@@ -0,0 +1,265 @@
|
|||||||
|
"""Tests for the dedicated events journal (write path and read path)."""
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
from hbd.server import journal
|
||||||
|
|
||||||
|
|
||||||
|
def _make_journal(tmp_path, **overrides):
|
||||||
|
cfg = {"journal_dir": str(tmp_path), "journal_file": "events.journal"}
|
||||||
|
cfg.update(overrides)
|
||||||
|
j = journal.MessageJournal(cfg)
|
||||||
|
assert asyncio.run(j.initialize())
|
||||||
|
return j
|
||||||
|
|
||||||
|
|
||||||
|
def _read_lines(tmp_path, name="events.journal"):
|
||||||
|
return (tmp_path / name).read_text(encoding="utf-8").splitlines()
|
||||||
|
|
||||||
|
|
||||||
|
EV1 = {"ts": 1000.0, "host": "h1", "level": "INFO", "service": None, "message": "host up"}
|
||||||
|
EV2 = {"ts": 2000.0, "host": "h2", "level": "CRITICAL", "service": "cpu", "message": "cpu high"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_event_writes_one_json_line(tmp_path):
|
||||||
|
j = _make_journal(tmp_path)
|
||||||
|
asyncio.run(j.log_event(EV1))
|
||||||
|
asyncio.run(j.close())
|
||||||
|
lines = _read_lines(tmp_path)
|
||||||
|
assert len(lines) == 1
|
||||||
|
assert json.loads(lines[0]) == EV1
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_event_appends_in_order(tmp_path):
|
||||||
|
j = _make_journal(tmp_path)
|
||||||
|
asyncio.run(j.log_event(EV1))
|
||||||
|
asyncio.run(j.log_event(EV2))
|
||||||
|
asyncio.run(j.close())
|
||||||
|
lines = _read_lines(tmp_path)
|
||||||
|
assert [json.loads(ln)["ts"] for ln in lines] == [1000.0, 2000.0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_event_rotates_at_max_size(tmp_path):
|
||||||
|
# max_size fits one serialized event line (~77 bytes) but not two, so the
|
||||||
|
# second write triggers exactly one rotation
|
||||||
|
j = _make_journal(tmp_path, journal_max_size=120)
|
||||||
|
asyncio.run(j.log_event(EV1))
|
||||||
|
asyncio.run(j.log_event(EV2))
|
||||||
|
asyncio.run(j.close())
|
||||||
|
backups = list(tmp_path.glob("events.journal.*"))
|
||||||
|
assert len(backups) == 1
|
||||||
|
assert json.loads(backups[0].read_text().splitlines()[0]) == EV1
|
||||||
|
assert json.loads(_read_lines(tmp_path)[0]) == EV2
|
||||||
|
|
||||||
|
|
||||||
|
def test_log_event_noop_when_disabled(tmp_path):
|
||||||
|
j = journal.MessageJournal(
|
||||||
|
{"journal_dir": str(tmp_path), "journal_file": "events.journal", "journal_enabled": False}
|
||||||
|
)
|
||||||
|
asyncio.run(j.initialize())
|
||||||
|
asyncio.run(j.log_event(EV1))
|
||||||
|
assert not (tmp_path / "events.journal").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_seeds_empty_journal(tmp_path):
|
||||||
|
j = _make_journal(tmp_path)
|
||||||
|
asyncio.run(j.backfill([EV1, EV2]))
|
||||||
|
asyncio.run(j.close())
|
||||||
|
lines = _read_lines(tmp_path)
|
||||||
|
assert len(lines) == 2
|
||||||
|
assert json.loads(lines[0]) == EV1
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_skipped_when_journal_nonempty(tmp_path):
|
||||||
|
(tmp_path / "events.journal").write_text(json.dumps(EV1) + "\n")
|
||||||
|
j = _make_journal(tmp_path) # initialize() picks up the existing size
|
||||||
|
asyncio.run(j.backfill([EV2]))
|
||||||
|
asyncio.run(j.close())
|
||||||
|
assert len(_read_lines(tmp_path)) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_events_journal_uses_events_config_keys(tmp_path):
|
||||||
|
journal._events_journal_instance = None
|
||||||
|
try:
|
||||||
|
ej = journal.get_events_journal(
|
||||||
|
{
|
||||||
|
"journal_dir": str(tmp_path),
|
||||||
|
"events_journal_file": "ev.jsonl",
|
||||||
|
"events_journal_max_size": 12345,
|
||||||
|
"events_journal_max_backups": 3,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert ej.journal_file == "ev.jsonl"
|
||||||
|
assert ej.max_size == 12345
|
||||||
|
assert ej.max_backups == 3
|
||||||
|
assert ej.journal_dir == tmp_path
|
||||||
|
# singleton: second call returns the same instance
|
||||||
|
assert journal.get_events_journal() is ej
|
||||||
|
finally:
|
||||||
|
journal._events_journal_instance = None
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_events_journal_defaults(tmp_path):
|
||||||
|
journal._events_journal_instance = None
|
||||||
|
try:
|
||||||
|
ej = journal.get_events_journal({"journal_dir": str(tmp_path)})
|
||||||
|
assert ej.journal_file == "events.journal"
|
||||||
|
assert ej.max_size == 10 * 1024 * 1024
|
||||||
|
assert ej.max_backups == 10
|
||||||
|
finally:
|
||||||
|
journal._events_journal_instance = None
|
||||||
|
|
||||||
|
|
||||||
|
# ---- read path -------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _write_journal(path, events):
|
||||||
|
path.write_text("".join(json.dumps(e) + "\n" for e in events), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _evts(*ts_list):
|
||||||
|
return [
|
||||||
|
{"ts": float(t), "host": f"host{i}", "level": "INFO", "service": None, "message": f"msg {t}"}
|
||||||
|
for i, t in enumerate(ts_list)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_newest_first(tmp_path):
|
||||||
|
_write_journal(tmp_path / "events.journal", _evts(1, 2, 3))
|
||||||
|
events, more = journal.read_events(tmp_path)
|
||||||
|
assert [e["ts"] for e in events] == [3.0, 2.0, 1.0]
|
||||||
|
assert more is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_limit_and_more(tmp_path):
|
||||||
|
_write_journal(tmp_path / "events.journal", _evts(1, 2, 3))
|
||||||
|
events, more = journal.read_events(tmp_path, limit=2)
|
||||||
|
assert [e["ts"] for e in events] == [3.0, 2.0]
|
||||||
|
assert more is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_before_cursor(tmp_path):
|
||||||
|
_write_journal(tmp_path / "events.journal", _evts(1, 2, 3))
|
||||||
|
events, _ = journal.read_events(tmp_path, before=3.0)
|
||||||
|
assert [e["ts"] for e in events] == [2.0, 1.0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_spans_rotated_files(tmp_path):
|
||||||
|
# rotated backup holds the oldest events; current file the newest
|
||||||
|
_write_journal(tmp_path / "events.journal.20260101-000000", _evts(1, 2))
|
||||||
|
_write_journal(tmp_path / "events.journal.20260201-000000", _evts(3, 4))
|
||||||
|
_write_journal(tmp_path / "events.journal", _evts(5, 6))
|
||||||
|
events, more = journal.read_events(tmp_path, limit=10)
|
||||||
|
assert [e["ts"] for e in events] == [6.0, 5.0, 4.0, 3.0, 2.0, 1.0]
|
||||||
|
assert more is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_pagination_across_files(tmp_path):
|
||||||
|
_write_journal(tmp_path / "events.journal.20260101-000000", _evts(1, 2))
|
||||||
|
_write_journal(tmp_path / "events.journal", _evts(3, 4))
|
||||||
|
page1, more1 = journal.read_events(tmp_path, limit=3)
|
||||||
|
assert [e["ts"] for e in page1] == [4.0, 3.0, 2.0]
|
||||||
|
assert more1 is True
|
||||||
|
page2, more2 = journal.read_events(tmp_path, limit=3, before=page1[-1]["ts"])
|
||||||
|
assert [e["ts"] for e in page2] == [1.0]
|
||||||
|
assert more2 is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_host_filter_substring_case_insensitive(tmp_path):
|
||||||
|
evs = [
|
||||||
|
{"ts": 1.0, "host": "Wentworth", "level": "INFO", "service": None, "message": "a"},
|
||||||
|
{"ts": 2.0, "host": "winter", "level": "INFO", "service": None, "message": "b"},
|
||||||
|
]
|
||||||
|
_write_journal(tmp_path / "events.journal", evs)
|
||||||
|
events, _ = journal.read_events(tmp_path, host="went")
|
||||||
|
assert [e["host"] for e in events] == ["Wentworth"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_level_filter_exact_case_insensitive(tmp_path):
|
||||||
|
evs = [
|
||||||
|
{"ts": 1.0, "host": "h", "level": "CRITICAL", "service": None, "message": "a"},
|
||||||
|
{"ts": 2.0, "host": "h", "level": "INFO", "service": None, "message": "b"},
|
||||||
|
]
|
||||||
|
_write_journal(tmp_path / "events.journal", evs)
|
||||||
|
events, _ = journal.read_events(tmp_path, level="critical")
|
||||||
|
assert [e["level"] for e in events] == ["CRITICAL"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_message_filter(tmp_path):
|
||||||
|
evs = [
|
||||||
|
{"ts": 1.0, "host": "h", "level": "INFO", "service": None, "message": "disk almost full"},
|
||||||
|
{"ts": 2.0, "host": "h", "level": "INFO", "service": None, "message": "all quiet"},
|
||||||
|
]
|
||||||
|
_write_journal(tmp_path / "events.journal", evs)
|
||||||
|
events, _ = journal.read_events(tmp_path, q="Disk")
|
||||||
|
assert [e["ts"] for e in events] == [1.0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_skips_malformed_lines(tmp_path):
|
||||||
|
p = tmp_path / "events.journal"
|
||||||
|
p.write_text('{"ts": 1.0, "host": "h", "level": "INFO", "message": "ok"}\nnot json\n[1,2]\n')
|
||||||
|
events, _ = journal.read_events(tmp_path)
|
||||||
|
assert [e["ts"] for e in events] == [1.0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_predicate(tmp_path):
|
||||||
|
_write_journal(tmp_path / "events.journal", _evts(1, 2))
|
||||||
|
events, _ = journal.read_events(tmp_path, predicate=lambda e: e["host"] == "host0")
|
||||||
|
assert [e["host"] for e in events] == ["host0"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_missing_dir(tmp_path):
|
||||||
|
events, more = journal.read_events(tmp_path / "nope")
|
||||||
|
assert events == [] and more is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_filter_events_over_in_memory_ring():
|
||||||
|
ring = _evts(1, 2, 3) # oldest-first, like data.msgs
|
||||||
|
events, more = journal.filter_events(reversed(ring), limit=2)
|
||||||
|
assert [e["ts"] for e in events] == [3.0, 2.0]
|
||||||
|
assert more is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_filter_events_skips_non_dict_entries():
|
||||||
|
ring = [
|
||||||
|
{"ts": 1.0, "host": "h", "level": "INFO", "service": None, "message": "a"},
|
||||||
|
"legacy string entry",
|
||||||
|
None,
|
||||||
|
{"ts": 2.0, "host": "h", "level": "INFO", "service": None, "message": "b"},
|
||||||
|
]
|
||||||
|
events, more = journal.filter_events(reversed(ring), limit=10)
|
||||||
|
assert [e["ts"] for e in events] == [2.0, 1.0]
|
||||||
|
assert more is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---- eventlog wiring --------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_eventlog_writes_to_events_journal(tmp_path):
|
||||||
|
from hbd.server import data, notify
|
||||||
|
|
||||||
|
journal._events_journal_instance = None
|
||||||
|
saved_msgs = data.msgs
|
||||||
|
data.msgs = []
|
||||||
|
try:
|
||||||
|
ej = journal.get_events_journal({"journal_dir": str(tmp_path)})
|
||||||
|
|
||||||
|
async def scenario():
|
||||||
|
await ej.initialize()
|
||||||
|
notify.setup({}, loop=asyncio.get_running_loop())
|
||||||
|
notify.eventlog("h1", "INFO", "hello world")
|
||||||
|
await asyncio.sleep(0.05) # let the scheduled journal write run
|
||||||
|
await ej.close()
|
||||||
|
|
||||||
|
asyncio.run(scenario())
|
||||||
|
lines = _read_lines(tmp_path)
|
||||||
|
assert len(lines) == 1
|
||||||
|
ev = json.loads(lines[0])
|
||||||
|
assert ev["host"] == "h1"
|
||||||
|
assert ev["level"] == "INFO"
|
||||||
|
assert ev["message"] == "hello world"
|
||||||
|
assert isinstance(ev["ts"], float)
|
||||||
|
finally:
|
||||||
|
journal._events_journal_instance = None
|
||||||
|
data.msgs = saved_msgs
|
||||||
|
notify._loop = None
|
||||||
@@ -0,0 +1,197 @@
|
|||||||
|
"""Tests for flap detection (hbd.server.flap) and its hook in notify.send_notification."""
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from hbd.server import flap, hbdclass, notify, users as users_mod
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def clean_flap_state():
|
||||||
|
"""Every test starts with an empty state and a 3-in-10-minutes config."""
|
||||||
|
flap._state.clear()
|
||||||
|
flap.setup({"flap_count": 3, "flap_interval": 10})
|
||||||
|
yield
|
||||||
|
flap._state.clear()
|
||||||
|
flap.setup({})
|
||||||
|
|
||||||
|
|
||||||
|
def alerts(n, host="h1", service="cpu", level="CRITICAL"):
|
||||||
|
return [flap.observe(host, service, level) for _ in range(n)]
|
||||||
|
|
||||||
|
|
||||||
|
def advance(seconds, host="h1", service="cpu"):
|
||||||
|
"""Backdate a key's recorded timestamps to simulate the clock moving on."""
|
||||||
|
st = flap._state[(host, service)]
|
||||||
|
st["events"] = [t - seconds for t in st["events"]]
|
||||||
|
if st["ok_since"] is not None:
|
||||||
|
st["ok_since"] -= seconds
|
||||||
|
|
||||||
|
|
||||||
|
# --- tripping ---------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_alerts_up_to_flap_count_pass():
|
||||||
|
assert alerts(3) == [flap.PASS] * 3
|
||||||
|
assert flap.flapping_services("h1") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_exceeding_flap_count_trips_then_suppresses():
|
||||||
|
assert alerts(5) == [flap.PASS, flap.PASS, flap.PASS, flap.TRIP, flap.SUPPRESS]
|
||||||
|
assert flap.flapping_services("h1") == ["cpu"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_warning_and_critical_both_count():
|
||||||
|
flap.observe("h1", "cpu", "WARNING")
|
||||||
|
flap.observe("h1", "cpu", "CRITICAL")
|
||||||
|
flap.observe("h1", "cpu", "WARNING")
|
||||||
|
assert flap.observe("h1", "cpu", "CRITICAL") == flap.TRIP
|
||||||
|
|
||||||
|
|
||||||
|
def test_alerts_outside_the_window_do_not_count():
|
||||||
|
alerts(3)
|
||||||
|
advance(11 * 60) # older than flap_interval
|
||||||
|
assert alerts(3) == [flap.PASS] * 3
|
||||||
|
assert flap.flapping_services("h1") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_recover_alone_never_trips():
|
||||||
|
assert [flap.observe("h1", "cpu", "RECOVER") for _ in range(5)] == [flap.PASS] * 5
|
||||||
|
|
||||||
|
|
||||||
|
# --- suppression while flapping --------------------------------------------
|
||||||
|
|
||||||
|
def test_recover_and_info_are_suppressed_while_flapping():
|
||||||
|
alerts(4)
|
||||||
|
assert flap.observe("h1", "cpu", "RECOVER") == flap.SUPPRESS
|
||||||
|
assert flap.observe("h1", "cpu", "INFO") == flap.SUPPRESS
|
||||||
|
|
||||||
|
|
||||||
|
def test_info_passes_when_not_flapping():
|
||||||
|
assert flap.observe("h1", "cpu", "INFO") == flap.PASS
|
||||||
|
|
||||||
|
|
||||||
|
# --- clearing ---------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_clears_silently_one_interval_after_recover():
|
||||||
|
alerts(4)
|
||||||
|
flap.observe("h1", "cpu", "RECOVER")
|
||||||
|
advance(10 * 60)
|
||||||
|
assert flap.flapping_services("h1") == []
|
||||||
|
assert flap.observe("h1", "cpu", "CRITICAL") == flap.PASS
|
||||||
|
|
||||||
|
|
||||||
|
def test_still_flapping_before_the_interval_elapses():
|
||||||
|
alerts(4)
|
||||||
|
flap.observe("h1", "cpu", "RECOVER")
|
||||||
|
advance(9 * 60)
|
||||||
|
assert flap.flapping_services("h1") == ["cpu"]
|
||||||
|
assert flap.observe("h1", "cpu", "CRITICAL") == flap.SUPPRESS
|
||||||
|
|
||||||
|
|
||||||
|
def test_alert_during_the_quiet_window_keeps_it_flapping():
|
||||||
|
alerts(4)
|
||||||
|
flap.observe("h1", "cpu", "RECOVER")
|
||||||
|
advance(9 * 60)
|
||||||
|
flap.observe("h1", "cpu", "CRITICAL") # resets the quiet window
|
||||||
|
advance(2 * 60)
|
||||||
|
assert flap.flapping_services("h1") == ["cpu"]
|
||||||
|
|
||||||
|
|
||||||
|
# --- keying -----------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_services_and_hosts_are_tracked_independently():
|
||||||
|
alerts(4, service="cpu")
|
||||||
|
assert flap.observe("h1", "disk", "CRITICAL") == flap.PASS
|
||||||
|
assert flap.observe("h2", "cpu", "CRITICAL") == flap.PASS
|
||||||
|
assert flap.flapping_services("h1") == ["cpu"]
|
||||||
|
assert flap.flapping_services("h2") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_level_events_use_the_empty_service_key():
|
||||||
|
alerts(4, service="")
|
||||||
|
assert flap.flapping_services("h1") == [""]
|
||||||
|
|
||||||
|
|
||||||
|
# --- disabled ---------------------------------------------------------------
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("cfg", [
|
||||||
|
{"flap_count": 0, "flap_interval": 10},
|
||||||
|
{"flap_count": 3, "flap_interval": 0},
|
||||||
|
{},
|
||||||
|
])
|
||||||
|
def test_disabled_never_suppresses(cfg):
|
||||||
|
flap.setup(cfg)
|
||||||
|
assert alerts(20) == [flap.PASS] * 20
|
||||||
|
assert flap.flapping_services("h1") == []
|
||||||
|
|
||||||
|
|
||||||
|
# --- integration through the real dispatch path -----------------------------
|
||||||
|
|
||||||
|
NOTIFY_CFG = {
|
||||||
|
"flap_count": 3,
|
||||||
|
"flap_interval": 10,
|
||||||
|
"notification_channels": {"ch1": {"type": "pushover", "token": "t", "user": "u"}},
|
||||||
|
"users": {"alice": {"notification_channels": ["ch1"]}},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def delivered(monkeypatch):
|
||||||
|
"""Wire up a host, a user and a stub channel driver; collect what gets delivered."""
|
||||||
|
sent = []
|
||||||
|
monkeypatch.setattr(notify, "_config", dict(notify._config)) # restored on teardown
|
||||||
|
notify.setup(NOTIFY_CFG)
|
||||||
|
users_mod.load_users(NOTIFY_CFG)
|
||||||
|
monkeypatch.setitem(notify._DRIVERS, "pushover", lambda cfg, n: sent.append(n.body) or True)
|
||||||
|
host = hbdclass.Host("flaphost")
|
||||||
|
host.watched = True
|
||||||
|
host.owner = "alice"
|
||||||
|
yield sent, host
|
||||||
|
hbdclass.Host.hosts.pop("flaphost", None)
|
||||||
|
users_mod.load_users({})
|
||||||
|
|
||||||
|
|
||||||
|
def _notify(level, service, body):
|
||||||
|
n = notify.Notification(title=f"[{level}] flaphost", body=body, level=level, service=service)
|
||||||
|
asyncio.run(notify.send_notification("flaphost", n))
|
||||||
|
|
||||||
|
|
||||||
|
def test_marker_on_the_tripping_notification_then_nothing(delivered):
|
||||||
|
sent, host = delivered
|
||||||
|
for i in range(5):
|
||||||
|
_notify("CRITICAL", "cpu", f"cpu = 9{i}")
|
||||||
|
_notify("RECOVER", "cpu", "cpu = 12")
|
||||||
|
|
||||||
|
assert sent == ["cpu = 90", "cpu = 91", "cpu = 92", f"cpu = 93 {flap.FLAP_MARKER}"]
|
||||||
|
assert host.stateinfo()["flapping"] == ["cpu"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_flapping_service_does_not_silence_another(delivered):
|
||||||
|
sent, host = delivered
|
||||||
|
for i in range(5):
|
||||||
|
_notify("CRITICAL", "cpu", "cpu = 99")
|
||||||
|
sent.clear()
|
||||||
|
_notify("CRITICAL", "disk", "disk = 91")
|
||||||
|
assert sent == ["disk = 91"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_notifications_resume_after_the_state_clears(delivered):
|
||||||
|
sent, host = delivered
|
||||||
|
for i in range(5):
|
||||||
|
_notify("CRITICAL", "cpu", "cpu = 99")
|
||||||
|
_notify("RECOVER", "cpu", "cpu = 12")
|
||||||
|
advance(10 * 60, host="flaphost")
|
||||||
|
assert host.stateinfo()["flapping"] == []
|
||||||
|
sent.clear()
|
||||||
|
_notify("CRITICAL", "cpu", "cpu = 99")
|
||||||
|
assert sent == ["cpu = 99"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_stateinfo_flapping_survives_json_encoding(delivered):
|
||||||
|
sent, host = delivered
|
||||||
|
for i in range(5):
|
||||||
|
_notify("CRITICAL", "", "IPv4 overdue") # host-level, empty service key
|
||||||
|
info = host.stateinfo()
|
||||||
|
assert info["flapping"] == [""]
|
||||||
|
assert json.loads(json.dumps(info))["flapping"] == [""]
|
||||||
@@ -20,7 +20,7 @@ def test_handle_cmd_sends_command():
|
|||||||
import hbdclass
|
import hbdclass
|
||||||
|
|
||||||
ctx = {
|
ctx = {
|
||||||
"config": {"watchhosts": [], "dyndnshosts": []},
|
"config": {"watchhosts": []},
|
||||||
"hbdclass": hbdclass,
|
"hbdclass": hbdclass,
|
||||||
"log": dummy_noop,
|
"log": dummy_noop,
|
||||||
"email": dummy_noop,
|
"email": dummy_noop,
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
"""Tests for the config read/write API helpers in http.py."""
|
||||||
|
import pytest
|
||||||
|
from hbd.server import http
|
||||||
|
|
||||||
|
|
||||||
|
def test_mask_config_for_api_masks_user_passwords():
|
||||||
|
config = {
|
||||||
|
"hbd_port": 50004,
|
||||||
|
"interval": 20,
|
||||||
|
"users": {
|
||||||
|
"alice": {"full_name": "Alice", "admin": True, "password": "pbkdf2:sha256:abc"},
|
||||||
|
},
|
||||||
|
"oauth": {},
|
||||||
|
}
|
||||||
|
result = http._mask_config_for_api(config)
|
||||||
|
assert result["users"]["alice"]["password"] == "•••"
|
||||||
|
assert result["users"]["alice"]["full_name"] == "Alice"
|
||||||
|
|
||||||
|
|
||||||
|
def test_mask_config_for_api_masks_oauth_client_secret():
|
||||||
|
config = {
|
||||||
|
"hbd_port": 50004,
|
||||||
|
"interval": 20,
|
||||||
|
"users": {},
|
||||||
|
"oauth": {
|
||||||
|
"gitea": {"type": "gitea", "url": "https://git.example.com",
|
||||||
|
"client_id": "cid", "client_secret": "verysecret"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result = http._mask_config_for_api(config)
|
||||||
|
assert result["oauth"]["gitea"]["client_secret"] == "•••"
|
||||||
|
assert result["oauth"]["gitea"]["client_id"] == "cid"
|
||||||
|
|
||||||
|
|
||||||
|
def test_mask_config_for_api_includes_server_keys():
|
||||||
|
config = {"hbd_port": 50004, "interval": 20, "users": {}, "oauth": {}}
|
||||||
|
result = http._mask_config_for_api(config)
|
||||||
|
assert result["server"]["hbd_port"] == 50004
|
||||||
|
assert result["server"]["interval"] == 20
|
||||||
|
|
||||||
|
|
||||||
|
def test_mask_config_for_api_no_password_in_users_leaves_no_key():
|
||||||
|
config = {
|
||||||
|
"hbd_port": 50004,
|
||||||
|
"users": {"bob": {"full_name": "Bob", "admin": False}},
|
||||||
|
"oauth": {},
|
||||||
|
}
|
||||||
|
result = http._mask_config_for_api(config)
|
||||||
|
assert "password" not in result["users"]["bob"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---- configio integration for write path ----
|
||||||
|
|
||||||
|
def test_write_path_applies_server_section(tmp_path):
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text("hbd_port: 50004\ninterval: 20\nusers: {}\n")
|
||||||
|
from hbd.server import configio
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
configio.apply_structured_section(data, "server", {"interval": 60})
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
data2 = configio.read_roundtrip(str(cfg))
|
||||||
|
assert data2["interval"] == 60
|
||||||
|
assert data2["hbd_port"] == 50004 # unchanged
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_path_applies_yaml_section(tmp_path):
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text(
|
||||||
|
"hbd_port: 50004\nnotification_channels:\n old_ch:\n type: email\n"
|
||||||
|
)
|
||||||
|
from hbd.server import configio
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
configio.apply_yaml_section(data, "notification_channels", "new_ch:\n type: pushover\n")
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
data2 = configio.read_roundtrip(str(cfg))
|
||||||
|
assert "new_ch" in data2["notification_channels"]
|
||||||
|
assert "old_ch" not in data2["notification_channels"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_path_hashes_plaintext_password(tmp_path):
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text("hbd_port: 50004\nusers:\n alice:\n full_name: Alice\n admin: true\n password: pbkdf2:sha256:old\n")
|
||||||
|
from hbd.server import configio
|
||||||
|
from hbd.server import users as users_mod
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
# Simulate what the POST handler does: hash plaintext password
|
||||||
|
new_users = {"alice": {"full_name": "Alice", "admin": True, "password": "newplaintext"}}
|
||||||
|
for username, attrs in new_users.items():
|
||||||
|
pw = attrs.get("password", "")
|
||||||
|
if pw and not pw.startswith("pbkdf2:"):
|
||||||
|
attrs["password"] = users_mod.hash_password(pw)
|
||||||
|
configio.apply_structured_section(data, "users", new_users)
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
data2 = configio.read_roundtrip(str(cfg))
|
||||||
|
assert data2["users"]["alice"]["password"].startswith("pbkdf2:")
|
||||||
|
assert data2["users"]["alice"]["password"] != "newplaintext"
|
||||||
|
|
||||||
|
|
||||||
|
def test_rollback_restores_backup(tmp_path):
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text("hbd_port: 50004\ninterval: 20\n")
|
||||||
|
from hbd.server import configio
|
||||||
|
# Make a change to create a backup
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
data["interval"] = 99
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
backups = configio.list_backups(str(cfg))
|
||||||
|
assert len(backups) == 1
|
||||||
|
# Read the backup and write it back (simulating rollback)
|
||||||
|
backup_data = configio.read_roundtrip(backups[0])
|
||||||
|
configio.write_config(str(cfg), backup_data)
|
||||||
|
restored = configio.read_roundtrip(str(cfg))
|
||||||
|
assert restored["interval"] == 20
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_path_preserves_masked_password(tmp_path):
|
||||||
|
"""The "•••" sentinel must preserve the existing hash, not write "•••" to disk."""
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
original_hash = "pbkdf2:sha256:original_hash"
|
||||||
|
cfg.write_text(
|
||||||
|
f"hbd_port: 50004\nusers:\n alice:\n full_name: Alice\n admin: true\n password: {original_hash}\n"
|
||||||
|
)
|
||||||
|
from hbd.server import configio
|
||||||
|
from hbd.server import users as users_mod
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
# Simulate what api_config_post does when client sends "•••" back
|
||||||
|
existing_users = data.get("users") or {}
|
||||||
|
users_payload = {"alice": {"full_name": "Alice", "admin": True, "password": "•••"}}
|
||||||
|
for username, attrs in users_payload.items():
|
||||||
|
pw = attrs.get("password", "")
|
||||||
|
if pw and pw != "•••" and not pw.startswith("pbkdf2:"):
|
||||||
|
attrs["password"] = users_mod.hash_password(pw)
|
||||||
|
elif not pw or pw == "•••":
|
||||||
|
existing_hash = (existing_users.get(username) or {}).get("password", "")
|
||||||
|
if existing_hash:
|
||||||
|
attrs["password"] = existing_hash
|
||||||
|
else:
|
||||||
|
attrs.pop("password", None)
|
||||||
|
configio.apply_structured_section(data, "users", users_payload)
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
data2 = configio.read_roundtrip(str(cfg))
|
||||||
|
assert data2["users"]["alice"]["password"] == original_hash, (
|
||||||
|
f"Expected original hash preserved, got: {data2['users']['alice']['password']!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_path_preserves_oauth_client_secret(tmp_path):
|
||||||
|
"""The "•••" sentinel for oauth client_secret must preserve the existing secret."""
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
original_secret = "real_client_secret_value"
|
||||||
|
cfg.write_text(
|
||||||
|
f"hbd_port: 50004\noauth:\n gitea:\n type: gitea\n url: https://git.example.com\n"
|
||||||
|
f" client_id: cid123\n client_secret: {original_secret}\n"
|
||||||
|
)
|
||||||
|
from hbd.server import configio
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
# Simulate what api_config_post does when client sends "•••" back for client_secret
|
||||||
|
existing_oauth = data.get("oauth") or {}
|
||||||
|
new_oauth = {"gitea": {"type": "gitea", "url": "https://git.example.com", "client_id": "cid123", "client_secret": "•••"}}
|
||||||
|
for name, attrs in new_oauth.items():
|
||||||
|
cs = attrs.get("client_secret", "")
|
||||||
|
if not cs or cs == "•••":
|
||||||
|
existing_cs = (existing_oauth.get(name) or {}).get("client_secret", "")
|
||||||
|
if existing_cs:
|
||||||
|
attrs["client_secret"] = existing_cs
|
||||||
|
else:
|
||||||
|
attrs.pop("client_secret", None)
|
||||||
|
data["oauth"] = new_oauth
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
data2 = configio.read_roundtrip(str(cfg))
|
||||||
|
assert data2["oauth"]["gitea"]["client_secret"] == original_secret, (
|
||||||
|
f"Expected original secret preserved, got: {data2['oauth']['gitea']['client_secret']!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---- threshold form payload shape ----
|
||||||
|
|
||||||
|
def test_build_threshold_configs_new_shape_with_owner():
|
||||||
|
form = {
|
||||||
|
"servers": {
|
||||||
|
"owner": "alice",
|
||||||
|
"metrics": {"cpu_monitor.load_15min": {"operator": ">", "warning": 4.0,
|
||||||
|
"critical": 8.0, "enabled": True}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result = http._build_threshold_configs_from_form(form)
|
||||||
|
assert result["servers"]["owner"] == "alice"
|
||||||
|
assert result["servers"]["thresholds"]["cpu_monitor"]["load_15min"]["warning"] == 4.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_threshold_configs_empty_owner_means_global():
|
||||||
|
form = {"servers": {"owner": "", "metrics": {"rtt": {"warning": 100.0}}}}
|
||||||
|
result = http._build_threshold_configs_from_form(form)
|
||||||
|
assert "owner" not in result["servers"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_threshold_configs_ignores_entries_without_metrics():
|
||||||
|
assert http._build_threshold_configs_from_form({"bad": {"owner": "x"}}) == {}
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
"""Tests for _build_host_info helper in http.py."""
|
||||||
|
import pytest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
from hbd.server.http import _build_host_info
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeConn:
|
||||||
|
def __init__(self, lastbeat):
|
||||||
|
self.lastbeat = lastbeat
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeHost:
|
||||||
|
def __init__(self, name="myhost", owner=None, managers=None,
|
||||||
|
connections=None, os_data=None, plugin_data=None):
|
||||||
|
self.name = name
|
||||||
|
self.owner = owner
|
||||||
|
self.managers = managers or []
|
||||||
|
self.connections = connections or {}
|
||||||
|
self._os_data = os_data
|
||||||
|
self.plugin_data = plugin_data or {}
|
||||||
|
|
||||||
|
def get_latest_plugin_data(self, plugin_name):
|
||||||
|
if plugin_name == "os_info" and self._os_data is not None:
|
||||||
|
return (1234567890.0, self._os_data)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_basic_fields():
|
||||||
|
host = _FakeHost(owner="alice", managers=["bob", "carol"])
|
||||||
|
result = _build_host_info(host)
|
||||||
|
assert result["owner"] == "alice"
|
||||||
|
assert result["managers"] == ["bob", "carol"]
|
||||||
|
assert result["hbc_version"] is None
|
||||||
|
assert result["hbc_type"] is None
|
||||||
|
assert result["last_packet"] is None
|
||||||
|
assert result["thresholds"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_no_owner():
|
||||||
|
host = _FakeHost()
|
||||||
|
result = _build_host_info(host)
|
||||||
|
assert result["owner"] is None
|
||||||
|
assert result["managers"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_reads_hbc_from_os_info():
|
||||||
|
host = _FakeHost(os_data={"hbc_version": "5.3.0", "hbc_type": "full"})
|
||||||
|
result = _build_host_info(host)
|
||||||
|
assert result["hbc_version"] == "5.3.0"
|
||||||
|
assert result["hbc_type"] == "full"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_hbc_none_when_no_os_info():
|
||||||
|
host = _FakeHost(os_data=None)
|
||||||
|
result = _build_host_info(host)
|
||||||
|
assert result["hbc_version"] is None
|
||||||
|
assert result["hbc_type"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_last_packet_is_max_lastbeat():
|
||||||
|
host = _FakeHost(connections={
|
||||||
|
"IPv4": _FakeConn(1000.0),
|
||||||
|
"IPv6": _FakeConn(2000.0),
|
||||||
|
})
|
||||||
|
result = _build_host_info(host)
|
||||||
|
assert result["last_packet"] == 2000.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_last_packet_none_when_no_connections():
|
||||||
|
host = _FakeHost(connections={})
|
||||||
|
result = _build_host_info(host)
|
||||||
|
assert result["last_packet"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_thresholds_none_without_checker():
|
||||||
|
host = _FakeHost()
|
||||||
|
result = _build_host_info(host, threshold_checker=None)
|
||||||
|
assert result["thresholds"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_thresholds_sorted_by_metric():
|
||||||
|
from hbd.server.threshold import ThresholdConfig
|
||||||
|
tc_cpu = ThresholdConfig("cpu_monitor.cpu_percent", warning=80.0, critical=95.0)
|
||||||
|
tc_mem = ThresholdConfig("memory_monitor.memory_percent", warning=85.0, critical=98.0)
|
||||||
|
|
||||||
|
checker = MagicMock()
|
||||||
|
checker.get_thresholds_for_host.return_value = {
|
||||||
|
"memory_monitor.memory_percent": tc_mem,
|
||||||
|
"cpu_monitor.cpu_percent": tc_cpu,
|
||||||
|
}
|
||||||
|
|
||||||
|
host = _FakeHost()
|
||||||
|
result = _build_host_info(host, threshold_checker=checker)
|
||||||
|
|
||||||
|
assert result["thresholds"] is not None
|
||||||
|
assert len(result["thresholds"]) == 2
|
||||||
|
assert result["thresholds"][0]["metric"] == "cpu_monitor.cpu_percent"
|
||||||
|
assert result["thresholds"][0]["warning"] == 80.0
|
||||||
|
assert result["thresholds"][0]["critical"] == 95.0
|
||||||
|
assert result["thresholds"][0]["operator"] == ">"
|
||||||
|
assert result["thresholds"][1]["metric"] == "memory_monitor.memory_percent"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_thresholds_empty_list_when_no_thresholds():
|
||||||
|
checker = MagicMock()
|
||||||
|
checker.get_thresholds_for_host.return_value = {}
|
||||||
|
host = _FakeHost()
|
||||||
|
result = _build_host_info(host, threshold_checker=checker)
|
||||||
|
assert result["thresholds"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_threshold_null_warning_critical():
|
||||||
|
from hbd.server.threshold import ThresholdConfig
|
||||||
|
tc = ThresholdConfig("rtt.myhost", warning=None, critical=500.0)
|
||||||
|
checker = MagicMock()
|
||||||
|
checker.get_thresholds_for_host.return_value = {"rtt.myhost": tc}
|
||||||
|
host = _FakeHost()
|
||||||
|
result = _build_host_info(host, threshold_checker=checker)
|
||||||
|
assert result["thresholds"][0]["warning"] is None
|
||||||
|
assert result["thresholds"][0]["critical"] == 500.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_nagios_operator_serialized():
|
||||||
|
from hbd.server.threshold import ThresholdConfig
|
||||||
|
tc = ThresholdConfig("nagios_runner.check_http", operator="nagios")
|
||||||
|
checker = MagicMock()
|
||||||
|
checker.get_thresholds_for_host.return_value = {"nagios_runner.check_http": tc}
|
||||||
|
host = _FakeHost()
|
||||||
|
result = _build_host_info(host, threshold_checker=checker)
|
||||||
|
assert result["thresholds"][0]["operator"] == "nagios"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_covers_suffix_matched_metrics():
|
||||||
|
"""memory_monitor.percent threshold covers swap_percent via suffix match."""
|
||||||
|
from hbd.server.threshold import ThresholdConfig
|
||||||
|
tc_pct = ThresholdConfig("memory_monitor.percent", warning=85.0, critical=95.0)
|
||||||
|
checker = MagicMock()
|
||||||
|
checker.get_thresholds_for_host.return_value = {"memory_monitor.percent": tc_pct}
|
||||||
|
|
||||||
|
host = _FakeHost(
|
||||||
|
connections={},
|
||||||
|
os_data=None,
|
||||||
|
)
|
||||||
|
# Simulate plugin_data with both percent and swap_percent fields
|
||||||
|
host.plugin_data = {
|
||||||
|
"memory_monitor": [(1234567890.0, {
|
||||||
|
"percent": 80.0,
|
||||||
|
"swap_percent": 25.0,
|
||||||
|
"available_mb": 2000,
|
||||||
|
})]
|
||||||
|
}
|
||||||
|
|
||||||
|
result = _build_host_info(host, threshold_checker=checker)
|
||||||
|
assert result["thresholds"] is not None
|
||||||
|
t = result["thresholds"][0]
|
||||||
|
assert t["metric"] == "memory_monitor.percent"
|
||||||
|
assert t["covers"] == ["memory_monitor.swap_percent"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_covers_empty_when_exact_matches_only():
|
||||||
|
"""No covers when all plugin fields match their threshold exactly."""
|
||||||
|
from hbd.server.threshold import ThresholdConfig
|
||||||
|
tc_pct = ThresholdConfig("memory_monitor.percent", warning=85.0, critical=95.0)
|
||||||
|
checker = MagicMock()
|
||||||
|
checker.get_thresholds_for_host.return_value = {"memory_monitor.percent": tc_pct}
|
||||||
|
|
||||||
|
host = _FakeHost()
|
||||||
|
host.plugin_data = {
|
||||||
|
"memory_monitor": [(1234567890.0, {"percent": 80.0})]
|
||||||
|
}
|
||||||
|
|
||||||
|
result = _build_host_info(host, threshold_checker=checker)
|
||||||
|
t = result["thresholds"][0]
|
||||||
|
assert t["covers"] == []
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeConnFull:
|
||||||
|
def __init__(self, afam, addr, state, rtt, statetime, lastbeat):
|
||||||
|
self.afam = afam
|
||||||
|
self.addr = addr
|
||||||
|
self.state = state
|
||||||
|
self.rtts = [rtt]
|
||||||
|
self.statetime = statetime
|
||||||
|
self.lastbeat = lastbeat
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_includes_connections():
|
||||||
|
host = _FakeHost(connections={
|
||||||
|
"IPv4": _FakeConnFull("IPv4", "10.0.0.5", "up", 14.2, 1000.0, 2000.0),
|
||||||
|
"IPv6": _FakeConnFull("IPv6", "fd00::5", "overdue", 0, 1500.0, 1900.0),
|
||||||
|
})
|
||||||
|
result = _build_host_info(host)
|
||||||
|
conns = {c["family"]: c for c in result["connections"]}
|
||||||
|
assert conns["IPv4"] == {"family": "IPv4", "addr": "10.0.0.5", "state": "up",
|
||||||
|
"rtt": 14.2, "statetime": 1000.0, "lastbeat": 2000.0}
|
||||||
|
assert conns["IPv6"]["state"] == "overdue"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_host_info_connections_empty():
|
||||||
|
host = _FakeHost()
|
||||||
|
result = _build_host_info(host)
|
||||||
|
assert result["connections"] == []
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
"""Tests for PUT /api/0/users/me logic."""
|
||||||
|
import pytest
|
||||||
|
from hbd.server import users as users_mod
|
||||||
|
|
||||||
|
|
||||||
|
def test_hash_password_roundtrip():
|
||||||
|
h = users_mod.hash_password("mysecret")
|
||||||
|
assert h.startswith("pbkdf2:sha256:")
|
||||||
|
assert users_mod.authenticate.__doc__ is not None # module loaded
|
||||||
|
|
||||||
|
|
||||||
|
def test_password_change_requires_correct_current(tmp_path):
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
initial_hash = users_mod.hash_password("oldpass")
|
||||||
|
cfg.write_text(
|
||||||
|
f"hbd_port: 50004\nusers:\n alice:\n full_name: Alice\n admin: true\n password: {initial_hash}\n"
|
||||||
|
)
|
||||||
|
users_mod.load_users({"users": {"alice": {"full_name": "Alice", "admin": True, "password": initial_hash}}})
|
||||||
|
|
||||||
|
# Correct current password authenticates
|
||||||
|
assert users_mod.authenticate("alice", "oldpass") is not None
|
||||||
|
# Wrong current password does not authenticate
|
||||||
|
assert users_mod.authenticate("alice", "wrongpass") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_put_users_me_writes_new_fields(tmp_path):
|
||||||
|
"""Simulate the write path: read config, update user, write back."""
|
||||||
|
initial_hash = users_mod.hash_password("secret")
|
||||||
|
yaml_content = (
|
||||||
|
"hbd_port: 50004\n"
|
||||||
|
f"users:\n alice:\n full_name: Old Name\n admin: true\n password: {initial_hash}\n"
|
||||||
|
)
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text(yaml_content)
|
||||||
|
|
||||||
|
from hbd.server import configio
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
|
||||||
|
# Simulate handler updating full_name and avatar
|
||||||
|
user_entry = dict(data["users"]["alice"])
|
||||||
|
user_entry["full_name"] = "New Name"
|
||||||
|
user_entry["avatar"] = "/img/alice.png"
|
||||||
|
data["users"]["alice"] = user_entry
|
||||||
|
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
result = configio.read_roundtrip(str(cfg))
|
||||||
|
assert result["users"]["alice"]["full_name"] == "New Name"
|
||||||
|
assert result["users"]["alice"]["avatar"] == "/img/alice.png"
|
||||||
|
assert result["users"]["alice"]["password"] == initial_hash # unchanged
|
||||||
|
|
||||||
|
|
||||||
|
def test_put_users_me_changes_password(tmp_path):
|
||||||
|
initial_hash = users_mod.hash_password("oldpass")
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text(
|
||||||
|
f"hbd_port: 50004\nusers:\n alice:\n full_name: Alice\n password: {initial_hash}\n"
|
||||||
|
)
|
||||||
|
from hbd.server import configio
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
|
||||||
|
new_hash = users_mod.hash_password("newpass")
|
||||||
|
data["users"]["alice"]["password"] = new_hash
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
|
||||||
|
result = configio.read_roundtrip(str(cfg))
|
||||||
|
# Load users from new config and authenticate with new password
|
||||||
|
new_config = {"users": dict(result["users"])}
|
||||||
|
users_mod.load_users(new_config)
|
||||||
|
assert users_mod.authenticate("alice", "newpass") is not None
|
||||||
|
assert users_mod.authenticate("alice", "oldpass") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_put_users_me_notification_channels(tmp_path):
|
||||||
|
cfg = tmp_path / ".hb.yaml"
|
||||||
|
cfg.write_text(
|
||||||
|
"hbd_port: 50004\n"
|
||||||
|
"notification_channels:\n pushover_ops:\n type: pushover\n"
|
||||||
|
"users:\n alice:\n full_name: Alice\n notification_channels: []\n"
|
||||||
|
)
|
||||||
|
from hbd.server import configio
|
||||||
|
data = configio.read_roundtrip(str(cfg))
|
||||||
|
data["users"]["alice"]["notification_channels"] = ["pushover_ops"]
|
||||||
|
configio.write_config(str(cfg), data)
|
||||||
|
result = configio.read_roundtrip(str(cfg))
|
||||||
|
assert result["users"]["alice"]["notification_channels"] == ["pushover_ops"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_visible_channels_excludes_private_from_others():
|
||||||
|
"""Private channels owned by another user must not appear in the visible set."""
|
||||||
|
from hbd.server import settings as settings_mod
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"notification_channels": {
|
||||||
|
"public_ch": {"type": "pushover", "token": "t", "user": "u"},
|
||||||
|
"alice_priv": {"type": "email", "owner": "alice", "private": True,
|
||||||
|
"recipients": ["a@b.com"], "sender": "s@b.com", "smtp_server": "s"},
|
||||||
|
"bob_priv": {"type": "email", "owner": "bob", "private": True,
|
||||||
|
"recipients": ["b@b.com"], "sender": "s@b.com", "smtp_server": "s"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeUser:
|
||||||
|
def __init__(self, username, admin=False):
|
||||||
|
self.username = username
|
||||||
|
self.admin = admin
|
||||||
|
|
||||||
|
alice = FakeUser("alice")
|
||||||
|
bob = FakeUser("bob")
|
||||||
|
admin = FakeUser("admin", admin=True)
|
||||||
|
|
||||||
|
# Simulate _visible_channels_for_user logic (mirrors http.py implementation)
|
||||||
|
def visible(user):
|
||||||
|
all_channels = config.get("notification_channels") or {}
|
||||||
|
if user.admin:
|
||||||
|
return set(all_channels.keys())
|
||||||
|
return {
|
||||||
|
name for name, cfg in all_channels.items()
|
||||||
|
if not cfg.get("private") or cfg.get("owner") == user.username
|
||||||
|
}
|
||||||
|
|
||||||
|
assert visible(alice) == {"public_ch", "alice_priv"}
|
||||||
|
assert visible(bob) == {"public_ch", "bob_priv"}
|
||||||
|
assert visible(admin) == {"public_ch", "alice_priv", "bob_priv"}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""Tests for notification channel CRUD via configio helpers and visibility logic."""
|
||||||
|
import pytest
|
||||||
|
from hbd.server import configio, settings as settings_mod
|
||||||
|
|
||||||
|
|
||||||
|
SAMPLE_YAML = """\
|
||||||
|
hbd_port: 50004
|
||||||
|
notification_channels:
|
||||||
|
pushover_ops:
|
||||||
|
type: pushover
|
||||||
|
token: abc123
|
||||||
|
user: usr456
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# configio helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_apply_channel_adds_new_entry(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.apply_channel(data, "email_ops", {"type": "email", "recipients": ["ops@example.com"]})
|
||||||
|
assert "email_ops" in data["notification_channels"]
|
||||||
|
assert data["notification_channels"]["email_ops"]["type"] == "email"
|
||||||
|
# Existing channel preserved
|
||||||
|
assert "pushover_ops" in data["notification_channels"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_channel_updates_existing(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.apply_channel(data, "pushover_ops", {"type": "pushover", "token": "new_tok", "user": "new_usr"})
|
||||||
|
assert data["notification_channels"]["pushover_ops"]["token"] == "new_tok"
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_channel_creates_section_if_absent():
|
||||||
|
data = {"hbd_port": 50004}
|
||||||
|
configio.apply_channel(data, "test_ch", {"type": "pushover", "token": "t", "user": "u"})
|
||||||
|
assert "notification_channels" in data
|
||||||
|
assert "test_ch" in data["notification_channels"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_channel_removes_entry(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.delete_channel(data, "pushover_ops")
|
||||||
|
assert "pushover_ops" not in data["notification_channels"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_channel_noop_for_missing():
|
||||||
|
data = {"notification_channels": {"ch1": {"type": "pushover"}}}
|
||||||
|
configio.delete_channel(data, "nonexistent") # must not raise
|
||||||
|
assert "ch1" in data["notification_channels"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_channel_noop_when_no_section():
|
||||||
|
data = {}
|
||||||
|
configio.delete_channel(data, "anything") # must not raise
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_channel_persisted_after_write(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.apply_channel(data, "signal_ops", {"type": "signal", "user": "+1", "recipient": "+2"})
|
||||||
|
configio.write_config(str(f), data)
|
||||||
|
result = configio.read_roundtrip(str(f))
|
||||||
|
assert "signal_ops" in result["notification_channels"]
|
||||||
|
assert result["notification_channels"]["signal_ops"]["user"] == "+1"
|
||||||
|
# Original channel preserved
|
||||||
|
assert "pushover_ops" in result["notification_channels"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_channel_persisted_after_write(tmp_path):
|
||||||
|
f = tmp_path / ".hb.yaml"
|
||||||
|
f.write_text(SAMPLE_YAML)
|
||||||
|
data = configio.read_roundtrip(str(f))
|
||||||
|
configio.delete_channel(data, "pushover_ops")
|
||||||
|
configio.write_config(str(f), data)
|
||||||
|
result = configio.read_roundtrip(str(f))
|
||||||
|
assert "pushover_ops" not in (result.get("notification_channels") or {})
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Visibility logic (owner-presence rule, hbd.server.config_access)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
from hbd.server import config_access as ca # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _visible(config, user):
|
||||||
|
all_channels = config.get("notification_channels") or {}
|
||||||
|
if user.get("admin"):
|
||||||
|
return set(all_channels.keys())
|
||||||
|
return set(ca.user_channels(all_channels, user["username"]))
|
||||||
|
|
||||||
|
|
||||||
|
CONFIG_VISIBILITY = {
|
||||||
|
"notification_channels": {
|
||||||
|
"pub_ch": {"type": "pushover", "token": "t", "user": "u"},
|
||||||
|
"alice_priv": {"type": "email", "owner": "alice",
|
||||||
|
"recipients": ["a@a.com"], "sender": "s@a.com", "smtp_server": "s"},
|
||||||
|
"bob_priv": {"type": "signal", "owner": "bob", "user": "+1", "recipient": "+2"},
|
||||||
|
"stale_flag": {"type": "pushover", "token": "t2", "user": "u2", "private": True},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_global_channel_visible_to_all():
|
||||||
|
for uname in ("alice", "bob", "carol"):
|
||||||
|
assert "pub_ch" in _visible(CONFIG_VISIBILITY, {"username": uname, "admin": False})
|
||||||
|
|
||||||
|
|
||||||
|
def test_owned_channel_visible_only_to_owner():
|
||||||
|
alice = {"username": "alice", "admin": False}
|
||||||
|
bob = {"username": "bob", "admin": False}
|
||||||
|
assert "alice_priv" in _visible(CONFIG_VISIBILITY, alice)
|
||||||
|
assert "alice_priv" not in _visible(CONFIG_VISIBILITY, bob)
|
||||||
|
assert "bob_priv" in _visible(CONFIG_VISIBILITY, bob)
|
||||||
|
assert "bob_priv" not in _visible(CONFIG_VISIBILITY, alice)
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_sees_all_channels():
|
||||||
|
admin = {"username": "adminuser", "admin": True}
|
||||||
|
assert _visible(CONFIG_VISIBILITY, admin) == {"pub_ch", "alice_priv", "bob_priv", "stale_flag"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_private_flag_without_owner_is_global():
|
||||||
|
"""Owner-presence is the single signal; a leftover private flag is ignored."""
|
||||||
|
alice = {"username": "alice", "admin": False}
|
||||||
|
assert "stale_flag" in _visible(CONFIG_VISIBILITY, alice)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Channel type schemas
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_all_required_types_in_schema():
|
||||||
|
for t in ("pushover", "email", "signal", "matrix", "sms_voipms"):
|
||||||
|
assert t in settings_mod.CHANNEL_TYPE_SCHEMAS
|
||||||
|
|
||||||
|
|
||||||
|
def test_schema_fields_have_required_keys():
|
||||||
|
for type_id, schema in settings_mod.CHANNEL_TYPE_SCHEMAS.items():
|
||||||
|
assert "label" in schema, f"{type_id} missing label"
|
||||||
|
assert "fields" in schema, f"{type_id} missing fields"
|
||||||
|
for f in schema["fields"]:
|
||||||
|
for k in ("key", "label", "type", "required"):
|
||||||
|
assert k in f, f"{type_id} field missing {k!r}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_secret_fields_use_secret_type():
|
||||||
|
"""Known secret fields must be typed 'secret' so the UI masks them."""
|
||||||
|
secret_keys = {"token", "user_key", "api_key", "api_password",
|
||||||
|
"smtp_password", "access_token"}
|
||||||
|
for type_id, schema in settings_mod.CHANNEL_TYPE_SCHEMAS.items():
|
||||||
|
for f in schema["fields"]:
|
||||||
|
if f["key"] in secret_keys:
|
||||||
|
assert f["type"] == "secret", (
|
||||||
|
f"{type_id}.{f['key']} should be type 'secret'"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_channel_labels_not_empty():
|
||||||
|
for type_id, schema in settings_mod.CHANNEL_TYPE_SCHEMAS.items():
|
||||||
|
assert schema["label"].strip(), f"{type_id} has empty label"
|
||||||
@@ -0,0 +1,602 @@
|
|||||||
|
import logging
|
||||||
|
import time as time_mod
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
from urllib.parse import urlparse, parse_qs
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from hbd.server import oauth
|
||||||
|
from hbd.server import users as users_mod
|
||||||
|
from hbd.server.users import User
|
||||||
|
|
||||||
|
|
||||||
|
CFG_OFF = {}
|
||||||
|
CFG_ON = {
|
||||||
|
"oauth": {
|
||||||
|
"gitea": {
|
||||||
|
"url": "https://git.example.com",
|
||||||
|
"client_id": "cid",
|
||||||
|
"client_secret": "csec",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
CFG_PARTIAL = {"oauth": {"gitea": {"url": "https://git.example.com"}}}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def clear_oauth_states():
|
||||||
|
oauth._states.clear()
|
||||||
|
yield
|
||||||
|
oauth._states.clear()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def reset_users_dict():
|
||||||
|
original = dict(users_mod.users)
|
||||||
|
yield
|
||||||
|
users_mod.users = original
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_state_returns_unique_tokens():
|
||||||
|
s1 = oauth.make_state()
|
||||||
|
s2 = oauth.make_state()
|
||||||
|
assert s1 != s2
|
||||||
|
assert len(s1) == 64 # 32 bytes hex
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_state_valid():
|
||||||
|
state = oauth.make_state()
|
||||||
|
assert oauth.validate_state(state) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_state_consumed_on_use():
|
||||||
|
state = oauth.make_state()
|
||||||
|
oauth.validate_state(state)
|
||||||
|
assert oauth.validate_state(state) is False # replay rejected
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_state_unknown():
|
||||||
|
assert oauth.validate_state("notastate") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_state_expired(monkeypatch):
|
||||||
|
state = oauth.make_state()
|
||||||
|
# Wind expiry into the past
|
||||||
|
monkeypatch.setitem(oauth._states, state, time_mod.time() - 1000)
|
||||||
|
assert oauth.validate_state(state) is False
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_users(entries=None):
|
||||||
|
users_mod.users = entries or {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_provision_oauth_user_new():
|
||||||
|
_reset_users()
|
||||||
|
user = users_mod.provision_oauth_user("gituser", "Git User", "https://example.com/avatar.png")
|
||||||
|
assert user.username == "gituser"
|
||||||
|
assert user.full_name == "Git User"
|
||||||
|
assert user.avatar == "https://example.com/avatar.png"
|
||||||
|
assert user.admin is False
|
||||||
|
assert user.password_hash == ""
|
||||||
|
assert "gituser" in users_mod.users
|
||||||
|
|
||||||
|
|
||||||
|
def test_provision_oauth_user_no_password_login():
|
||||||
|
_reset_users()
|
||||||
|
user = users_mod.provision_oauth_user("gituser", "Git User", "")
|
||||||
|
assert user.check_password("anything") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_provision_oauth_user_existing_updates_profile():
|
||||||
|
existing = User(
|
||||||
|
username="alice",
|
||||||
|
full_name="Old Name",
|
||||||
|
avatar="old.png",
|
||||||
|
password_hash="pbkdf2:sha256:1:salt:abc",
|
||||||
|
admin=True,
|
||||||
|
notification_channels=["chan1"],
|
||||||
|
)
|
||||||
|
_reset_users({"alice": existing})
|
||||||
|
user = users_mod.provision_oauth_user("alice", "New Name", "new.png")
|
||||||
|
assert user.full_name == "New Name"
|
||||||
|
assert user.avatar == "new.png"
|
||||||
|
# Preserved
|
||||||
|
assert user.admin is True
|
||||||
|
assert user.password_hash == "pbkdf2:sha256:1:salt:abc"
|
||||||
|
assert user.notification_channels == ["chan1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_provision_oauth_user_does_not_overwrite_with_empty():
|
||||||
|
existing = User(username="bob", full_name="Bob", avatar="bob.png")
|
||||||
|
_reset_users({"bob": existing})
|
||||||
|
user = users_mod.provision_oauth_user("bob", "", "")
|
||||||
|
assert user.full_name == "Bob"
|
||||||
|
assert user.avatar == "bob.png"
|
||||||
|
|
||||||
|
|
||||||
|
def test_provision_oauth_user_survives_config_reload():
|
||||||
|
_reset_users()
|
||||||
|
users_mod.provision_oauth_user("oauthonly", "OAuth Only", "https://example.com/a.png")
|
||||||
|
assert "oauthonly" in users_mod.users
|
||||||
|
# Reload with empty config — OAuth user should survive
|
||||||
|
users_mod.load_users({})
|
||||||
|
assert "oauthonly" in users_mod.users
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Integration-style tests: callback logic chain
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_callback_invalid_state_rejects():
|
||||||
|
"""Verify validate_state returns False for unknown state tokens."""
|
||||||
|
fake_state = "this-is-not-a-real-state"
|
||||||
|
assert oauth.validate_state(fake_state) is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_full_oauth_flow_chain():
|
||||||
|
"""Integration-style test: state → exchange → fetch → provision chain."""
|
||||||
|
p = _gitea_provider()
|
||||||
|
redirect_uri = "https://hbd.example.com/login/oauth/gitea/callback"
|
||||||
|
|
||||||
|
state = oauth.make_state()
|
||||||
|
assert oauth.validate_state(state) is True
|
||||||
|
|
||||||
|
mock_token_response = AsyncMock()
|
||||||
|
mock_token_response.status = 200
|
||||||
|
mock_token_response.json = AsyncMock(return_value={"access_token": "flow_token"})
|
||||||
|
|
||||||
|
mock_user_response = AsyncMock()
|
||||||
|
mock_user_response.status = 200
|
||||||
|
mock_user_response.json = AsyncMock(return_value={
|
||||||
|
"login": "flowuser",
|
||||||
|
"full_name": "Flow User",
|
||||||
|
"avatar_url": "https://git.example.com/avatars/flow.png",
|
||||||
|
})
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.post = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_token_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
mock_session.get = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_user_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
token = await oauth.exchange_code(p, "authcode", redirect_uri)
|
||||||
|
profile = await oauth.fetch_user(p, token)
|
||||||
|
|
||||||
|
assert token == "flow_token"
|
||||||
|
assert profile["login"] == "flowuser"
|
||||||
|
|
||||||
|
_reset_users()
|
||||||
|
user = users_mod.provision_oauth_user(
|
||||||
|
profile["login"], profile["full_name"], profile["avatar_url"]
|
||||||
|
)
|
||||||
|
assert user.username == "flowuser"
|
||||||
|
assert user.check_password("anything") is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# get_providers()
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
CFG_GITHUB = {
|
||||||
|
"oauth": {
|
||||||
|
"github": {"type": "github", "client_id": "ghid", "client_secret": "ghs"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
CFG_NEXTCLOUD = {
|
||||||
|
"oauth": {
|
||||||
|
"nc": {
|
||||||
|
"type": "nextcloud",
|
||||||
|
"url": "https://nc.example.com",
|
||||||
|
"client_id": "ncid",
|
||||||
|
"client_secret": "ncs",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
CFG_MULTI = {
|
||||||
|
"oauth": {
|
||||||
|
"mygitea": {
|
||||||
|
"type": "gitea",
|
||||||
|
"url": "https://git.example.com",
|
||||||
|
"client_id": "cid",
|
||||||
|
"client_secret": "cs",
|
||||||
|
"label": "Work Gitea",
|
||||||
|
"logo": "https://example.com/logo.png",
|
||||||
|
},
|
||||||
|
"github": {"type": "github", "client_id": "ghid", "client_secret": "ghs"},
|
||||||
|
"nc": {
|
||||||
|
"type": "nextcloud",
|
||||||
|
"url": "https://nc.example.com",
|
||||||
|
"client_id": "ncid",
|
||||||
|
"client_secret": "ncs",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_backward_compat_no_type_field():
|
||||||
|
"""Old config without 'type' defaults to gitea."""
|
||||||
|
providers = oauth.get_providers(CFG_ON)
|
||||||
|
assert len(providers) == 1
|
||||||
|
p = providers[0]
|
||||||
|
assert p.name == "gitea"
|
||||||
|
assert p.type == "gitea"
|
||||||
|
assert p.label == "Gitea"
|
||||||
|
assert p.client_id == "cid"
|
||||||
|
assert p.authorize_url == "https://git.example.com/login/oauth/authorize"
|
||||||
|
assert p.token_url == "https://git.example.com/login/oauth/access_token"
|
||||||
|
assert p.profile_url == "https://git.example.com/api/v1/user"
|
||||||
|
assert p.scope == "user:email"
|
||||||
|
assert p.profile_data_path == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_multiple():
|
||||||
|
providers = oauth.get_providers(CFG_MULTI)
|
||||||
|
assert len(providers) == 3
|
||||||
|
names = [p.name for p in providers]
|
||||||
|
assert "mygitea" in names
|
||||||
|
assert "github" in names
|
||||||
|
assert "nc" in names
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_custom_label_and_logo():
|
||||||
|
providers = oauth.get_providers(CFG_MULTI)
|
||||||
|
gitea = next(p for p in providers if p.name == "mygitea")
|
||||||
|
assert gitea.label == "Work Gitea"
|
||||||
|
assert gitea.logo == "https://example.com/logo.png"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_github_default_label():
|
||||||
|
providers = oauth.get_providers(CFG_GITHUB)
|
||||||
|
assert providers[0].label == "GitHub"
|
||||||
|
assert providers[0].logo == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_github_fixed_urls():
|
||||||
|
providers = oauth.get_providers(CFG_GITHUB)
|
||||||
|
p = providers[0]
|
||||||
|
assert p.authorize_url == "https://github.com/login/oauth/authorize"
|
||||||
|
assert p.token_url == "https://github.com/login/oauth/access_token"
|
||||||
|
assert p.profile_url == "https://api.github.com/user"
|
||||||
|
assert p.scope == "read:user"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_nextcloud_urls_and_path():
|
||||||
|
providers = oauth.get_providers(CFG_NEXTCLOUD)
|
||||||
|
p = providers[0]
|
||||||
|
assert p.authorize_url == "https://nc.example.com/apps/oauth2/authorize"
|
||||||
|
assert p.token_url == "https://nc.example.com/apps/oauth2/api/v1/token"
|
||||||
|
assert p.profile_url == "https://nc.example.com/ocs/v2.php/cloud/user?format=json"
|
||||||
|
assert p.profile_data_path == ["ocs", "data"]
|
||||||
|
assert p.scope == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_skips_missing_client_id(caplog):
|
||||||
|
cfg = {"oauth": {"gitea": {"url": "https://git.example.com", "client_secret": "cs"}}}
|
||||||
|
with caplog.at_level(logging.WARNING, logger="hbd.server.oauth"):
|
||||||
|
result = oauth.get_providers(cfg)
|
||||||
|
assert result == []
|
||||||
|
assert "missing" in caplog.text.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_skips_missing_client_secret(caplog):
|
||||||
|
cfg = {"oauth": {"gitea": {"url": "https://git.example.com", "client_id": "cid"}}}
|
||||||
|
with caplog.at_level(logging.WARNING, logger="hbd.server.oauth"):
|
||||||
|
result = oauth.get_providers(cfg)
|
||||||
|
assert result == []
|
||||||
|
assert "missing" in caplog.text.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_skips_missing_url_for_gitea(caplog):
|
||||||
|
cfg = {"oauth": {"gitea": {"type": "gitea", "client_id": "cid", "client_secret": "cs"}}}
|
||||||
|
with caplog.at_level(logging.WARNING, logger="hbd.server.oauth"):
|
||||||
|
result = oauth.get_providers(cfg)
|
||||||
|
assert result == []
|
||||||
|
assert "url" in caplog.text.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_skips_missing_url_for_nextcloud(caplog):
|
||||||
|
cfg = {"oauth": {"nc": {"type": "nextcloud", "client_id": "cid", "client_secret": "cs"}}}
|
||||||
|
with caplog.at_level(logging.WARNING, logger="hbd.server.oauth"):
|
||||||
|
result = oauth.get_providers(cfg)
|
||||||
|
assert result == []
|
||||||
|
assert "url" in caplog.text.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_github_no_url_required():
|
||||||
|
providers = oauth.get_providers(CFG_GITHUB)
|
||||||
|
assert len(providers) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_skips_unknown_type(caplog):
|
||||||
|
cfg = {"oauth": {"mystery": {"type": "saml", "client_id": "cid", "client_secret": "cs"}}}
|
||||||
|
import logging
|
||||||
|
with caplog.at_level(logging.WARNING, logger="hbd.server.oauth"):
|
||||||
|
result = oauth.get_providers(cfg)
|
||||||
|
assert result == []
|
||||||
|
assert "saml" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_providers_empty_config():
|
||||||
|
assert oauth.get_providers({}) == []
|
||||||
|
assert oauth.get_providers(CFG_OFF) == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# build_auth_url / exchange_code / fetch_user (generic, ResolvedProvider-based)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _gitea_provider() -> oauth.ResolvedProvider:
|
||||||
|
return oauth.get_providers(CFG_ON)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _github_provider() -> oauth.ResolvedProvider:
|
||||||
|
return oauth.get_providers(CFG_GITHUB)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _nextcloud_provider() -> oauth.ResolvedProvider:
|
||||||
|
return oauth.get_providers(CFG_NEXTCLOUD)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_auth_url_gitea():
|
||||||
|
p = _gitea_provider()
|
||||||
|
url = oauth.build_auth_url(p, "teststate", "https://hbd.example.com/login/oauth/gitea/callback")
|
||||||
|
parsed = urlparse(url)
|
||||||
|
qs = parse_qs(parsed.query)
|
||||||
|
assert parsed.netloc == "git.example.com"
|
||||||
|
assert parsed.path == "/login/oauth/authorize"
|
||||||
|
assert qs["client_id"] == ["cid"]
|
||||||
|
assert qs["state"] == ["teststate"]
|
||||||
|
assert qs["scope"] == ["user:email"]
|
||||||
|
assert qs["response_type"] == ["code"]
|
||||||
|
assert qs["redirect_uri"] == ["https://hbd.example.com/login/oauth/gitea/callback"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_auth_url_github():
|
||||||
|
p = _github_provider()
|
||||||
|
url = oauth.build_auth_url(p, "st", "https://hbd.example.com/login/oauth/github/callback")
|
||||||
|
parsed = urlparse(url)
|
||||||
|
qs = parse_qs(parsed.query)
|
||||||
|
assert parsed.netloc == "github.com"
|
||||||
|
assert qs["scope"] == ["read:user"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_auth_url_nextcloud_no_scope_param():
|
||||||
|
"""Nextcloud scope is empty — the 'scope' key must be absent from the URL."""
|
||||||
|
p = _nextcloud_provider()
|
||||||
|
url = oauth.build_auth_url(p, "st", "https://hbd.example.com/login/oauth/nc/callback")
|
||||||
|
qs = parse_qs(urlparse(url).query)
|
||||||
|
assert "scope" not in qs
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_exchange_code_generic_returns_token():
|
||||||
|
p = _gitea_provider()
|
||||||
|
redirect_uri = "https://hbd.example.com/login/oauth/gitea/callback"
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 200
|
||||||
|
mock_response.json = AsyncMock(return_value={"access_token": "tok123"})
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.post = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
token = await oauth.exchange_code(p, "mycode", redirect_uri)
|
||||||
|
assert token == "tok123"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_exchange_code_sends_accept_json():
|
||||||
|
"""Accept: application/json must be present for all providers (required by GitHub)."""
|
||||||
|
p = _github_provider()
|
||||||
|
captured_headers = {}
|
||||||
|
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 200
|
||||||
|
mock_response.json = AsyncMock(return_value={"access_token": "ghtoken"})
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
|
||||||
|
def capture_post(url, **kwargs):
|
||||||
|
captured_headers.update(kwargs.get("headers", {}))
|
||||||
|
return AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_session.post = capture_post
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
await oauth.exchange_code(p, "code", "https://hbd.example.com/login/oauth/github/callback")
|
||||||
|
|
||||||
|
assert captured_headers.get("Accept") == "application/json"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_exchange_code_raises_on_error_status():
|
||||||
|
p = _gitea_provider()
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 401
|
||||||
|
mock_response.text = AsyncMock(return_value="unauthorized")
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.post = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
with pytest.raises(oauth.OAuthError):
|
||||||
|
await oauth.exchange_code(p, "badcode", "https://hbd.example.com/login/oauth/gitea/callback")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_exchange_code_raises_when_no_access_token():
|
||||||
|
p = _gitea_provider()
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 200
|
||||||
|
mock_response.json = AsyncMock(return_value={"error": "bad_request"})
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.post = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
with pytest.raises(oauth.OAuthError):
|
||||||
|
await oauth.exchange_code(p, "mycode", "https://hbd.example.com/login/oauth/gitea/callback")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_user_gitea_returns_profile():
|
||||||
|
p = _gitea_provider()
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 200
|
||||||
|
mock_response.json = AsyncMock(return_value={
|
||||||
|
"login": "alice",
|
||||||
|
"full_name": "Alice Smith",
|
||||||
|
"avatar_url": "https://git.example.com/avatars/alice.png",
|
||||||
|
})
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.get = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
profile = await oauth.fetch_user(p, "tok123")
|
||||||
|
|
||||||
|
assert profile == {
|
||||||
|
"login": "alice",
|
||||||
|
"full_name": "Alice Smith",
|
||||||
|
"avatar_url": "https://git.example.com/avatars/alice.png",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_user_github_maps_name_field():
|
||||||
|
p = _github_provider()
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 200
|
||||||
|
mock_response.json = AsyncMock(return_value={
|
||||||
|
"login": "bobgh",
|
||||||
|
"name": "Bob GitHub",
|
||||||
|
"avatar_url": "https://avatars.githubusercontent.com/u/1",
|
||||||
|
})
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.get = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
profile = await oauth.fetch_user(p, "ghtoken")
|
||||||
|
|
||||||
|
assert profile["login"] == "bobgh"
|
||||||
|
assert profile["full_name"] == "Bob GitHub"
|
||||||
|
assert profile["avatar_url"] == "https://avatars.githubusercontent.com/u/1"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_user_nextcloud_nested_extraction():
|
||||||
|
"""Nextcloud profile is nested under ocs.data; avatar is absent."""
|
||||||
|
p = _nextcloud_provider()
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 200
|
||||||
|
mock_response.json = AsyncMock(return_value={
|
||||||
|
"ocs": {
|
||||||
|
"meta": {"status": "ok", "statuscode": 200},
|
||||||
|
"data": {
|
||||||
|
"id": "ncuser",
|
||||||
|
"display-name": "NC User",
|
||||||
|
"email": "nc@example.com",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.get = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
profile = await oauth.fetch_user(p, "nctoken")
|
||||||
|
|
||||||
|
assert profile["login"] == "ncuser"
|
||||||
|
assert profile["full_name"] == "NC User"
|
||||||
|
assert profile["avatar_url"] == "" # Nextcloud has no avatar field
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_user_raises_on_error_status():
|
||||||
|
p = _gitea_provider()
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 401
|
||||||
|
mock_response.text = AsyncMock(return_value="unauthorized")
|
||||||
|
|
||||||
|
mock_session = MagicMock()
|
||||||
|
mock_session.get = MagicMock(return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_response),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
))
|
||||||
|
|
||||||
|
with patch("hbd.server.oauth.aiohttp.ClientSession", return_value=AsyncMock(
|
||||||
|
__aenter__=AsyncMock(return_value=mock_session),
|
||||||
|
__aexit__=AsyncMock(return_value=False),
|
||||||
|
)):
|
||||||
|
with pytest.raises(oauth.OAuthError):
|
||||||
|
await oauth.fetch_user(p, "badtoken")
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_enabled_with_valid_provider():
|
||||||
|
assert oauth.is_enabled(CFG_ON) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_enabled_false_when_no_providers():
|
||||||
|
assert oauth.is_enabled(CFG_OFF) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_enabled_false_partial_config():
|
||||||
|
assert oauth.is_enabled(CFG_PARTIAL) is False
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
import pytest
|
||||||
|
from hbd.server import settings as settings_mod
|
||||||
|
|
||||||
|
CFG = {
|
||||||
|
"hbd_port": 50004,
|
||||||
|
"interval": 20,
|
||||||
|
"grace": 2,
|
||||||
|
"users": {
|
||||||
|
"alice": {"full_name": "Alice Smith", "admin": True, "password": "pbkdf2:sha256:abc",
|
||||||
|
"notification_channels": ["pushover_ops"]},
|
||||||
|
},
|
||||||
|
"oauth": {
|
||||||
|
"gitea": {"type": "gitea", "url": "https://git.example.com",
|
||||||
|
"client_id": "cid", "client_secret": "csec", "label": "Sign in with Gitea"},
|
||||||
|
},
|
||||||
|
"notification_channels": {
|
||||||
|
"pushover_ops": {"type": "pushover", "token": "tok", "user": "usr"},
|
||||||
|
},
|
||||||
|
"hosts": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_sections_have_section_mode():
|
||||||
|
sections = settings_mod.get_settings_sections(CFG)
|
||||||
|
for s in sections:
|
||||||
|
assert "section_mode" in s, f"Section {s['id']} missing section_mode"
|
||||||
|
assert s["section_mode"] in ("form", "yaml", "channels", "hosts", "thresholds")
|
||||||
|
|
||||||
|
|
||||||
|
def test_sections_have_api_section():
|
||||||
|
sections = settings_mod.get_settings_sections(CFG)
|
||||||
|
for s in sections:
|
||||||
|
assert "api_section" in s, f"Section {s['id']} missing api_section"
|
||||||
|
|
||||||
|
|
||||||
|
def test_network_section_has_editable_fields():
|
||||||
|
sections = settings_mod.get_settings_sections(CFG)
|
||||||
|
network = next(s for s in sections if s["id"] == "network")
|
||||||
|
assert network["section_mode"] == "form"
|
||||||
|
assert network["api_section"] == "server"
|
||||||
|
editable = [f for f in network["fields"] if f["editable"]]
|
||||||
|
assert len(editable) >= 2 # hbd_port, ws_port at minimum
|
||||||
|
|
||||||
|
|
||||||
|
def test_thresholds_and_dns_section_modes():
|
||||||
|
sections = settings_mod.get_settings_sections(CFG)
|
||||||
|
by_id = {s["id"]: s for s in sections}
|
||||||
|
assert by_id["thresholds"]["section_mode"] == "thresholds"
|
||||||
|
assert by_id["thresholds"]["api_section"] == "thresholds"
|
||||||
|
assert by_id["dns"]["section_mode"] == "form"
|
||||||
|
assert by_id["dns"]["api_section"] == "dns"
|
||||||
|
|
||||||
|
|
||||||
|
def test_hosts_section_uses_hosts_mode():
|
||||||
|
sections = settings_mod.get_settings_sections(CFG)
|
||||||
|
hosts_sec = next(s for s in sections if s["id"] == "hosts")
|
||||||
|
assert hosts_sec["section_mode"] == "hosts"
|
||||||
|
assert hosts_sec["api_section"] == "hosts"
|
||||||
|
|
||||||
|
|
||||||
|
def test_channels_section_uses_channels_mode():
|
||||||
|
sections = settings_mod.get_settings_sections(CFG)
|
||||||
|
ch_sec = next(s for s in sections if s["id"] == "channels")
|
||||||
|
assert ch_sec["section_mode"] == "channels"
|
||||||
|
assert ch_sec["api_section"] == "notification_channels"
|
||||||
|
assert len(ch_sec["channels"]) == 1
|
||||||
|
ch = ch_sec["channels"][0]
|
||||||
|
assert ch["name"] == "pushover_ops"
|
||||||
|
assert ch["type"] == "pushover"
|
||||||
|
assert "owner" in ch
|
||||||
|
assert "editable" in ch
|
||||||
|
|
||||||
|
|
||||||
|
def test_channel_type_schemas_exported():
|
||||||
|
assert hasattr(settings_mod, "CHANNEL_TYPE_SCHEMAS")
|
||||||
|
for required_type in ("pushover", "email", "signal", "matrix", "sms_voipms"):
|
||||||
|
assert required_type in settings_mod.CHANNEL_TYPE_SCHEMAS
|
||||||
|
schema = settings_mod.CHANNEL_TYPE_SCHEMAS[required_type]
|
||||||
|
assert "label" in schema
|
||||||
|
assert "fields" in schema
|
||||||
|
for f in schema["fields"]:
|
||||||
|
assert "key" in f
|
||||||
|
assert "type" in f
|
||||||
|
assert "required" in f
|
||||||
|
|
||||||
|
|
||||||
|
def test_oauth_section_exists():
|
||||||
|
sections = settings_mod.get_settings_sections(CFG)
|
||||||
|
oauth = next((s for s in sections if s["id"] == "oauth"), None)
|
||||||
|
assert oauth is not None
|
||||||
|
assert oauth["section_mode"] == "form"
|
||||||
|
assert oauth["api_section"] == "oauth"
|
||||||
|
assert len(oauth["providers"]) == 1
|
||||||
|
assert oauth["providers"][0]["name"] == "gitea"
|
||||||
|
assert oauth["providers"][0]["client_secret"] == "•••"
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_channel_names_returned():
|
||||||
|
result = settings_mod.get_settings_data(CFG)
|
||||||
|
assert "all_channel_names" in result
|
||||||
|
assert "pushover_ops" in result["all_channel_names"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_users_section_has_user_list():
|
||||||
|
sections = settings_mod.get_settings_sections(CFG)
|
||||||
|
users_sec = next(s for s in sections if s["id"] == "users")
|
||||||
|
assert users_sec["section_mode"] == "form"
|
||||||
|
assert users_sec["api_section"] == "users"
|
||||||
|
assert len(users_sec["users"]) == 1
|
||||||
|
assert users_sec["users"][0]["username"] == "alice"
|
||||||
|
# Password hash never exposed
|
||||||
|
assert "password" not in users_sec["users"][0]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Per-user filtering (owners/managers on the settings page)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
from types import SimpleNamespace # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
MULTI_CFG = {
|
||||||
|
**CFG,
|
||||||
|
"users": {
|
||||||
|
"alice": {"full_name": "Alice", "admin": True, "password": "x"},
|
||||||
|
"bob": {"full_name": "Bob", "admin": False, "password": "x"},
|
||||||
|
},
|
||||||
|
"notification_channels": {
|
||||||
|
"global_ch": {"type": "pushover", "token": "t", "user": "u"},
|
||||||
|
"bob_ch": {"type": "pushover", "token": "t", "user": "u", "owner": "bob"},
|
||||||
|
"carol_ch": {"type": "pushover", "token": "t", "user": "u", "owner": "carol"},
|
||||||
|
},
|
||||||
|
"threshold_configs": {
|
||||||
|
"bob_tc": {"owner": "bob", "thresholds": {}},
|
||||||
|
"carol_tc": {"owner": "carol", "thresholds": {}},
|
||||||
|
},
|
||||||
|
"hosts": {
|
||||||
|
"bobhost": {"owner": "bob"},
|
||||||
|
"managedhost": {"owner": "carol", "managers": ["bob"]},
|
||||||
|
"carolhost": {"owner": "carol"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
BOB = SimpleNamespace(username="bob", admin=False)
|
||||||
|
ADMIN = SimpleNamespace(username="alice", admin=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_nonadmin_sees_only_three_sections():
|
||||||
|
sections = settings_mod.get_settings_sections(MULTI_CFG, user=BOB)
|
||||||
|
assert [s["id"] for s in sections] == ["channels", "hosts", "thresholds"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_nonadmin_sections_hide_admin_fields():
|
||||||
|
sections = settings_mod.get_settings_sections(MULTI_CFG, user=BOB)
|
||||||
|
for s in sections:
|
||||||
|
assert s["fields"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_nonadmin_hosts_filtered_with_is_owner():
|
||||||
|
sections = settings_mod.get_settings_sections(MULTI_CFG, user=BOB)
|
||||||
|
hosts = next(s for s in sections if s["id"] == "hosts")["hosts"]
|
||||||
|
by_name = {h["name"]: h for h in hosts}
|
||||||
|
assert set(by_name) == {"bobhost", "managedhost"}
|
||||||
|
assert by_name["bobhost"]["is_owner"] is True
|
||||||
|
assert by_name["managedhost"]["is_owner"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_nonadmin_channels_filtered_with_editable():
|
||||||
|
sections = settings_mod.get_settings_sections(MULTI_CFG, user=BOB)
|
||||||
|
chans = {c["name"]: c for c in next(s for s in sections if s["id"] == "channels")["channels"]}
|
||||||
|
assert set(chans) == {"global_ch", "bob_ch"}
|
||||||
|
assert chans["bob_ch"]["editable"] is True
|
||||||
|
assert chans["global_ch"]["editable"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_sees_everything_with_editable():
|
||||||
|
sections = settings_mod.get_settings_sections(MULTI_CFG, user=ADMIN)
|
||||||
|
ids = [s["id"] for s in sections]
|
||||||
|
assert "network" in ids and "users" in ids
|
||||||
|
chans = {c["name"]: c for c in next(s for s in sections if s["id"] == "channels")["channels"]}
|
||||||
|
assert set(chans) == {"global_ch", "bob_ch", "carol_ch"}
|
||||||
|
assert all(c["editable"] for c in chans.values())
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_data_pickers_filtered_for_nonadmin():
|
||||||
|
data = settings_mod.get_settings_data(MULTI_CFG, user=BOB)
|
||||||
|
assert data["all_channel_names"] == ["bob_ch", "global_ch"]
|
||||||
|
assert data["all_threshold_configs"] == ["bob_tc"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_user_means_admin_view():
|
||||||
|
sections = settings_mod.get_settings_sections(MULTI_CFG) # auth disabled
|
||||||
|
assert len(sections) > 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_filtering_unaffected_by_other_users_in_config():
|
||||||
|
"""The users-section loop must not clobber the requesting username (regression)."""
|
||||||
|
cfg = dict(MULTI_CFG)
|
||||||
|
cfg["users"] = {
|
||||||
|
"alice": {"full_name": "Alice", "admin": True, "password": "x"},
|
||||||
|
"bob": {"full_name": "Bob", "admin": False, "password": "x"},
|
||||||
|
"zed": {"full_name": "Zed", "admin": False, "password": "x"}, # bob is not last
|
||||||
|
}
|
||||||
|
sections = settings_mod.get_settings_sections(cfg, user=BOB)
|
||||||
|
hosts = {h["name"] for h in next(s for s in sections if s["id"] == "hosts")["hosts"]}
|
||||||
|
assert hosts == {"bobhost", "managedhost"}
|
||||||
Reference in New Issue
Block a user