diff --git a/src/netmover/app_state.py b/src/netmover/app_state.py index a89dfeb..f00b1dc 100644 --- a/src/netmover/app_state.py +++ b/src/netmover/app_state.py @@ -9,6 +9,7 @@ from __future__ import annotations from datetime import datetime +from netmover import kuma from netmover.config import Settings from netmover.crypto import SecretBox, load_or_create_key from netmover.db import make_engine, make_session_factory, upgrade @@ -23,6 +24,7 @@ from netmover.mover.runner import RuleRunner from netmover.notify import Notifier from netmover.runs import RunService from netmover.scheduler import RuleScheduler +from netmover.settings_store import load_settings log = get_logger(__name__) @@ -59,7 +61,10 @@ class NetMover: self.runner = RuleRunner(self.session_factory, self.mounts) self.runs = RunService(self.session_factory, self.runner, self.notifier) self.scheduler = RuleScheduler( - self.session_factory, self.runs, retry_mounts=self.mounts.retry_pending + self.session_factory, + self.runs, + retry_mounts=self.mounts.retry_pending, + push_health=self.push_health, ) self.started_at: datetime = utcnow() self.restore_results: list[RestoreResult] = [] @@ -76,6 +81,12 @@ class NetMover: self.scheduler.shutdown() self.db_engine.dispose() + def push_health(self) -> None: + """Scheduler-Job (jede Minute): Zustand an Uptime Kuma melden, falls eingerichtet.""" + url = load_settings(self.session_factory).kuma_push_url + if url: + kuma.push(url, self.health()) + def health(self) -> Health: return check_health( self.session_factory, diff --git a/src/netmover/health.py b/src/netmover/health.py index 69d2b15..3dbcb4b 100644 --- a/src/netmover/health.py +++ b/src/netmover/health.py @@ -1,18 +1,22 @@ -"""Gesundheitszustand für ``/healthz`` und die Statusseite (Fallstrick 9). +"""Gesundheitszustand für ``/healthz``, Uptime Kuma und die Statusseite (Fallstrick 9). Ein Werkzeug, das nichts mehr tut, darf nicht wie eines aussehen, das nichts zu -tun hat. Deshalb: +tun hat. Geprüft wird: -* der Scheduler muss leben (Herzschlag nicht älter als 3 Minuten), -* keine aktive Anbindung im Fehlerzustand, -* keine aktive Regel, deren letzter erfolgreicher Lauf länger als das Dreifache - ihres Intervalls zurückliegt. Bezugspunkt ist der späteste von letztem Erfolg, - Anlage der Regel und Dienststart – sonst wäre direkt nach einem Neustart alles rot. +* der Scheduler lebt (Herzschlag nicht älter als 3 Minuten), +* jede aktive Anbindung ist nicht im Fehlerzustand, +* jede aktive Regel hatte vor höchstens dem Dreifachen ihres Intervalls einen + erfolgreichen Lauf (Bezug: spätester von letztem Erfolg, Anlage der Regel und + Dienststart – sonst wäre direkt nach einem Neustart alles rot) und ist nicht + mehrmals hintereinander gescheitert. + +Jeder Punkt ist ein eigener :class:`Check` – so kann Uptime Kuma auch einzelne +Anbindungen und Regeln überwachen (``/healthz/mount/``, ``/healthz/rule/``). """ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from datetime import datetime, timedelta from sqlalchemy import select @@ -22,18 +26,41 @@ from netmover.db import session_scope from netmover.models import Mount, MountState, Rule, utcnow STALE_FACTOR = 3 +FAILURES_ALERT = 3 HEARTBEAT_MAX_AGE = timedelta(minutes=3) +@dataclass +class Check: + kind: str # "scheduler" | "mount" | "rule" + name: str + ok: bool + message: str + id: int | None = None + slug: str | None = None + + def as_dict(self) -> dict[str, object]: + return {k: v for k, v in asdict(self).items() if v is not None} + + @dataclass class Health: ok: bool = True problems: list[str] = field(default_factory=list) stale_rules: set[int] = field(default_factory=set) + checks: list[Check] = field(default_factory=list) - def fail(self, problem: str) -> None: - self.ok = False - self.problems.append(problem) + def add(self, check: Check) -> None: + self.checks.append(check) + if not check.ok: + self.ok = False + self.problems.append(check.message) + + def find(self, kind: str, *, slug: str | None = None, id: int | None = None) -> Check | None: + for c in self.checks: + if c.kind == kind and (slug is None or c.slug == slug) and (id is None or c.id == id): + return c + return None def rule_is_stale(rule: Rule, service_started: datetime, now: datetime) -> bool: @@ -45,6 +72,40 @@ def rule_is_stale(rule: Rule, service_started: datetime, now: datetime) -> bool: return now - reference > timedelta(seconds=STALE_FACTOR * rule.interval_seconds) +def _mount_check(m: Mount) -> Check: + def mk(ok: bool, message: str) -> Check: + return Check("mount", m.name, ok, message, id=m.id, slug=m.slug) + + if not m.enabled: + return mk(True, f"Anbindung „{m.name}“ deaktiviert") + if m.state is MountState.ERROR: + detail = (m.last_error or "").strip() + return mk(False, f"Anbindung „{m.name}“ im Fehlerzustand: {detail}".strip()) + if m.state is MountState.DEGRADED: + return mk(True, f"Anbindung „{m.name}“ gestört, wird neu eingehängt") + return mk(True, f"Anbindung „{m.name}“ in Ordnung") + + +def _rule_check(r: Rule, service_started: datetime, now: datetime) -> Check: + def mk(ok: bool, message: str) -> Check: + return Check("rule", r.name, ok, message, id=r.id) + + if not r.enabled: + return mk(True, f"Regel „{r.name}“ deaktiviert") + if rule_is_stale(r, service_started, now): + ago = ( + "noch nie" + if r.last_success_at is None + else f"seit {r.last_success_at:%Y-%m-%d %H:%M} UTC" + ) + return mk(False, f"Regel „{r.name}“: kein erfolgreicher Lauf {ago}") + if r.consecutive_failures >= FAILURES_ALERT: + return mk( + False, f"Regel „{r.name}“: {r.consecutive_failures} fehlgeschlagene Läufe in Folge" + ) + return mk(True, f"Regel „{r.name}“ in Ordnung") + + def check_health( session_factory: sessionmaker[Session], *, @@ -56,20 +117,19 @@ def check_health( now = now or utcnow() health = Health() if not scheduler_running: - health.fail("Scheduler läuft nicht") + health.add(Check("scheduler", "Scheduler", False, "Scheduler läuft nicht")) elif last_heartbeat is None or now - last_heartbeat > HEARTBEAT_MAX_AGE: - health.fail("Scheduler reagiert nicht (kein Herzschlag)") + health.add( + Check("scheduler", "Scheduler", False, "Scheduler reagiert nicht (kein Herzschlag)") + ) + else: + health.add(Check("scheduler", "Scheduler", True, "Scheduler läuft")) with session_scope(session_factory) as s: - for m in s.scalars(select(Mount).where(Mount.enabled.is_(True))): - if m.state is MountState.ERROR: - health.fail(f"Anbindung „{m.name}“ im Fehlerzustand: {m.last_error or ''}".strip()) - for r in s.scalars(select(Rule).where(Rule.enabled.is_(True))): - if rule_is_stale(r, service_started, now): + for m in s.scalars(select(Mount).order_by(Mount.name)): + health.add(_mount_check(m)) + for r in s.scalars(select(Rule).order_by(Rule.name)): + check = _rule_check(r, service_started, now) + if r.enabled and rule_is_stale(r, service_started, now): health.stale_rules.add(r.id) - ago = ( - "noch nie" - if r.last_success_at is None - else f"seit {r.last_success_at:%Y-%m-%d %H:%M} UTC" - ) - health.fail(f"Regel „{r.name}“: kein erfolgreicher Lauf {ago}") + health.add(check) return health diff --git a/src/netmover/kuma.py b/src/netmover/kuma.py new file mode 100644 index 0000000..e8120b9 --- /dev/null +++ b/src/netmover/kuma.py @@ -0,0 +1,50 @@ +"""Uptime Kuma: NetMover meldet sich regelmäßig bei einem „Push“-Monitor. + +Vorteil gegenüber dem Abfragen von ``/healthz``: Es funktioniert auch, wenn Kuma +NetMover nicht erreichen kann (z. B. Bindung an 127.0.0.1 hinter einem Proxy). Und +steht der Dienst oder die ganze VM, bleibt die Meldung aus – dann schlägt Kuma +von selbst Alarm. + +Die Push-URL kopiert man aus Kuma (``https://kuma/api/push/?status=up&msg=OK&ping=``); +NetMover ersetzt ``status`` und ``msg`` bei jeder Meldung. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import httpx + +from netmover.health import Health +from netmover.logging_setup import event, get_logger + +log = get_logger(__name__) + +MAX_MESSAGE = 250 + +Getter = Callable[..., Any] + + +def push_url(url: str, health: Health) -> str: + """Push-URL mit aktuellem Zustand (vorhandene Parameter werden ersetzt).""" + msg = "OK" if health.ok else "; ".join(health.problems) + if len(msg) > MAX_MESSAGE: + msg = msg[: MAX_MESSAGE - 1] + "…" + target = httpx.URL(url) + for key, value in (("status", "up" if health.ok else "down"), ("msg", msg), ("ping", "")): + target = target.copy_set_param(key, value) + return str(target) + + +def push(url: str, health: Health, getter: Getter | None = None) -> bool: + """Meldet den Zustand an Kuma. Fehler werden protokolliert, nie weitergereicht – + die URL nicht, weil sie das Token enthält.""" + try: + resp = (getter or httpx.get)(push_url(url, health), timeout=10) + resp.raise_for_status() + except Exception as exc: + event(log, "kuma.push_failed", level=30, error=type(exc).__name__) + return False + event(log, "kuma.pushed", level=10, status="up" if health.ok else "down") + return True diff --git a/src/netmover/scheduler.py b/src/netmover/scheduler.py index 7ecba05..2d72719 100644 --- a/src/netmover/scheduler.py +++ b/src/netmover/scheduler.py @@ -33,6 +33,7 @@ MAX_START_OFFSET = 10.0 HEARTBEAT_SECONDS = 30 CLEANUP_SECONDS = 3600 RETRY_SECONDS = 60 +PUSH_SECONDS = 60 def _job_id(rule_id: int) -> str: @@ -46,12 +47,14 @@ class RuleScheduler: service: RunService, *, retry_mounts: Callable[[], object] | None = None, + push_health: Callable[[], object] | None = None, rng: random.Random | None = None, workers: int = 8, ) -> None: self.session_factory = session_factory self.service = service self._retry_mounts = retry_mounts + self._push_health = push_health self._rng = rng or random.Random() # noqa: S311 – nur Startversatz self._executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="nm-run") self.scheduler = AsyncIOScheduler( @@ -78,6 +81,10 @@ class RuleScheduler: self.scheduler.add_job( self._retry, IntervalTrigger(seconds=RETRY_SECONDS), id="mount-retry" ) + if self._push_health is not None: + self.scheduler.add_job( + self._push, IntervalTrigger(seconds=PUSH_SECONDS), id="kuma-push" + ) self.sync() event(log, "scheduler.started", rules=len(self._intervals)) @@ -154,6 +161,11 @@ class RuleScheduler: loop = asyncio.get_running_loop() await loop.run_in_executor(self._executor, self._retry_mounts) + async def _push(self) -> None: + if self._push_health is not None: + loop = asyncio.get_running_loop() + await loop.run_in_executor(self._executor, self._push_health) + async def _cleanup(self) -> None: loop = asyncio.get_running_loop() await loop.run_in_executor(self._executor, cleanup, self.session_factory) diff --git a/src/netmover/settings_store.py b/src/netmover/settings_store.py index adef882..d753784 100644 --- a/src/netmover/settings_store.py +++ b/src/netmover/settings_store.py @@ -34,6 +34,7 @@ class AppSettings: notify_recovery: bool = True webhook_template: str = DEFAULT_TEMPLATE webhook_content_type: str = "application/json" + kuma_push_url: str = "" _PREFIX = "app." diff --git a/tests/test_monitoring.py b/tests/test_monitoring.py new file mode 100644 index 0000000..f6623ee --- /dev/null +++ b/tests/test_monitoring.py @@ -0,0 +1,218 @@ +"""Uptime Kuma: /healthz, Einzel-Endpunkte, Push; Hilfe-Seite und Menü.""" + +from __future__ import annotations + +from datetime import timedelta +from typing import Any + +import httpx +import pytest +from fastapi.testclient import TestClient + +from netmover.db import session_scope +from netmover.health import Check, Health +from netmover.kuma import push, push_url +from netmover.models import Mount, MountState, MountType, Rule, utcnow +from netmover.settings_store import AppSettings, load_settings, save_settings +from tests.webutil import post + +# -- push_url / push --------------------------------------------------------------------------- + + +def _health(ok: bool, *problems: str) -> Health: + h = Health() + h.add(Check("scheduler", "Scheduler", True, "läuft")) + for p in problems: + h.add(Check("mount", "x", False, p)) + assert h.ok is ok + return h + + +def test_push_url_replaces_kuma_defaults() -> None: + url = push_url( + "https://kuma.lan/api/push/AbC?status=up&msg=OK&ping=", _health(False, "NAS weg") + ) + parsed = httpx.URL(url) + assert parsed.path == "/api/push/AbC" + assert parsed.params["status"] == "down" and parsed.params["msg"] == "NAS weg" + assert push_url("https://kuma.lan/api/push/AbC", _health(True)).endswith( + "status=up&msg=OK&ping=" + ) + + +def test_push_message_is_truncated() -> None: + url = push_url("https://k/api/push/x", _health(False, "x" * 1000)) + assert len(httpx.URL(url).params["msg"]) == 250 + + +def test_push_swallows_errors_and_reports_result() -> None: + calls: list[str] = [] + + def ok_get(url: str, **kw: Any) -> httpx.Response: + calls.append(url) + return httpx.Response(200, request=httpx.Request("GET", url)) + + def bad_get(url: str, **kw: Any) -> httpx.Response: + raise httpx.ConnectError("weg") + + assert push("https://k/api/push/x", _health(True), getter=ok_get) is True + assert calls and "status=up" in calls[0] + assert push("https://k/api/push/x", _health(True), getter=bad_get) is False + + +# -- Endpunkte ---------------------------------------------------------------------------------- + + +def _add_mount( + state: Any, name: str, slug: str, mstate: MountState, error: str | None = None +) -> int: + with session_scope(state.session_factory) as s: + m = Mount( + name=name, + slug=slug, + type=MountType.CIFS, + server="nas", + share=slug, + mountpoint=f"/mnt/netmover/{slug}", + state=mstate, + last_error=error, + ) + s.add(m) + s.flush() + return m.id + + +def test_healthz_details_and_single_endpoints(client: TestClient, state: Any) -> None: + ok_id = _add_mount(state, "NAS", "nas", MountState.MOUNTED) + _add_mount(state, "Archiv", "archiv", MountState.ERROR, "Host is down") + with session_scope(state.session_factory) as s: + r = Rule(name="Scans", src_mount_id=ok_id, dst_path="/tmp", consecutive_failures=3) + s.add(r) + s.flush() + rid = r.id + + r = client.get("/healthz") + assert r.status_code == 503 + body = r.json() + assert body["status"] == "error" and body["version"] + assert body["scheduler"]["running"] is True + kinds = {(c["kind"], c["name"], c["ok"]) for c in body["checks"]} + assert ("mount", "NAS", True) in kinds and ("mount", "Archiv", False) in kinds + assert ("rule", "Scans", False) in kinds # drei Fehlschläge in Folge + + assert client.get("/healthz/mount/nas").status_code == 200 + r = client.get("/healthz/mount/archiv") + assert r.status_code == 503 and "Host is down" in r.json()["message"] + r = client.get(f"/healthz/rule/{rid}") + assert r.status_code == 503 and "3 fehlgeschlagene Läufe" in r.json()["message"] + assert client.get("/healthz/mount/gibtsnicht").status_code == 404 + assert client.get("/healthz/rule/4711").status_code == 404 + + +def test_healthz_needs_no_login(raw_client: TestClient) -> None: + assert raw_client.get("/healthz").status_code == 200 + assert raw_client.get("/healthz/mount/x").status_code == 404 + + +def test_disabled_rule_and_degraded_mount_are_not_alarms(client: TestClient, state: Any) -> None: + mid = _add_mount(state, "NAS", "nas", MountState.DEGRADED) + with session_scope(state.session_factory) as s: + s.add( + Rule( + name="aus", + src_mount_id=mid, + dst_path="/tmp", + enabled=False, + created_at=utcnow() - timedelta(days=3), + ) + ) + assert client.get("/healthz").status_code == 200 + + +def test_kuma_push_job_uses_saved_url(state: Any, monkeypatch: pytest.MonkeyPatch) -> None: + sent: list[str] = [] + monkeypatch.setattr( + "netmover.kuma.httpx.get", + lambda url, **kw: ( + sent.append(url), + httpx.Response(200, request=httpx.Request("GET", url)), + )[1], + ) + state.push_health() # ohne URL: nichts + assert sent == [] + save_settings( + state.session_factory, AppSettings(kuma_push_url="https://kuma.lan/api/push/T0k3n") + ) + state.push_health() + assert len(sent) == 1 and sent[0].startswith("https://kuma.lan/api/push/T0k3n?status=") + + +def test_settings_kuma_url_masked_and_monitor_urls_listed(client: TestClient, state: Any) -> None: + _add_mount(state, "NAS", "nas", MountState.MOUNTED) + post( + client, + "/settings", + { + "retention_days": "30", + "notify_mode": "off", + "notify_after": "3", + "webhook_url": "", + "webhook_content_type": "application/json", + "webhook_template": "$message", + "kuma_push_url": "https://kuma.lan/api/push/GeHeIm?status=up", + }, + ) + assert load_settings(state.session_factory).kuma_push_url.endswith("GeHeIm?status=up") + page = client.get("/settings").text + assert "GeHeIm" not in page and "kuma.lan" in page + assert "/healthz/mount/nas" in page and "/healthz" in page + + +# -- Hilfe und Menü --------------------------------------------------------------------------- + + +def test_help_page_renders_guide_with_anchors(client: TestClient) -> None: + html = client.get("/help").text + for anchor in ( + "begriffe", + "anbindungen", + "schluessel", + "regeln", + "regel-sicherheit", + "probelauf", + "status", + "protokoll", + "uptime-kuma", + "fragen", + ): + assert f'id="{anchor}"' in html, anchor + assert "" in html and "{#" not in html + assert 'src="/help/img/' in html + + +def test_help_images_and_path_safety(client: TestClient) -> None: + from netmover.web.app import _help_dir + + if (_help_dir() / "img" / "status.png").is_file(): + r = client.get("/help/img/status.png") + assert r.status_code == 200 and r.headers["content-type"] == "image/png" + for bad in ("../ANLEITUNG.md", "..%2F..%2Fpyproject.toml", "x.svg"): + assert client.get(f"/help/img/{bad}").status_code == 404 + + +def test_navigation_groups_health_and_help_links(client: TestClient, state: Any) -> None: + _add_mount(state, "Archiv", "archiv", MountState.ERROR, "weg") + html = client.get("/mounts").text + for group in ("Übersicht", "Verbindungen", "Automatisierung", "System"): + assert group in html + assert 'class="health bad"' in html and "1 Problem" in html + assert "1 gestört" in html + assert 'href="/help#anbindungen"' in html # Hilfe zu dieser Seite + assert 'aria-current="page"' in html and 'class="menu-toggle"' in html + form = client.get("/rules/new").text + assert 'href="/help#regel-sicherheit"' in form + + +def test_logged_out_pages_have_no_menu(raw_client: TestClient) -> None: + html = raw_client.get("/setup").text + assert 'id="sidebar"' not in html