feat: Uptime Kuma – Einzel-Endpunkte, ausführliches /healthz, Push-Monitor

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/netmover/app_state.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 from datetime import datetime
 
+from netmover import kuma
 from netmover.config import Settings
 from netmover.crypto import SecretBox, load_or_create_key
 from netmover.db import make_engine, make_session_factory, upgrade
@@ -23,6 +24,7 @@ from netmover.mover.runner import RuleRunner
 from netmover.notify import Notifier
 from netmover.runs import RunService
 from netmover.scheduler import RuleScheduler
+from netmover.settings_store import load_settings
 
 log = get_logger(__name__)
 
@@ -59,7 +61,10 @@ class NetMover:
         self.runner = RuleRunner(self.session_factory, self.mounts)
         self.runs = RunService(self.session_factory, self.runner, self.notifier)
         self.scheduler = RuleScheduler(
-            self.session_factory, self.runs, retry_mounts=self.mounts.retry_pending
+            self.session_factory,
+            self.runs,
+            retry_mounts=self.mounts.retry_pending,
+            push_health=self.push_health,
         )
         self.started_at: datetime = utcnow()
         self.restore_results: list[RestoreResult] = []
@@ -76,6 +81,12 @@ class NetMover:
         self.scheduler.shutdown()
         self.db_engine.dispose()
 
+    def push_health(self) -> None:
+        """Scheduler-Job (jede Minute): Zustand an Uptime Kuma melden, falls eingerichtet."""
+        url = load_settings(self.session_factory).kuma_push_url
+        if url:
+            kuma.push(url, self.health())
+
     def health(self) -> Health:
         return check_health(
             self.session_factory,

src/netmover/health.py

@@ -1,18 +1,22 @@
-"""Gesundheitszustand für ``/healthz`` und die Statusseite (Fallstrick 9).
+"""Gesundheitszustand für ``/healthz``, Uptime Kuma und die Statusseite (Fallstrick 9).
 
 Ein Werkzeug, das nichts mehr tut, darf nicht wie eines aussehen, das nichts zu
-tun hat. Deshalb:
+tun hat. Geprüft wird:
 
-* der Scheduler muss leben (Herzschlag nicht älter als 3 Minuten),
-* keine aktive Anbindung im Fehlerzustand,
-* keine aktive Regel, deren letzter erfolgreicher Lauf länger als das Dreifache
-  ihres Intervalls zurückliegt. Bezugspunkt ist der späteste von letztem Erfolg,
-  Anlage der Regel und Dienststart – sonst wäre direkt nach einem Neustart alles rot.
+* der Scheduler lebt (Herzschlag nicht älter als 3 Minuten),
+* jede aktive Anbindung ist nicht im Fehlerzustand,
+* jede aktive Regel hatte vor höchstens dem Dreifachen ihres Intervalls einen
+  erfolgreichen Lauf (Bezug: spätester von letztem Erfolg, Anlage der Regel und
+  Dienststart – sonst wäre direkt nach einem Neustart alles rot) und ist nicht
+  mehrmals hintereinander gescheitert.
+
+Jeder Punkt ist ein eigener :class:`Check` – so kann Uptime Kuma auch einzelne
+Anbindungen und Regeln überwachen (``/healthz/mount/<slug>``, ``/healthz/rule/<id>``).
 """
 
 from __future__ import annotations
 
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from datetime import datetime, timedelta
 
 from sqlalchemy import select
@@ -22,18 +26,41 @@ from netmover.db import session_scope
 from netmover.models import Mount, MountState, Rule, utcnow
 
 STALE_FACTOR = 3
+FAILURES_ALERT = 3
 HEARTBEAT_MAX_AGE = timedelta(minutes=3)
 
 
+@dataclass
+class Check:
+    kind: str  # "scheduler" | "mount" | "rule"
+    name: str
+    ok: bool
+    message: str
+    id: int | None = None
+    slug: str | None = None
+
+    def as_dict(self) -> dict[str, object]:
+        return {k: v for k, v in asdict(self).items() if v is not None}
+
+
 @dataclass
 class Health:
     ok: bool = True
     problems: list[str] = field(default_factory=list)
     stale_rules: set[int] = field(default_factory=set)
+    checks: list[Check] = field(default_factory=list)
 
-    def fail(self, problem: str) -> None:
+    def add(self, check: Check) -> None:
+        self.checks.append(check)
+        if not check.ok:
             self.ok = False
-        self.problems.append(problem)
+            self.problems.append(check.message)
+
+    def find(self, kind: str, *, slug: str | None = None, id: int | None = None) -> Check | None:
+        for c in self.checks:
+            if c.kind == kind and (slug is None or c.slug == slug) and (id is None or c.id == id):
+                return c
+        return None
 
 
 def rule_is_stale(rule: Rule, service_started: datetime, now: datetime) -> bool:
@@ -45,6 +72,40 @@ def rule_is_stale(rule: Rule, service_started: datetime, now: datetime) -> bool:
     return now - reference > timedelta(seconds=STALE_FACTOR * rule.interval_seconds)
 
 
+def _mount_check(m: Mount) -> Check:
+    def mk(ok: bool, message: str) -> Check:
+        return Check("mount", m.name, ok, message, id=m.id, slug=m.slug)
+
+    if not m.enabled:
+        return mk(True, f"Anbindung „{m.name}“ deaktiviert")
+    if m.state is MountState.ERROR:
+        detail = (m.last_error or "").strip()
+        return mk(False, f"Anbindung „{m.name}“ im Fehlerzustand: {detail}".strip())
+    if m.state is MountState.DEGRADED:
+        return mk(True, f"Anbindung „{m.name}“ gestört, wird neu eingehängt")
+    return mk(True, f"Anbindung „{m.name}“ in Ordnung")
+
+
+def _rule_check(r: Rule, service_started: datetime, now: datetime) -> Check:
+    def mk(ok: bool, message: str) -> Check:
+        return Check("rule", r.name, ok, message, id=r.id)
+
+    if not r.enabled:
+        return mk(True, f"Regel „{r.name}“ deaktiviert")
+    if rule_is_stale(r, service_started, now):
+        ago = (
+            "noch nie"
+            if r.last_success_at is None
+            else f"seit {r.last_success_at:%Y-%m-%d %H:%M} UTC"
+        )
+        return mk(False, f"Regel „{r.name}“: kein erfolgreicher Lauf {ago}")
+    if r.consecutive_failures >= FAILURES_ALERT:
+        return mk(
+            False, f"Regel „{r.name}“: {r.consecutive_failures} fehlgeschlagene Läufe in Folge"
+        )
+    return mk(True, f"Regel „{r.name}“ in Ordnung")
+
+
 def check_health(
     session_factory: sessionmaker[Session],
     *,
@@ -56,20 +117,19 @@ def check_health(
     now = now or utcnow()
     health = Health()
     if not scheduler_running:
-        health.fail("Scheduler läuft nicht")
+        health.add(Check("scheduler", "Scheduler", False, "Scheduler läuft nicht"))
     elif last_heartbeat is None or now - last_heartbeat > HEARTBEAT_MAX_AGE:
-        health.fail("Scheduler reagiert nicht (kein Herzschlag)")
-    with session_scope(session_factory) as s:
-        for m in s.scalars(select(Mount).where(Mount.enabled.is_(True))):
-            if m.state is MountState.ERROR:
-                health.fail(f"Anbindung „{m.name}“ im Fehlerzustand: {m.last_error or ''}".strip())
-        for r in s.scalars(select(Rule).where(Rule.enabled.is_(True))):
-            if rule_is_stale(r, service_started, now):
-                health.stale_rules.add(r.id)
-                ago = (
-                    "noch nie"
-                    if r.last_success_at is None
-                    else f"seit {r.last_success_at:%Y-%m-%d %H:%M} UTC"
+        health.add(
+            Check("scheduler", "Scheduler", False, "Scheduler reagiert nicht (kein Herzschlag)")
         )
-                health.fail(f"Regel „{r.name}“: kein erfolgreicher Lauf {ago}")
+    else:
+        health.add(Check("scheduler", "Scheduler", True, "Scheduler läuft"))
+    with session_scope(session_factory) as s:
+        for m in s.scalars(select(Mount).order_by(Mount.name)):
+            health.add(_mount_check(m))
+        for r in s.scalars(select(Rule).order_by(Rule.name)):
+            check = _rule_check(r, service_started, now)
+            if r.enabled and rule_is_stale(r, service_started, now):
+                health.stale_rules.add(r.id)
+            health.add(check)
     return health

src/netmover/kuma.py

@@ -0,0 +1,50 @@
+"""Uptime Kuma: NetMover meldet sich regelmäßig bei einem „Push“-Monitor.
+
+Vorteil gegenüber dem Abfragen von ``/healthz``: Es funktioniert auch, wenn Kuma
+NetMover nicht erreichen kann (z. B. Bindung an 127.0.0.1 hinter einem Proxy). Und
+steht der Dienst oder die ganze VM, bleibt die Meldung aus – dann schlägt Kuma
+von selbst Alarm.
+
+Die Push-URL kopiert man aus Kuma (``https://kuma/api/push/<token>?status=up&msg=OK&ping=``);
+NetMover ersetzt ``status`` und ``msg`` bei jeder Meldung.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+import httpx
+
+from netmover.health import Health
+from netmover.logging_setup import event, get_logger
+
+log = get_logger(__name__)
+
+MAX_MESSAGE = 250
+
+Getter = Callable[..., Any]
+
+
+def push_url(url: str, health: Health) -> str:
+    """Push-URL mit aktuellem Zustand (vorhandene Parameter werden ersetzt)."""
+    msg = "OK" if health.ok else "; ".join(health.problems)
+    if len(msg) > MAX_MESSAGE:
+        msg = msg[: MAX_MESSAGE - 1] + "…"
+    target = httpx.URL(url)
+    for key, value in (("status", "up" if health.ok else "down"), ("msg", msg), ("ping", "")):
+        target = target.copy_set_param(key, value)
+    return str(target)
+
+
+def push(url: str, health: Health, getter: Getter | None = None) -> bool:
+    """Meldet den Zustand an Kuma. Fehler werden protokolliert, nie weitergereicht –
+    die URL nicht, weil sie das Token enthält."""
+    try:
+        resp = (getter or httpx.get)(push_url(url, health), timeout=10)
+        resp.raise_for_status()
+    except Exception as exc:
+        event(log, "kuma.push_failed", level=30, error=type(exc).__name__)
+        return False
+    event(log, "kuma.pushed", level=10, status="up" if health.ok else "down")
+    return True

src/netmover/scheduler.py

@@ -33,6 +33,7 @@ MAX_START_OFFSET = 10.0
 HEARTBEAT_SECONDS = 30
 CLEANUP_SECONDS = 3600
 RETRY_SECONDS = 60
+PUSH_SECONDS = 60
 
 
 def _job_id(rule_id: int) -> str:
@@ -46,12 +47,14 @@ class RuleScheduler:
         service: RunService,
         *,
         retry_mounts: Callable[[], object] | None = None,
+        push_health: Callable[[], object] | None = None,
         rng: random.Random | None = None,
         workers: int = 8,
     ) -> None:
         self.session_factory = session_factory
         self.service = service
         self._retry_mounts = retry_mounts
+        self._push_health = push_health
         self._rng = rng or random.Random()  # noqa: S311 – nur Startversatz
         self._executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="nm-run")
         self.scheduler = AsyncIOScheduler(
@@ -78,6 +81,10 @@ class RuleScheduler:
             self.scheduler.add_job(
                 self._retry, IntervalTrigger(seconds=RETRY_SECONDS), id="mount-retry"
             )
+        if self._push_health is not None:
+            self.scheduler.add_job(
+                self._push, IntervalTrigger(seconds=PUSH_SECONDS), id="kuma-push"
+            )
         self.sync()
         event(log, "scheduler.started", rules=len(self._intervals))
 
@@ -154,6 +161,11 @@ class RuleScheduler:
             loop = asyncio.get_running_loop()
             await loop.run_in_executor(self._executor, self._retry_mounts)
 
+    async def _push(self) -> None:
+        if self._push_health is not None:
+            loop = asyncio.get_running_loop()
+            await loop.run_in_executor(self._executor, self._push_health)
+
     async def _cleanup(self) -> None:
         loop = asyncio.get_running_loop()
         await loop.run_in_executor(self._executor, cleanup, self.session_factory)

src/netmover/settings_store.py

@@ -34,6 +34,7 @@ class AppSettings:
     notify_recovery: bool = True
     webhook_template: str = DEFAULT_TEMPLATE
     webhook_content_type: str = "application/json"
+    kuma_push_url: str = ""
 
 
 _PREFIX = "app."

tests/test_monitoring.py

@@ -0,0 +1,218 @@
+"""Uptime Kuma: /healthz, Einzel-Endpunkte, Push; Hilfe-Seite und Menü."""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any
+
+import httpx
+import pytest
+from fastapi.testclient import TestClient
+
+from netmover.db import session_scope
+from netmover.health import Check, Health
+from netmover.kuma import push, push_url
+from netmover.models import Mount, MountState, MountType, Rule, utcnow
+from netmover.settings_store import AppSettings, load_settings, save_settings
+from tests.webutil import post
+
+# -- push_url / push ---------------------------------------------------------------------------
+
+
+def _health(ok: bool, *problems: str) -> Health:
+    h = Health()
+    h.add(Check("scheduler", "Scheduler", True, "läuft"))
+    for p in problems:
+        h.add(Check("mount", "x", False, p))
+    assert h.ok is ok
+    return h
+
+
+def test_push_url_replaces_kuma_defaults() -> None:
+    url = push_url(
+        "https://kuma.lan/api/push/AbC?status=up&msg=OK&ping=", _health(False, "NAS weg")
+    )
+    parsed = httpx.URL(url)
+    assert parsed.path == "/api/push/AbC"
+    assert parsed.params["status"] == "down" and parsed.params["msg"] == "NAS weg"
+    assert push_url("https://kuma.lan/api/push/AbC", _health(True)).endswith(
+        "status=up&msg=OK&ping="
+    )
+
+
+def test_push_message_is_truncated() -> None:
+    url = push_url("https://k/api/push/x", _health(False, "x" * 1000))
+    assert len(httpx.URL(url).params["msg"]) == 250
+
+
+def test_push_swallows_errors_and_reports_result() -> None:
+    calls: list[str] = []
+
+    def ok_get(url: str, **kw: Any) -> httpx.Response:
+        calls.append(url)
+        return httpx.Response(200, request=httpx.Request("GET", url))
+
+    def bad_get(url: str, **kw: Any) -> httpx.Response:
+        raise httpx.ConnectError("weg")
+
+    assert push("https://k/api/push/x", _health(True), getter=ok_get) is True
+    assert calls and "status=up" in calls[0]
+    assert push("https://k/api/push/x", _health(True), getter=bad_get) is False
+
+
+# -- Endpunkte ----------------------------------------------------------------------------------
+
+
+def _add_mount(
+    state: Any, name: str, slug: str, mstate: MountState, error: str | None = None
+) -> int:
+    with session_scope(state.session_factory) as s:
+        m = Mount(
+            name=name,
+            slug=slug,
+            type=MountType.CIFS,
+            server="nas",
+            share=slug,
+            mountpoint=f"/mnt/netmover/{slug}",
+            state=mstate,
+            last_error=error,
+        )
+        s.add(m)
+        s.flush()
+        return m.id
+
+
+def test_healthz_details_and_single_endpoints(client: TestClient, state: Any) -> None:
+    ok_id = _add_mount(state, "NAS", "nas", MountState.MOUNTED)
+    _add_mount(state, "Archiv", "archiv", MountState.ERROR, "Host is down")
+    with session_scope(state.session_factory) as s:
+        r = Rule(name="Scans", src_mount_id=ok_id, dst_path="/tmp", consecutive_failures=3)
+        s.add(r)
+        s.flush()
+        rid = r.id
+
+    r = client.get("/healthz")
+    assert r.status_code == 503
+    body = r.json()
+    assert body["status"] == "error" and body["version"]
+    assert body["scheduler"]["running"] is True
+    kinds = {(c["kind"], c["name"], c["ok"]) for c in body["checks"]}
+    assert ("mount", "NAS", True) in kinds and ("mount", "Archiv", False) in kinds
+    assert ("rule", "Scans", False) in kinds  # drei Fehlschläge in Folge
+
+    assert client.get("/healthz/mount/nas").status_code == 200
+    r = client.get("/healthz/mount/archiv")
+    assert r.status_code == 503 and "Host is down" in r.json()["message"]
+    r = client.get(f"/healthz/rule/{rid}")
+    assert r.status_code == 503 and "3 fehlgeschlagene Läufe" in r.json()["message"]
+    assert client.get("/healthz/mount/gibtsnicht").status_code == 404
+    assert client.get("/healthz/rule/4711").status_code == 404
+
+
+def test_healthz_needs_no_login(raw_client: TestClient) -> None:
+    assert raw_client.get("/healthz").status_code == 200
+    assert raw_client.get("/healthz/mount/x").status_code == 404
+
+
+def test_disabled_rule_and_degraded_mount_are_not_alarms(client: TestClient, state: Any) -> None:
+    mid = _add_mount(state, "NAS", "nas", MountState.DEGRADED)
+    with session_scope(state.session_factory) as s:
+        s.add(
+            Rule(
+                name="aus",
+                src_mount_id=mid,
+                dst_path="/tmp",
+                enabled=False,
+                created_at=utcnow() - timedelta(days=3),
+            )
+        )
+    assert client.get("/healthz").status_code == 200
+
+
+def test_kuma_push_job_uses_saved_url(state: Any, monkeypatch: pytest.MonkeyPatch) -> None:
+    sent: list[str] = []
+    monkeypatch.setattr(
+        "netmover.kuma.httpx.get",
+        lambda url, **kw: (
+            sent.append(url),
+            httpx.Response(200, request=httpx.Request("GET", url)),
+        )[1],
+    )
+    state.push_health()  # ohne URL: nichts
+    assert sent == []
+    save_settings(
+        state.session_factory, AppSettings(kuma_push_url="https://kuma.lan/api/push/T0k3n")
+    )
+    state.push_health()
+    assert len(sent) == 1 and sent[0].startswith("https://kuma.lan/api/push/T0k3n?status=")
+
+
+def test_settings_kuma_url_masked_and_monitor_urls_listed(client: TestClient, state: Any) -> None:
+    _add_mount(state, "NAS", "nas", MountState.MOUNTED)
+    post(
+        client,
+        "/settings",
+        {
+            "retention_days": "30",
+            "notify_mode": "off",
+            "notify_after": "3",
+            "webhook_url": "",
+            "webhook_content_type": "application/json",
+            "webhook_template": "$message",
+            "kuma_push_url": "https://kuma.lan/api/push/GeHeIm?status=up",
+        },
+    )
+    assert load_settings(state.session_factory).kuma_push_url.endswith("GeHeIm?status=up")
+    page = client.get("/settings").text
+    assert "GeHeIm" not in page and "kuma.lan" in page
+    assert "/healthz/mount/nas" in page and "/healthz" in page
+
+
+# -- Hilfe und Menü ---------------------------------------------------------------------------
+
+
+def test_help_page_renders_guide_with_anchors(client: TestClient) -> None:
+    html = client.get("/help").text
+    for anchor in (
+        "begriffe",
+        "anbindungen",
+        "schluessel",
+        "regeln",
+        "regel-sicherheit",
+        "probelauf",
+        "status",
+        "protokoll",
+        "uptime-kuma",
+        "fragen",
+    ):
+        assert f'id="{anchor}"' in html, anchor
+    assert "<table>" in html and "{#" not in html
+    assert 'src="/help/img/' in html
+
+
+def test_help_images_and_path_safety(client: TestClient) -> None:
+    from netmover.web.app import _help_dir
+
+    if (_help_dir() / "img" / "status.png").is_file():
+        r = client.get("/help/img/status.png")
+        assert r.status_code == 200 and r.headers["content-type"] == "image/png"
+    for bad in ("../ANLEITUNG.md", "..%2F..%2Fpyproject.toml", "x.svg"):
+        assert client.get(f"/help/img/{bad}").status_code == 404
+
+
+def test_navigation_groups_health_and_help_links(client: TestClient, state: Any) -> None:
+    _add_mount(state, "Archiv", "archiv", MountState.ERROR, "weg")
+    html = client.get("/mounts").text
+    for group in ("Übersicht", "Verbindungen", "Automatisierung", "System"):
+        assert group in html
+    assert 'class="health bad"' in html and "1 Problem" in html
+    assert "1 gestört" in html
+    assert 'href="/help#anbindungen"' in html  # Hilfe zu dieser Seite
+    assert 'aria-current="page"' in html and 'class="menu-toggle"' in html
+    form = client.get("/rules/new").text
+    assert 'href="/help#regel-sicherheit"' in form
+
+
+def test_logged_out_pages_have_no_menu(raw_client: TestClient) -> None:
+    html = raw_client.get("/setup").text
+    assert 'id="sidebar"' not in html