feat: Regeln pausieren automatisch, solange eine Anbindung fehlt; Wartungsmodus

Wartezustand mit einem Protokolleintrag statt eines Fehlers alle 30 s,
automatisches Fortsetzen; Meldung einmal pro Anbindung mit betroffenen
Regeln (auch beim Aushängen von außen). Migration 0003 mit Tests auf
befüllten Datenbanken.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Florian Wilde
2026-10-11 00:47:02 +02:00
co-authored by Claude Opus 5.5
parent e1a69c5e5f
commit b6b6bf6bd7
9 changed files with 359 additions and 17 deletions
+16 -3
View File
@@ -49,6 +49,7 @@ class Health:
problems: list[str] = field(default_factory=list) problems: list[str] = field(default_factory=list)
stale_rules: set[int] = field(default_factory=set) stale_rules: set[int] = field(default_factory=set)
checks: list[Check] = field(default_factory=list) checks: list[Check] = field(default_factory=list)
maintenance: bool = False
def add(self, check: Check) -> None: def add(self, check: Check) -> None:
self.checks.append(check) self.checks.append(check)
@@ -86,12 +87,22 @@ def _mount_check(m: Mount) -> Check:
return mk(True, f"Anbindung „{m.name}“ in Ordnung") return mk(True, f"Anbindung „{m.name}“ in Ordnung")
def _rule_check(r: Rule, service_started: datetime, now: datetime) -> Check: def _rule_check(
r: Rule, service_started: datetime, now: datetime, maintenance: bool = False
) -> Check:
def mk(ok: bool, message: str) -> Check: def mk(ok: bool, message: str) -> Check:
return Check("rule", r.name, ok, message, id=r.id) return Check("rule", r.name, ok, message, id=r.id)
if not r.enabled: if not r.enabled:
return mk(True, f"Regel „{r.name}“ deaktiviert") return mk(True, f"Regel „{r.name}“ deaktiviert")
if maintenance:
return mk(True, f"Regel „{r.name}“ pausiert (Wartungsmodus)")
if r.waiting_since is not None:
return mk(
False,
f"Regel „{r.name}“ wartet seit {r.waiting_since:%Y-%m-%d %H:%M} UTC auf eine "
f"Anbindung: {r.waiting_reason or ''}".strip(),
)
if rule_is_stale(r, service_started, now): if rule_is_stale(r, service_started, now):
ago = ( ago = (
"noch nie" "noch nie"
@@ -113,9 +124,11 @@ def check_health(
last_heartbeat: datetime | None, last_heartbeat: datetime | None,
service_started: datetime, service_started: datetime,
now: datetime | None = None, now: datetime | None = None,
maintenance: bool = False,
) -> Health: ) -> Health:
now = now or utcnow() now = now or utcnow()
health = Health() health = Health()
health.maintenance = maintenance
if not scheduler_running: if not scheduler_running:
health.add(Check("scheduler", "Scheduler", False, "Scheduler läuft nicht")) health.add(Check("scheduler", "Scheduler", False, "Scheduler läuft nicht"))
elif last_heartbeat is None or now - last_heartbeat > HEARTBEAT_MAX_AGE: elif last_heartbeat is None or now - last_heartbeat > HEARTBEAT_MAX_AGE:
@@ -128,8 +141,8 @@ def check_health(
for m in s.scalars(select(Mount).order_by(Mount.name)): for m in s.scalars(select(Mount).order_by(Mount.name)):
health.add(_mount_check(m)) health.add(_mount_check(m))
for r in s.scalars(select(Rule).order_by(Rule.name)): for r in s.scalars(select(Rule).order_by(Rule.name)):
check = _rule_check(r, service_started, now) check = _rule_check(r, service_started, now, maintenance)
if r.enabled and rule_is_stale(r, service_started, now): if r.enabled and not maintenance and rule_is_stale(r, service_started, now):
health.stale_rules.add(r.id) health.stale_rules.add(r.id)
health.add(check) health.add(check)
return health return health
@@ -0,0 +1,32 @@
"""rule waiting state
Revision ID: 0003
Revises: 0002
Create Date: 2026-10-11
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
import netmover.models
revision: str = "0003"
down_revision: str | None = "0002"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
with op.batch_alter_table("rules", schema=None) as batch_op:
batch_op.add_column(
sa.Column("waiting_since", netmover.models.UTCDateTime(), nullable=True)
)
batch_op.add_column(sa.Column("waiting_reason", sa.Text(), nullable=True))
def downgrade() -> None:
with op.batch_alter_table("rules", schema=None) as batch_op:
batch_op.drop_column("waiting_reason")
batch_op.drop_column("waiting_since")
+5
View File
@@ -10,6 +10,8 @@ Abweichungen bzw. Ergänzungen gegenüber dem Mindestumfang aus der Spezifikatio
* ``credentials.managed`` + ``mounts.username`` (Migration 0002) – * ``credentials.managed`` + ``mounts.username`` (Migration 0002) –
Schlüsselverwaltung: gespeicherte SSH-Schlüssel, die mehrere Anbindungen mit Schlüsselverwaltung: gespeicherte SSH-Schlüssel, die mehrere Anbindungen mit
jeweils eigenem Benutzernamen nutzen. jeweils eigenem Benutzernamen nutzen.
* ``rules.waiting_since`` / ``rules.waiting_reason`` (Migration 0003) – Regel wartet
auf eine nicht erreichbare Anbindung (automatisch pausiert).
* ``mounts.state_since`` – für „hängt seit …“ / „gestört seit …“ auf der Statusseite. * ``mounts.state_since`` – für „hängt seit …“ / „gestört seit …“ auf der Statusseite.
* ``mounts.write_ok`` / ``mounts.root_squash`` / ``mounts.last_test_*`` – Ergebnis * ``mounts.write_ok`` / ``mounts.root_squash`` / ``mounts.last_test_*`` – Ergebnis
des letzten Mount-Tests; ``root_squash`` entscheidet, ob ``chown`` auf NFS-Zielen des letzten Mount-Tests; ``root_squash`` entscheidet, ob ``chown`` auf NFS-Zielen
@@ -260,6 +262,9 @@ class Rule(Base):
last_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime) last_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime)
last_success_at: Mapped[datetime | None] = mapped_column(UTCDateTime) last_success_at: Mapped[datetime | None] = mapped_column(UTCDateTime)
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0) consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
# Automatisch pausiert, solange Quelle oder Ziel nicht erreichbar ist (Migration 0003)
waiting_since: Mapped[datetime | None] = mapped_column(UTCDateTime)
waiting_reason: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow)
src_mount: Mapped[Mount] = relationship(foreign_keys=[src_mount_id]) src_mount: Mapped[Mount] = relationship(foreign_keys=[src_mount_id])
+30 -7
View File
@@ -32,7 +32,7 @@ from sqlalchemy.orm import Session, sessionmaker
from netmover.crypto import SecretBox, redact from netmover.crypto import SecretBox, redact
from netmover.db import session_scope from netmover.db import session_scope
from netmover.logging_setup import event, get_logger from netmover.logging_setup import event, get_logger
from netmover.models import CredentialKind, Mount, MountState, MountType, utcnow from netmover.models import CredentialKind, Mount, MountState, MountType, Rule, utcnow
from netmover.mounting.backend import CifsCredentials, MountBackend, MountError, MountSpec from netmover.mounting.backend import CifsCredentials, MountBackend, MountError, MountSpec
from netmover.mounting.report import CheckReport, assess_write from netmover.mounting.report import CheckReport, assess_write
from netmover.mounting.sftp import SftpConfig, SftpError, check_connection, connect from netmover.mounting.sftp import SftpConfig, SftpError, check_connection, connect
@@ -203,13 +203,32 @@ class MountManager:
with self._locks_guard: with self._locks_guard:
return self._locks.setdefault(mount_id, threading.RLock()) return self._locks.setdefault(mount_id, threading.RLock())
def _notify(self, kind: str, name: str, message: str) -> None: def _notify(self, kind: str, name: str, message: str, mount_id: int | None = None) -> None:
if self._notifier is not None: if self._notifier is None:
return
if mount_id is not None:
rules = self._affected_rules(mount_id)
if rules and kind != "mount.recovered":
waiting = "\nBetroffene Regeln (warten, bis die Anbindung zurück ist): "
message += waiting + ", ".join(rules)
elif rules:
message += f"\nRegeln laufen wieder: {', '.join(rules)}"
try: try:
self._notifier(kind, name, message) self._notifier(kind, name, message)
except Exception: except Exception:
log.exception("notifier.failed") log.exception("notifier.failed")
def _affected_rules(self, mount_id: int) -> list[str]:
with session_scope(self.session_factory) as s:
return list(
s.scalars(
select(Rule.name)
.where(Rule.enabled.is_(True))
.where((Rule.src_mount_id == mount_id) | (Rule.dst_mount_id == mount_id))
.order_by(Rule.name)
)
)
def mountpoint_for(self, slug: str) -> Path: def mountpoint_for(self, slug: str) -> Path:
if slugify(slug) != slug: if slugify(slug) != slug:
raise ValidationError("slug", f"Ungültiger Slug: {slug!r}") raise ValidationError("slug", f"Ungültiger Slug: {slug!r}")
@@ -347,7 +366,7 @@ class MountManager:
# z. B. Stromausfall: die VM bootet schneller als das NAS # z. B. Stromausfall: die VM bootet schneller als das NAS
self._retry_at[mount_id] = self._clock() + self.retry_interval self._retry_at[mount_id] = self._clock() + self.retry_interval
results.append(RestoreResult(name, False, str(exc))) results.append(RestoreResult(name, False, str(exc)))
self._notify("mount.restore_failed", name, str(exc)) self._notify("mount.restore_failed", name, str(exc), mount_id)
for r in results: for r in results:
event( event(
log, log,
@@ -531,6 +550,8 @@ class MountManager:
self._retry_mount(mount_id, m, cred, mp) self._retry_mount(mount_id, m, cred, mp)
return mp return mp
msg = f"„{m.name}“ ist nicht eingehängt – Lauf abgebrochen, nichts angefasst." msg = f"„{m.name}“ ist nicht eingehängt – Lauf abgebrochen, nichts angefasst."
if m.state is not MountState.ERROR:
self._notify("mount.error", m.name, msg, mount_id)
self._set_state(mount_id, MountState.ERROR, msg) self._set_state(mount_id, MountState.ERROR, msg)
raise MountUnavailable(msg) raise MountUnavailable(msg)
try: try:
@@ -563,8 +584,10 @@ class MountManager:
msg = f"„{m.name}“ nicht erreichbar: {exc}" msg = f"„{m.name}“ nicht erreichbar: {exc}"
self._set_state(mount_id, MountState.ERROR, msg) self._set_state(mount_id, MountState.ERROR, msg)
if m.state is not MountState.ERROR: if m.state is not MountState.ERROR:
self._notify("mount.error", m.name, msg) self._notify("mount.error", m.name, msg, mount_id)
raise MountUnavailable(msg) from None raise MountUnavailable(msg) from None
if m.state is MountState.ERROR:
self._notify("mount.recovered", m.name, f"„{m.name}“ ist wieder erreichbar.", mount_id)
if m.state is not MountState.MOUNTED: if m.state is not MountState.MOUNTED:
self._set_state(mount_id, MountState.MOUNTED) self._set_state(mount_id, MountState.MOUNTED)
@@ -601,7 +624,7 @@ class MountManager:
self._retry_at[mount_id] = self._clock() + self.retry_interval self._retry_at[mount_id] = self._clock() + self.retry_interval
msg += f"\n→ Neuer Versuch automatisch alle {self.retry_interval:g} s." msg += f"\n→ Neuer Versuch automatisch alle {self.retry_interval:g} s."
self._set_state(mount_id, MountState.ERROR, msg) self._set_state(mount_id, MountState.ERROR, msg)
self._notify("mount.error", m.name, msg) self._notify("mount.error", m.name, msg, mount_id)
raise MountUnavailable(msg) raise MountUnavailable(msg)
def retry_pending(self) -> list[str]: def retry_pending(self) -> list[str]:
@@ -663,7 +686,7 @@ class MountManager:
del self._retry_at[mount_id] del self._retry_at[mount_id]
self._set_state(mount_id, MountState.MOUNTED, mounted=True) self._set_state(mount_id, MountState.MOUNTED, mounted=True)
event(log, "mount.recovered", mount=m.name, attempt="retry") event(log, "mount.recovered", mount=m.name, attempt="retry")
self._notify("mount.recovered", m.name, f"„{m.name}“ ist wieder eingehängt.") self._notify("mount.recovered", m.name, f"„{m.name}“ ist wieder eingehängt.", mount_id)
def usage(self, mount_id: int, timeout: float | None = None) -> tuple[int, int] | None: def usage(self, mount_id: int, timeout: float | None = None) -> tuple[int, int] | None:
"""(gesamt, frei) in Bytes – nur bei eingehängten Kernel-Mounts.""" """(gesamt, frei) in Bytes – nur bei eingehängten Kernel-Mounts."""
+2
View File
@@ -92,6 +92,8 @@ class RunResult:
failed: int = 0 failed: int = 0
message: str | None = None message: str | None = None
items: list[ItemResult] = field(default_factory=list) items: list[ItemResult] = field(default_factory=list)
# Quelle oder Ziel nicht erreichbar → Regel wartet (automatisch pausiert)
unavailable: bool = False
def add(self, item: ItemResult) -> None: def add(self, item: ItemResult) -> None:
self.items.append(item) self.items.append(item)
+1 -1
View File
@@ -178,7 +178,7 @@ class RuleRunner:
else None else None
) )
except MountUnavailable as exc: except MountUnavailable as exc:
return RunResult(RunStatus.ERROR, dry, message=str(exc)) return RunResult(RunStatus.ERROR, dry, message=str(exc), unavailable=True)
if src_mp is None: # nur bei SFTP, oben bereits ausgeschlossen if src_mp is None: # nur bei SFTP, oben bereits ausgeschlossen
return RunResult(RunStatus.ERROR, dry, message="Quelle ist kein Kernel-Mount") return RunResult(RunStatus.ERROR, dry, message="Quelle ist kein Kernel-Mount")
src_root = src_mp / plan.src_subpath if plan.src_subpath else src_mp src_root = src_mp / plan.src_subpath if plan.src_subpath else src_mp
+42
View File
@@ -8,6 +8,14 @@ die Einzeldateien in ``run_items``. Danach die Regelzähler:
* ``consecutive_failures`` – ``error`` und ``partial`` zählen hoch, ``ok`` setzt zurück. * ``consecutive_failures`` – ``error`` und ``partial`` zählen hoch, ``ok`` setzt zurück.
Ein manuell gestarteter Probelauf verändert keine Zähler. Ein manuell gestarteter Probelauf verändert keine Zähler.
Automatisch pausieren (0.4.0): Ist Quelle oder Ziel nicht erreichbar, wechselt die
Regel in „wartet auf Anbindung“. Protokolliert wird nur der Beginn, nicht jeder
weitere Termin, und „Fehler in Folge“ zählt nicht hoch – die Benachrichtigung
kommt einmal pro Anbindung vom MountManager. Sobald ein Lauf wieder durchkommt,
endet das Warten von selbst, und der Lauf vermerkt die Wartezeit.
Wartungsmodus: geplante Läufe werden gar nicht erst ausgeführt.
""" """
from __future__ import annotations from __future__ import annotations
@@ -51,6 +59,8 @@ class RunService:
) -> int | None: ) -> int | None:
"""Führt die Regel aus, protokolliert den Lauf und liefert die Run-ID """Führt die Regel aus, protokolliert den Lauf und liefert die Run-ID
(None, wenn die Regel inzwischen gelöscht wurde).""" (None, wenn die Regel inzwischen gelöscht wurde)."""
if trigger is RunTrigger.SCHEDULE and load_settings(self.session_factory).maintenance:
return None
started = utcnow() started = utcnow()
result = self.runner.run(rule_id, dry_run=dry_run) result = self.runner.run(rule_id, dry_run=dry_run)
finished = utcnow() finished = utcnow()
@@ -73,6 +83,29 @@ class RunService:
if rule is None: if rule is None:
return None return None
message = result.message message = result.message
if update_counters and result.unavailable:
if rule.waiting_since is not None:
# wartet bereits – nur den Grund aktualisieren, kein neuer Eintrag
rule.waiting_reason = result.message
event(log, "rule.still_waiting", level=10, rule=rule.name)
return None
rule.waiting_since = finished
rule.waiting_reason = result.message
rule.last_run_at = finished
message = f"Pausiert, bis die Anbindung zurück ist: {result.message}"
update_counters = False # „Fehler in Folge“ nicht hochzählen
event(log, "rule.waiting", level=30, rule=rule.name, reason=result.message or "")
elif (
update_counters
and rule.waiting_since is not None
and result.status is not RunStatus.SKIPPED
):
waited = _human(finished - rule.waiting_since)
rule.waiting_since = None
rule.waiting_reason = None
note = f"Fortgesetzt nach {waited} Wartezeit"
message = f"{note}; {message}" if message else note
event(log, "rule.resumed", rule=rule.name, waited=waited)
items = result.items items = result.items
if len(items) > MAX_ITEMS_PER_RUN: if len(items) > MAX_ITEMS_PER_RUN:
extra = len(items) - MAX_ITEMS_PER_RUN extra = len(items) - MAX_ITEMS_PER_RUN
@@ -153,6 +186,15 @@ class RunService:
return "\n".join(lines) return "\n".join(lines)
def _human(delta: timedelta) -> str:
seconds = int(delta.total_seconds())
if seconds < 90:
return f"{seconds} s"
if seconds < 5400:
return f"{round(seconds / 60)} min"
return f"{seconds / 3600:.1f} h".replace(".", ",")
def cleanup(session_factory: sessionmaker[Session], now: datetime | None = None) -> dict[str, int]: def cleanup(session_factory: sessionmaker[Session], now: datetime | None = None) -> dict[str, int]:
"""Protokolle älter als die Aufbewahrungsfrist löschen (Items per CASCADE), """Protokolle älter als die Aufbewahrungsfrist löschen (Items per CASCADE),
dazu Fehlerzähler, die seitdem nicht mehr angefasst wurden.""" dazu Fehlerzähler, die seitdem nicht mehr angefasst wurden."""
+43 -2
View File
@@ -18,6 +18,14 @@ from alembic import command
from netmover.db import alembic_config, upgrade from netmover.db import alembic_config, upgrade
def _head() -> str:
from alembic.script import ScriptDirectory
head = ScriptDirectory.from_config(alembic_config("sqlite://")).get_current_head()
assert head is not None
return head
def _seed(db: Path) -> None: def _seed(db: Path) -> None:
"""Datenbank auf Stand 0001 mit verknüpften Daten, wie auf einem echten System.""" """Datenbank auf Stand 0001 mit verknüpften Daten, wie auf einem echten System."""
c = sqlite3.connect(db) c = sqlite3.connect(db)
@@ -80,7 +88,7 @@ def test_upgrade_with_linked_data(db_0001: tuple[str, Path]) -> None:
url, db = db_0001 url, db = db_0001
upgrade(url) upgrade(url)
st = _state(db) st = _state(db)
assert st["version"] == "0002" assert st["version"] == _head()
assert st["tmp_tables"] == [] and st["fk_violations"] == [] assert st["tmp_tables"] == [] and st["fk_violations"] == []
assert "managed" in st["cred_columns"] # type: ignore[operator] assert "managed" in st["cred_columns"] # type: ignore[operator]
assert st["mounts"] == [("NAS", 1), ("paperless", 2)] assert st["mounts"] == [("NAS", 1), ("paperless", 2)]
@@ -122,7 +130,7 @@ def test_failed_migration_leaves_database_untouched(
monkeypatch.setattr(DefaultImpl, "drop_table", real_drop) monkeypatch.setattr(DefaultImpl, "drop_table", real_drop)
upgrade(url) # danach klappt es ohne Handarbeit upgrade(url) # danach klappt es ohne Handarbeit
assert _state(db)["version"] == "0002" assert _state(db)["version"] == _head()
def test_app_engine_still_enforces_foreign_keys(db_url: str) -> None: def test_app_engine_still_enforces_foreign_keys(db_url: str) -> None:
@@ -134,3 +142,36 @@ def test_app_engine_still_enforces_foreign_keys(db_url: str) -> None:
with engine.connect() as conn: with engine.connect() as conn:
assert conn.execute(text("PRAGMA foreign_keys")).scalar() == 1 assert conn.execute(text("PRAGMA foreign_keys")).scalar() == 1
engine.dispose() engine.dispose()
def test_upgrade_from_0002_keeps_everything_linked_to_rules(tmp_path: Path) -> None:
"""0003 baut ``rules`` neu auf – Läufe, Buchführung und Fehlerzähler verweisen darauf."""
db = tmp_path / "v02.db"
url = f"sqlite:///{db}"
upgrade(url, "0001")
_seed(db)
upgrade(url, "0002")
c = sqlite3.connect(db)
c.execute("PRAGMA foreign_keys=ON")
c.executescript(
"""
UPDATE credentials SET managed = 1 WHERE id = 2;
UPDATE mounts SET username = 'paperless' WHERE id = 2;
INSERT INTO copied_files (rule_id, src_relpath, size, mtime_ns, dst_path, copied_at)
VALUES (1, 'a.pdf', 1, 1, '/x/a.pdf', '2026-10-10 20:00:00');
INSERT INTO file_failures (rule_id, src_relpath, size, mtime_ns, attempts, last_attempt_at)
VALUES (1, 'b.pdf', 1, 1, 2, '2026-10-10 20:00:00');
"""
)
c.commit()
c.close()
upgrade(url)
st = _state(db)
assert st["version"] == _head() and st["fk_violations"] == [] and st["tmp_tables"] == []
assert st["rules"] == [("Scans → Paperless", 1, 2)] and st["run_items"] == 1
c = sqlite3.connect(db)
assert c.execute("SELECT COUNT(*) FROM copied_files").fetchone()[0] == 1
assert c.execute("SELECT attempts FROM file_failures").fetchone()[0] == 2
assert c.execute("SELECT waiting_since FROM rules").fetchone()[0] is None
assert c.execute("SELECT username FROM mounts WHERE id = 2").fetchone()[0] == "paperless"
c.close()
+184
View File
@@ -0,0 +1,184 @@
"""0.4.0: automatisch pausieren/fortsetzen, Wartungsmodus, eine Meldung pro Anbindung."""
from __future__ import annotations
from datetime import timedelta
from pathlib import Path
import pytest
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from netmover.db import session_scope
from netmover.health import check_health
from netmover.models import Rule, Run, RunStatus, RunTrigger, utcnow
from netmover.mounting.fake import FakeMountBackend
from netmover.mounting.manager import MountManager
from netmover.mover.engine import RunResult
from netmover.mover.runner import RuleRunner
from netmover.runs import RunService
from netmover.settings_store import AppSettings, save_settings
from tests.conftest import MountFactory
class FakeRunner:
def __init__(self) -> None:
self.results: list[RunResult] = []
self.calls = 0
def run(self, rule_id: int, *, dry_run: bool | None = None) -> RunResult:
self.calls += 1
return self.results.pop(0)
@pytest.fixture
def rule_id(make_mount: MountFactory, session_factory: sessionmaker[Session]) -> int:
mid = make_mount()
with session_scope(session_factory) as s:
r = Rule(name="Scans", src_mount_id=mid, dst_path="/x", interval_seconds=30)
s.add(r)
s.flush()
return r.id
def _rule(sf: sessionmaker[Session], rid: int) -> Rule:
with sf() as s:
r = s.get(Rule, rid)
assert r is not None
s.expunge(r)
return r
def _runs(sf: sessionmaker[Session]) -> list[Run]:
with sf() as s:
return list(s.scalars(select(Run).order_by(Run.id)))
def unavailable() -> RunResult:
return RunResult(
RunStatus.ERROR,
message="„NAS“ ist nicht eingehängt – Lauf abgebrochen, nichts angefasst.",
unavailable=True,
)
def test_waiting_logs_once_and_resumes_by_itself(
session_factory: sessionmaker[Session], rule_id: int
) -> None:
runner = FakeRunner()
service = RunService(session_factory, runner) # type: ignore[arg-type]
runner.results += [unavailable(), unavailable(), unavailable(), RunResult(RunStatus.OK)]
first = service.execute(rule_id)
assert first is not None
r = _rule(session_factory, rule_id)
assert r.waiting_since is not None and "nicht eingehängt" in (r.waiting_reason or "")
assert r.consecutive_failures == 0 # Warten ist kein Fehlschlag
assert service.execute(rule_id) is None # kein neuer Eintrag alle 30 s
assert service.execute(rule_id) is None
assert len(_runs(session_factory)) == 1
assert _runs(session_factory)[0].message.startswith("Pausiert, bis die Anbindung zurück ist")
service.execute(rule_id) # Anbindung wieder da
r = _rule(session_factory, rule_id)
assert r.waiting_since is None and r.waiting_reason is None and r.last_success_at is not None
runs = _runs(session_factory)
assert len(runs) == 2 and runs[1].message and runs[1].message.startswith("Fortgesetzt nach")
assert r.consecutive_failures == 0
def test_real_errors_still_count(session_factory: sessionmaker[Session], rule_id: int) -> None:
runner = FakeRunner()
service = RunService(session_factory, runner) # type: ignore[arg-type]
runner.results += [RunResult(RunStatus.ERROR, message="kaputt")] * 2
service.execute(rule_id)
service.execute(rule_id)
r = _rule(session_factory, rule_id)
assert r.consecutive_failures == 2 and r.waiting_since is None
assert len(_runs(session_factory)) == 2
def test_manual_dry_run_does_not_start_waiting(
session_factory: sessionmaker[Session], rule_id: int
) -> None:
runner = FakeRunner()
service = RunService(session_factory, runner) # type: ignore[arg-type]
runner.results.append(unavailable())
service.execute(rule_id, RunTrigger.MANUAL, dry_run=True)
assert _rule(session_factory, rule_id).waiting_since is None
assert len(_runs(session_factory)) == 1 # Probelauf sieht man trotzdem
def test_maintenance_skips_scheduled_runs_but_not_manual(
session_factory: sessionmaker[Session], rule_id: int
) -> None:
runner = FakeRunner()
service = RunService(session_factory, runner) # type: ignore[arg-type]
save_settings(session_factory, AppSettings(maintenance=True))
assert service.execute(rule_id) is None
assert runner.calls == 0 and _runs(session_factory) == []
runner.results.append(RunResult(RunStatus.OK))
assert service.execute(rule_id, RunTrigger.MANUAL) is not None
assert runner.calls == 1
def _health(sf: sessionmaker[Session], **kw: object): # type: ignore[no-untyped-def]
now = utcnow()
return check_health(
sf,
scheduler_running=True,
last_heartbeat=now,
service_started=now - timedelta(hours=1),
now=now,
**kw,
) # type: ignore[arg-type]
def test_health_waiting_and_maintenance(
session_factory: sessionmaker[Session], rule_id: int
) -> None:
with session_scope(session_factory) as s:
r = s.get(Rule, rule_id)
assert r is not None
r.created_at = utcnow() - timedelta(days=1)
r.last_success_at = utcnow() - timedelta(hours=2)
r.waiting_since = utcnow() - timedelta(minutes=12)
r.waiting_reason = "„NAS“ nicht verfügbar"
h = _health(session_factory)
assert not h.ok and "wartet seit" in h.problems[0] and "NAS" in h.problems[0]
h = _health(session_factory, maintenance=True)
assert h.ok and h.maintenance and h.stale_rules == set()
check = h.find("rule", id=rule_id)
assert check is not None and "Wartungsmodus" in check.message
# -- Integration mit MountManager und Runner ------------------------------------------------
def test_vanished_mount_notifies_once_with_affected_rules(
manager: MountManager,
fake_backend: FakeMountBackend,
make_mount: MountFactory,
session_factory: sessionmaker[Session],
notifications: list[tuple[str, str, str]],
tmp_path: Path,
) -> None:
fake_backend.export("nas", "scans")
mid = make_mount("nas", share="scans")
manager.mount(mid)
with session_scope(session_factory) as s:
s.add(Rule(name="Scans → Paperless", src_mount_id=mid, dst_path=str(tmp_path)))
s.add(Rule(name="Archiv", src_mount_id=mid, dst_path=str(tmp_path), enabled=False))
fake_backend.vanish(manager.mountpoint_for("nas"))
runner = RuleRunner(session_factory, manager, sleep=lambda _: None)
with session_factory() as s:
rid = s.scalars(select(Rule.id).where(Rule.name == "Scans → Paperless")).one()
for _ in range(3):
result = runner.run(rid)
assert result.unavailable and result.status is RunStatus.ERROR
errors = [n for n in notifications if n[0] == "mount.error"]
assert len(errors) == 1
assert "Betroffene Regeln" in errors[0][2] and "Scans → Paperless" in errors[0][2]
assert "Archiv" not in errors[0][2] # deaktivierte Regel nicht erwähnt