feat: Regeln pausieren automatisch, solange eine Anbindung fehlt; Wartungsmodus
Wartezustand mit einem Protokolleintrag statt eines Fehlers alle 30 s, automatisches Fortsetzen; Meldung einmal pro Anbindung mit betroffenen Regeln (auch beim Aushängen von außen). Migration 0003 mit Tests auf befüllten Datenbanken. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e1a69c5e5f
commit
b6b6bf6bd7
+16
-3
@@ -49,6 +49,7 @@ class Health:
|
|||||||
problems: list[str] = field(default_factory=list)
|
problems: list[str] = field(default_factory=list)
|
||||||
stale_rules: set[int] = field(default_factory=set)
|
stale_rules: set[int] = field(default_factory=set)
|
||||||
checks: list[Check] = field(default_factory=list)
|
checks: list[Check] = field(default_factory=list)
|
||||||
|
maintenance: bool = False
|
||||||
|
|
||||||
def add(self, check: Check) -> None:
|
def add(self, check: Check) -> None:
|
||||||
self.checks.append(check)
|
self.checks.append(check)
|
||||||
@@ -86,12 +87,22 @@ def _mount_check(m: Mount) -> Check:
|
|||||||
return mk(True, f"Anbindung „{m.name}“ in Ordnung")
|
return mk(True, f"Anbindung „{m.name}“ in Ordnung")
|
||||||
|
|
||||||
|
|
||||||
def _rule_check(r: Rule, service_started: datetime, now: datetime) -> Check:
|
def _rule_check(
|
||||||
|
r: Rule, service_started: datetime, now: datetime, maintenance: bool = False
|
||||||
|
) -> Check:
|
||||||
def mk(ok: bool, message: str) -> Check:
|
def mk(ok: bool, message: str) -> Check:
|
||||||
return Check("rule", r.name, ok, message, id=r.id)
|
return Check("rule", r.name, ok, message, id=r.id)
|
||||||
|
|
||||||
if not r.enabled:
|
if not r.enabled:
|
||||||
return mk(True, f"Regel „{r.name}“ deaktiviert")
|
return mk(True, f"Regel „{r.name}“ deaktiviert")
|
||||||
|
if maintenance:
|
||||||
|
return mk(True, f"Regel „{r.name}“ pausiert (Wartungsmodus)")
|
||||||
|
if r.waiting_since is not None:
|
||||||
|
return mk(
|
||||||
|
False,
|
||||||
|
f"Regel „{r.name}“ wartet seit {r.waiting_since:%Y-%m-%d %H:%M} UTC auf eine "
|
||||||
|
f"Anbindung: {r.waiting_reason or ''}".strip(),
|
||||||
|
)
|
||||||
if rule_is_stale(r, service_started, now):
|
if rule_is_stale(r, service_started, now):
|
||||||
ago = (
|
ago = (
|
||||||
"noch nie"
|
"noch nie"
|
||||||
@@ -113,9 +124,11 @@ def check_health(
|
|||||||
last_heartbeat: datetime | None,
|
last_heartbeat: datetime | None,
|
||||||
service_started: datetime,
|
service_started: datetime,
|
||||||
now: datetime | None = None,
|
now: datetime | None = None,
|
||||||
|
maintenance: bool = False,
|
||||||
) -> Health:
|
) -> Health:
|
||||||
now = now or utcnow()
|
now = now or utcnow()
|
||||||
health = Health()
|
health = Health()
|
||||||
|
health.maintenance = maintenance
|
||||||
if not scheduler_running:
|
if not scheduler_running:
|
||||||
health.add(Check("scheduler", "Scheduler", False, "Scheduler läuft nicht"))
|
health.add(Check("scheduler", "Scheduler", False, "Scheduler läuft nicht"))
|
||||||
elif last_heartbeat is None or now - last_heartbeat > HEARTBEAT_MAX_AGE:
|
elif last_heartbeat is None or now - last_heartbeat > HEARTBEAT_MAX_AGE:
|
||||||
@@ -128,8 +141,8 @@ def check_health(
|
|||||||
for m in s.scalars(select(Mount).order_by(Mount.name)):
|
for m in s.scalars(select(Mount).order_by(Mount.name)):
|
||||||
health.add(_mount_check(m))
|
health.add(_mount_check(m))
|
||||||
for r in s.scalars(select(Rule).order_by(Rule.name)):
|
for r in s.scalars(select(Rule).order_by(Rule.name)):
|
||||||
check = _rule_check(r, service_started, now)
|
check = _rule_check(r, service_started, now, maintenance)
|
||||||
if r.enabled and rule_is_stale(r, service_started, now):
|
if r.enabled and not maintenance and rule_is_stale(r, service_started, now):
|
||||||
health.stale_rules.add(r.id)
|
health.stale_rules.add(r.id)
|
||||||
health.add(check)
|
health.add(check)
|
||||||
return health
|
return health
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
"""rule waiting state
|
||||||
|
|
||||||
|
Revision ID: 0003
|
||||||
|
Revises: 0002
|
||||||
|
Create Date: 2026-10-11
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
import netmover.models
|
||||||
|
|
||||||
|
revision: str = "0003"
|
||||||
|
down_revision: str | None = "0002"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.batch_alter_table("rules", schema=None) as batch_op:
|
||||||
|
batch_op.add_column(
|
||||||
|
sa.Column("waiting_since", netmover.models.UTCDateTime(), nullable=True)
|
||||||
|
)
|
||||||
|
batch_op.add_column(sa.Column("waiting_reason", sa.Text(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.batch_alter_table("rules", schema=None) as batch_op:
|
||||||
|
batch_op.drop_column("waiting_reason")
|
||||||
|
batch_op.drop_column("waiting_since")
|
||||||
@@ -10,6 +10,8 @@ Abweichungen bzw. Ergänzungen gegenüber dem Mindestumfang aus der Spezifikatio
|
|||||||
* ``credentials.managed`` + ``mounts.username`` (Migration 0002) –
|
* ``credentials.managed`` + ``mounts.username`` (Migration 0002) –
|
||||||
Schlüsselverwaltung: gespeicherte SSH-Schlüssel, die mehrere Anbindungen mit
|
Schlüsselverwaltung: gespeicherte SSH-Schlüssel, die mehrere Anbindungen mit
|
||||||
jeweils eigenem Benutzernamen nutzen.
|
jeweils eigenem Benutzernamen nutzen.
|
||||||
|
* ``rules.waiting_since`` / ``rules.waiting_reason`` (Migration 0003) – Regel wartet
|
||||||
|
auf eine nicht erreichbare Anbindung (automatisch pausiert).
|
||||||
* ``mounts.state_since`` – für „hängt seit …“ / „gestört seit …“ auf der Statusseite.
|
* ``mounts.state_since`` – für „hängt seit …“ / „gestört seit …“ auf der Statusseite.
|
||||||
* ``mounts.write_ok`` / ``mounts.root_squash`` / ``mounts.last_test_*`` – Ergebnis
|
* ``mounts.write_ok`` / ``mounts.root_squash`` / ``mounts.last_test_*`` – Ergebnis
|
||||||
des letzten Mount-Tests; ``root_squash`` entscheidet, ob ``chown`` auf NFS-Zielen
|
des letzten Mount-Tests; ``root_squash`` entscheidet, ob ``chown`` auf NFS-Zielen
|
||||||
@@ -260,6 +262,9 @@ class Rule(Base):
|
|||||||
last_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime)
|
last_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime)
|
||||||
last_success_at: Mapped[datetime | None] = mapped_column(UTCDateTime)
|
last_success_at: Mapped[datetime | None] = mapped_column(UTCDateTime)
|
||||||
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
|
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
# Automatisch pausiert, solange Quelle oder Ziel nicht erreichbar ist (Migration 0003)
|
||||||
|
waiting_since: Mapped[datetime | None] = mapped_column(UTCDateTime)
|
||||||
|
waiting_reason: Mapped[str | None] = mapped_column(Text)
|
||||||
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow)
|
created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow)
|
||||||
|
|
||||||
src_mount: Mapped[Mount] = relationship(foreign_keys=[src_mount_id])
|
src_mount: Mapped[Mount] = relationship(foreign_keys=[src_mount_id])
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
|||||||
from netmover.crypto import SecretBox, redact
|
from netmover.crypto import SecretBox, redact
|
||||||
from netmover.db import session_scope
|
from netmover.db import session_scope
|
||||||
from netmover.logging_setup import event, get_logger
|
from netmover.logging_setup import event, get_logger
|
||||||
from netmover.models import CredentialKind, Mount, MountState, MountType, utcnow
|
from netmover.models import CredentialKind, Mount, MountState, MountType, Rule, utcnow
|
||||||
from netmover.mounting.backend import CifsCredentials, MountBackend, MountError, MountSpec
|
from netmover.mounting.backend import CifsCredentials, MountBackend, MountError, MountSpec
|
||||||
from netmover.mounting.report import CheckReport, assess_write
|
from netmover.mounting.report import CheckReport, assess_write
|
||||||
from netmover.mounting.sftp import SftpConfig, SftpError, check_connection, connect
|
from netmover.mounting.sftp import SftpConfig, SftpError, check_connection, connect
|
||||||
@@ -203,13 +203,32 @@ class MountManager:
|
|||||||
with self._locks_guard:
|
with self._locks_guard:
|
||||||
return self._locks.setdefault(mount_id, threading.RLock())
|
return self._locks.setdefault(mount_id, threading.RLock())
|
||||||
|
|
||||||
def _notify(self, kind: str, name: str, message: str) -> None:
|
def _notify(self, kind: str, name: str, message: str, mount_id: int | None = None) -> None:
|
||||||
if self._notifier is not None:
|
if self._notifier is None:
|
||||||
|
return
|
||||||
|
if mount_id is not None:
|
||||||
|
rules = self._affected_rules(mount_id)
|
||||||
|
if rules and kind != "mount.recovered":
|
||||||
|
waiting = "\nBetroffene Regeln (warten, bis die Anbindung zurück ist): "
|
||||||
|
message += waiting + ", ".join(rules)
|
||||||
|
elif rules:
|
||||||
|
message += f"\nRegeln laufen wieder: {', '.join(rules)}"
|
||||||
try:
|
try:
|
||||||
self._notifier(kind, name, message)
|
self._notifier(kind, name, message)
|
||||||
except Exception:
|
except Exception:
|
||||||
log.exception("notifier.failed")
|
log.exception("notifier.failed")
|
||||||
|
|
||||||
|
def _affected_rules(self, mount_id: int) -> list[str]:
|
||||||
|
with session_scope(self.session_factory) as s:
|
||||||
|
return list(
|
||||||
|
s.scalars(
|
||||||
|
select(Rule.name)
|
||||||
|
.where(Rule.enabled.is_(True))
|
||||||
|
.where((Rule.src_mount_id == mount_id) | (Rule.dst_mount_id == mount_id))
|
||||||
|
.order_by(Rule.name)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def mountpoint_for(self, slug: str) -> Path:
|
def mountpoint_for(self, slug: str) -> Path:
|
||||||
if slugify(slug) != slug:
|
if slugify(slug) != slug:
|
||||||
raise ValidationError("slug", f"Ungültiger Slug: {slug!r}")
|
raise ValidationError("slug", f"Ungültiger Slug: {slug!r}")
|
||||||
@@ -347,7 +366,7 @@ class MountManager:
|
|||||||
# z. B. Stromausfall: die VM bootet schneller als das NAS
|
# z. B. Stromausfall: die VM bootet schneller als das NAS
|
||||||
self._retry_at[mount_id] = self._clock() + self.retry_interval
|
self._retry_at[mount_id] = self._clock() + self.retry_interval
|
||||||
results.append(RestoreResult(name, False, str(exc)))
|
results.append(RestoreResult(name, False, str(exc)))
|
||||||
self._notify("mount.restore_failed", name, str(exc))
|
self._notify("mount.restore_failed", name, str(exc), mount_id)
|
||||||
for r in results:
|
for r in results:
|
||||||
event(
|
event(
|
||||||
log,
|
log,
|
||||||
@@ -531,6 +550,8 @@ class MountManager:
|
|||||||
self._retry_mount(mount_id, m, cred, mp)
|
self._retry_mount(mount_id, m, cred, mp)
|
||||||
return mp
|
return mp
|
||||||
msg = f"„{m.name}“ ist nicht eingehängt – Lauf abgebrochen, nichts angefasst."
|
msg = f"„{m.name}“ ist nicht eingehängt – Lauf abgebrochen, nichts angefasst."
|
||||||
|
if m.state is not MountState.ERROR:
|
||||||
|
self._notify("mount.error", m.name, msg, mount_id)
|
||||||
self._set_state(mount_id, MountState.ERROR, msg)
|
self._set_state(mount_id, MountState.ERROR, msg)
|
||||||
raise MountUnavailable(msg)
|
raise MountUnavailable(msg)
|
||||||
try:
|
try:
|
||||||
@@ -563,8 +584,10 @@ class MountManager:
|
|||||||
msg = f"„{m.name}“ nicht erreichbar: {exc}"
|
msg = f"„{m.name}“ nicht erreichbar: {exc}"
|
||||||
self._set_state(mount_id, MountState.ERROR, msg)
|
self._set_state(mount_id, MountState.ERROR, msg)
|
||||||
if m.state is not MountState.ERROR:
|
if m.state is not MountState.ERROR:
|
||||||
self._notify("mount.error", m.name, msg)
|
self._notify("mount.error", m.name, msg, mount_id)
|
||||||
raise MountUnavailable(msg) from None
|
raise MountUnavailable(msg) from None
|
||||||
|
if m.state is MountState.ERROR:
|
||||||
|
self._notify("mount.recovered", m.name, f"„{m.name}“ ist wieder erreichbar.", mount_id)
|
||||||
if m.state is not MountState.MOUNTED:
|
if m.state is not MountState.MOUNTED:
|
||||||
self._set_state(mount_id, MountState.MOUNTED)
|
self._set_state(mount_id, MountState.MOUNTED)
|
||||||
|
|
||||||
@@ -601,7 +624,7 @@ class MountManager:
|
|||||||
self._retry_at[mount_id] = self._clock() + self.retry_interval
|
self._retry_at[mount_id] = self._clock() + self.retry_interval
|
||||||
msg += f"\n→ Neuer Versuch automatisch alle {self.retry_interval:g} s."
|
msg += f"\n→ Neuer Versuch automatisch alle {self.retry_interval:g} s."
|
||||||
self._set_state(mount_id, MountState.ERROR, msg)
|
self._set_state(mount_id, MountState.ERROR, msg)
|
||||||
self._notify("mount.error", m.name, msg)
|
self._notify("mount.error", m.name, msg, mount_id)
|
||||||
raise MountUnavailable(msg)
|
raise MountUnavailable(msg)
|
||||||
|
|
||||||
def retry_pending(self) -> list[str]:
|
def retry_pending(self) -> list[str]:
|
||||||
@@ -663,7 +686,7 @@ class MountManager:
|
|||||||
del self._retry_at[mount_id]
|
del self._retry_at[mount_id]
|
||||||
self._set_state(mount_id, MountState.MOUNTED, mounted=True)
|
self._set_state(mount_id, MountState.MOUNTED, mounted=True)
|
||||||
event(log, "mount.recovered", mount=m.name, attempt="retry")
|
event(log, "mount.recovered", mount=m.name, attempt="retry")
|
||||||
self._notify("mount.recovered", m.name, f"„{m.name}“ ist wieder eingehängt.")
|
self._notify("mount.recovered", m.name, f"„{m.name}“ ist wieder eingehängt.", mount_id)
|
||||||
|
|
||||||
def usage(self, mount_id: int, timeout: float | None = None) -> tuple[int, int] | None:
|
def usage(self, mount_id: int, timeout: float | None = None) -> tuple[int, int] | None:
|
||||||
"""(gesamt, frei) in Bytes – nur bei eingehängten Kernel-Mounts."""
|
"""(gesamt, frei) in Bytes – nur bei eingehängten Kernel-Mounts."""
|
||||||
|
|||||||
@@ -92,6 +92,8 @@ class RunResult:
|
|||||||
failed: int = 0
|
failed: int = 0
|
||||||
message: str | None = None
|
message: str | None = None
|
||||||
items: list[ItemResult] = field(default_factory=list)
|
items: list[ItemResult] = field(default_factory=list)
|
||||||
|
# Quelle oder Ziel nicht erreichbar → Regel wartet (automatisch pausiert)
|
||||||
|
unavailable: bool = False
|
||||||
|
|
||||||
def add(self, item: ItemResult) -> None:
|
def add(self, item: ItemResult) -> None:
|
||||||
self.items.append(item)
|
self.items.append(item)
|
||||||
|
|||||||
@@ -178,7 +178,7 @@ class RuleRunner:
|
|||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
except MountUnavailable as exc:
|
except MountUnavailable as exc:
|
||||||
return RunResult(RunStatus.ERROR, dry, message=str(exc))
|
return RunResult(RunStatus.ERROR, dry, message=str(exc), unavailable=True)
|
||||||
if src_mp is None: # nur bei SFTP, oben bereits ausgeschlossen
|
if src_mp is None: # nur bei SFTP, oben bereits ausgeschlossen
|
||||||
return RunResult(RunStatus.ERROR, dry, message="Quelle ist kein Kernel-Mount")
|
return RunResult(RunStatus.ERROR, dry, message="Quelle ist kein Kernel-Mount")
|
||||||
src_root = src_mp / plan.src_subpath if plan.src_subpath else src_mp
|
src_root = src_mp / plan.src_subpath if plan.src_subpath else src_mp
|
||||||
|
|||||||
@@ -8,6 +8,14 @@ die Einzeldateien in ``run_items``. Danach die Regelzähler:
|
|||||||
* ``consecutive_failures`` – ``error`` und ``partial`` zählen hoch, ``ok`` setzt zurück.
|
* ``consecutive_failures`` – ``error`` und ``partial`` zählen hoch, ``ok`` setzt zurück.
|
||||||
|
|
||||||
Ein manuell gestarteter Probelauf verändert keine Zähler.
|
Ein manuell gestarteter Probelauf verändert keine Zähler.
|
||||||
|
|
||||||
|
Automatisch pausieren (0.4.0): Ist Quelle oder Ziel nicht erreichbar, wechselt die
|
||||||
|
Regel in „wartet auf Anbindung“. Protokolliert wird nur der Beginn, nicht jeder
|
||||||
|
weitere Termin, und „Fehler in Folge“ zählt nicht hoch – die Benachrichtigung
|
||||||
|
kommt einmal pro Anbindung vom MountManager. Sobald ein Lauf wieder durchkommt,
|
||||||
|
endet das Warten von selbst, und der Lauf vermerkt die Wartezeit.
|
||||||
|
|
||||||
|
Wartungsmodus: geplante Läufe werden gar nicht erst ausgeführt.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -51,6 +59,8 @@ class RunService:
|
|||||||
) -> int | None:
|
) -> int | None:
|
||||||
"""Führt die Regel aus, protokolliert den Lauf und liefert die Run-ID
|
"""Führt die Regel aus, protokolliert den Lauf und liefert die Run-ID
|
||||||
(None, wenn die Regel inzwischen gelöscht wurde)."""
|
(None, wenn die Regel inzwischen gelöscht wurde)."""
|
||||||
|
if trigger is RunTrigger.SCHEDULE and load_settings(self.session_factory).maintenance:
|
||||||
|
return None
|
||||||
started = utcnow()
|
started = utcnow()
|
||||||
result = self.runner.run(rule_id, dry_run=dry_run)
|
result = self.runner.run(rule_id, dry_run=dry_run)
|
||||||
finished = utcnow()
|
finished = utcnow()
|
||||||
@@ -73,6 +83,29 @@ class RunService:
|
|||||||
if rule is None:
|
if rule is None:
|
||||||
return None
|
return None
|
||||||
message = result.message
|
message = result.message
|
||||||
|
if update_counters and result.unavailable:
|
||||||
|
if rule.waiting_since is not None:
|
||||||
|
# wartet bereits – nur den Grund aktualisieren, kein neuer Eintrag
|
||||||
|
rule.waiting_reason = result.message
|
||||||
|
event(log, "rule.still_waiting", level=10, rule=rule.name)
|
||||||
|
return None
|
||||||
|
rule.waiting_since = finished
|
||||||
|
rule.waiting_reason = result.message
|
||||||
|
rule.last_run_at = finished
|
||||||
|
message = f"Pausiert, bis die Anbindung zurück ist: {result.message}"
|
||||||
|
update_counters = False # „Fehler in Folge“ nicht hochzählen
|
||||||
|
event(log, "rule.waiting", level=30, rule=rule.name, reason=result.message or "")
|
||||||
|
elif (
|
||||||
|
update_counters
|
||||||
|
and rule.waiting_since is not None
|
||||||
|
and result.status is not RunStatus.SKIPPED
|
||||||
|
):
|
||||||
|
waited = _human(finished - rule.waiting_since)
|
||||||
|
rule.waiting_since = None
|
||||||
|
rule.waiting_reason = None
|
||||||
|
note = f"Fortgesetzt nach {waited} Wartezeit"
|
||||||
|
message = f"{note}; {message}" if message else note
|
||||||
|
event(log, "rule.resumed", rule=rule.name, waited=waited)
|
||||||
items = result.items
|
items = result.items
|
||||||
if len(items) > MAX_ITEMS_PER_RUN:
|
if len(items) > MAX_ITEMS_PER_RUN:
|
||||||
extra = len(items) - MAX_ITEMS_PER_RUN
|
extra = len(items) - MAX_ITEMS_PER_RUN
|
||||||
@@ -153,6 +186,15 @@ class RunService:
|
|||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def _human(delta: timedelta) -> str:
|
||||||
|
seconds = int(delta.total_seconds())
|
||||||
|
if seconds < 90:
|
||||||
|
return f"{seconds} s"
|
||||||
|
if seconds < 5400:
|
||||||
|
return f"{round(seconds / 60)} min"
|
||||||
|
return f"{seconds / 3600:.1f} h".replace(".", ",")
|
||||||
|
|
||||||
|
|
||||||
def cleanup(session_factory: sessionmaker[Session], now: datetime | None = None) -> dict[str, int]:
|
def cleanup(session_factory: sessionmaker[Session], now: datetime | None = None) -> dict[str, int]:
|
||||||
"""Protokolle älter als die Aufbewahrungsfrist löschen (Items per CASCADE),
|
"""Protokolle älter als die Aufbewahrungsfrist löschen (Items per CASCADE),
|
||||||
dazu Fehlerzähler, die seitdem nicht mehr angefasst wurden."""
|
dazu Fehlerzähler, die seitdem nicht mehr angefasst wurden."""
|
||||||
|
|||||||
@@ -18,6 +18,14 @@ from alembic import command
|
|||||||
from netmover.db import alembic_config, upgrade
|
from netmover.db import alembic_config, upgrade
|
||||||
|
|
||||||
|
|
||||||
|
def _head() -> str:
|
||||||
|
from alembic.script import ScriptDirectory
|
||||||
|
|
||||||
|
head = ScriptDirectory.from_config(alembic_config("sqlite://")).get_current_head()
|
||||||
|
assert head is not None
|
||||||
|
return head
|
||||||
|
|
||||||
|
|
||||||
def _seed(db: Path) -> None:
|
def _seed(db: Path) -> None:
|
||||||
"""Datenbank auf Stand 0001 mit verknüpften Daten, wie auf einem echten System."""
|
"""Datenbank auf Stand 0001 mit verknüpften Daten, wie auf einem echten System."""
|
||||||
c = sqlite3.connect(db)
|
c = sqlite3.connect(db)
|
||||||
@@ -80,7 +88,7 @@ def test_upgrade_with_linked_data(db_0001: tuple[str, Path]) -> None:
|
|||||||
url, db = db_0001
|
url, db = db_0001
|
||||||
upgrade(url)
|
upgrade(url)
|
||||||
st = _state(db)
|
st = _state(db)
|
||||||
assert st["version"] == "0002"
|
assert st["version"] == _head()
|
||||||
assert st["tmp_tables"] == [] and st["fk_violations"] == []
|
assert st["tmp_tables"] == [] and st["fk_violations"] == []
|
||||||
assert "managed" in st["cred_columns"] # type: ignore[operator]
|
assert "managed" in st["cred_columns"] # type: ignore[operator]
|
||||||
assert st["mounts"] == [("NAS", 1), ("paperless", 2)]
|
assert st["mounts"] == [("NAS", 1), ("paperless", 2)]
|
||||||
@@ -122,7 +130,7 @@ def test_failed_migration_leaves_database_untouched(
|
|||||||
|
|
||||||
monkeypatch.setattr(DefaultImpl, "drop_table", real_drop)
|
monkeypatch.setattr(DefaultImpl, "drop_table", real_drop)
|
||||||
upgrade(url) # danach klappt es ohne Handarbeit
|
upgrade(url) # danach klappt es ohne Handarbeit
|
||||||
assert _state(db)["version"] == "0002"
|
assert _state(db)["version"] == _head()
|
||||||
|
|
||||||
|
|
||||||
def test_app_engine_still_enforces_foreign_keys(db_url: str) -> None:
|
def test_app_engine_still_enforces_foreign_keys(db_url: str) -> None:
|
||||||
@@ -134,3 +142,36 @@ def test_app_engine_still_enforces_foreign_keys(db_url: str) -> None:
|
|||||||
with engine.connect() as conn:
|
with engine.connect() as conn:
|
||||||
assert conn.execute(text("PRAGMA foreign_keys")).scalar() == 1
|
assert conn.execute(text("PRAGMA foreign_keys")).scalar() == 1
|
||||||
engine.dispose()
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_from_0002_keeps_everything_linked_to_rules(tmp_path: Path) -> None:
|
||||||
|
"""0003 baut ``rules`` neu auf – Läufe, Buchführung und Fehlerzähler verweisen darauf."""
|
||||||
|
db = tmp_path / "v02.db"
|
||||||
|
url = f"sqlite:///{db}"
|
||||||
|
upgrade(url, "0001")
|
||||||
|
_seed(db)
|
||||||
|
upgrade(url, "0002")
|
||||||
|
c = sqlite3.connect(db)
|
||||||
|
c.execute("PRAGMA foreign_keys=ON")
|
||||||
|
c.executescript(
|
||||||
|
"""
|
||||||
|
UPDATE credentials SET managed = 1 WHERE id = 2;
|
||||||
|
UPDATE mounts SET username = 'paperless' WHERE id = 2;
|
||||||
|
INSERT INTO copied_files (rule_id, src_relpath, size, mtime_ns, dst_path, copied_at)
|
||||||
|
VALUES (1, 'a.pdf', 1, 1, '/x/a.pdf', '2026-10-10 20:00:00');
|
||||||
|
INSERT INTO file_failures (rule_id, src_relpath, size, mtime_ns, attempts, last_attempt_at)
|
||||||
|
VALUES (1, 'b.pdf', 1, 1, 2, '2026-10-10 20:00:00');
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
c.commit()
|
||||||
|
c.close()
|
||||||
|
upgrade(url)
|
||||||
|
st = _state(db)
|
||||||
|
assert st["version"] == _head() and st["fk_violations"] == [] and st["tmp_tables"] == []
|
||||||
|
assert st["rules"] == [("Scans → Paperless", 1, 2)] and st["run_items"] == 1
|
||||||
|
c = sqlite3.connect(db)
|
||||||
|
assert c.execute("SELECT COUNT(*) FROM copied_files").fetchone()[0] == 1
|
||||||
|
assert c.execute("SELECT attempts FROM file_failures").fetchone()[0] == 2
|
||||||
|
assert c.execute("SELECT waiting_since FROM rules").fetchone()[0] is None
|
||||||
|
assert c.execute("SELECT username FROM mounts WHERE id = 2").fetchone()[0] == "paperless"
|
||||||
|
c.close()
|
||||||
|
|||||||
@@ -0,0 +1,184 @@
|
|||||||
|
"""0.4.0: automatisch pausieren/fortsetzen, Wartungsmodus, eine Meldung pro Anbindung."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
||||||
|
from netmover.db import session_scope
|
||||||
|
from netmover.health import check_health
|
||||||
|
from netmover.models import Rule, Run, RunStatus, RunTrigger, utcnow
|
||||||
|
from netmover.mounting.fake import FakeMountBackend
|
||||||
|
from netmover.mounting.manager import MountManager
|
||||||
|
from netmover.mover.engine import RunResult
|
||||||
|
from netmover.mover.runner import RuleRunner
|
||||||
|
from netmover.runs import RunService
|
||||||
|
from netmover.settings_store import AppSettings, save_settings
|
||||||
|
from tests.conftest import MountFactory
|
||||||
|
|
||||||
|
|
||||||
|
class FakeRunner:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.results: list[RunResult] = []
|
||||||
|
self.calls = 0
|
||||||
|
|
||||||
|
def run(self, rule_id: int, *, dry_run: bool | None = None) -> RunResult:
|
||||||
|
self.calls += 1
|
||||||
|
return self.results.pop(0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def rule_id(make_mount: MountFactory, session_factory: sessionmaker[Session]) -> int:
|
||||||
|
mid = make_mount()
|
||||||
|
with session_scope(session_factory) as s:
|
||||||
|
r = Rule(name="Scans", src_mount_id=mid, dst_path="/x", interval_seconds=30)
|
||||||
|
s.add(r)
|
||||||
|
s.flush()
|
||||||
|
return r.id
|
||||||
|
|
||||||
|
|
||||||
|
def _rule(sf: sessionmaker[Session], rid: int) -> Rule:
|
||||||
|
with sf() as s:
|
||||||
|
r = s.get(Rule, rid)
|
||||||
|
assert r is not None
|
||||||
|
s.expunge(r)
|
||||||
|
return r
|
||||||
|
|
||||||
|
|
||||||
|
def _runs(sf: sessionmaker[Session]) -> list[Run]:
|
||||||
|
with sf() as s:
|
||||||
|
return list(s.scalars(select(Run).order_by(Run.id)))
|
||||||
|
|
||||||
|
|
||||||
|
def unavailable() -> RunResult:
|
||||||
|
return RunResult(
|
||||||
|
RunStatus.ERROR,
|
||||||
|
message="„NAS“ ist nicht eingehängt – Lauf abgebrochen, nichts angefasst.",
|
||||||
|
unavailable=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_waiting_logs_once_and_resumes_by_itself(
|
||||||
|
session_factory: sessionmaker[Session], rule_id: int
|
||||||
|
) -> None:
|
||||||
|
runner = FakeRunner()
|
||||||
|
service = RunService(session_factory, runner) # type: ignore[arg-type]
|
||||||
|
runner.results += [unavailable(), unavailable(), unavailable(), RunResult(RunStatus.OK)]
|
||||||
|
|
||||||
|
first = service.execute(rule_id)
|
||||||
|
assert first is not None
|
||||||
|
r = _rule(session_factory, rule_id)
|
||||||
|
assert r.waiting_since is not None and "nicht eingehängt" in (r.waiting_reason or "")
|
||||||
|
assert r.consecutive_failures == 0 # Warten ist kein Fehlschlag
|
||||||
|
|
||||||
|
assert service.execute(rule_id) is None # kein neuer Eintrag alle 30 s
|
||||||
|
assert service.execute(rule_id) is None
|
||||||
|
assert len(_runs(session_factory)) == 1
|
||||||
|
assert _runs(session_factory)[0].message.startswith("Pausiert, bis die Anbindung zurück ist")
|
||||||
|
|
||||||
|
service.execute(rule_id) # Anbindung wieder da
|
||||||
|
r = _rule(session_factory, rule_id)
|
||||||
|
assert r.waiting_since is None and r.waiting_reason is None and r.last_success_at is not None
|
||||||
|
runs = _runs(session_factory)
|
||||||
|
assert len(runs) == 2 and runs[1].message and runs[1].message.startswith("Fortgesetzt nach")
|
||||||
|
assert r.consecutive_failures == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_errors_still_count(session_factory: sessionmaker[Session], rule_id: int) -> None:
|
||||||
|
runner = FakeRunner()
|
||||||
|
service = RunService(session_factory, runner) # type: ignore[arg-type]
|
||||||
|
runner.results += [RunResult(RunStatus.ERROR, message="kaputt")] * 2
|
||||||
|
service.execute(rule_id)
|
||||||
|
service.execute(rule_id)
|
||||||
|
r = _rule(session_factory, rule_id)
|
||||||
|
assert r.consecutive_failures == 2 and r.waiting_since is None
|
||||||
|
assert len(_runs(session_factory)) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_manual_dry_run_does_not_start_waiting(
|
||||||
|
session_factory: sessionmaker[Session], rule_id: int
|
||||||
|
) -> None:
|
||||||
|
runner = FakeRunner()
|
||||||
|
service = RunService(session_factory, runner) # type: ignore[arg-type]
|
||||||
|
runner.results.append(unavailable())
|
||||||
|
service.execute(rule_id, RunTrigger.MANUAL, dry_run=True)
|
||||||
|
assert _rule(session_factory, rule_id).waiting_since is None
|
||||||
|
assert len(_runs(session_factory)) == 1 # Probelauf sieht man trotzdem
|
||||||
|
|
||||||
|
|
||||||
|
def test_maintenance_skips_scheduled_runs_but_not_manual(
|
||||||
|
session_factory: sessionmaker[Session], rule_id: int
|
||||||
|
) -> None:
|
||||||
|
runner = FakeRunner()
|
||||||
|
service = RunService(session_factory, runner) # type: ignore[arg-type]
|
||||||
|
save_settings(session_factory, AppSettings(maintenance=True))
|
||||||
|
assert service.execute(rule_id) is None
|
||||||
|
assert runner.calls == 0 and _runs(session_factory) == []
|
||||||
|
runner.results.append(RunResult(RunStatus.OK))
|
||||||
|
assert service.execute(rule_id, RunTrigger.MANUAL) is not None
|
||||||
|
assert runner.calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
def _health(sf: sessionmaker[Session], **kw: object): # type: ignore[no-untyped-def]
|
||||||
|
now = utcnow()
|
||||||
|
return check_health(
|
||||||
|
sf,
|
||||||
|
scheduler_running=True,
|
||||||
|
last_heartbeat=now,
|
||||||
|
service_started=now - timedelta(hours=1),
|
||||||
|
now=now,
|
||||||
|
**kw,
|
||||||
|
) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_health_waiting_and_maintenance(
|
||||||
|
session_factory: sessionmaker[Session], rule_id: int
|
||||||
|
) -> None:
|
||||||
|
with session_scope(session_factory) as s:
|
||||||
|
r = s.get(Rule, rule_id)
|
||||||
|
assert r is not None
|
||||||
|
r.created_at = utcnow() - timedelta(days=1)
|
||||||
|
r.last_success_at = utcnow() - timedelta(hours=2)
|
||||||
|
r.waiting_since = utcnow() - timedelta(minutes=12)
|
||||||
|
r.waiting_reason = "„NAS“ nicht verfügbar"
|
||||||
|
h = _health(session_factory)
|
||||||
|
assert not h.ok and "wartet seit" in h.problems[0] and "NAS" in h.problems[0]
|
||||||
|
h = _health(session_factory, maintenance=True)
|
||||||
|
assert h.ok and h.maintenance and h.stale_rules == set()
|
||||||
|
check = h.find("rule", id=rule_id)
|
||||||
|
assert check is not None and "Wartungsmodus" in check.message
|
||||||
|
|
||||||
|
|
||||||
|
# -- Integration mit MountManager und Runner ------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_vanished_mount_notifies_once_with_affected_rules(
|
||||||
|
manager: MountManager,
|
||||||
|
fake_backend: FakeMountBackend,
|
||||||
|
make_mount: MountFactory,
|
||||||
|
session_factory: sessionmaker[Session],
|
||||||
|
notifications: list[tuple[str, str, str]],
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
fake_backend.export("nas", "scans")
|
||||||
|
mid = make_mount("nas", share="scans")
|
||||||
|
manager.mount(mid)
|
||||||
|
with session_scope(session_factory) as s:
|
||||||
|
s.add(Rule(name="Scans → Paperless", src_mount_id=mid, dst_path=str(tmp_path)))
|
||||||
|
s.add(Rule(name="Archiv", src_mount_id=mid, dst_path=str(tmp_path), enabled=False))
|
||||||
|
fake_backend.vanish(manager.mountpoint_for("nas"))
|
||||||
|
|
||||||
|
runner = RuleRunner(session_factory, manager, sleep=lambda _: None)
|
||||||
|
with session_factory() as s:
|
||||||
|
rid = s.scalars(select(Rule.id).where(Rule.name == "Scans → Paperless")).one()
|
||||||
|
for _ in range(3):
|
||||||
|
result = runner.run(rid)
|
||||||
|
assert result.unavailable and result.status is RunStatus.ERROR
|
||||||
|
errors = [n for n in notifications if n[0] == "mount.error"]
|
||||||
|
assert len(errors) == 1
|
||||||
|
assert "Betroffene Regeln" in errors[0][2] and "Scans → Paperless" in errors[0][2]
|
||||||
|
assert "Archiv" not in errors[0][2] # deaktivierte Regel nicht erwähnt
|
||||||
Reference in New Issue
Block a user