From c836f5818901b40a488eb47bcb80eed376a5aab8 Mon Sep 17 00:00:00 2001 From: Florian Wilde Date: Sat, 10 Oct 2026 21:50:06 +0200 Subject: [PATCH] =?UTF-8?q?fix(mounts):=20nach=20gescheiterter=20Wiederher?= =?UTF-8?q?stellung=20min=C3=BCtlich=20neu=20versuchen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ein NAS-Neustart dauert länger als die drei Wiederherstellungsversuche. Von NetMover selbst ausgehängte Mounts mit auto_mount werden höchstens einmal pro Minute erneut eingehängt, mit Entwarnung per Webhook. Von außen ausgehängte bleiben unangetastet. Hinweistexte werden nicht mehr mitgeschwärzt. Co-Authored-By: Claude Opus 5.5 --- src/netmover/mounting/manager.py | 59 +++++++++++++++- src/netmover/notify.py | 7 +- tests/test_manager.py | 118 +++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 2 deletions(-) diff --git a/src/netmover/mounting/manager.py b/src/netmover/mounting/manager.py index 0fba6c3..d6b52fd 100644 --- a/src/netmover/mounting/manager.py +++ b/src/netmover/mounting/manager.py @@ -15,7 +15,9 @@ Fallstricke: from __future__ import annotations +import contextlib import errno +import math import os import threading import time @@ -104,6 +106,13 @@ class RestoreResult: message: str +def _redact_output(message: str, secrets: Sequence[str]) -> str: + """Schwärzt nur die Ausgabe von ``mount``, nicht den angehängten Hinweistext + (sonst würde z. B. das Passwort „falsch“ auch im Hinweis geschwärzt).""" + output, sep, hint = message.partition("\n→ ") + return redact(output, secrets) + sep + hint + + def build_spec(cfg: MountConfig, mountpoint: Path, cred: CredentialData | None) -> MountSpec: """Baut den ``mount``-Aufruf. Validiert erneut (Verteidigung in der Tiefe) – auch Daten aus der Datenbank laufen hier noch einmal durch die Prüfung.""" @@ -166,6 +175,8 @@ class MountManager: sleep: Callable[[float], None] = time.sleep, euid: Callable[[], int] = os.geteuid, notifier: Notifier | None = None, + retry_interval: float = 60, + clock: Callable[[], float] = time.monotonic, ) -> None: self.session_factory = session_factory self.backend = backend @@ -177,6 +188,12 @@ class MountManager: self._sleep = sleep self._euid = euid self._notifier = notifier + self.retry_interval = retry_interval + self._clock = clock + # Mounts, die NetMover nach gescheiterter Wiederherstellung selbst ausgehängt + # hat: nächster erlaubter Versuch (monotone Zeit). Nur diese werden + # automatisch neu eingehängt – von außen ausgehängte nie (Fallstrick 2). + self._retry_at: dict[int, float] = {} self._locks: dict[int, threading.RLock] = {} self._locks_guard = threading.Lock() @@ -267,6 +284,7 @@ class MountManager: def mount(self, mount_id: int) -> None: with self._lock(mount_id): + self._retry_at.pop(mount_id, None) m, cred = self._load(mount_id) if not m.type.is_kernel_mount: raise MountError("SFTP-Anbindungen werden nicht eingehängt, nur getestet") @@ -292,6 +310,7 @@ class MountManager: def unmount(self, mount_id: int) -> None: with self._lock(mount_id): + self._retry_at.pop(mount_id, None) m, _ = self._load(mount_id) if not m.type.is_kernel_mount: self._set_state(mount_id, MountState.UNMOUNTED) @@ -391,7 +410,7 @@ class MountManager: self.backend.mount(build_spec(cfg, temp, cred), timeout=self.mount_timeout) except (MountError, ValidationError) as exc: self._remove_mountpoint(temp) - report.step("mount", False, redact(str(exc), secrets)) + report.step("mount", False, _redact_output(str(exc), secrets)) report.step("read", None) report.step("write", None) return report @@ -501,6 +520,9 @@ class MountManager: return None mp = self.mountpoint_for(m.slug) if not self.backend.is_mounted(mp): + if mount_id in self._retry_at: + self._retry_mount(mount_id, m, cred, mp) + return mp msg = f"„{m.name}“ ist nicht eingehängt – Lauf abgebrochen, nichts angefasst." self._set_state(mount_id, MountState.ERROR, msg) raise MountUnavailable(msg) @@ -568,10 +590,45 @@ class MountManager: f"„{m.name}“ gestört ({reason}); Neu-Einhängen nach " f"{len(self.recover_backoff)} Versuchen gescheitert: {last}" ) + if m.auto_mount: + self._retry_at[mount_id] = self._clock() + self.retry_interval + msg += f"\n→ Neuer Versuch automatisch alle {self.retry_interval:g} s." self._set_state(mount_id, MountState.ERROR, msg) self._notify("mount.error", m.name, msg) raise MountUnavailable(msg) + def _retry_mount(self, mount_id: int, m: Mount, cred: CredentialData | None, mp: Path) -> None: + """Ein Versuch, einen von NetMover selbst ausgehängten Mount zurückzuholen – + höchstens einmal pro ``retry_interval``, damit kein Lauf alle 10 s hängt.""" + wait = self._retry_at[mount_id] - self._clock() + if wait > 0: + raise MountUnavailable( + f"„{m.name}“ nicht verfügbar – nächster Versuch, neu einzuhängen, " + f"in {math.ceil(wait)} s. Lauf abgebrochen, nichts angefasst." + ) + try: + self._prepare_mountpoint(mp) + self.backend.mount( + build_spec(MountConfig.from_model(m), mp, cred), timeout=self.mount_timeout + ) + self._probe(m.name, mp) + except (MountError, OSError, OperationTimeout, ValidationError) as exc: + if self.backend.is_mounted(mp): + with contextlib.suppress(MountError): + self.backend.unmount(mp, lazy=True, timeout=self.mount_timeout) + self._retry_at[mount_id] = self._clock() + self.retry_interval + msg = ( + f"„{m.name}“ weiterhin nicht verfügbar: {exc}\n" + f"→ Neuer Versuch in {self.retry_interval:g} s." + ) + self._set_state(mount_id, MountState.ERROR, msg) + event(log, "mount.retry_failed", level=30, mount=m.name, error=str(exc)) + raise MountUnavailable(msg) from None + del self._retry_at[mount_id] + self._set_state(mount_id, MountState.MOUNTED, mounted=True) + event(log, "mount.recovered", mount=m.name, attempt="retry") + self._notify("mount.recovered", m.name, f"„{m.name}“ ist wieder eingehängt.") + def usage(self, mount_id: int, timeout: float | None = None) -> tuple[int, int] | None: """(gesamt, frei) in Bytes – nur bei eingehängten Kernel-Mounts.""" m, _ = self._load(mount_id) diff --git a/src/netmover/notify.py b/src/netmover/notify.py index bb28dfd..fc44602 100644 --- a/src/netmover/notify.py +++ b/src/netmover/notify.py @@ -125,4 +125,9 @@ class Notifier: def mount_event(self, kind: str, mount: str, message: str) -> None: """Passt zur ``Notifier``-Signatur des :class:`MountManager`.""" - self.send(Notification(kind, f"NetMover: Anbindung „{mount}“ gestört", message)) + title = ( + f"NetMover: Anbindung „{mount}“ wieder verfügbar" + if kind == "mount.recovered" + else f"NetMover: Anbindung „{mount}“ gestört" + ) + self.send(Notification(kind, title, message)) diff --git a/tests/test_manager.py b/tests/test_manager.py index d15be07..f498070 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -367,3 +367,121 @@ def test_usage( manager.mount(mid) usage = manager.usage(mid) assert usage is not None and usage[0] >= usage[1] > 0 + + +# -- Praxisbefund: NAS-Neustart dauert länger als die Wiederherstellung ---------------------- + + +class Clock: + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + +@pytest.fixture +def clock() -> Clock: + return Clock() + + +@pytest.fixture +def retry_manager( + session_factory: sessionmaker[Session], + fake_backend: FakeMountBackend, + secretbox: object, + mount_base: Path, + notifications: list[tuple[str, str, str]], + clock: Clock, +) -> MountManager: + return MountManager( + session_factory, + fake_backend, + secretbox, + mount_base, # type: ignore[arg-type] + probe_timeout=0.5, + recover_backoff=(0, 0, 0), + sleep=lambda _: None, + notifier=lambda *a: notifications.append(a), + retry_interval=60, + clock=clock, + ) + + +def test_failed_recovery_retries_every_interval_until_nas_is_back( + retry_manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + session_factory: sessionmaker[Session], + notifications: list[tuple[str, str, str]], + clock: Clock, +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + retry_manager.mount(mid) + mp = retry_manager.mountpoint_for("scans") + fake_backend.inject_fault(mp, errno.ESTALE) + fake_backend.fail_next_mount(times=3) # NAS startet neu + with pytest.raises(MountUnavailable, match="alle 60 s"): + retry_manager.check_alive(mid) + assert not fake_backend.is_mounted(mp) + + mounts_before = sum(1 for c in fake_backend.calls if c.action == "mount") + clock.now += 30 # zu früh: kein Versuch, Lauf abgebrochen + with pytest.raises(MountUnavailable, match="in 30 s"): + retry_manager.check_alive(mid) + assert sum(1 for c in fake_backend.calls if c.action == "mount") == mounts_before + + clock.now += 30 # NAS noch nicht zurück + fake_backend.fail_next_mount() + with pytest.raises(MountUnavailable, match="weiterhin nicht verfügbar"): + retry_manager.check_alive(mid) + + clock.now += 60 # NAS wieder da + assert retry_manager.check_alive(mid) == mp + assert fake_backend.is_mounted(mp) + assert _state(session_factory, mid).state is MountState.MOUNTED + assert notifications[-1][0] == "mount.recovered" + + +def test_no_retry_without_auto_mount( + retry_manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + clock: Clock, +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount(auto_mount=False) + retry_manager.mount(mid) + fake_backend.inject_fault(retry_manager.mountpoint_for("scans"), errno.ESTALE) + fake_backend.fail_next_mount(times=3) + with pytest.raises(MountUnavailable): + retry_manager.check_alive(mid) + clock.now += 600 + with pytest.raises(MountUnavailable, match="nicht eingehängt"): + retry_manager.check_alive(mid) + + +def test_externally_unmounted_is_still_never_remounted( + retry_manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + clock: Clock, +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + retry_manager.mount(mid) + fake_backend.vanish(retry_manager.mountpoint_for("scans")) + clock.now += 600 + with pytest.raises(MountUnavailable, match="nicht eingehängt"): + retry_manager.check_alive(mid) + assert [c.action for c in fake_backend.calls] == ["mount"] + + +def test_hint_text_is_not_redacted() -> None: + from netmover.mounting.manager import _redact_output + + msg = "mount error(13): Permission denied falsch\n→ Benutzername/Passwort falsch oder …" + out = _redact_output(msg, ["falsch"]) + assert out.startswith("mount error(13): Permission denied ••••••") + assert out.endswith("→ Benutzername/Passwort falsch oder …")