From 1bf0fae2fcc683f56e423e2465e209b60e98a3da Mon Sep 17 00:00:00 2001 From: Florian Wilde Date: Sat, 10 Oct 2026 21:51:58 +0200 Subject: [PATCH] =?UTF-8?q?fix(mounts):=20gescheiterte=20Start-Wiederherst?= =?UTF-8?q?ellung=20ebenfalls=20min=C3=BCtlich=20neu=20versuchen?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bootet die VM schneller als das NAS (Stromausfall), käme die Freigabe sonst nie zurück. Ein Scheduler-Job prüft fällige Neuversuche jede Minute, auch für Anbindungen ohne Regel. Co-Authored-By: Claude Opus 5.5 --- src/netmover/app_state.py | 4 +++- src/netmover/mounting/manager.py | 31 ++++++++++++++++++++++++++ src/netmover/scheduler.py | 13 +++++++++++ tests/test_manager.py | 38 ++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/netmover/app_state.py b/src/netmover/app_state.py index 2693ffe..a89dfeb 100644 --- a/src/netmover/app_state.py +++ b/src/netmover/app_state.py @@ -58,7 +58,9 @@ class NetMover: ) self.runner = RuleRunner(self.session_factory, self.mounts) self.runs = RunService(self.session_factory, self.runner, self.notifier) - self.scheduler = RuleScheduler(self.session_factory, self.runs) + self.scheduler = RuleScheduler( + self.session_factory, self.runs, retry_mounts=self.mounts.retry_pending + ) self.started_at: datetime = utcnow() self.restore_results: list[RestoreResult] = [] diff --git a/src/netmover/mounting/manager.py b/src/netmover/mounting/manager.py index d6b52fd..aeb263f 100644 --- a/src/netmover/mounting/manager.py +++ b/src/netmover/mounting/manager.py @@ -344,6 +344,8 @@ class MountManager: self.mount(mount_id) results.append(RestoreResult(name, True, "eingehängt")) except (MountError, ValidationError, LookupError) as exc: + # z. B. Stromausfall: die VM bootet schneller als das NAS + self._retry_at[mount_id] = self._clock() + self.retry_interval results.append(RestoreResult(name, False, str(exc))) self._notify("mount.restore_failed", name, str(exc)) for r in results: @@ -597,6 +599,35 @@ class MountManager: self._notify("mount.error", m.name, msg) raise MountUnavailable(msg) + def retry_pending(self) -> list[str]: + """Fällige Neuversuche ausführen (Scheduler-Job, jede Minute) – auch für + Anbindungen, die gerade keine Regel benutzt. Liefert die zurückgeholten Namen.""" + recovered: list[str] = [] + for mount_id in list(self._retry_at): + if self._retry_at.get(mount_id, float("inf")) > self._clock(): + continue + with self._lock(mount_id): + if mount_id not in self._retry_at: + continue + try: + m, cred = self._load(mount_id) + except LookupError: + self._retry_at.pop(mount_id, None) + continue + if not (m.enabled and m.auto_mount and m.type.is_kernel_mount): + self._retry_at.pop(mount_id, None) + continue + mp = self.mountpoint_for(m.slug) + if self.backend.is_mounted(mp): + self._retry_at.pop(mount_id, None) + continue + try: + self._retry_mount(mount_id, m, cred, mp) + recovered.append(m.name) + except MountUnavailable: + pass + return recovered + def _retry_mount(self, mount_id: int, m: Mount, cred: CredentialData | None, mp: Path) -> None: """Ein Versuch, einen von NetMover selbst ausgehängten Mount zurückzuholen – höchstens einmal pro ``retry_interval``, damit kein Lauf alle 10 s hängt.""" diff --git a/src/netmover/scheduler.py b/src/netmover/scheduler.py index 940d6e3..7ecba05 100644 --- a/src/netmover/scheduler.py +++ b/src/netmover/scheduler.py @@ -13,6 +13,7 @@ from __future__ import annotations import asyncio import random +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta @@ -31,6 +32,7 @@ log = get_logger(__name__) MAX_START_OFFSET = 10.0 HEARTBEAT_SECONDS = 30 CLEANUP_SECONDS = 3600 +RETRY_SECONDS = 60 def _job_id(rule_id: int) -> str: @@ -43,11 +45,13 @@ class RuleScheduler: session_factory: sessionmaker[Session], service: RunService, *, + retry_mounts: Callable[[], object] | None = None, rng: random.Random | None = None, workers: int = 8, ) -> None: self.session_factory = session_factory self.service = service + self._retry_mounts = retry_mounts self._rng = rng or random.Random() # noqa: S311 – nur Startversatz self._executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="nm-run") self.scheduler = AsyncIOScheduler( @@ -70,6 +74,10 @@ class RuleScheduler: IntervalTrigger(seconds=CLEANUP_SECONDS, start_date=utcnow() + timedelta(minutes=1)), id="cleanup", ) + if self._retry_mounts is not None: + self.scheduler.add_job( + self._retry, IntervalTrigger(seconds=RETRY_SECONDS), id="mount-retry" + ) self.sync() event(log, "scheduler.started", rules=len(self._intervals)) @@ -141,6 +149,11 @@ class RuleScheduler: self._executor, self.service.execute, rule_id, RunTrigger.MANUAL, dry_run ) + async def _retry(self) -> None: + if self._retry_mounts is not None: + loop = asyncio.get_running_loop() + await loop.run_in_executor(self._executor, self._retry_mounts) + async def _cleanup(self) -> None: loop = asyncio.get_running_loop() await loop.run_in_executor(self._executor, cleanup, self.session_factory) diff --git a/tests/test_manager.py b/tests/test_manager.py index f498070..164737d 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -485,3 +485,41 @@ def test_hint_text_is_not_redacted() -> None: out = _redact_output(msg, ["falsch"]) assert out.startswith("mount error(13): Permission denied ••••••") assert out.endswith("→ Benutzername/Passwort falsch oder …") + + +def test_restore_failure_at_boot_is_retried_by_background_job( + retry_manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + session_factory: sessionmaker[Session], + notifications: list[tuple[str, str, str]], + clock: Clock, +) -> None: + """Stromausfall: die VM bootet schneller als das NAS.""" + fake_backend.export("nas", "scans") + mid = make_mount() # keine Regel benutzt die Anbindung + fake_backend.fail_next_mount() + assert [r.ok for r in retry_manager.restore_all()] == [False] + assert retry_manager.retry_pending() == [] # noch nicht fällig + clock.now += 60 + assert retry_manager.retry_pending() == ["scans"] + assert fake_backend.is_mounted(retry_manager.mountpoint_for("scans")) + assert _state(session_factory, mid).state is MountState.MOUNTED + assert notifications[-1][0] == "mount.recovered" + clock.now += 60 + assert retry_manager.retry_pending() == [] # erledigt + + +def test_manual_unmount_stops_retrying( + retry_manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + clock: Clock, +) -> None: + mid = make_mount() # nicht exportiert → Wiederherstellung scheitert + retry_manager.restore_all() + retry_manager.unmount(mid) + fake_backend.export("nas", "scans") + clock.now += 120 + assert retry_manager.retry_pending() == [] + assert not fake_backend.mounted