From e5ed8c4d88bd58aa16fd55f422a17ef7d1834109 Mon Sep 17 00:00:00 2001 From: Florian Wilde Date: Sat, 10 Oct 2026 18:46:50 +0200 Subject: [PATCH] feat: Mount-Schicht mit Linux-/Fake-Backend, Test und Wiederherstellung MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MountBackend-Interface mit LinuxMountBackend (Argumentliste, kurzlebige Credentials-Datei) und FakeMountBackend; MountManager für Einhängen, Aushängen, Start-Wiederherstellung, Drei-Schritt-Test mit Root-Squash- Erkennung, Lebenszeichen und umount -l/Neu-Einhängen; SFTP-Anbindung mit festgehaltenem Hostschlüssel. Tests inkl. echtem SFTP-Server im Prozess. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 18 + DEVLOG.md | 46 ++- README.md | 4 +- src/netmover/mounting/__init__.py | 1 + src/netmover/mounting/backend.py | 53 +++ src/netmover/mounting/fake.py | 126 +++++++ src/netmover/mounting/linux.py | 132 +++++++ src/netmover/mounting/manager.py | 587 +++++++++++++++++++++++++++++ src/netmover/mounting/mountinfo.py | 68 ++++ src/netmover/mounting/report.py | 134 +++++++ src/netmover/mounting/sftp.py | 260 +++++++++++++ tests/conftest.py | 115 +++++- tests/sftp_stub.py | 202 ++++++++++ tests/test_linux_backend.py | 117 ++++++ tests/test_manager.py | 369 ++++++++++++++++++ tests/test_mountinfo.py | 40 ++ tests/test_report.py | 61 +++ tests/test_sftp.py | 154 ++++++++ 18 files changed, 2478 insertions(+), 9 deletions(-) create mode 100644 src/netmover/mounting/__init__.py create mode 100644 src/netmover/mounting/backend.py create mode 100644 src/netmover/mounting/fake.py create mode 100644 src/netmover/mounting/linux.py create mode 100644 src/netmover/mounting/manager.py create mode 100644 src/netmover/mounting/mountinfo.py create mode 100644 src/netmover/mounting/report.py create mode 100644 src/netmover/mounting/sftp.py create mode 100644 tests/sftp_stub.py create mode 100644 tests/test_linux_backend.py create mode 100644 tests/test_manager.py create mode 100644 tests/test_mountinfo.py create mode 100644 tests/test_report.py create mode 100644 tests/test_sftp.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a61883b..786a7ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,24 @@ Die Version steht an genau einer Stelle: `src/netmover/__init__.py`. - Zeitzonenbewusste Zeitstempel (UTC) auch auf SQLite. - Strukturierte Logs (logfmt oder JSON, ein Ereignis pro Zeile). - CLI `netmover migrate` / `netmover version`. +- SFTP als Zieltyp (`sftp`) für Ziele, die nur per SSH erreichbar sind; Anmeldung + per selbst erzeugtem Ed25519-Schlüssel oder Passwort, Hostschlüssel wird beim + ersten Test festgehalten und danach erzwungen. +- Eingabeprüfung für Server, Freigaben, Pfade und Mount-Optionen (Allowlist pro + Typ; `credentials`, `username`, `password`, `pass`, `domain` gesperrt). +- Fernet-Verschlüsselung der Zugangsdaten (`secret.key`, 0600, beim ersten Start + erzeugt) und Schwärzung bekannter Geheimnisse in `mount`-Ausgaben. +- Mount-Schicht hinter `MountBackend` mit `LinuxMountBackend` (`mount`/`umount` + nur mit Argumentliste, CIFS-Zugangsdaten über kurzlebige Datei unter + `/run/netmover`) und `FakeMountBackend` für Tests ohne root. +- Mount-Test in drei Schritten (einhängen, lesen, schreiben) mit Rollen + („nur Quelle, kopieren“ / „Quelle mit Löschen“ / „Ziel“) und Root-Squash-Erkennung + inklusive Synology-Hinweis. +- Lebenszeichen mit Zeitlimit vor jedem Lauf; gestörte Mounts (ESTALE, EIO, + ENOTCONN, Zeitüberschreitung) werden per `umount -l` bis zu dreimal mit + wachsendem Abstand neu eingehängt. +- Mountpoints ausschließlich unter `/mnt/netmover/`, müssen vor dem + Einhängen leer sein und werden nach dem Aushängen entfernt. ### Behoben - Löschen von Zugangsdaten, die noch von einem Mount benutzt werden, trennt den diff --git a/DEVLOG.md b/DEVLOG.md index e5ef2a1..75f1368 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -8,10 +8,54 @@ Zeiten in Ortszeit (Europe/Berlin). | Datum | Von | Bis | Dauer | Schritt | Thema | |------------|-------|-------|-------|---------|----------------------------------------| | 2026-10-10 | 17:21 | 17:27 | 0:06 | 1 | Gerüst, Datenmodell, Migrationen | -| **Summe** | | | **0:06** | | | +| 2026-10-10 | 18:35 | 18:48 | 0:13 | 2 | Mount-Schicht, Validierung, SFTP | +| **Summe** | | | **0:19** | | | --- +## 2026-10-10 · 18:35–18:48 · Schritt 2: Mount-Schicht, Validierung, SFTP + +**Anforderungsänderung (Rücksprache)** +- Ziel ist voraussichtlich ein LXC, der nur per SSH erreichbar ist → neuer + Anbindungstyp `sftp` (nur als Ziel). Host: VM mit Debian 13 empfohlen. +- `/healthz` zählt nur aktive Anbindungen; Python ≥ 3.12 (Debian 13: 3.13). + +**Erledigt** +- `validation.py`: strikte Muster für Server/Freigabe/Pfade, Options-Allowlist + pro Typ, gesperrte Schlüssel, Werte gegen `^[A-Za-z0-9._:/@-]+$`. +- `crypto.py`: Fernet-Schlüssel (0600, O_EXCL), Ed25519-Schlüsselerzeugung, + Schwärzung. +- `mounting/`: `MountBackend`, `LinuxMountBackend`, `FakeMountBackend`, + `mountinfo`-Parser (Oktal-Escapes, übereinanderliegende Mounts), + `MountManager` (einhängen, aushängen, Wiederherstellung beim Start, Test, + Lebenszeichen, Wiederherstellung), SFTP-Anbindung mit TOFU-Hostschlüssel. +- `timeouts.py`: Zugriffe in Daemon-Threads mit Zeitlimit (kein Pool, damit + hängende Threads nichts blockieren). +- 113 neue Tests (123 gesamt), u. a. gegen einen echten SFTP-Server im Prozess. +- Mutationsprobe: 5 absichtlich eingebaute Fehler (Credentials-Datei bleibt liegen, + fehlende ismount-Prüfung, kein `umount -l`, nicht-leerer Mountpoint, gesperrte + Optionen) werden alle von Tests erkannt. + +**Entscheidungen** +- Migration 0001 angepasst statt 0002 – noch kein Release, keine Datenbank im + Einsatz. Ab v0.1.0 sind veröffentlichte Migrationen unveränderlich. +- Mount verschwunden (nicht eingehängt) → Lauf abbrechen, **kein** automatisches + Neu-Einhängen (Fallstrick 2). Gestört (ESTALE & Co.) → Neu-Einhängen (Fallstrick 13). +- Root-Squash-Auswertung als reine Funktion (`assess_write`), damit testbar ohne + root und ohne NAS. +- `statvfs`/`fsync` per SFTP über die OpenSSH-Erweiterungen; fehlen sie, gibt es + einen Hinweis statt eines Fehlers. +- Ruff-Regel `PTH` abgeschaltet: die bewussten `os.open`/`os.fsync`-Aufrufe + sind hier richtig. + +**Stolpersteine** +- Funktionen und Klassen mit dem Präfix `test_`/`Test` in Produktionscode würden + von pytest beim Import gesammelt → `check_connection`, `CheckReport`. + +**Offen / nächster Schritt** +- Schritt 3: Mover (Stabilität, Atomarität, Kollisionen, Kopier-Buchführung, + Worker mit Zeitlimit, Quarantäne) mit Zielen lokal/Kernel-Mount und SFTP. + ## 2026-10-10 · 17:21–17:27 · Schritt 1: Gerüst, Datenmodell, Migrationen **Erledigt** diff --git a/README.md b/README.md index 86d0d60..60e479e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Selbst gehosteter Dienst mit Web-Oberfläche, der SMB/CIFS- und NFS-Freigaben verwaltet und Dateien zeitgesteuert zwischen ihnen verschiebt. -> Status: in Entwicklung (Schritt 1 von 6). Vollständige Doku folgt mit Schritt 6. +> Status: in Entwicklung (Schritt 2 von 6). Vollständige Doku folgt mit Schritt 6. ## Entwicklung @@ -21,6 +21,8 @@ Neue Migration nach Modelländerung: NETMOVER_DATA_DIR=$PWD/data .venv/bin/alembic revision --autogenerate -m "…" ``` +Veröffentlichte Migrationen (ab v0.1.0) werden nie mehr geändert, nur ergänzt. + ## Arbeitsweise - Commits nach [Conventional Commits](https://www.conventionalcommits.org/de/) diff --git a/src/netmover/mounting/__init__.py b/src/netmover/mounting/__init__.py new file mode 100644 index 0000000..b738d56 --- /dev/null +++ b/src/netmover/mounting/__init__.py @@ -0,0 +1 @@ +"""Mount-Schicht: Backends (Linux/Fake), Verwaltung, Tests, SFTP-Anbindung.""" diff --git a/src/netmover/mounting/backend.py b/src/netmover/mounting/backend.py new file mode 100644 index 0000000..a121d7b --- /dev/null +++ b/src/netmover/mounting/backend.py @@ -0,0 +1,53 @@ +"""Interface der Mount-Schicht. + +Alles, was den Kernel betrifft (einhängen, aushängen, „ist eingehängt?“, Lebens- +zeichen), läuft über :class:`MountBackend`. Dadurch ist die gesamte Mover-Logik +mit :class:`~netmover.mounting.fake.FakeMountBackend` ohne root testbar. +""" + +from __future__ import annotations + +import os +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass(frozen=True) +class CifsCredentials: + username: str + password: str + domain: str | None = None + + def __repr__(self) -> str: # nie das Passwort in Logs/Tracebacks + return f"CifsCredentials(username={self.username!r}, password='***')" + + +@dataclass(frozen=True) +class MountSpec: + fstype: str # Kernel-Dateisystemtyp: "cifs", "nfs", "nfs4" + source: str # "//server/share" bzw. "server:/export" + mountpoint: Path + options: tuple[str, ...] = () + credentials: CifsCredentials | None = field(default=None, repr=True) + + +class MountError(RuntimeError): + """``mount``/``umount`` gescheitert. Die Meldung ist bereits geschwärzt.""" + + +class MountBackend(ABC): + @abstractmethod + def mount(self, spec: MountSpec, timeout: float = 60) -> None: ... + + @abstractmethod + def unmount(self, mountpoint: Path, lazy: bool = False, timeout: float = 60) -> None: ... + + @abstractmethod + def is_mounted(self, mountpoint: Path) -> bool: ... + + def probe(self, path: Path) -> None: + """Lebenszeichen: ``stat`` + ``listdir``. Kann bei hängenden Mounts blockieren + – Aufrufer verwenden :func:`netmover.timeouts.run_with_timeout`.""" + os.stat(path) + os.listdir(path) diff --git a/src/netmover/mounting/fake.py b/src/netmover/mounting/fake.py new file mode 100644 index 0000000..611a2ee --- /dev/null +++ b/src/netmover/mounting/fake.py @@ -0,0 +1,126 @@ +"""Test-Implementierung: simuliert Mounts auf einem temporären Verzeichnis. + +* „Entfernte“ Freigaben sind Verzeichnisse unter ``/remote//`` + (anlegen mit :meth:`FakeMountBackend.export`). +* Einhängen ersetzt das leere Mountpoint-Verzeichnis durch einen Symlink auf die + Freigabe; Aushängen stellt das leere Verzeichnis wieder her. Ein ausgehängter + Mountpoint ist also – genau wie in echt – ein ganz normales leeres Verzeichnis. +* :meth:`inject_fault` lässt das Lebenszeichen eines Mounts mit ``ESTALE``, + ``EIO``, ``ENOTCONN`` scheitern oder hängen (Fallstrick 13). +""" + +from __future__ import annotations + +import errno +import os +import threading +from collections import deque +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from netmover.mounting.backend import MountBackend, MountError, MountSpec + +Fault = int | Literal["hang"] + + +@dataclass +class FakeCall: + action: str + mountpoint: Path + spec: MountSpec | None = None + lazy: bool = False + + +class FakeMountBackend(MountBackend): + def __init__(self, root: Path) -> None: + self.remote_root = root / "remote" + self.remote_root.mkdir(parents=True, exist_ok=True) + self.mounted: dict[Path, Path] = {} + self.faults: dict[Path, Fault] = {} + self.mount_failures: deque[str] = deque() + self.heal_on_remount = True + self.busy: set[Path] = set() + self.calls: list[FakeCall] = [] + self._release = threading.Event() + + # -- Steuerung für Tests ----------------------------------------------------- + + def export(self, server: str, share: str) -> Path: + path = self.remote_root / server / share.strip("/") + path.mkdir(parents=True, exist_ok=True) + return path + + def fail_next_mount( + self, message: str = "mount error(112): Host is down", times: int = 1 + ) -> None: + self.mount_failures.extend([message] * times) + + def inject_fault(self, mountpoint: Path, fault: Fault) -> None: + self.faults[Path(mountpoint)] = fault + + def clear_fault(self, mountpoint: Path) -> None: + self.faults.pop(Path(mountpoint), None) + + def vanish(self, mountpoint: Path) -> None: + """Mount verschwindet von außen (z. B. ``umount`` durch Dritte).""" + mp = Path(mountpoint) + if mp in self.mounted: + mp.unlink() + mp.mkdir() + del self.mounted[mp] + + def release_hangs(self) -> None: + self._release.set() + + def _remote_for(self, spec: MountSpec) -> Path: + if spec.source.startswith("//"): + server, _, share = spec.source[2:].partition("/") + else: + server, _, share = spec.source.partition(":") + return self.remote_root / server / share.strip("/") + + # -- Interface ----------------------------------------------------------------- + + def mount(self, spec: MountSpec, timeout: float = 60) -> None: + mp = Path(spec.mountpoint) + self.calls.append(FakeCall("mount", mp, spec=spec)) + if self.mount_failures: + raise MountError(self.mount_failures.popleft()) + if mp in self.mounted: + raise MountError(f"mount: {mp}: already mounted") + remote = self._remote_for(spec) + if not remote.is_dir(): + raise MountError("mount error(2): No such file or directory") + if not mp.is_dir() or mp.is_symlink(): + raise MountError(f"mount: {mp}: mount point does not exist") + mp.rmdir() # scheitert, wenn nicht leer – wie ein echter Mount, der verdeckt + mp.symlink_to(remote, target_is_directory=True) + self.mounted[mp] = remote + if self.heal_on_remount: + self.faults.pop(mp, None) + + def unmount(self, mountpoint: Path, lazy: bool = False, timeout: float = 60) -> None: + mp = Path(mountpoint) + self.calls.append(FakeCall("unmount", mp, lazy=lazy)) + if mp not in self.mounted: + raise MountError(f"umount: {mp}: not mounted.") + if mp in self.busy and not lazy: + raise MountError(f"umount: {mp}: target is busy.") + mp.unlink() + mp.mkdir() + del self.mounted[mp] + self.busy.discard(mp) + + def is_mounted(self, mountpoint: Path) -> bool: + return Path(mountpoint) in self.mounted + + def probe(self, path: Path) -> None: + p = Path(path) + for mp, fault in self.faults.items(): + if p == mp or mp in p.parents: + if fault == "hang": + self._release.wait() + raise OSError(errno.EIO, os.strerror(errno.EIO), str(p)) + raise OSError(fault, os.strerror(fault), str(p)) + super().probe(p) diff --git a/src/netmover/mounting/linux.py b/src/netmover/mounting/linux.py new file mode 100644 index 0000000..8f0c546 --- /dev/null +++ b/src/netmover/mounting/linux.py @@ -0,0 +1,132 @@ +"""Echte Implementierung: ruft ``mount``/``umount`` auf. + +Fallstrick 5: ``subprocess`` nur mit Argumentliste, nie ``shell=True``. +Fallstrick 7: Zugangsdaten für ``mount.cifs`` nur über eine kurzlebige +Credentials-Datei (0600) unter ``/run/netmover``, nie als ``password=`` in der +Optionszeile – die stünde sonst in der Prozessliste. +""" + +from __future__ import annotations + +import os +import subprocess +import uuid +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Any + +from netmover.crypto import redact +from netmover.logging_setup import event, get_logger +from netmover.mounting.backend import CifsCredentials, MountBackend, MountError, MountSpec +from netmover.mounting.mountinfo import MOUNTINFO, find_mount, read_mountinfo + +log = get_logger(__name__) + +Runner = Callable[..., "subprocess.CompletedProcess[str]"] + +_ENV = {"LC_ALL": "C", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin"} + + +def explain_mount_error(fstype: str, message: str) -> str: + """Hängt an typische ``mount``-Meldungen einen Hinweis im Klartext an.""" + low = message.lower() + hint = "" + if "helper program" in low or "unknown filesystem type" in low: + pkg = "cifs-utils" if fstype == "cifs" else "nfs-common" + hint = f"Das Paket „{pkg}“ fehlt: apt install {pkg}" + elif "permission denied" in low and fstype == "cifs": + hint = "Benutzername/Passwort falsch oder der Benutzer hat keine Rechte auf die Freigabe." + elif "permission denied" in low or "access denied by server" in low: + hint = ( + "Der NFS-Server erlaubt diesem Host den Zugriff nicht. Synology: Gemeinsamer " + "Ordner → Bearbeiten → NFS-Berechtigungen → Regel mit der IP dieses Hosts anlegen." + ) + elif "operation not permitted" in low or "only root can" in low: + hint = ( + "Keine Berechtigung zum Einhängen. NetMover muss als root mit CAP_SYS_ADMIN laufen " + "(in unprivilegierten LXC-Containern ist Einhängen nicht möglich)." + ) + elif "no such file or directory" in low or "mount error(2)" in low: + hint = "Freigabe bzw. Exportpfad existiert auf dem Server nicht – Schreibweise prüfen." + elif "host is down" in low or "no route to host" in low or "timed out" in low: + hint = "Server nicht erreichbar – Adresse, Netzwerk und Firewall prüfen." + elif "connection refused" in low: + hint = "Server lehnt die Verbindung ab – ist SMB bzw. NFS am NAS aktiviert?" + return f"{message}\n→ {hint}" if hint else message + + +class LinuxMountBackend(MountBackend): + def __init__( + self, + run_dir: Path, + mountinfo_path: Path = MOUNTINFO, + runner: Runner = subprocess.run, + ) -> None: + self.run_dir = run_dir + self.mountinfo_path = mountinfo_path + self._runner = runner + + # -- Hilfen ------------------------------------------------------------------ + + def _write_credentials(self, creds: CifsCredentials) -> Path: + self.run_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + path = self.run_dir / f"cred-{uuid.uuid4().hex}" + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(f"username={creds.username}\n") + fh.write(f"password={creds.password}\n") + if creds.domain: + fh.write(f"domain={creds.domain}\n") + return path + + def _run(self, argv: Sequence[str], timeout: float, secrets: Sequence[str]) -> None: + kwargs: dict[str, Any] = { + "capture_output": True, + "text": True, + "timeout": timeout, + "env": _ENV, + "check": False, + "stdin": subprocess.DEVNULL, + } + try: + proc = self._runner(list(argv), **kwargs) + except subprocess.TimeoutExpired: + raise MountError(f"{argv[0]}: keine Antwort nach {timeout:g} s") from None + except FileNotFoundError: + raise MountError(f"{argv[0]}: Programm nicht gefunden") from None + if proc.returncode != 0: + output = (proc.stderr or "").strip() or (proc.stdout or "").strip() + output = redact(output or f"Exit-Code {proc.returncode}", secrets) + raise MountError(output) + + # -- Interface ----------------------------------------------------------------- + + def mount(self, spec: MountSpec, timeout: float = 60) -> None: + options = list(spec.options) + secrets: list[str] = [] + cred_file: Path | None = None + try: + if spec.credentials is not None: + cred_file = self._write_credentials(spec.credentials) + options.append(f"credentials={cred_file}") + secrets.append(spec.credentials.password) + argv = ["mount", "-t", spec.fstype] + if options: + argv += ["-o", ",".join(options)] + argv += [spec.source, str(spec.mountpoint)] + try: + self._run(argv, timeout, secrets) + except MountError as exc: + raise MountError(explain_mount_error(spec.fstype, str(exc))) from None + finally: + if cred_file is not None: + cred_file.unlink(missing_ok=True) + event(log, "mount.mounted", source=spec.source, mountpoint=str(spec.mountpoint)) + + def unmount(self, mountpoint: Path, lazy: bool = False, timeout: float = 60) -> None: + argv = ["umount"] + (["-l"] if lazy else []) + [str(mountpoint)] + self._run(argv, timeout, []) + event(log, "mount.unmounted", mountpoint=str(mountpoint), lazy=lazy) + + def is_mounted(self, mountpoint: Path) -> bool: + return find_mount(mountpoint, read_mountinfo(self.mountinfo_path)) is not None diff --git a/src/netmover/mounting/manager.py b/src/netmover/mounting/manager.py new file mode 100644 index 0000000..04e5cd8 --- /dev/null +++ b/src/netmover/mounting/manager.py @@ -0,0 +1,587 @@ +"""Verwaltung der Anbindungen: einhängen, aushängen, testen, wiederherstellen. + +Verbindet Datenbank, Geheimnisse und :class:`MountBackend`. Alle Methoden sind +synchron und blockierend; der Web-Teil ruft sie über ``asyncio.to_thread`` auf, +die Regel-Worker direkt. + +Fallstricke: +* 2 – :meth:`MountManager.check_alive` prüft „eingehängt“ über das Backend + (``/proc/self/mountinfo``) und bricht ab, statt in ein leeres Verzeichnis zu schreiben. +* 6 – Mountpoints entstehen nur als ``/``; vor dem Einhängen + existiert das Verzeichnis und ist leer, nach dem Aushängen wird es entfernt. +* 13 – Lebenszeichen mit Zeitlimit; bei ESTALE/EIO/ENOTCONN/Zeitüberschreitung + ``umount -l`` und neu einhängen, höchstens N Versuche mit wachsendem Abstand. +""" + +from __future__ import annotations + +import errno +import os +import threading +import time +import uuid +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.orm import Session, sessionmaker + +from netmover.crypto import SecretBox, redact +from netmover.db import session_scope +from netmover.logging_setup import event, get_logger +from netmover.models import CredentialKind, Mount, MountState, MountType, utcnow +from netmover.mounting.backend import CifsCredentials, MountBackend, MountError, MountSpec +from netmover.mounting.report import CheckReport, assess_write +from netmover.mounting.sftp import SftpConfig, SftpError, check_connection, connect +from netmover.timeouts import OperationTimeout, run_with_timeout +from netmover.validation import ( + ValidationError, + parse_extra_options, + slugify, + validate_server, + validate_share, +) + +log = get_logger(__name__) + +# Fehler, bei denen ein Neu-Einhängen hilft (Fallstrick 13). +RECOVERABLE_ERRNOS = frozenset({errno.ESTALE, errno.EIO, errno.ENOTCONN, errno.EHOSTDOWN}) + +Notifier = Callable[[str, str, str], None] # (ereignis, anbindung, meldung) + + +class MountUnavailable(RuntimeError): + """Anbindung für einen Lauf nicht nutzbar – Lauf abbrechen, nichts anfassen.""" + + +@dataclass(frozen=True) +class CredentialData: + kind: CredentialKind + username: str + secret: str + domain: str | None = None + + def __repr__(self) -> str: + return f"CredentialData(kind={self.kind}, username={self.username!r}, secret='***')" + + +@dataclass(frozen=True) +class MountConfig: + name: str + type: MountType + server: str + share: str + port: int | None = None + host_key: str | None = None + uid: int | None = None + gid: int | None = None + file_mode: str | None = None + dir_mode: str | None = None + extra_options: str = "" + + @classmethod + def from_model(cls, m: Mount) -> MountConfig: + return cls( + name=m.name, + type=m.type, + server=m.server, + share=m.share, + port=m.port, + host_key=m.host_key, + uid=m.uid, + gid=m.gid, + file_mode=m.file_mode, + dir_mode=m.dir_mode, + extra_options=m.extra_options, + ) + + +@dataclass(frozen=True) +class RestoreResult: + name: str + ok: bool + message: str + + +def build_spec(cfg: MountConfig, mountpoint: Path, cred: CredentialData | None) -> MountSpec: + """Baut den ``mount``-Aufruf. Validiert erneut (Verteidigung in der Tiefe) – + auch Daten aus der Datenbank laufen hier noch einmal durch die Prüfung.""" + if not cfg.type.is_kernel_mount: + raise MountError("SFTP-Anbindungen werden nicht eingehängt") + server = validate_server(cfg.server) + share = validate_share(cfg.type, cfg.share) + options = [o.render() for o in parse_extra_options(cfg.type, cfg.extra_options)] + credentials: CifsCredentials | None = None + + if cfg.type is MountType.CIFS: + fstype, source = "cifs", f"//{server}/{share}" + for key, value in ( + ("uid", cfg.uid), + ("gid", cfg.gid), + ("file_mode", cfg.file_mode), + ("dir_mode", cfg.dir_mode), + ): + if value is not None: + options.append(f"{key}={value}") + if cred is None: + options.append("guest") + else: + if cred.kind is not CredentialKind.PASSWORD: + raise MountError("CIFS braucht Zugangsdaten mit Passwort, keinen SSH-Schlüssel") + credentials = CifsCredentials(cred.username, cred.secret, cred.domain) + elif cfg.type is MountType.NFS4: + fstype, source = "nfs4", f"{server}:{share}" + else: + fstype, source = "nfs", f"{server}:{share}" + options.insert(0, "vers=3") + return MountSpec(fstype, source, mountpoint, tuple(options), credentials) + + +def sftp_config(cfg: MountConfig, cred: CredentialData | None) -> SftpConfig: + if cred is None: + raise SftpError("SFTP braucht Zugangsdaten (SSH-Schlüssel oder Passwort)") + return SftpConfig( + host=validate_server(cfg.server), + port=cfg.port or 22, + username=cred.username, + base_path=validate_share(MountType.SFTP, cfg.share), + password=cred.secret if cred.kind is CredentialKind.PASSWORD else None, + private_key=cred.secret if cred.kind is CredentialKind.SSH_KEY else None, + host_key=cfg.host_key, + ) + + +class MountManager: + def __init__( + self, + session_factory: sessionmaker[Session], + backend: MountBackend, + secrets: SecretBox, + mount_base: Path, + *, + mount_timeout: float = 60, + probe_timeout: float = 15, + recover_backoff: Sequence[float] = (0, 5, 15), + sleep: Callable[[float], None] = time.sleep, + euid: Callable[[], int] = os.geteuid, + notifier: Notifier | None = None, + ) -> None: + self.session_factory = session_factory + self.backend = backend + self.secrets = secrets + self.mount_base = mount_base + self.mount_timeout = mount_timeout + self.probe_timeout = probe_timeout + self.recover_backoff = tuple(recover_backoff) + self._sleep = sleep + self._euid = euid + self._notifier = notifier + self._locks: dict[int, threading.RLock] = {} + self._locks_guard = threading.Lock() + + # -- Hilfen -------------------------------------------------------------------- + + def _lock(self, mount_id: int) -> threading.RLock: + with self._locks_guard: + return self._locks.setdefault(mount_id, threading.RLock()) + + def _notify(self, kind: str, name: str, message: str) -> None: + if self._notifier is not None: + try: + self._notifier(kind, name, message) + except Exception: + log.exception("notifier.failed") + + def mountpoint_for(self, slug: str) -> Path: + if slugify(slug) != slug: + raise ValidationError("slug", f"Ungültiger Slug: {slug!r}") + base = self.mount_base.resolve() + mp = base / slug + if mp.parent != base: + raise ValidationError("slug", "Mountpoint außerhalb des Basisverzeichnisses") + return mp + + def _load(self, mount_id: int) -> tuple[Mount, CredentialData | None]: + with session_scope(self.session_factory) as s: + m = s.get(Mount, mount_id) + if m is None: + raise LookupError(f"Anbindung {mount_id} existiert nicht") + cred: CredentialData | None = None + if m.credential is not None: + c = m.credential + cred = CredentialData( + c.kind, c.username, self.secrets.decrypt(c.secret_enc), c.domain + ) + s.expunge(m) + return m, cred + + def _set_state( + self, + mount_id: int, + state: MountState, + error: str | None = None, + *, + mounted: bool = False, + ) -> None: + with session_scope(self.session_factory) as s: + m = s.get(Mount, mount_id) + if m is None: + return + if m.state is not state: + m.state_since = utcnow() + event(log, "mount.state", mount=m.name, old=m.state.value, new=state.value) + m.state = state + m.last_error = error + if mounted: + m.last_mounted_at = utcnow() + + def _prepare_mountpoint(self, mp: Path) -> bool: + """Legt den Mountpoint an bzw. prüft, dass er leer ist. True = neu angelegt.""" + self.mount_base.mkdir(mode=0o755, parents=True, exist_ok=True) + if mp.is_symlink() or (mp.exists() and not mp.is_dir()): + raise MountError(f"{mp} ist kein Verzeichnis – bitte von Hand prüfen") + if mp.is_dir(): + if any(mp.iterdir()): + raise MountError( + f"{mp} ist nicht leer. Vermutlich wurde dort geschrieben, während nichts " + "eingehängt war. Inhalt prüfen und sichern, dann Verzeichnis leeren." + ) + return False + mp.mkdir(mode=0o755) + return True + + @staticmethod + def _remove_mountpoint(mp: Path) -> None: + try: + mp.rmdir() + except FileNotFoundError: + pass + except OSError as exc: + event(log, "mount.rmdir_failed", level=30, mountpoint=str(mp), error=str(exc)) + + def _probe(self, name: str, mp: Path) -> None: + run_with_timeout(lambda: self.backend.probe(mp), self.probe_timeout, f"probe:{name}") + + # -- Einhängen / Aushängen ----------------------------------------------------- + + def mount(self, mount_id: int) -> None: + with self._lock(mount_id): + m, cred = self._load(mount_id) + if not m.type.is_kernel_mount: + raise MountError("SFTP-Anbindungen werden nicht eingehängt, nur getestet") + mp = self.mountpoint_for(m.slug) + if self.backend.is_mounted(mp): + self._set_state(mount_id, MountState.MOUNTED) + return + try: + spec = build_spec(MountConfig.from_model(m), mp, cred) + created = self._prepare_mountpoint(mp) + except (MountError, ValidationError) as exc: + self._set_state(mount_id, MountState.ERROR, str(exc)) + raise MountError(str(exc)) from None + try: + self.backend.mount(spec, timeout=self.mount_timeout) + except MountError as exc: + if created: + self._remove_mountpoint(mp) + self._set_state(mount_id, MountState.ERROR, str(exc)) + event(log, "mount.failed", level=40, mount=m.name, error=str(exc)) + raise + self._set_state(mount_id, MountState.MOUNTED, mounted=True) + + def unmount(self, mount_id: int) -> None: + with self._lock(mount_id): + m, _ = self._load(mount_id) + if not m.type.is_kernel_mount: + self._set_state(mount_id, MountState.UNMOUNTED) + return + mp = self.mountpoint_for(m.slug) + if self.backend.is_mounted(mp): + try: + self.backend.unmount(mp, timeout=self.mount_timeout) + except MountError as exc: + msg = str(exc) + if "busy" in msg.lower(): + msg += "\n→ Ein Prozess benutzt den Mount noch (z. B. ein Regellauf)." + raise MountError(msg) from None + if not self.backend.is_mounted(mp): + self._remove_mountpoint(mp) + self._set_state(mount_id, MountState.UNMOUNTED) + + def restore_all(self) -> list[RestoreResult]: + """Beim Dienststart: alle Anbindungen mit ``auto_mount`` wiederherstellen.""" + with session_scope(self.session_factory) as s: + rows = s.execute( + select(Mount.id, Mount.name, Mount.type).where( + Mount.auto_mount.is_(True), Mount.enabled.is_(True) + ) + ).all() + results: list[RestoreResult] = [] + for mount_id, name, mtype in rows: + if not MountType(mtype).is_kernel_mount: + continue + try: + self.mount(mount_id) + results.append(RestoreResult(name, True, "eingehängt")) + except (MountError, ValidationError, LookupError) as exc: + results.append(RestoreResult(name, False, str(exc))) + self._notify("mount.restore_failed", name, str(exc)) + for r in results: + event( + log, + "mount.restore", + level=20 if r.ok else 40, + mount=r.name, + ok=r.ok, + message=r.message, + ) + return results + + # -- Test ------------------------------------------------------------------------- + + def check( + self, + cfg: MountConfig, + cred: CredentialData | None, + mount_id: int | None = None, + ) -> CheckReport: + """Testet eine (ggf. noch nicht gespeicherte) Anbindung in drei Schritten.""" + if cfg.type is MountType.SFTP: + report = self._check_sftp(cfg, cred) + else: + report = self._check_kernel(cfg, cred, mount_id) + if mount_id is not None: + self._store_check(mount_id, report, cfg) + return report + + def _check_sftp(self, cfg: MountConfig, cred: CredentialData | None) -> CheckReport: + try: + scfg = sftp_config(cfg, cred) + except (SftpError, ValidationError) as exc: + report = CheckReport() + report.step("connect", False, str(exc)) + return report + return run_with_timeout( + lambda: check_connection(scfg, timeout=self.probe_timeout), + self.probe_timeout * 4, + f"check:{cfg.name}", + ) + + def _check_kernel( + self, cfg: MountConfig, cred: CredentialData | None, mount_id: int | None + ) -> CheckReport: + report = CheckReport() + secrets = [cred.secret] if cred else [] + target: Path | None = None + temp: Path | None = None + + if mount_id is not None: + m, _ = self._load(mount_id) + mp = self.mountpoint_for(m.slug) + if self.backend.is_mounted(mp): + target = mp + report.step("mount", True, "bereits eingehängt") + + if target is None: + self.mount_base.mkdir(mode=0o755, parents=True, exist_ok=True) + temp = self.mount_base / f".test-{uuid.uuid4().hex[:12]}" + temp.mkdir(mode=0o700) + try: + self.backend.mount(build_spec(cfg, temp, cred), timeout=self.mount_timeout) + except (MountError, ValidationError) as exc: + self._remove_mountpoint(temp) + report.step("mount", False, redact(str(exc), secrets)) + report.step("read", None) + report.step("write", None) + return report + target = temp + report.step("mount", True, "Einhängen erfolgreich") + + try: + self._check_read_write(cfg, target, report) + finally: + if temp is not None: + try: + self.backend.unmount(temp, timeout=self.mount_timeout) + except MountError: + try: + self.backend.unmount(temp, lazy=True, timeout=self.mount_timeout) + except MountError as exc: + event(log, "check.unmount_failed", level=40, path=str(temp), error=str(exc)) + if not self.backend.is_mounted(temp): + self._remove_mountpoint(temp) + return report + + def _check_read_write(self, cfg: MountConfig, target: Path, report: CheckReport) -> None: + name = cfg.name + try: + count = run_with_timeout( + lambda: len(os.listdir(target)), self.probe_timeout, f"read:{name}" + ) + report.can_read = True + report.step("read", True, f"{count} Einträge") + except (OSError, OperationTimeout) as exc: + report.step("read", False, str(exc)) + report.step("write", None) + return + + def write_probe() -> tuple[OSError | None, int | None, int | None, OSError | None]: + # Punkt am Anfang + .tmp: ein parallel laufender Mover fasst sie nie an. + path = target / f".netmover-test-{uuid.uuid4().hex}.tmp" + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + try: + os.write(fd, b"netmover") + os.fsync(fd) + finally: + os.close(fd) + except OSError as exc: + return exc, None, None, None + st = os.stat(path) + try: + path.unlink() + except OSError as exc: + return None, st.st_uid, st.st_gid, exc + return None, st.st_uid, st.st_gid, None + + try: + werr, uid, gid, derr = run_with_timeout( + write_probe, self.probe_timeout, f"write:{name}" + ) + except OperationTimeout as exc: + report.step("write", False, str(exc)) + return + result = assess_write(cfg.type, self._euid(), werr, uid, derr) + report.can_write = result.write_ok + report.can_delete = result.delete_ok + report.root_squash = result.root_squash + report.hints.extend(result.hints) + if uid is not None: + report.owner = f"{uid}:{gid}" + report.step("write", result.write_ok, result.message) + if cfg.type is MountType.CIFS and result.write_ok: + report.hints.append( + "CIFS: Eigentümer und Rechte bestimmen die Mount-Optionen uid/gid/file_mode – " + "chown/chmod nach dem Verschieben werden übersprungen." + ) + try: + st = run_with_timeout(lambda: os.statvfs(target), self.probe_timeout, f"statvfs:{name}") + report.total_bytes = st.f_blocks * st.f_frsize + report.free_bytes = st.f_bavail * st.f_frsize + except (OSError, OperationTimeout): + pass + + def _store_check(self, mount_id: int, report: CheckReport, cfg: MountConfig) -> None: + with session_scope(self.session_factory) as s: + m = s.get(Mount, mount_id) + if m is None: + return + m.last_test_at = utcnow() + m.last_test_result = report.to_json() + m.write_ok = report.can_write + if report.root_squash is not None: + m.root_squash = report.root_squash + # SFTP: Hostschlüssel beim ersten erfolgreichen Test festhalten (TOFU). + if cfg.type is MountType.SFTP and m.host_key is None and report.can_read: + m.host_key = report.host_key + event(log, "sftp.host_key_pinned", mount=m.name, fp=report.host_key_fingerprint) + + # -- Lebenszeichen und Wiederherstellung -------------------------------------- + + def check_alive(self, mount_id: int) -> Path | None: + """Vor jedem Lauf. Liefert den Mountpoint (None bei SFTP) oder wirft + :class:`MountUnavailable`. Stellt gestörte Mounts selbst wieder her.""" + with self._lock(mount_id): + m, cred = self._load(mount_id) + if not m.enabled: + raise MountUnavailable(f"Anbindung „{m.name}“ ist deaktiviert") + if m.type is MountType.SFTP: + self._check_alive_sftp(mount_id, m, cred) + return None + mp = self.mountpoint_for(m.slug) + if not self.backend.is_mounted(mp): + msg = f"„{m.name}“ ist nicht eingehängt – Lauf abgebrochen, nichts angefasst." + self._set_state(mount_id, MountState.ERROR, msg) + raise MountUnavailable(msg) + try: + self._probe(m.name, mp) + except OperationTimeout as exc: + self._recover(mount_id, m, cred, f"hängt: {exc}") + except OSError as exc: + if exc.errno not in RECOVERABLE_ERRNOS: + msg = f"„{m.name}“: {exc.strerror} ({exc.filename})" + self._set_state(mount_id, MountState.ERROR, msg) + raise MountUnavailable(msg) from None + self._recover( + mount_id, m, cred, f"{errno.errorcode.get(exc.errno or 0, '?')}: {exc.strerror}" + ) + else: + if m.state is not MountState.MOUNTED: + self._set_state(mount_id, MountState.MOUNTED) + return mp + + def _check_alive_sftp(self, mount_id: int, m: Mount, cred: CredentialData | None) -> None: + try: + scfg = sftp_config(MountConfig.from_model(m), cred) + + def probe() -> None: + with connect(scfg, timeout=self.probe_timeout) as sess: + sess.client.stat(scfg.base_path) + + run_with_timeout(probe, self.probe_timeout * 2, f"probe:{m.name}") + except (SftpError, ValidationError, OSError, OperationTimeout) as exc: + msg = f"„{m.name}“ nicht erreichbar: {exc}" + self._set_state(mount_id, MountState.ERROR, msg) + if m.state is not MountState.ERROR: + self._notify("mount.error", m.name, msg) + raise MountUnavailable(msg) from None + if m.state is not MountState.MOUNTED: + self._set_state(mount_id, MountState.MOUNTED) + + def _recover(self, mount_id: int, m: Mount, cred: CredentialData | None, reason: str) -> None: + mp = self.mountpoint_for(m.slug) + self._set_state(mount_id, MountState.DEGRADED, f"gestört ({reason}) – wird neu eingehängt") + event(log, "mount.degraded", level=30, mount=m.name, reason=reason) + last: str = reason + for attempt, delay in enumerate(self.recover_backoff, start=1): + if delay: + self._sleep(delay) + try: + if self.backend.is_mounted(mp): + self.backend.unmount(mp, lazy=True, timeout=self.mount_timeout) + self._prepare_mountpoint(mp) + self.backend.mount( + build_spec(MountConfig.from_model(m), mp, cred), timeout=self.mount_timeout + ) + self._probe(m.name, mp) + except (MountError, OSError, OperationTimeout, ValidationError) as exc: + last = str(exc) + event( + log, "mount.recover_failed", level=30, mount=m.name, attempt=attempt, error=last + ) + continue + self._set_state(mount_id, MountState.MOUNTED, mounted=True) + event(log, "mount.recovered", mount=m.name, attempt=attempt) + return + msg = ( + f"„{m.name}“ gestört ({reason}); Neu-Einhängen nach " + f"{len(self.recover_backoff)} Versuchen gescheitert: {last}" + ) + self._set_state(mount_id, MountState.ERROR, msg) + self._notify("mount.error", m.name, msg) + raise MountUnavailable(msg) + + def usage(self, mount_id: int) -> tuple[int, int] | None: + """(gesamt, frei) in Bytes – nur bei eingehängten Kernel-Mounts.""" + m, _ = self._load(mount_id) + if not m.type.is_kernel_mount: + return None + mp = self.mountpoint_for(m.slug) + if not self.backend.is_mounted(mp): + return None + try: + st = run_with_timeout(lambda: os.statvfs(mp), self.probe_timeout, f"statvfs:{m.name}") + except (OSError, OperationTimeout): + return None + return st.f_blocks * st.f_frsize, st.f_bavail * st.f_frsize diff --git a/src/netmover/mounting/mountinfo.py b/src/netmover/mounting/mountinfo.py new file mode 100644 index 0000000..40808a5 --- /dev/null +++ b/src/netmover/mounting/mountinfo.py @@ -0,0 +1,68 @@ +"""``/proc/self/mountinfo`` lesen (Fallstrick 2). + +Ob ein Verzeichnis eingehängt ist, entscheidet ausschließlich der Kernel – nie +das Vorhandensein von Dateien. Ein leerer Mountpoint sieht sonst genauso aus wie +eine leere Freigabe. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from pathlib import Path + +MOUNTINFO = Path("/proc/self/mountinfo") + +_OCTAL_RE = re.compile(r"\\([0-7]{3})") + + +def _unescape(field: str) -> str: + # Der Kernel kodiert Leerzeichen, Tab, Zeilenumbruch und Backslash oktal. + return _OCTAL_RE.sub(lambda m: chr(int(m.group(1), 8)), field) + + +@dataclass(frozen=True) +class MountEntry: + mountpoint: str + fstype: str + source: str + options: str + + +def parse_mountinfo(text: str) -> list[MountEntry]: + entries: list[MountEntry] = [] + for line in text.splitlines(): + if not line.strip(): + continue + left, sep, right = line.partition(" - ") + if not sep: + continue + lf = left.split(" ") + rf = right.split(" ") + if len(lf) < 6 or len(rf) < 3: + continue + entries.append( + MountEntry( + mountpoint=_unescape(lf[4]), + fstype=rf[0], + source=_unescape(rf[1]), + options=rf[2], + ) + ) + return entries + + +def read_mountinfo(path: Path = MOUNTINFO) -> list[MountEntry]: + return parse_mountinfo(path.read_text(encoding="utf-8", errors="surrogateescape")) + + +def find_mount(mountpoint: Path, entries: list[MountEntry]) -> MountEntry | None: + """Liefert den obersten Eintrag für genau diesen Mountpoint (bei Übereinander- + Mounts gewinnt der zuletzt eingehängte).""" + target = os.path.normpath(str(mountpoint)) + found: MountEntry | None = None + for entry in entries: + if entry.mountpoint == target: + found = entry + return found diff --git a/src/netmover/mounting/report.py b/src/netmover/mounting/report.py new file mode 100644 index 0000000..743d49b --- /dev/null +++ b/src/netmover/mounting/report.py @@ -0,0 +1,134 @@ +"""Ergebnis eines Mount-Tests: drei getrennte Schritte plus abgeleitete Rollen. + +Fallstrick 12: Der Schreibtest erkennt Root-Squash auf NFS und nennt Ursache +und Abhilfe im Klartext. +""" + +from __future__ import annotations + +import errno +import json +from dataclasses import asdict, dataclass, field +from enum import StrEnum + +from netmover.models import MountType + + +class Role(StrEnum): + SOURCE_COPY = "nur Quelle, kopieren" + SOURCE_MOVE = "Quelle mit Löschen" + TARGET = "Ziel" + + +SYNOLOGY_SQUASH_HINT = ( + "Synology: Systemsteuerung → Gemeinsamer Ordner → Ordner wählen → Bearbeiten → " + "NFS-Berechtigungen → Regel bearbeiten → Squash: „Keine Zuordnung“ " + "(No mapping) oder „Alle Benutzer zu admin zuordnen“." +) + + +@dataclass +class CheckStep: + name: str # "mount" | "read" | "write" (bei sftp: "connect") + ok: bool | None # None = nicht ausgeführt + message: str = "" + + +@dataclass +class CheckReport: + steps: list[CheckStep] = field(default_factory=list) + can_read: bool = False + can_write: bool = False + can_delete: bool = False + root_squash: bool | None = None + owner: str | None = None + total_bytes: int | None = None + free_bytes: int | None = None + hints: list[str] = field(default_factory=list) + host_key: str | None = None # sftp: festgestellter Hostschlüssel + host_key_fingerprint: str | None = None + + @property + def ok(self) -> bool: + return all(s.ok is not False for s in self.steps) and self.can_read + + @property + def roles(self) -> list[Role]: + roles: list[Role] = [] + if self.can_read: + roles.append(Role.SOURCE_MOVE if self.can_delete else Role.SOURCE_COPY) + if self.can_write: + roles.append(Role.TARGET) + return roles + + def step(self, name: str, ok: bool | None, message: str = "") -> CheckStep: + s = CheckStep(name, ok, message) + self.steps.append(s) + return s + + def to_json(self) -> str: + data = asdict(self) + data["roles"] = [r.value for r in self.roles] + data["ok"] = self.ok + return json.dumps(data, ensure_ascii=False) + + +@dataclass +class WriteAssessment: + write_ok: bool + delete_ok: bool + root_squash: bool | None + message: str + hints: list[str] + + +def assess_write( + mount_type: MountType, + euid: int, + write_error: OSError | None, + created_uid: int | None, + delete_error: OSError | None, +) -> WriteAssessment: + """Reine Auswertung des Schreibtests – getrennt von den Dateizugriffen, damit + sich Root-Squash ohne root und ohne NAS testen lässt.""" + is_nfs = mount_type in (MountType.NFS3, MountType.NFS4) + hints: list[str] = [] + + if write_error is not None: + denied = write_error.errno in (errno.EACCES, errno.EPERM) + if is_nfs and euid == 0 and denied: + hints.append( + "Schreiben als root wurde abgelehnt – vermutlich Root-Squash: Der NFS-Server " + "bildet root auf „nobody“ ab, und „nobody“ darf hier nicht schreiben. " + + SYNOLOGY_SQUASH_HINT + ) + return WriteAssessment(False, False, True, "Schreiben verweigert (Root-Squash?)", hints) + if write_error.errno == errno.EROFS: + return WriteAssessment(False, False, None, "Freigabe ist schreibgeschützt", hints) + if denied: + hints.append("Keine Schreibrechte – als Quelle nur zum Kopieren geeignet.") + return WriteAssessment( + False, False, None, f"Schreiben fehlgeschlagen: {write_error.strerror}", hints + ) + + root_squash: bool | None = None + message = "Datei angelegt und gelöscht" + if is_nfs and euid == 0 and created_uid is not None: + root_squash = created_uid != 0 + if root_squash: + message = f"Schreiben möglich, Datei gehört aber uid {created_uid} (Root-Squash)" + hints.append( + "Root-Squash aktiv: Dateien gehören nicht root, „chown“ nach dem Verschieben " + "ist nicht möglich und wird übersprungen. Wer Eigentümer setzen will: " + + SYNOLOGY_SQUASH_HINT + ) + + if delete_error is not None: + return WriteAssessment( + True, + False, + root_squash, + f"Angelegt, aber Löschen fehlgeschlagen: {delete_error.strerror}", + [*hints, "Testdatei konnte nicht gelöscht werden – bitte von Hand entfernen."], + ) + return WriteAssessment(True, True, root_squash, message, hints) diff --git a/src/netmover/mounting/sftp.py b/src/netmover/mounting/sftp.py new file mode 100644 index 0000000..3ff6b33 --- /dev/null +++ b/src/netmover/mounting/sftp.py @@ -0,0 +1,260 @@ +"""SFTP-Anbindung für Ziele, die nur per SSH erreichbar sind (z. B. ein LXC). + +Kein Kernel-Mount – daher kein Hängen im Kernel, aber auch kein ``ismount``. +Der Hostschlüssel wird beim ersten Test festgehalten (Trust on first use) und ist +danach verbindlich; eine Abweichung bricht jede Verbindung ab. +""" + +from __future__ import annotations + +import base64 +import contextlib +import hashlib +import io +import posixpath +import socket +import stat +import uuid +from collections.abc import Iterator +from dataclasses import dataclass +from types import TracebackType + +import paramiko +from paramiko.sftp import CMD_EXTENDED, CMD_EXTENDED_REPLY + +from netmover.crypto import redact +from netmover.mounting.report import CheckReport + + +class SftpError(RuntimeError): + pass + + +class HostKeyMismatch(SftpError): + pass + + +class HostKeyUnknown(SftpError): + pass + + +@dataclass(frozen=True) +class SftpConfig: + host: str + port: int + username: str + base_path: str + password: str | None = None + private_key: str | None = None + host_key: str | None = None # " " + + def __repr__(self) -> str: + return ( + f"SftpConfig(host={self.host!r}, port={self.port}, username={self.username!r}, " + f"base_path={self.base_path!r})" + ) + + @property + def secrets(self) -> list[str | None]: + return [self.password, self.private_key] + + +def fingerprint(key: paramiko.PKey) -> str: + digest = hashlib.sha256(key.asbytes()).digest() + return "SHA256:" + base64.b64encode(digest).decode().rstrip("=") + + +def host_key_string(key: paramiko.PKey) -> str: + return f"{key.get_name()} {key.get_base64()}" + + +def load_private_key(pem: str) -> paramiko.PKey: + errors: list[str] = [] + key_classes: list[type[paramiko.PKey]] = [ + paramiko.Ed25519Key, + paramiko.ECDSAKey, + paramiko.RSAKey, + ] + for cls in key_classes: + try: + return cls.from_private_key(io.StringIO(pem)) + except (paramiko.SSHException, ValueError) as exc: + errors.append(f"{cls.__name__}: {exc}") + raise SftpError("Privater Schlüssel nicht lesbar (Ed25519, ECDSA oder RSA ohne Passphrase)") + + +class SftpSession: + def __init__(self, transport: paramiko.Transport, client: paramiko.SFTPClient) -> None: + self.transport = transport + self.client = client + key = transport.get_remote_server_key() + self.host_key = host_key_string(key) + self.fingerprint = fingerprint(key) + + def close(self) -> None: + with contextlib.suppress(Exception): + self.client.close() + with contextlib.suppress(Exception): + self.transport.close() + + def __enter__(self) -> SftpSession: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + self.close() + + # -- Erweiterungen von OpenSSH (paramiko bietet sie nicht direkt an) ----------- + + def statvfs(self, path: str) -> tuple[int, int] | None: + """(gesamt, frei für Nutzer) in Bytes über ``statvfs@openssh.com``, sonst None.""" + try: + t, msg = self.client._request( # type: ignore[attr-defined] + CMD_EXTENDED, "statvfs@openssh.com", path + ) + except (OSError, paramiko.SSHException): + return None + if t != CMD_EXTENDED_REPLY or msg is None: + return None + _bsize, frsize, blocks, _bfree, bavail = (msg.get_int64() for _ in range(5)) + return blocks * frsize, bavail * frsize + + def fsync(self, handle: paramiko.SFTPFile) -> bool: + """``fsync@openssh.com`` auf eine offene Datei; False, wenn nicht unterstützt.""" + try: + self.client._request( # type: ignore[attr-defined] + CMD_EXTENDED, "fsync@openssh.com", handle.handle + ) + except (OSError, paramiko.SSHException): + return False + return True + + +def connect( + cfg: SftpConfig, timeout: float = 20, allow_unknown_host_key: bool = False +) -> SftpSession: + try: + sock = socket.create_connection((cfg.host, cfg.port), timeout=timeout) + except OSError as exc: + raise SftpError(f"Verbindung zu {cfg.host}:{cfg.port} fehlgeschlagen: {exc}") from None + transport = paramiko.Transport(sock) + transport.banner_timeout = timeout + transport.auth_timeout = timeout + try: + transport.start_client(timeout=timeout) + server_key = transport.get_remote_server_key() + seen = host_key_string(server_key) + if cfg.host_key is None: + if not allow_unknown_host_key: + raise HostKeyUnknown( + "Hostschlüssel noch nicht bestätigt – bitte die Anbindung zuerst testen." + ) + elif cfg.host_key.split()[:2] != seen.split()[:2]: + raise HostKeyMismatch( + f"Hostschlüssel von {cfg.host} hat sich geändert " + f"(jetzt {fingerprint(server_key)}). " + "Wurde der Server neu aufgesetzt? Dann Hostschlüssel in der Anbindung " + "zurücksetzen und neu testen – sonst nicht verbinden!" + ) + if cfg.private_key: + transport.auth_publickey(cfg.username, load_private_key(cfg.private_key)) + elif cfg.password is not None: + transport.auth_password(cfg.username, cfg.password) + else: + raise SftpError("Keine Zugangsdaten hinterlegt") + client = paramiko.SFTPClient.from_transport(transport) + if client is None: + raise SftpError("SFTP-Subsystem nicht verfügbar") + client.get_channel().settimeout(timeout) # type: ignore[union-attr] + return SftpSession(transport, client) + except paramiko.AuthenticationException: + transport.close() + raise SftpError( + "Anmeldung abgelehnt – Benutzername prüfen und den öffentlichen Schlüssel in " + "~/.ssh/authorized_keys des Zielbenutzers eintragen." + ) from None + except (paramiko.SSHException, OSError, EOFError) as exc: + transport.close() + raise SftpError(redact(f"SSH-Fehler: {exc}", cfg.secrets)) from None + except SftpError: + transport.close() + raise + + +@contextlib.contextmanager +def session(cfg: SftpConfig, timeout: float = 20) -> Iterator[SftpSession]: + s = connect(cfg, timeout=timeout) + try: + yield s + finally: + s.close() + + +def check_connection(cfg: SftpConfig, timeout: float = 20) -> CheckReport: + """Verbinden, Basisordner lesen, Testdatei schreiben/umbenennen/löschen.""" + report = CheckReport() + try: + sess = connect(cfg, timeout=timeout, allow_unknown_host_key=True) + except SftpError as exc: + report.step("connect", False, str(exc)) + report.step("read", None) + report.step("write", None) + return report + with sess: + report.host_key = sess.host_key + report.host_key_fingerprint = sess.fingerprint + new = " (neu – bitte Fingerabdruck prüfen)" if cfg.host_key is None else "" + report.step("connect", True, f"Hostschlüssel {sess.fingerprint}{new}") + + try: + attrs = sess.client.stat(cfg.base_path) + if attrs.st_mode is None or not stat.S_ISDIR(attrs.st_mode): + raise SftpError(f"{cfg.base_path} ist kein Verzeichnis") + count = len(sess.client.listdir(cfg.base_path)) + report.can_read = True + report.step("read", True, f"{count} Einträge") + except (OSError, SftpError) as exc: + report.step("read", False, f"{cfg.base_path}: {exc}") + report.step("write", None) + return report + + token = uuid.uuid4().hex + part = posixpath.join(cfg.base_path, f".netmover-test-{token}.part") + final = posixpath.join(cfg.base_path, f".netmover-test-{token}.tmp") + try: + with sess.client.open(part, "wb") as fh: + fh.write(b"netmover") + fh.flush() + synced = sess.fsync(fh) + sess.client.posix_rename(part, final) + st = sess.client.stat(final) + report.owner = f"{st.st_uid}:{st.st_gid}" + report.can_write = True + except OSError as exc: + report.step("write", False, f"Schreiben fehlgeschlagen: {exc}") + with contextlib.suppress(OSError): + sess.client.remove(part) + return report + try: + sess.client.remove(final) + report.can_delete = True + report.step( + "write", + True, + "Datei angelegt, atomar umbenannt und gelöscht" + + ("" if synced else " (Server unterstützt kein fsync – Hinweis)"), + ) + except OSError as exc: + report.step("write", True, f"Angelegt, aber Löschen fehlgeschlagen: {exc}") + usage = sess.statvfs(cfg.base_path) + if usage: + report.total_bytes, report.free_bytes = usage + else: + report.hints.append( + "Server meldet keinen freien Speicher (statvfs) – Platzprüfung wird übersprungen." + ) + return report diff --git a/tests/conftest.py b/tests/conftest.py index 309ac32..b44a359 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,17 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path import pytest -from sqlalchemy.orm import Session +from cryptography.fernet import Fernet +from sqlalchemy.orm import Session, sessionmaker -from netmover.db import make_engine, make_session_factory, upgrade +from netmover.crypto import SecretBox +from netmover.db import make_engine, make_session_factory, session_scope, upgrade +from netmover.models import Credential, CredentialKind, Mount, MountType +from netmover.mounting.fake import FakeMountBackend +from netmover.mounting.manager import MountManager @pytest.fixture @@ -17,9 +22,105 @@ def db_url(tmp_path: Path) -> str: @pytest.fixture -def session(db_url: str) -> Iterator[Session]: +def session_factory(db_url: str) -> Iterator[sessionmaker[Session]]: engine = make_engine(db_url) - factory = make_session_factory(engine) - with factory() as s: - yield s + yield make_session_factory(engine) engine.dispose() + + +@pytest.fixture +def session(session_factory: sessionmaker[Session]) -> Iterator[Session]: + with session_factory() as s: + yield s + + +@pytest.fixture +def secretbox() -> SecretBox: + return SecretBox(Fernet.generate_key()) + + +@pytest.fixture +def fake_backend(tmp_path: Path) -> Iterator[FakeMountBackend]: + backend = FakeMountBackend(tmp_path / "fake") + yield backend + backend.release_hangs() + + +@pytest.fixture +def mount_base(tmp_path: Path) -> Path: + return tmp_path / "mnt" + + +@pytest.fixture +def sleeps() -> list[float]: + return [] + + +@pytest.fixture +def notifications() -> list[tuple[str, str, str]]: + return [] + + +@pytest.fixture +def manager( + session_factory: sessionmaker[Session], + fake_backend: FakeMountBackend, + secretbox: SecretBox, + mount_base: Path, + sleeps: list[float], + notifications: list[tuple[str, str, str]], +) -> MountManager: + return MountManager( + session_factory, + fake_backend, + secretbox, + mount_base, + probe_timeout=0.5, + recover_backoff=(0, 1, 2), + sleep=sleeps.append, + notifier=lambda *a: notifications.append(a), + ) + + +MountFactory = Callable[..., int] + + +@pytest.fixture +def make_mount( + session_factory: sessionmaker[Session], secretbox: SecretBox, mount_base: Path +) -> MountFactory: + def factory( + name: str = "scans", + type: MountType = MountType.CIFS, + server: str = "nas", + share: str = "scans", + password: str | None = "geheim123", + **kw: object, + ) -> int: + with session_scope(session_factory) as s: + cred_id = None + if password is not None: + c = Credential( + name=f"cred-{name}", + kind=CredentialKind.PASSWORD, + username="scanner", + secret_enc=secretbox.encrypt(password), + ) + s.add(c) + s.flush() + cred_id = c.id + m = Mount( + name=name, + slug=name, + type=type, + server=server, + share=share, + mountpoint=None if type is MountType.SFTP else str(mount_base.resolve() / name), + cred_id=cred_id, + **kw, + ) + s.add(m) + s.flush() + return m.id + + return factory diff --git a/tests/sftp_stub.py b/tests/sftp_stub.py new file mode 100644 index 0000000..8385f63 --- /dev/null +++ b/tests/sftp_stub.py @@ -0,0 +1,202 @@ +"""Minimaler SFTP-Server für Tests (paramiko-Serverseite, auf ein Verzeichnis beschränkt). + +Unterstützt ``posix-rename@openssh.com`` (paramiko leitet es an ``posix_rename`` +weiter), aber bewusst nicht ``statvfs@openssh.com`` – damit ist auch der +Rückfallweg ohne Platzprüfung abgedeckt. +""" + +from __future__ import annotations + +import contextlib +import io +import os +import socket +import threading +from pathlib import Path +from typing import Any + +import paramiko +from paramiko import SFTPAttributes, SFTPHandle, SFTPServer, SFTPServerInterface +from paramiko.sftp import SFTP_FAILURE, SFTP_OK + +from netmover.crypto import generate_ssh_keypair + + +class _Server(paramiko.ServerInterface): + def __init__(self, username: str, password: str | None, pubkey: paramiko.PKey | None) -> None: + self.username = username + self.password = password + self.pubkey = pubkey + + def check_auth_password(self, username: str, password: str) -> int: + if username == self.username and self.password is not None and password == self.password: + return paramiko.AUTH_SUCCESSFUL + return paramiko.AUTH_FAILED + + def check_auth_publickey(self, username: str, key: paramiko.PKey) -> int: + if username == self.username and self.pubkey is not None and key == self.pubkey: + return paramiko.AUTH_SUCCESSFUL + return paramiko.AUTH_FAILED + + def get_allowed_auths(self, username: str) -> str: + return "password,publickey" + + def check_channel_request(self, kind: str, chanid: int) -> int: + if kind == "session": + return paramiko.OPEN_SUCCEEDED + return paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + +class _Handle(SFTPHandle): + pass + + +class _RootedSFTP(SFTPServerInterface): + def __init__(self, server: Any, *args: Any, root: Path, **kwargs: Any) -> None: + super().__init__(server, *args, **kwargs) + self.root = root + + def _real(self, path: str) -> str: + return str(self.root / self.canonicalize(path).lstrip("/")) + + def list_folder(self, path: str) -> Any: + try: + real = self._real(path) + out = [] + for name in os.listdir(real): + attr = SFTPAttributes.from_stat(os.stat(os.path.join(real, name))) + attr.filename = name + out.append(attr) + return out + except OSError as e: + return SFTPServer.convert_errno(e.errno) + + def stat(self, path: str) -> Any: + try: + return SFTPAttributes.from_stat(os.stat(self._real(path))) + except OSError as e: + return SFTPServer.convert_errno(e.errno) + + lstat = stat + + def open(self, path: str, flags: int, attr: SFTPAttributes) -> Any: + real = self._real(path) + try: + fd = os.open(real, flags | getattr(os, "O_BINARY", 0), 0o644) + except OSError as e: + return SFTPServer.convert_errno(e.errno) + if flags & os.O_WRONLY: + mode = "ab" if flags & os.O_APPEND else "wb" + elif flags & os.O_RDWR: + mode = "a+b" if flags & os.O_APPEND else "r+b" + else: + mode = "rb" + f = os.fdopen(fd, mode) + h = _Handle(flags) + h.filename = real + h.readfile = f + h.writefile = f + return h + + def remove(self, path: str) -> int: + try: + os.remove(self._real(path)) + except OSError as e: + return SFTPServer.convert_errno(e.errno) + return SFTP_OK + + def rename(self, oldpath: str, newpath: str) -> int: + new = self._real(newpath) + if os.path.exists(new): + return SFTP_FAILURE # SFTPv3-rename überschreibt nicht + try: + os.rename(self._real(oldpath), new) + except OSError as e: + return SFTPServer.convert_errno(e.errno) + return SFTP_OK + + def posix_rename(self, oldpath: str, newpath: str) -> int: + try: + os.replace(self._real(oldpath), self._real(newpath)) + except OSError as e: + return SFTPServer.convert_errno(e.errno) + return SFTP_OK + + def mkdir(self, path: str, attr: SFTPAttributes) -> int: + try: + os.mkdir(self._real(path)) + except OSError as e: + return SFTPServer.convert_errno(e.errno) + return SFTP_OK + + def rmdir(self, path: str) -> int: + try: + os.rmdir(self._real(path)) + except OSError as e: + return SFTPServer.convert_errno(e.errno) + return SFTP_OK + + def chattr(self, path: str, attr: SFTPAttributes) -> int: + try: + if attr.st_mode is not None: + os.chmod(self._real(path), attr.st_mode & 0o7777) + if attr.st_atime is not None and attr.st_mtime is not None: + os.utime(self._real(path), (attr.st_atime, attr.st_mtime)) + except OSError as e: + return SFTPServer.convert_errno(e.errno) + return SFTP_OK + + +def _load_key(pem: str) -> paramiko.PKey: + return paramiko.Ed25519Key.from_private_key(io.StringIO(pem)) + + +class StubSFTPServer: + def __init__( + self, root: Path, username: str = "paperless", password: str | None = None + ) -> None: + self.root = root + self.username = username + self.password = password + host_pem, _ = generate_ssh_keypair("host") + self.host_key = _load_key(host_pem) + self.client_private_key, self.client_public_key = generate_ssh_keypair("netmover") + self.authorized = _load_key(self.client_private_key) + self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._sock.bind(("127.0.0.1", 0)) + self._sock.listen(8) + self.port: int = self._sock.getsockname()[1] + self._stop = threading.Event() + self._transports: list[paramiko.Transport] = [] + self._thread = threading.Thread(target=self._serve, daemon=True) + + @property + def host_key_string(self) -> str: + return f"{self.host_key.get_name()} {self.host_key.get_base64()}" + + def start(self) -> StubSFTPServer: + self._thread.start() + return self + + def _serve(self) -> None: + self._sock.settimeout(0.2) + while not self._stop.is_set(): + try: + conn, _ = self._sock.accept() + except (TimeoutError, OSError): + continue + t = paramiko.Transport(conn) + t.add_server_key(self.host_key) + t.set_subsystem_handler("sftp", SFTPServer, _RootedSFTP, root=self.root) + self._transports.append(t) + with contextlib.suppress(Exception): + t.start_server(server=_Server(self.username, self.password, self.authorized)) + + def stop(self) -> None: + self._stop.set() + for t in self._transports: + with contextlib.suppress(Exception): + t.close() + self._sock.close() + self._thread.join(timeout=2) diff --git a/tests/test_linux_backend.py b/tests/test_linux_backend.py new file mode 100644 index 0000000..a31e2b9 --- /dev/null +++ b/tests/test_linux_backend.py @@ -0,0 +1,117 @@ +"""Fallstricke 5 und 7 am echten Backend – ``subprocess`` wird nur abgefangen.""" + +from __future__ import annotations + +import stat +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from netmover.mounting.backend import CifsCredentials, MountError, MountSpec +from netmover.mounting.linux import LinuxMountBackend, explain_mount_error + + +class Recorder: + def __init__(self, returncode: int = 0, stderr: str = "") -> None: + self.returncode = returncode + self.stderr = stderr + self.calls: list[tuple[list[str], dict[str, Any]]] = [] + self.cred_snapshot: dict[str, Any] = {} + + def __call__(self, argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + self.calls.append((argv, kwargs)) + for opt in ",".join(argv).split(","): + if opt.startswith("credentials="): + p = Path(opt.removeprefix("credentials=")) + self.cred_snapshot = { + "path": p, + "mode": stat.S_IMODE(p.stat().st_mode), + "content": p.read_text(), + } + return subprocess.CompletedProcess(argv, self.returncode, "", self.stderr) + + +def _spec(tmp_path: Path, password: str = "S3cr3t,pw=x") -> MountSpec: + return MountSpec( + fstype="cifs", + source="//nas/scans", + mountpoint=tmp_path / "mnt" / "scans", + options=("vers=3.0", "uid=1000"), + credentials=CifsCredentials("scanner", password, "WORKGROUP"), + ) + + +def test_mount_uses_argv_and_credentials_file(tmp_path: Path) -> None: + rec = Recorder() + backend = LinuxMountBackend(tmp_path / "run", runner=rec) + backend.mount(_spec(tmp_path)) + + argv, kwargs = rec.calls[0] + assert "shell" not in kwargs + assert argv[:4] == ["mount", "-t", "cifs", "-o"] + assert argv[-2:] == ["//nas/scans", str(tmp_path / "mnt" / "scans")] + assert argv[4].startswith("vers=3.0,uid=1000,credentials=") + # Passwort nie in der Prozessliste + assert not any("S3cr3t" in a for a in argv) + assert "password" not in argv[4] + # Credentials-Datei: 0600, Inhalt korrekt, danach gelöscht + snap = rec.cred_snapshot + assert snap["mode"] == 0o600 + assert snap["content"] == "username=scanner\npassword=S3cr3t,pw=x\ndomain=WORKGROUP\n" + assert snap["path"].parent == tmp_path / "run" + assert not snap["path"].exists() + assert kwargs["env"]["LC_ALL"] == "C" + + +def test_credentials_file_removed_on_failure_and_output_redacted(tmp_path: Path) -> None: + rec = Recorder(returncode=32, stderr="mount error(13): Permission denied for S3cr3t,pw=x") + backend = LinuxMountBackend(tmp_path / "run", runner=rec) + with pytest.raises(MountError) as exc: + backend.mount(_spec(tmp_path)) + assert "S3cr3t" not in str(exc.value) + assert "Benutzername/Passwort" in str(exc.value) + assert list((tmp_path / "run").iterdir()) == [] + + +def test_mount_timeout(tmp_path: Path) -> None: + def runner(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + raise subprocess.TimeoutExpired(argv, kwargs["timeout"]) + + backend = LinuxMountBackend(tmp_path / "run", runner=runner) + with pytest.raises(MountError, match="keine Antwort nach 5 s"): + backend.mount(_spec(tmp_path), timeout=5) + assert list((tmp_path / "run").iterdir()) == [] + + +def test_unmount_lazy_argv(tmp_path: Path) -> None: + rec = Recorder() + backend = LinuxMountBackend(tmp_path / "run", runner=rec) + backend.unmount(Path("/mnt/netmover/scans"), lazy=True) + assert rec.calls[0][0] == ["umount", "-l", "/mnt/netmover/scans"] + + +def test_is_mounted_reads_mountinfo(tmp_path: Path) -> None: + info = tmp_path / "mountinfo" + info.write_text("98 22 0:52 / /mnt/netmover/scans rw - cifs //nas/scans rw\n") + backend = LinuxMountBackend(tmp_path / "run", mountinfo_path=info, runner=Recorder()) + assert backend.is_mounted(Path("/mnt/netmover/scans")) + assert not backend.is_mounted(Path("/mnt/netmover/other")) + + +@pytest.mark.parametrize( + ("fstype", "msg", "expect"), + [ + ( + "cifs", + "mount: wrong fs type, bad option; you might need a /sbin/mount. helper program", + "cifs-utils", + ), + ("nfs4", "mount.nfs4: access denied by server while mounting nas:/x", "NFS-Berechtigungen"), + ("cifs", "mount error(112): Host is down", "nicht erreichbar"), + ("cifs", "mount: only root can do that", "CAP_SYS_ADMIN"), + ], +) +def test_explain(fstype: str, msg: str, expect: str) -> None: + assert expect in explain_mount_error(fstype, msg) diff --git a/tests/test_manager.py b/tests/test_manager.py new file mode 100644 index 0000000..d15be07 --- /dev/null +++ b/tests/test_manager.py @@ -0,0 +1,369 @@ +"""Mount-Verwaltung mit FakeMountBackend – Fallstricke 2, 6, 7 und 13.""" + +from __future__ import annotations + +import errno +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from netmover.models import CredentialKind, Mount, MountState, MountType +from netmover.mounting.backend import MountError +from netmover.mounting.fake import FakeMountBackend +from netmover.mounting.manager import ( + CredentialData, + MountConfig, + MountManager, + MountUnavailable, + build_spec, +) +from netmover.validation import ValidationError +from tests.conftest import MountFactory + + +def _state(sf: sessionmaker[Session], mount_id: int) -> Mount: + with sf() as s: + m = s.get(Mount, mount_id) + assert m is not None + s.expunge(m) + return m + + +# -- build_spec ------------------------------------------------------------------- + + +def test_build_spec_cifs(tmp_path: Path) -> None: + cfg = MountConfig( + "scans", + MountType.CIFS, + "nas", + "scans", + uid=1000, + gid=100, + file_mode="0644", + extra_options="vers=3.0", + ) + cred = CredentialData(CredentialKind.PASSWORD, "scanner", "pw", "WG") + spec = build_spec(cfg, tmp_path, cred) + assert spec.fstype == "cifs" + assert spec.source == "//nas/scans" + assert spec.options == ("vers=3.0", "uid=1000", "gid=100", "file_mode=0644") + assert spec.credentials is not None and spec.credentials.password == "pw" + assert "pw" not in repr(spec) + + +def test_build_spec_nfs() -> None: + nfs3 = build_spec(MountConfig("a", MountType.NFS3, "nas", "/volume1/a"), Path("/x"), None) + assert (nfs3.fstype, nfs3.source, nfs3.options) == ("nfs", "nas:/volume1/a", ("vers=3",)) + nfs4 = build_spec(MountConfig("a", MountType.NFS4, "nas", "/volume1/a"), Path("/x"), None) + assert (nfs4.fstype, nfs4.options) == ("nfs4", ()) + + +def test_build_spec_guest_and_revalidation() -> None: + spec = build_spec(MountConfig("a", MountType.CIFS, "nas", "pub"), Path("/x"), None) + assert spec.options == ("guest",) + # Auch Werte „aus der Datenbank“ laufen erneut durch die Validierung + with pytest.raises(ValidationError): + build_spec(MountConfig("a", MountType.CIFS, "nas;id", "pub"), Path("/x"), None) + with pytest.raises(ValidationError): + build_spec( + MountConfig("a", MountType.CIFS, "nas", "pub", extra_options="password=x"), + Path("/x"), + None, + ) + + +# -- Einhängen / Aushängen (Fallstrick 6) -------------------------------------------- + + +def test_mount_creates_mountpoint_under_base_and_unmount_removes_it( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + mount_base: Path, + session_factory: sessionmaker[Session], +) -> None: + remote = fake_backend.export("nas", "scans") + (remote / "a.pdf").write_text("x") + mid = make_mount() + manager.mount(mid) + mp = mount_base.resolve() / "scans" + assert fake_backend.is_mounted(mp) + assert (mp / "a.pdf").read_text() == "x" + m = _state(session_factory, mid) + assert m.state is MountState.MOUNTED and m.last_mounted_at is not None + # Zugangsdaten wurden entschlüsselt übergeben + spec = fake_backend.calls[0].spec + assert spec is not None and spec.credentials is not None + assert spec.credentials.password == "geheim123" + + manager.unmount(mid) + assert not mp.exists() + assert _state(session_factory, mid).state is MountState.UNMOUNTED + + +def test_mount_refuses_non_empty_mountpoint( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + mount_base: Path, + session_factory: sessionmaker[Session], +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + mp = mount_base.resolve() / "scans" + mp.mkdir(parents=True) + (mp / "verirrt.pdf").write_text("lokal geschrieben") + with pytest.raises(MountError, match="nicht leer"): + manager.mount(mid) + assert not fake_backend.calls + assert (mp / "verirrt.pdf").exists() + assert _state(session_factory, mid).state is MountState.ERROR + + +def test_failed_mount_cleans_up_and_records_error( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + mount_base: Path, + session_factory: sessionmaker[Session], +) -> None: + mid = make_mount() # nicht exportiert → „No such file or directory“ + with pytest.raises(MountError): + manager.mount(mid) + assert not (mount_base.resolve() / "scans").exists() + m = _state(session_factory, mid) + assert m.state is MountState.ERROR + assert m.last_error and "No such file" in m.last_error + + +def test_mountpoint_for_rejects_escape(manager: MountManager) -> None: + for bad in ["../etc", "a/b", "", "Scans"]: + with pytest.raises(ValidationError): + manager.mountpoint_for(bad) + + +def test_sftp_is_never_mounted(manager: MountManager, make_mount: MountFactory) -> None: + mid = make_mount("paperless", type=MountType.SFTP, share="/consume") + with pytest.raises(MountError, match="SFTP"): + manager.mount(mid) + + +def test_restore_all_only_auto_mount( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + notifications: list[tuple[str, str, str]], +) -> None: + fake_backend.export("nas", "a") + fake_backend.export("nas", "b") + a = make_mount("a", share="a") + make_mount("b", share="b", auto_mount=False) + make_mount("c", share="fehlt") + make_mount("d", share="a", enabled=False) + results = {r.name: r.ok for r in manager.restore_all()} + assert results == {"a": True, "c": False} + assert fake_backend.is_mounted(manager.mountpoint_for("a")) + assert notifications and notifications[0][:2] == ("mount.restore_failed", "c") + # zweiter Aufruf ist idempotent + manager.mount(a) + assert ( + sum(1 for c in fake_backend.calls if c.action == "mount" and c.mountpoint.name == "a") == 1 + ) + + +# -- Test (drei Schritte) ----------------------------------------------------------- + + +def test_check_unsaved_config_mounts_temporarily( + manager: MountManager, fake_backend: FakeMountBackend, mount_base: Path +) -> None: + fake_backend.export("nas", "scans") + cfg = MountConfig("neu", MountType.CIFS, "nas", "scans") + cred = CredentialData(CredentialKind.PASSWORD, "u", "pw") + report = manager.check(cfg, cred) + assert [(s.name, s.ok) for s in report.steps] == [ + ("mount", True), + ("read", True), + ("write", True), + ] + assert report.can_write and report.can_delete + assert report.free_bytes is not None + assert [r.value for r in report.roles] == ["Quelle mit Löschen", "Ziel"] + # temporärer Mountpoint wieder weg, Testdatei gelöscht + assert list(mount_base.iterdir()) == [] + assert list(fake_backend.remote_root.joinpath("nas", "scans").iterdir()) == [] + assert not fake_backend.mounted + + +def test_check_mount_failure_reports_steps(manager: MountManager, mount_base: Path) -> None: + report = manager.check(MountConfig("x", MountType.CIFS, "nas", "fehlt"), None) + assert [(s.name, s.ok) for s in report.steps] == [ + ("mount", False), + ("read", None), + ("write", None), + ] + assert not report.ok + assert list(mount_base.iterdir()) == [] + + +def test_check_read_only_share_is_source_copy_only( + manager: MountManager, fake_backend: FakeMountBackend +) -> None: + remote = fake_backend.export("nas", "ro") + remote.chmod(0o555) + try: + report = manager.check(MountConfig("ro", MountType.CIFS, "nas", "ro"), None) + finally: + remote.chmod(0o755) + assert report.can_read and not report.can_write + assert [r.value for r in report.roles] == ["nur Quelle, kopieren"] + assert report.steps[2].ok is False + + +def test_check_stores_result_on_saved_mount( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + session_factory: sessionmaker[Session], +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + manager.mount(mid) + m = _state(session_factory, mid) + report = manager.check(MountConfig.from_model(m), None, mount_id=mid) + assert report.steps[0].message == "bereits eingehängt" + m = _state(session_factory, mid) + assert m.write_ok is True and m.last_test_result and '"ok": true' in m.last_test_result + assert fake_backend.is_mounted(manager.mountpoint_for("scans")) # bleibt eingehängt + + +# -- Fallstrick 2: nicht eingehängt --------------------------------------------------- + + +def test_check_alive_refuses_unmounted_mountpoint( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + session_factory: sessionmaker[Session], +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + manager.mount(mid) + mp = manager.check_alive(mid) + assert mp is not None + fake_backend.vanish(mp) + assert mp.is_dir() and list(mp.iterdir()) == [] # sieht aus wie eine leere Freigabe + with pytest.raises(MountUnavailable, match="nicht eingehängt"): + manager.check_alive(mid) + m = _state(session_factory, mid) + assert m.state is MountState.ERROR + # kein Wiederherstellungsversuch, kein Schreiben + assert [c.action for c in fake_backend.calls] == ["mount"] + assert list(mp.iterdir()) == [] + + +# -- Fallstrick 13: gestörte und hängende Mounts --------------------------------------- + + +@pytest.mark.parametrize("code", [errno.ESTALE, errno.EIO, errno.ENOTCONN]) +def test_stale_mount_is_lazily_unmounted_and_remounted( + code: int, + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + session_factory: sessionmaker[Session], +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + manager.mount(mid) + mp = manager.mountpoint_for("scans") + fake_backend.inject_fault(mp, code) + assert manager.check_alive(mid) == mp + actions = [(c.action, c.lazy) for c in fake_backend.calls] + assert actions == [("mount", False), ("unmount", True), ("mount", False)] + m = _state(session_factory, mid) + assert m.state is MountState.MOUNTED and m.last_error is None + + +def test_recovery_gives_up_after_three_attempts_with_growing_delay( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + session_factory: sessionmaker[Session], + sleeps: list[float], + notifications: list[tuple[str, str, str]], +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + manager.mount(mid) + mp = manager.mountpoint_for("scans") + fake_backend.inject_fault(mp, errno.ESTALE) + fake_backend.fail_next_mount(times=3) + with pytest.raises(MountUnavailable, match="3 Versuchen"): + manager.check_alive(mid) + assert sleeps == [1, 2] # 1. Versuch sofort, dann wachsender Abstand + assert sum(1 for c in fake_backend.calls if c.action == "mount") == 4 + m = _state(session_factory, mid) + assert m.state is MountState.ERROR + assert notifications[-1][0] == "mount.error" + + +def test_hanging_mount_times_out_and_is_recovered( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + session_factory: sessionmaker[Session], +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + manager.mount(mid) + mp = manager.mountpoint_for("scans") + fake_backend.inject_fault(mp, "hang") + assert manager.check_alive(mid) == mp # blockiert nicht, Zeitlimit 0.5 s + assert ("unmount", True) in [(c.action, c.lazy) for c in fake_backend.calls] + assert _state(session_factory, mid).state is MountState.MOUNTED + + +def test_hanging_mount_without_recovery_shows_since( + manager: MountManager, + fake_backend: FakeMountBackend, + make_mount: MountFactory, + session_factory: sessionmaker[Session], +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + manager.mount(mid) + mp = manager.mountpoint_for("scans") + fake_backend.heal_on_remount = False + fake_backend.inject_fault(mp, "hang") + with pytest.raises(MountUnavailable): + manager.check_alive(mid) + m = _state(session_factory, mid) + assert m.state is MountState.ERROR + assert m.state_since is not None + assert m.last_error and "keine Antwort" in m.last_error + + +def test_non_recoverable_error_does_not_remount( + manager: MountManager, fake_backend: FakeMountBackend, make_mount: MountFactory +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + manager.mount(mid) + fake_backend.inject_fault(manager.mountpoint_for("scans"), errno.EACCES) + with pytest.raises(MountUnavailable): + manager.check_alive(mid) + assert [c.action for c in fake_backend.calls] == ["mount"] + + +def test_usage( + manager: MountManager, fake_backend: FakeMountBackend, make_mount: MountFactory +) -> None: + fake_backend.export("nas", "scans") + mid = make_mount() + assert manager.usage(mid) is None + manager.mount(mid) + usage = manager.usage(mid) + assert usage is not None and usage[0] >= usage[1] > 0 diff --git a/tests/test_mountinfo.py b/tests/test_mountinfo.py new file mode 100644 index 0000000..c5a9602 --- /dev/null +++ b/tests/test_mountinfo.py @@ -0,0 +1,40 @@ +"""Fallstrick 2: „eingehängt?“ entscheidet /proc/self/mountinfo.""" + +from __future__ import annotations + +from pathlib import Path + +from netmover.mounting.mountinfo import find_mount, parse_mountinfo + +SAMPLE = """\ +22 1 8:1 / / rw,relatime shared:1 - ext4 /dev/sda1 rw +98 22 0:52 / /mnt/netmover/scans rw,relatime shared:50 - cifs //nas/scans rw,vers=3.1.1 +99 22 0:53 / /mnt/netmover/mit\\040leer rw,relatime - nfs4 nas:/volume1/a\\040b rw +100 22 0:54 / /mnt/netmover/scans rw,relatime - cifs //nas2/scans rw +""" + + +def test_parse_with_octal_escapes() -> None: + entries = parse_mountinfo(SAMPLE) + assert len(entries) == 4 + assert entries[2].mountpoint == "/mnt/netmover/mit leer" + assert entries[2].source == "nas:/volume1/a b" + assert entries[2].fstype == "nfs4" + + +def test_find_mount_uses_topmost_entry() -> None: + entries = parse_mountinfo(SAMPLE) + found = find_mount(Path("/mnt/netmover/scans/"), entries) + assert found is not None + assert found.source == "//nas2/scans" + + +def test_unmounted_directory_is_not_mounted() -> None: + entries = parse_mountinfo(SAMPLE) + assert find_mount(Path("/mnt/netmover/archiv"), entries) is None + # Unterverzeichnis eines Mounts ist selbst kein Mountpoint + assert find_mount(Path("/mnt/netmover/scans/eingang"), entries) is None + + +def test_garbage_lines_are_ignored() -> None: + assert parse_mountinfo("kaputt\n\n1 2 3\n") == [] diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..512857a --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,61 @@ +"""Fallstrick 12: Schreibtest erkennt Root-Squash und leitet Rollen ab.""" + +from __future__ import annotations + +import errno + +from netmover.models import MountType +from netmover.mounting.report import CheckReport, Role, assess_write + + +def _err(code: int) -> OSError: + return OSError(code, "x") + + +def test_nfs_root_write_denied_is_root_squash_with_synology_hint() -> None: + r = assess_write(MountType.NFS4, 0, _err(errno.EACCES), None, None) + assert not r.write_ok + assert r.root_squash is True + assert any("Synology" in h and "Squash" in h for h in r.hints) + + +def test_nfs_file_owned_by_nobody_is_root_squash() -> None: + r = assess_write(MountType.NFS3, 0, None, 65534, None) + assert r.write_ok and r.delete_ok + assert r.root_squash is True + assert "65534" in r.message + assert any("chown" in h for h in r.hints) + + +def test_nfs_no_root_squash() -> None: + r = assess_write(MountType.NFS4, 0, None, 0, None) + assert r.root_squash is False + assert r.hints == [] + + +def test_cifs_owner_never_counts_as_squash() -> None: + r = assess_write(MountType.CIFS, 0, None, 1000, None) + assert r.root_squash is None + + +def test_not_root_cannot_judge_squash() -> None: + assert assess_write(MountType.NFS4, 1000, None, 1000, None).root_squash is None + + +def test_read_only() -> None: + r = assess_write(MountType.CIFS, 0, _err(errno.EROFS), None, None) + assert not r.write_ok + assert "schreibgeschützt" in r.message + + +def test_delete_failed() -> None: + r = assess_write(MountType.CIFS, 0, None, 0, _err(errno.EACCES)) + assert r.write_ok and not r.delete_ok + + +def test_roles() -> None: + rep = CheckReport(can_read=True) + assert rep.roles == [Role.SOURCE_COPY] + rep.can_write = rep.can_delete = True + assert rep.roles == [Role.SOURCE_MOVE, Role.TARGET] + assert '"Quelle mit Löschen"' in rep.to_json() diff --git a/tests/test_sftp.py b/tests/test_sftp.py new file mode 100644 index 0000000..30c3e5e --- /dev/null +++ b/tests/test_sftp.py @@ -0,0 +1,154 @@ +"""SFTP-Ziel gegen einen echten (im Prozess laufenden) SFTP-Server.""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from netmover.crypto import SecretBox +from netmover.db import session_scope +from netmover.models import Credential, CredentialKind, Mount, MountState, MountType +from netmover.mounting.manager import MountManager, MountUnavailable +from netmover.mounting.sftp import ( + HostKeyMismatch, + HostKeyUnknown, + SftpConfig, + SftpError, + check_connection, + connect, +) +from tests.sftp_stub import StubSFTPServer + + +@pytest.fixture +def server(tmp_path: Path) -> Iterator[StubSFTPServer]: + root = tmp_path / "sftp-root" + (root / "consume").mkdir(parents=True) + srv = StubSFTPServer(root, password="pw").start() + yield srv + srv.stop() + + +def _cfg(server: StubSFTPServer, **kw: object) -> SftpConfig: + data: dict[str, object] = { + "host": "127.0.0.1", + "port": server.port, + "username": server.username, + "base_path": "/consume", + "private_key": server.client_private_key, + } + data.update(kw) + return SftpConfig(**data) # type: ignore[arg-type] + + +def test_check_connection_with_key_tofu(server: StubSFTPServer) -> None: + report = check_connection(_cfg(server)) + assert [(s.name, s.ok) for s in report.steps] == [ + ("connect", True), + ("read", True), + ("write", True), + ] + assert report.host_key == server.host_key_string + assert report.host_key_fingerprint and report.host_key_fingerprint.startswith("SHA256:") + assert "neu" in report.steps[0].message + assert report.can_write and report.can_delete + # Stub kennt kein statvfs → Hinweis statt Fehler + assert report.free_bytes is None + assert any("statvfs" in h for h in report.hints) + assert list((server.root / "consume").iterdir()) == [] + + +def test_password_auth(server: StubSFTPServer) -> None: + report = check_connection( + _cfg(server, private_key=None, password="pw", host_key=server.host_key_string) + ) + assert report.ok + + +def test_wrong_credentials(server: StubSFTPServer) -> None: + report = check_connection(_cfg(server, private_key=None, password="falsch")) + assert report.steps[0].ok is False + assert "authorized_keys" in report.steps[0].message + assert "falsch" not in report.steps[0].message + + +def test_unknown_host_key_refused_outside_test(server: StubSFTPServer) -> None: + with pytest.raises(HostKeyUnknown): + connect(_cfg(server)) + + +def test_host_key_mismatch(server: StubSFTPServer) -> None: + other = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOmvG9t0hJ7oC2Y3Fq6m0eKZ1x2y3z4A5B6C7D8E9F0G" + with pytest.raises(HostKeyMismatch, match="geändert"): + connect(_cfg(server, host_key=other)) + report = check_connection(_cfg(server, host_key=other)) + assert report.steps[0].ok is False + + +def test_missing_base_path(server: StubSFTPServer) -> None: + report = check_connection(_cfg(server, base_path="/fehlt")) + assert [(s.name, s.ok) for s in report.steps] == [ + ("connect", True), + ("read", False), + ("write", None), + ] + + +def test_unreachable() -> None: + with pytest.raises(SftpError, match="fehlgeschlagen"): + connect(SftpConfig("127.0.0.1", 1, "x", "/", password="x"), timeout=2) + + +def test_manager_pins_host_key_and_checks_alive( + server: StubSFTPServer, + manager: MountManager, + session_factory: sessionmaker[Session], + secretbox: SecretBox, +) -> None: + with session_scope(session_factory) as s: + c = Credential( + name="paperless-key", + kind=CredentialKind.SSH_KEY, + username=server.username, + secret_enc=secretbox.encrypt(server.client_private_key), + public_key=server.client_public_key, + ) + s.add(c) + s.flush() + m = Mount( + name="paperless", + slug="paperless", + type=MountType.SFTP, + server="127.0.0.1", + share="/consume", + port=server.port, + cred_id=c.id, + ) + s.add(m) + s.flush() + mid = m.id + + # vor dem ersten Test: Hostschlüssel unbekannt → kein Lauf möglich + with pytest.raises(MountUnavailable, match="Hostschlüssel"): + manager.check_alive(mid) + + from netmover.mounting.manager import CredentialData, MountConfig + + with session_factory() as s: + mm = s.get(Mount, mid) + assert mm is not None + cfg = MountConfig.from_model(mm) + cred = CredentialData(CredentialKind.SSH_KEY, server.username, server.client_private_key) + assert manager.check(cfg, cred, mount_id=mid).ok + + with session_factory() as s: + mm = s.get(Mount, mid) + assert mm is not None and mm.host_key == server.host_key_string + + assert manager.check_alive(mid) is None + with session_factory() as s: + mm = s.get(Mount, mid) + assert mm is not None and mm.state is MountState.MOUNTED