feat: Archiv je Regel – erst Archiv, dann Ziel, dann Quelle löschen

Zusätzliche Kopie ins Archiv (Anbindung oder lokaler Pfad, Ablage wie Quelle
oder nach Datum). Buchführung verhindert doppelte Archivkopien bei Neuversuch;
im Archiv wird nie überschrieben. Migration 0004 mit Tests auf befüllten
Datenbanken (inkl. Downgrade).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Florian Wilde
2026-10-12 00:22:39 +02:00
co-authored by Claude Opus 5.5
parent cf7a3bcc88
commit e59932ab75
6 changed files with 701 additions and 34 deletions
@@ -0,0 +1,103 @@
"""archive
Revision ID: 0004
Revises: 0003
Create Date: 2026-10-12
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
import netmover.models
revision: str = "0004"
down_revision: str | None = "0003"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"archived_files",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("rule_id", sa.Integer(), nullable=False),
sa.Column("src_relpath", sa.String(length=2048), nullable=False),
sa.Column("size", sa.BigInteger(), nullable=False),
sa.Column("mtime_ns", sa.BigInteger(), nullable=False),
sa.Column("archive_path", sa.String(length=2048), nullable=False),
sa.Column("archived_at", netmover.models.UTCDateTime(), nullable=False),
sa.ForeignKeyConstraint(
["rule_id"],
["rules.id"],
name=op.f("fk_archived_files_rule_id_rules"),
ondelete="CASCADE",
),
sa.PrimaryKeyConstraint("id", name=op.f("pk_archived_files")),
sa.UniqueConstraint(
"rule_id",
"src_relpath",
"size",
"mtime_ns",
name=op.f("uq_archived_files_rule_id_src_relpath_size_mtime_ns"),
),
)
with op.batch_alter_table("rules", schema=None) as batch_op:
batch_op.add_column(
sa.Column(
"archive_kind",
sa.Enum(
"none",
"mount",
"local",
name="archive_kind",
native_enum=False,
create_constraint=True,
length=24,
),
server_default="none",
nullable=False,
)
)
batch_op.add_column(sa.Column("archive_mount_id", sa.Integer(), nullable=True))
batch_op.add_column(
sa.Column("archive_path", sa.String(length=1024), server_default="", nullable=False)
)
batch_op.add_column(
sa.Column(
"archive_layout",
sa.Enum(
"flat",
"date",
name="archive_layout",
native_enum=False,
create_constraint=True,
length=24,
),
server_default="flat",
nullable=False,
)
)
batch_op.create_foreign_key(
batch_op.f("fk_rules_archive_mount_id_mounts"),
"mounts",
["archive_mount_id"],
["id"],
ondelete="RESTRICT",
)
def downgrade() -> None:
with op.batch_alter_table("rules", schema=None) as batch_op:
batch_op.drop_constraint(batch_op.f("fk_rules_archive_mount_id_mounts"), type_="foreignkey")
# CHECK-Regeln der Auswahlfelder zuerst entfernen – sonst verweisen sie beim
# Neuaufbau der Tabelle auf Spalten, die es nicht mehr gibt.
batch_op.drop_constraint(batch_op.f("ck_rules_archive_kind"), type_="check")
batch_op.drop_constraint(batch_op.f("ck_rules_archive_layout"), type_="check")
batch_op.drop_column("archive_layout")
batch_op.drop_column("archive_path")
batch_op.drop_column("archive_mount_id")
batch_op.drop_column("archive_kind")
op.drop_table("archived_files")
+41
View File
@@ -10,6 +10,7 @@ Abweichungen bzw. Ergänzungen gegenüber dem Mindestumfang aus der Spezifikatio
* ``credentials.managed`` + ``mounts.username`` (Migration 0002) – * ``credentials.managed`` + ``mounts.username`` (Migration 0002) –
Schlüsselverwaltung: gespeicherte SSH-Schlüssel, die mehrere Anbindungen mit Schlüsselverwaltung: gespeicherte SSH-Schlüssel, die mehrere Anbindungen mit
jeweils eigenem Benutzernamen nutzen. jeweils eigenem Benutzernamen nutzen.
* ``rules.archive_*`` + Tabelle ``archived_files`` (Migration 0004) – Archivkopie.
* ``rules.waiting_since`` / ``rules.waiting_reason`` (Migration 0003) – Regel wartet * ``rules.waiting_since`` / ``rules.waiting_reason`` (Migration 0003) – Regel wartet
auf eine nicht erreichbare Anbindung (automatisch pausiert). auf eine nicht erreichbare Anbindung (automatisch pausiert).
* ``mounts.state_since`` – für „hängt seit …“ / „gestört seit …“ auf der Statusseite. * ``mounts.state_since`` – für „hängt seit …“ / „gestört seit …“ auf der Statusseite.
@@ -138,6 +139,17 @@ class CollisionPolicy(enum.StrEnum):
OVERWRITE = "overwrite" OVERWRITE = "overwrite"
class ArchiveKind(enum.StrEnum):
NONE = "none"
MOUNT = "mount"
LOCAL = "local"
class ArchiveLayout(enum.StrEnum):
FLAT = "flat" # wie in der Quelle
DATE = "date" # Jahr/Monat/…, nach Tag der Verarbeitung
class RunStatus(enum.StrEnum): class RunStatus(enum.StrEnum):
RUNNING = "running" RUNNING = "running"
OK = "ok" # keine Dateifehler OK = "ok" # keine Dateifehler
@@ -259,6 +271,18 @@ class Rule(Base):
quarantine_path: Mapped[str | None] = mapped_column(String(1024)) quarantine_path: Mapped[str | None] = mapped_column(String(1024))
quarantine_after: Mapped[int] = mapped_column(Integer, default=3) quarantine_after: Mapped[int] = mapped_column(Integer, default=3)
# Archiv (Migration 0004): zusätzliche Kopie; Quelle erst löschen, wenn Ziel UND Archiv da
archive_kind: Mapped[ArchiveKind] = mapped_column(
_enum(ArchiveKind, "archive_kind"), default=ArchiveKind.NONE, server_default="none"
)
archive_mount_id: Mapped[int | None] = mapped_column(
ForeignKey("mounts.id", ondelete="RESTRICT")
)
archive_path: Mapped[str] = mapped_column(String(1024), default="", server_default="")
archive_layout: Mapped[ArchiveLayout] = mapped_column(
_enum(ArchiveLayout, "archive_layout"), default=ArchiveLayout.FLAT, server_default="flat"
)
last_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime) last_run_at: Mapped[datetime | None] = mapped_column(UTCDateTime)
last_success_at: Mapped[datetime | None] = mapped_column(UTCDateTime) last_success_at: Mapped[datetime | None] = mapped_column(UTCDateTime)
consecutive_failures: Mapped[int] = mapped_column(Integer, default=0) consecutive_failures: Mapped[int] = mapped_column(Integer, default=0)
@@ -269,6 +293,7 @@ class Rule(Base):
src_mount: Mapped[Mount] = relationship(foreign_keys=[src_mount_id]) src_mount: Mapped[Mount] = relationship(foreign_keys=[src_mount_id])
dst_mount: Mapped[Mount | None] = relationship(foreign_keys=[dst_mount_id]) dst_mount: Mapped[Mount | None] = relationship(foreign_keys=[dst_mount_id])
archive_mount: Mapped[Mount | None] = relationship(foreign_keys=[archive_mount_id])
runs: Mapped[list[Run]] = relationship( runs: Mapped[list[Run]] = relationship(
back_populates="rule", cascade="all, delete-orphan", passive_deletes=True back_populates="rule", cascade="all, delete-orphan", passive_deletes=True
) )
@@ -330,6 +355,22 @@ class CopiedFile(Base):
copied_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) copied_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow)
class ArchivedFile(Base):
"""Welche Dateiversion schon im Archiv liegt – damit ein Neuversuch nach einem
gescheiterten Ziel keine zweite Archivkopie anlegt."""
__tablename__ = "archived_files"
__table_args__ = (UniqueConstraint("rule_id", "src_relpath", "size", "mtime_ns"),)
id: Mapped[int] = mapped_column(Integer, primary_key=True)
rule_id: Mapped[int] = mapped_column(ForeignKey("rules.id", ondelete="CASCADE"))
src_relpath: Mapped[str] = mapped_column(String(2048))
size: Mapped[int] = mapped_column(BigInteger)
mtime_ns: Mapped[int] = mapped_column(BigInteger)
archive_path: Mapped[str] = mapped_column(String(2048))
archived_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow)
class FileFailure(Base): class FileFailure(Base):
"""Fehlversuche pro Datei, Grundlage für die Quarantäne.""" """Fehlversuche pro Datei, Grundlage für die Quarantäne."""
+85 -9
View File
@@ -19,6 +19,7 @@ import posixpath
import time import time
from collections.abc import Callable from collections.abc import Callable
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -72,6 +73,7 @@ class RuleSnapshot:
set_mode: int | None = None set_mode: int | None = None
quarantine_after: int = 3 quarantine_after: int = 3
dst_root_squash: bool | None = None # nur NFS-Ziele: Ergebnis des Schreibtests dst_root_squash: bool | None = None # nur NFS-Ziele: Ergebnis des Schreibtests
archive_layout: str = "flat" # "flat" | "date" (Jahr/Monat/…)
@dataclass @dataclass
@@ -159,6 +161,7 @@ class Engine:
dry_run: bool = False, dry_run: bool = False,
quarantine_rel: str | None = None, quarantine_rel: str | None = None,
quarantine_root: Path | None = None, quarantine_root: Path | None = None,
archive: Destination | None = None,
) -> RunResult: ) -> RunResult:
"""``source``: eine :class:`Source` oder (Kurzform) ein lokales Verzeichnis. """``source``: eine :class:`Source` oder (Kurzform) ein lokales Verzeichnis.
Quarantäne relativ zur Wurzel der Anbindung (``quarantine_rel``) oder – bei Quarantäne relativ zur Wurzel der Anbindung (``quarantine_rel``) oder – bei
@@ -168,7 +171,7 @@ class Engine:
if quarantine_root is not None and isinstance(src, LocalSource): if quarantine_root is not None and isinstance(src, LocalSource):
quarantine_rel = str(quarantine_root.relative_to(src.root)) quarantine_rel = str(quarantine_root.relative_to(src.root))
try: try:
self._execute(rule, src, dest, token, result, dry_run, quarantine_rel) self._execute(rule, src, dest, token, result, dry_run, quarantine_rel, archive)
except Cancelled: except Cancelled:
result.status = RunStatus.ERROR result.status = RunStatus.ERROR
result.message = "Lauf abgebrochen" result.message = "Lauf abgebrochen"
@@ -195,10 +198,13 @@ class Engine:
result: RunResult, result: RunResult,
dry_run: bool, dry_run: bool,
quarantine_rel: str | None, quarantine_rel: str | None,
archive: Destination | None = None,
) -> None: ) -> None:
src.check_root() src.check_root()
if not dry_run: if not dry_run:
dest.prepare() dest.prepare()
if archive is not None:
archive.prepare()
token.check() token.check()
exclude = frozenset({quarantine_rel}) if quarantine_rel else frozenset() exclude = frozenset({quarantine_rel}) if quarantine_rel else frozenset()
@@ -212,8 +218,12 @@ class Engine:
# Kopier-Buchführung (Fallstrick 11). Gilt auch für „verschieben“, falls # Kopier-Buchführung (Fallstrick 11). Gilt auch für „verschieben“, falls
# dort einmal die Quelle nicht gelöscht werden konnte. # dort einmal die Quelle nicht gelöscht werden konnte.
done = self.ledger.copied_keys(rule.id) done = self.ledger.copied_keys(rule.id)
archived = self.ledger.archived_keys(rule.id) if archive is not None else set()
if not dry_run: if not dry_run:
self.ledger.prune_copied(rule.id, {c.relpath for c in candidates}) present = {c.relpath for c in candidates}
self.ledger.prune_copied(rule.id, present)
if archive is not None:
self.ledger.prune_archived(rule.id, present)
fresh = [c for c in candidates if (c.relpath, c.state.size, c.state.mtime_ns) not in done] fresh = [c for c in candidates if (c.relpath, c.state.size, c.state.mtime_ns) not in done]
unchanged = len(candidates) - len(fresh) unchanged = len(candidates) - len(fresh)
@@ -271,7 +281,11 @@ class Engine:
failures = self.ledger.failures(rule.id) failures = self.ledger.failures(rule.id)
for c in stable: for c in stable:
token.check() token.check()
result.add(self._process(rule, c, src, dest, token, dry_run, quarantine_rel, failures)) result.add(
self._process(
rule, c, src, dest, token, dry_run, quarantine_rel, failures, archive, archived
)
)
if unchanged: if unchanged:
result.message = f"{unchanged} bereits übertragen" result.message = f"{unchanged} bereits übertragen"
@@ -309,8 +323,14 @@ class Engine:
dry_run: bool, dry_run: bool,
quarantine_rel: str | None, quarantine_rel: str | None,
failures: dict[str, FailureInfo], failures: dict[str, FailureInfo],
archive: Destination | None = None,
archived: set[tuple[str, int, int]] | None = None,
) -> ItemResult: ) -> ItemResult:
size = c.state.size size = c.state.size
arch_rel = self._archive_rel(c.relpath, rule.archive_layout)
need_archive = archive is not None and (
(c.relpath, size, c.state.mtime_ns) not in (archived or set())
)
failure = failures.get(c.relpath) failure = failures.get(c.relpath)
# Quarantäne für wiederholt scheiternde Dateien # Quarantäne für wiederholt scheiternde Dateien
@@ -349,9 +369,14 @@ class Engine:
else None else None
) )
verb = "kopieren" if rule.action is RuleAction.COPY else "verschieben" verb = "kopieren" if rule.action is RuleAction.COPY else "verschieben"
return ItemResult( msg = msg or f"würde {verb}"
c.relpath, ItemStatus.PLANNED, dest.describe(planned), size, msg or f"würde {verb}" if archive is not None:
) if need_archive:
arch_planned = archive.plan(arch_rel, CollisionPolicy.RENAME)
msg += f"; Archiv: {archive.describe(arch_planned or arch_rel)}"
else:
msg += "; bereits archiviert"
return ItemResult(c.relpath, ItemStatus.PLANNED, dest.describe(planned), size, msg)
try: try:
free = dest.free_bytes() free = dest.free_bytes()
@@ -367,10 +392,51 @@ class Engine:
f"benötigt {_mb(int(size * SPACE_RESERVE))})", f"benötigt {_mb(int(size * SPACE_RESERVE))})",
) )
if need_archive and archive is not None:
try:
afree = archive.free_bytes()
except OSError:
afree = None
if afree is not None and afree < size * SPACE_RESERVE:
return ItemResult(
c.relpath,
ItemStatus.FAILED,
archive.describe(arch_rel),
size,
f"zu wenig Platz im Archiv (frei {_mb(afree)}, "
f"benötigt {_mb(int(size * SPACE_RESERVE))})",
)
def verify() -> None: def verify() -> None:
if src.state(c.relpath) != c.state: if src.state(c.relpath) != c.state:
raise SourceChanged("Quelle hat sich während der Übertragung geändert") raise SourceChanged("Quelle hat sich während der Übertragung geändert")
# Archiv ZUERST: scheitert es, ist noch nichts ans Ziel (z. B. Paperless)
# geliefert. Scheitert danach das Ziel, verhindert die Buchführung eine
# zweite Archivkopie beim nächsten Versuch.
archive_note: str | None = None
if need_archive and archive is not None:
try:
meta, _ = self._meta(rule, src, c.relpath, dest)
ares = archive.commit(
_SourceFile(src, c.relpath),
size,
arch_rel,
CollisionPolicy.RENAME,
MetaOptions(meta.atime_ns, meta.mtime_ns, meta.mode, None),
verify,
token,
)
except CollisionSkip: # kann bei RENAME nicht auftreten
return ItemResult(c.relpath, ItemStatus.SKIPPED, None, size, "Archiv belegt")
except SourceChanged as exc:
return ItemResult(c.relpath, ItemStatus.SKIPPED, None, size, str(exc))
except (TransferError, OSError) as exc:
return self._fail(rule, c, exc, dry_run, prefix="Archiv: ")
archive_path = archive.describe(ares.final_rel)
self.ledger.record_archive(rule.id, c.relpath, size, c.state.mtime_ns, archive_path)
archive_note = f"archiviert: {archive_path}"
try: try:
meta, notes = self._meta(rule, src, c.relpath, dest) meta, notes = self._meta(rule, src, c.relpath, dest)
res = None res = None
@@ -401,6 +467,8 @@ class Engine:
return self._fail(rule, c, exc, dry_run) return self._fail(rule, c, exc, dry_run)
notes += res.notes notes += res.notes
if archive_note:
notes.insert(0, archive_note)
target = dest.describe(res.final_rel) target = dest.describe(res.final_rel)
if c.relpath in failures: if c.relpath in failures:
self.ledger.clear_failure(rule.id, c.relpath) self.ledger.clear_failure(rule.id, c.relpath)
@@ -425,10 +493,12 @@ class Engine:
) )
return ItemResult(c.relpath, ItemStatus.MOVED, target, size, "; ".join(notes) or None) return ItemResult(c.relpath, ItemStatus.MOVED, target, size, "; ".join(notes) or None)
def _fail(self, rule: RuleSnapshot, c: Candidate, exc: Exception, dry_run: bool) -> ItemResult: def _fail(
self, rule: RuleSnapshot, c: Candidate, exc: Exception, dry_run: bool, prefix: str = ""
) -> ItemResult:
if isinstance(exc, OSError) and exc.errno in MOUNT_ERRNOS: if isinstance(exc, OSError) and exc.errno in MOUNT_ERRNOS:
raise RunAborted(f"Anbindung gestört: {_describe_os_error(exc)}") from exc raise RunAborted(f"{prefix}Anbindung gestört: {_describe_os_error(exc)}") from exc
msg = _describe_os_error(exc) if isinstance(exc, OSError) else str(exc) msg = prefix + (_describe_os_error(exc) if isinstance(exc, OSError) else str(exc))
if not dry_run: if not dry_run:
attempts = self.ledger.record_failure( attempts = self.ledger.record_failure(
rule.id, c.relpath, c.state.size, c.state.mtime_ns, msg rule.id, c.relpath, c.state.size, c.state.mtime_ns, msg
@@ -437,6 +507,12 @@ class Engine:
event(log, "file.failed", level=30, rule=rule.name, file=c.relpath, error=msg) event(log, "file.failed", level=30, rule=rule.name, file=c.relpath, error=msg)
return ItemResult(c.relpath, ItemStatus.FAILED, None, c.state.size, msg) return ItemResult(c.relpath, ItemStatus.FAILED, None, c.state.size, msg)
def _archive_rel(self, relpath: str, layout: str) -> str:
if layout == "date":
t = datetime.fromtimestamp(self._clock())
return f"{t:%Y}/{t:%m}/{relpath}"
return relpath
def _quarantine( def _quarantine(
self, rule: RuleSnapshot, c: Candidate, src: Source, q_rel: str, attempts: int self, rule: RuleSnapshot, c: Candidate, src: Source, q_rel: str, attempts: int
) -> ItemResult: ) -> ItemResult:
+48 -1
View File
@@ -14,7 +14,7 @@ from sqlalchemy import delete, select
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
from netmover.db import session_scope from netmover.db import session_scope
from netmover.models import CopiedFile, FileFailure, utcnow from netmover.models import ArchivedFile, CopiedFile, FileFailure, utcnow
CopyKey = tuple[str, int, int] # (relpath, size, mtime_ns) CopyKey = tuple[str, int, int] # (relpath, size, mtime_ns)
@@ -72,6 +72,53 @@ class Ledger:
s.execute(delete(CopiedFile).where(CopiedFile.id.in_(stale))) s.execute(delete(CopiedFile).where(CopiedFile.id.in_(stale)))
return len(stale) return len(stale)
# -- Archiv ------------------------------------------------------------------------
def archived_keys(self, rule_id: int) -> set[CopyKey]:
with session_scope(self.session_factory) as s:
rows = s.execute(
select(ArchivedFile.src_relpath, ArchivedFile.size, ArchivedFile.mtime_ns).where(
ArchivedFile.rule_id == rule_id
)
).all()
return {(r[0], r[1], r[2]) for r in rows}
def record_archive(
self, rule_id: int, relpath: str, size: int, mtime_ns: int, archive_path: str
) -> None:
with session_scope(self.session_factory) as s:
exists = s.scalar(
select(ArchivedFile.id).where(
ArchivedFile.rule_id == rule_id,
ArchivedFile.src_relpath == relpath,
ArchivedFile.size == size,
ArchivedFile.mtime_ns == mtime_ns,
)
)
if exists is None:
s.add(
ArchivedFile(
rule_id=rule_id,
src_relpath=relpath,
size=size,
mtime_ns=mtime_ns,
archive_path=archive_path,
)
)
def prune_archived(self, rule_id: int, present: set[str]) -> int:
"""Wie :meth:`prune_copied`: nur nach einem vollständigen Scan aufrufen."""
with session_scope(self.session_factory) as s:
rows = s.execute(
select(ArchivedFile.id, ArchivedFile.src_relpath).where(
ArchivedFile.rule_id == rule_id
)
).all()
stale = [r[0] for r in rows if r[1] not in present]
if stale:
s.execute(delete(ArchivedFile).where(ArchivedFile.id.in_(stale)))
return len(stale)
def failures(self, rule_id: int) -> dict[str, FailureInfo]: def failures(self, rule_id: int) -> dict[str, FailureInfo]:
with session_scope(self.session_factory) as s: with session_scope(self.session_factory) as s:
rows = s.scalars(select(FileFailure).where(FileFailure.rule_id == rule_id)).all() rows = s.scalars(select(FileFailure).where(FileFailure.rule_id == rule_id)).all()
+73 -24
View File
@@ -56,6 +56,11 @@ class RulePlan:
quarantine_path: str | None quarantine_path: str | None
op_timeout: int op_timeout: int
dry_run: bool dry_run: bool
archive_kind: str = "none"
archive_mount_id: int | None = None
archive_type: MountType | None = None
archive_name: str | None = None
archive_path: str = ""
def parse_owner(value: str | None) -> tuple[int, int] | None: def parse_owner(value: str | None) -> tuple[int, int] | None:
@@ -92,6 +97,7 @@ def load_plan(session: Session, rule_id: int) -> RulePlan:
set_mode=parse_mode(r.set_mode), set_mode=parse_mode(r.set_mode),
quarantine_after=r.quarantine_after, quarantine_after=r.quarantine_after,
dst_root_squash=dst.root_squash if dst is not None else None, dst_root_squash=dst.root_squash if dst is not None else None,
archive_layout=r.archive_layout.value,
) )
return RulePlan( return RulePlan(
snapshot=snap, snapshot=snap,
@@ -109,6 +115,11 @@ def load_plan(session: Session, rule_id: int) -> RulePlan:
else None, else None,
op_timeout=r.op_timeout_seconds, op_timeout=r.op_timeout_seconds,
dry_run=r.dry_run, dry_run=r.dry_run,
archive_kind=r.archive_kind.value,
archive_mount_id=r.archive_mount_id if r.archive_kind.value == "mount" else None,
archive_type=r.archive_mount.type if r.archive_mount is not None else None,
archive_name=r.archive_mount.name if r.archive_mount is not None else None,
archive_path=r.archive_path,
) )
@@ -176,24 +187,52 @@ class RuleRunner:
if plan.dst_mount_id is not None if plan.dst_mount_id is not None
else None else None
) )
arch_mp = (
self.mounts.check_alive(plan.archive_mount_id)
if plan.archive_mount_id is not None
else None
)
except MountUnavailable as exc: except MountUnavailable as exc:
return RunResult(RunStatus.ERROR, dry, message=str(exc), unavailable=True) return RunResult(RunStatus.ERROR, dry, message=str(exc), unavailable=True)
try: try:
make_source = self._source_factory(plan, src_mp) make_source = self._source_factory(plan, src_mp)
src_root = src_mp / plan.src_subpath if src_mp and plan.src_subpath else src_mp src_root = src_mp / plan.src_subpath if src_mp and plan.src_subpath else src_mp
make_dest = self._dest_factory(plan, dst_mp, src_root) make_dest = self._target(
"dst_path",
plan.dst_mount_id,
plan.dst_type,
plan.dst_name,
plan.dst_path,
dst_mp,
src_root,
)
make_archive = (
self._target(
"archive_path",
plan.archive_mount_id,
plan.archive_type,
plan.archive_name,
plan.archive_path,
arch_mp,
src_root,
)
if plan.archive_kind != "none"
else None
)
except (ValidationError, TransferError, SftpError) as exc: except (ValidationError, TransferError, SftpError) as exc:
return RunResult(RunStatus.ERROR, dry, message=str(exc)) return RunResult(RunStatus.ERROR, dry, message=str(exc))
def work(token: CancelToken) -> RunResult: def work(token: CancelToken) -> RunResult:
source = make_source() opened: list[Source | Destination] = []
try: try:
source = make_source()
opened.append(source)
dest = make_dest() dest = make_dest()
except BaseException: opened.append(dest)
source.close() archive = make_archive() if make_archive is not None else None
raise if archive is not None:
try: opened.append(archive)
return self.engine.execute( return self.engine.execute(
plan.snapshot, plan.snapshot,
source, source,
@@ -201,10 +240,11 @@ class RuleRunner:
token, token,
dry_run=dry, dry_run=dry,
quarantine_rel=plan.quarantine_path, quarantine_rel=plan.quarantine_path,
archive=archive,
) )
finally: finally:
dest.close() for item in reversed(opened):
source.close() item.close()
def finish() -> None: def finish() -> None:
self._started.pop(rule_id, None) self._started.pop(rule_id, None)
@@ -251,41 +291,50 @@ class RuleRunner:
return make_local return make_local
def _dest_factory( def _target(
self, plan: RulePlan, dst_mp: Path | None, src_root: Path | None self,
field: str,
mount_id: int | None,
mount_type: MountType | None,
mount_name: str | None,
path: str,
mp: Path | None,
src_root: Path | None,
) -> Callable[[], Destination]: ) -> Callable[[], Destination]:
if plan.dst_mount_id is None: """Ziel oder Archiv: lokaler Pfad (``mount_id`` None), SFTP oder Kernel-Mount."""
root = Path(safe_abspath("dst_path", plan.dst_path)) what = "Archiv" if field == "archive_path" else "Ziel"
if mount_id is None:
root = Path(safe_abspath(field, path))
if self._under_mount_base(root): if self._under_mount_base(root):
base = self.mounts.mount_base base = self.mounts.mount_base
raise ValidationError( raise ValidationError(
"dst_path", field,
f"Lokales Ziel liegt unter {base} – bitte die Anbindung als Ziel wählen, " f"Lokales {what} liegt unter {base} – bitte die Anbindung als {what} wählen, "
"damit geprüft wird, ob sie eingehängt ist.", "damit geprüft wird, ob sie eingehängt ist.",
) )
kind = "local" kind = "local"
local_root = root local_root = root
create = False create = False
elif plan.dst_type is MountType.SFTP: elif mount_type is MountType.SFTP:
scfg = self.mounts.sftp_config_for(plan.dst_mount_id) scfg = self.mounts.sftp_config_for(mount_id)
sub = safe_relpath("dst_path", plan.dst_path) sub = safe_relpath(field, path)
remote = posixpath.join(scfg.base_path, sub) if sub else scfg.base_path remote = posixpath.join(scfg.base_path, sub) if sub else scfg.base_path
label = plan.dst_name or "sftp" label = mount_name or "sftp"
def make_sftp() -> Destination: def make_sftp() -> Destination:
return SftpDestination(connect(scfg), remote, label=label) return SftpDestination(connect(scfg), remote, label=label)
return make_sftp return make_sftp
else: else:
if dst_mp is None: if mp is None:
raise TransferError("Ziel-Anbindung ist nicht eingehängt") raise TransferError(f"{what}-Anbindung ist nicht eingehängt")
sub = safe_relpath("dst_path", plan.dst_path) sub = safe_relpath(field, path)
local_root = dst_mp / sub if sub else dst_mp local_root = mp / sub if sub else mp
kind = "cifs" if plan.dst_type is MountType.CIFS else "nfs" kind = "cifs" if mount_type is MountType.CIFS else "nfs"
create = True create = True
if src_root is not None and local_root.resolve() == src_root.resolve(): if src_root is not None and local_root.resolve() == src_root.resolve():
raise ValidationError("dst_path", "Quelle und Ziel sind derselbe Ordner") raise ValidationError(field, f"Quelle und {what} sind derselbe Ordner")
def make_local() -> Destination: def make_local() -> Destination:
return LocalDestination(local_root, kind, create_root=create) # type: ignore[arg-type] return LocalDestination(local_root, kind, create_root=create) # type: ignore[arg-type]
+351
View File
@@ -0,0 +1,351 @@
"""0.7.0: Archiv – Kopie ans Ziel UND ins Archiv, Quelle erst danach löschen."""
from __future__ import annotations
import os
from collections.abc import Iterator
from datetime import datetime
from pathlib import Path
import pytest
from sqlalchemy import select
from sqlalchemy.orm import Session, sessionmaker
from netmover.db import session_scope
from netmover.models import (
ArchivedFile,
ArchiveKind,
CollisionPolicy,
ItemStatus,
Rule,
RuleAction,
RunStatus,
)
from netmover.mounting.fake import FakeMountBackend
from netmover.mounting.manager import MountManager
from netmover.mover.destinations import LocalDestination, TransferError
from netmover.mover.engine import Engine, RuleSnapshot, RunResult
from netmover.mover.ledger import Ledger
from netmover.mover.runner import RuleRunner
from netmover.timeouts import CancelToken
from tests.conftest import MountFactory
@pytest.fixture
def rule_id(make_mount: MountFactory, session_factory: sessionmaker[Session]) -> int:
mid = make_mount()
with session_scope(session_factory) as s:
r = Rule(name="r", src_mount_id=mid, dst_path="/x")
s.add(r)
s.flush()
return r.id
FIXED = datetime(2026, 10, 12, 9, 30).timestamp()
@pytest.fixture
def engine(session_factory: sessionmaker[Session]) -> Engine:
return Engine(Ledger(session_factory), sleep=lambda _: None, clock=lambda: FIXED)
@pytest.fixture
def dirs(tmp_path: Path) -> tuple[Path, Path, Path]:
src, dst, arch = tmp_path / "src", tmp_path / "dst", tmp_path / "archiv"
for d in (src, dst, arch):
d.mkdir()
return src, dst, arch
@pytest.fixture(autouse=True)
def cross_device(monkeypatch: pytest.MonkeyPatch) -> None:
"""tmp_path liegt auf einem Gerät – realistischer Weg über Kopie erzwingen."""
monkeypatch.setattr(LocalDestination, "move_same_device", lambda *a, **k: None)
def snap(rule_id: int, **kw: object) -> RuleSnapshot:
return RuleSnapshot(id=rule_id, name="r", stability_seconds=0, **kw) # type: ignore[arg-type]
def run(
engine: Engine, rule: RuleSnapshot, src: Path, dst: Path, arch: Path | None, **kw: object
) -> RunResult:
archive = LocalDestination(arch) if arch is not None else None
return engine.execute(rule, src, LocalDestination(dst), CancelToken(), archive=archive, **kw) # type: ignore[arg-type]
def write(p: Path, data: bytes = b"inhalt") -> Path:
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(data)
return p
def names(d: Path) -> list[str]:
return sorted(str(p.relative_to(d)) for p in d.rglob("*") if p.is_file())
def test_move_delivers_to_target_and_archive_then_deletes_source(
engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path]
) -> None:
src, dst, arch = dirs
data = os.urandom(200_000)
write(src / "scan001.pdf", data)
result = run(engine, snap(rule_id), src, dst, arch)
assert result.status is RunStatus.OK
item = result.items[0]
assert item.status is ItemStatus.MOVED and item.message and "archiviert:" in item.message
assert (dst / "scan001.pdf").read_bytes() == data == (arch / "scan001.pdf").read_bytes()
assert not (src / "scan001.pdf").exists()
assert names(dst) == ["scan001.pdf"] and names(arch) == ["scan001.pdf"] # keine .part-Reste
def test_date_layout(engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path]) -> None:
src, dst, arch = dirs
write(src / "max" / "a.pdf")
run(engine, snap(rule_id, recursive=True, archive_layout="date"), src, dst, arch)
assert names(arch) == ["2026/10/max/a.pdf"] and names(dst) == ["max/a.pdf"]
def test_archive_failure_delivers_nothing_and_keeps_source(
engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch
) -> None:
src, dst, arch = dirs
write(src / "a.pdf")
real = LocalDestination._link_noreplace
def fail_in_archive(self: LocalDestination, tmp: str, final: str) -> None:
if self.root == arch:
raise TransferError("Archiv voll")
real(self, tmp, final)
monkeypatch.setattr(LocalDestination, "_link_noreplace", fail_in_archive)
result = run(engine, snap(rule_id), src, dst, arch)
item = result.items[0]
assert item.status is ItemStatus.FAILED and item.message == "Archiv: Archiv voll (Versuch 1)"
assert (src / "a.pdf").exists() and names(dst) == [] and names(arch) == []
def test_target_failure_after_archive_never_archives_twice(
engine: Engine,
rule_id: int,
dirs: tuple[Path, Path, Path],
monkeypatch: pytest.MonkeyPatch,
session_factory: sessionmaker[Session],
) -> None:
src, dst, arch = dirs
write(src / "a.pdf")
real = LocalDestination._link_noreplace
def fail_in_target(self: LocalDestination, tmp: str, final: str) -> None:
if self.root == dst:
raise TransferError("Ziel gesperrt")
real(self, tmp, final)
monkeypatch.setattr(LocalDestination, "_link_noreplace", fail_in_target)
first = run(engine, snap(rule_id), src, dst, arch)
assert first.items[0].status is ItemStatus.FAILED
assert (src / "a.pdf").exists() and names(arch) == ["a.pdf"] and names(dst) == []
with session_factory() as s:
assert len(s.scalars(select(ArchivedFile)).all()) == 1
monkeypatch.setattr(LocalDestination, "_link_noreplace", real) # Ziel wieder frei
second = run(engine, snap(rule_id), src, dst, arch)
assert second.items[0].status is ItemStatus.MOVED
assert names(arch) == ["a.pdf"] # keine zweite Archivkopie (a-1.pdf)
assert names(dst) == ["a.pdf"] and not (src / "a.pdf").exists()
def test_archive_never_overwrites(
engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path]
) -> None:
src, dst, arch = dirs
write(arch / "scan.pdf", b"alt im Archiv")
write(src / "scan.pdf", b"neu")
run(engine, snap(rule_id, on_collision=CollisionPolicy.OVERWRITE), src, dst, arch)
assert (arch / "scan.pdf").read_bytes() == b"alt im Archiv"
assert (arch / "scan-1.pdf").read_bytes() == b"neu"
def test_no_space_in_archive_writes_nothing(
engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch
) -> None:
src, dst, arch = dirs
write(src / "a.pdf", b"x" * 1000)
real = LocalDestination.free_bytes
monkeypatch.setattr(
LocalDestination, "free_bytes", lambda self: 10 if self.root == arch else real(self)
)
result = run(engine, snap(rule_id), src, dst, arch)
assert "zu wenig Platz im Archiv" in (result.items[0].message or "")
assert names(dst) == [] and names(arch) == [] and (src / "a.pdf").exists()
def test_source_changed_during_archive_copy(
engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch
) -> None:
src, dst, arch = dirs
f = write(src / "a.pdf", b"original")
real = LocalDestination._apply_meta
def change(self: LocalDestination, rel: str, meta: object) -> list[str]:
if self.root == arch:
f.write_bytes(b"geaendert!!")
return real(self, rel, meta) # type: ignore[arg-type]
monkeypatch.setattr(LocalDestination, "_apply_meta", change)
result = run(engine, snap(rule_id), src, dst, arch)
assert result.items[0].status is ItemStatus.SKIPPED
assert names(dst) == [] and names(arch) == [] and f.exists()
def test_copy_mode_with_archive(
engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path]
) -> None:
src, dst, arch = dirs
f = write(src / "a.pdf", b"v1")
rule = snap(rule_id, action=RuleAction.COPY)
for _ in range(3):
run(engine, rule, src, dst, arch)
assert f.exists() and names(dst) == ["a.pdf"] and names(arch) == ["a.pdf"]
f.write_bytes(b"version 2")
run(engine, rule, src, dst, arch)
assert names(dst) == ["a-1.pdf", "a.pdf"] and names(arch) == ["a-1.pdf", "a.pdf"]
def test_dry_run_shows_archive_and_writes_nothing(
engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path]
) -> None:
src, dst, arch = dirs
write(src / "a.pdf")
result = run(engine, snap(rule_id, archive_layout="date"), src, dst, arch, dry_run=True)
assert result.items[0].status is ItemStatus.PLANNED
assert f"Archiv: {arch / '2026/10/a.pdf'}" in (result.items[0].message or "")
assert names(dst) == [] and names(arch) == [] and (src / "a.pdf").exists()
def test_without_archive_unchanged(
engine: Engine, rule_id: int, dirs: tuple[Path, Path, Path]
) -> None:
src, dst, arch = dirs
write(src / "a.pdf")
result = run(engine, snap(rule_id), src, dst, None)
assert result.items[0].message is None and names(arch) == []
# -- Regellauf ----------------------------------------------------------------------------------
@pytest.fixture
def setup(
manager: MountManager,
fake_backend: FakeMountBackend,
make_mount: MountFactory,
session_factory: sessionmaker[Session],
tmp_path: Path,
) -> Iterator[tuple[int, Path, Path, Path]]:
scans = fake_backend.export("nas", "scans")
arch_remote = fake_backend.export("nas", "archiv")
src_id = make_mount("scans", share="scans")
arch_id = make_mount("archiv", share="archiv")
manager.mount(src_id)
manager.mount(arch_id)
consume = tmp_path / "consume"
consume.mkdir()
with session_scope(session_factory) as s:
r = Rule(
name="Scans → Paperless + Archiv",
src_mount_id=src_id,
dst_path=str(consume),
stability_seconds=0,
archive_kind=ArchiveKind.MOUNT,
archive_mount_id=arch_id,
archive_path="scans",
)
s.add(r)
s.flush()
rid = r.id
yield rid, scans, consume, arch_remote
def test_runner_with_archive_mount(
setup: tuple[int, Path, Path, Path],
manager: MountManager,
session_factory: sessionmaker[Session],
) -> None:
rid, scans, consume, arch = setup
write(scans / "doc.pdf", b"%PDF")
result = RuleRunner(session_factory, manager, sleep=lambda _: None).run(rid)
assert result.status is RunStatus.OK, result.message
assert (
(consume / "doc.pdf").read_bytes() == b"%PDF" == (arch / "scans" / "doc.pdf").read_bytes()
)
assert not (scans / "doc.pdf").exists()
def test_unavailable_archive_makes_rule_wait(
setup: tuple[int, Path, Path, Path],
manager: MountManager,
fake_backend: FakeMountBackend,
session_factory: sessionmaker[Session],
) -> None:
rid, scans, consume, _arch = setup
write(scans / "doc.pdf")
fake_backend.vanish(manager.mountpoint_for("archiv"))
result = RuleRunner(session_factory, manager, sleep=lambda _: None).run(rid)
assert result.unavailable and result.status is RunStatus.ERROR
assert (scans / "doc.pdf").exists() and list(consume.iterdir()) == []
def test_mount_used_as_archive_cannot_be_deleted(
setup: tuple[int, Path, Path, Path],
session_factory: sessionmaker[Session],
manager: MountManager,
) -> None:
from netmover.admin import AdminError, delete_mount
from netmover.models import Mount
with session_scope(session_factory) as s:
arch_id = s.scalar(select(Mount.id).where(Mount.name == "archiv"))
assert arch_id is not None
with pytest.raises(AdminError, match="Wird noch von Regeln benutzt"):
delete_mount(s, manager, arch_id)
def test_rule_form_archive_validation(
session_factory: sessionmaker[Session], make_mount: MountFactory
) -> None:
from netmover.admin import parse_rule_form, save_rule
mid = make_mount()
base = {
"name": "x",
"src_mount_id": str(mid),
"dst_kind": "local",
"dst_path": "/srv/consume",
"interval_seconds": "30",
"stability_seconds": "5",
"op_timeout_seconds": "120",
}
with session_scope(session_factory) as s:
_, err = parse_rule_form(
s, {**base, "archive_kind": "local", "archive_path": "/srv/consume"}
)
assert "derselbe Ordner" in err["archive_path"]
_, err = parse_rule_form(s, {**base, "archive_kind": "mount", "archive_path": "a"})
assert "archive_mount_id" in err
_, err = parse_rule_form(s, {**base, "archive_kind": "local", "archive_path": "relativ"})
assert "absoluten Pfad" in err["archive_path"]
d, err = parse_rule_form(
s,
{
**base,
"archive_kind": "mount",
"archive_mount_id": str(mid),
"archive_path": "archiv/scans",
"archive_layout": "date",
},
)
assert err == {}
r, _ = save_rule(s, d)
assert r is not None and r.archive_kind is ArchiveKind.MOUNT
assert r.archive_path == "archiv/scans" and r.archive_layout.value == "date"