feat: Endname per renameat2(RENAME_NOREPLACE) – echtes rename ohne Überschreiben
Watcher, die nur auf IN_MOVED_TO/IN_CLOSE_WRITE lauschen, sehen damit neue Dateien wie bei os.replace. Dateisysteme ohne renameat2 (NFS, CIFS) nutzen weiter link() + unlink() bzw. prüfen + rename. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
2770c2e204
commit
00a0537085
@@ -7,7 +7,8 @@ verlieren. Ablauf für jede Datei:
|
||||
Größe prüfen,
|
||||
2. Metadaten setzen (mtime, Modus, Eigentümer – Fehler sind nur Hinweise),
|
||||
3. prüfen, dass sich die Quelle seit der Stabilitätsmessung nicht verändert hat,
|
||||
4. atomar auf den Endnamen umbenennen – ohne zu überschreiben (``link`` bzw.
|
||||
4. atomar auf den Endnamen umbenennen – ohne zu überschreiben
|
||||
(``renameat2(RENAME_NOREPLACE)``, sonst ``link`` + ``unlink`` bzw.
|
||||
SFTP-``rename``), außer „überschreiben“ ist ausdrücklich eingestellt,
|
||||
5. erst danach löscht der Aufrufer die Quelle.
|
||||
|
||||
@@ -33,6 +34,7 @@ from typing import Literal
|
||||
|
||||
from netmover.models import CollisionPolicy
|
||||
from netmover.mounting.sftp import SftpSession
|
||||
from netmover.mover.fsops import rename_noreplace
|
||||
from netmover.mover.naming import collision_candidates
|
||||
from netmover.timeouts import CancelToken
|
||||
|
||||
@@ -283,6 +285,9 @@ class LocalDestination(Destination):
|
||||
|
||||
def _link_noreplace(self, tmp: str, final: str) -> None:
|
||||
src, dst = self._p(tmp), self._p(final)
|
||||
# Bevorzugt ein echtes rename ohne Überschreiben (Watcher sehen IN_MOVED_TO).
|
||||
if rename_noreplace(src, dst):
|
||||
return
|
||||
try:
|
||||
os.link(src, dst)
|
||||
except FileExistsError:
|
||||
@@ -340,6 +345,9 @@ class LocalDestination(Destination):
|
||||
raise CollisionSkip
|
||||
continue
|
||||
try:
|
||||
if rename_noreplace(src, self._p(cand)):
|
||||
final = cand
|
||||
break
|
||||
os.link(src, self._p(cand))
|
||||
except FileExistsError:
|
||||
if policy is CollisionPolicy.SKIP:
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Atomares Umbenennen ohne Überschreiben.
|
||||
|
||||
``renameat2(..., RENAME_NOREPLACE)`` ist ein echtes ``rename`` (Watcher sehen
|
||||
``IN_MOVED_TO`` wie bei ``os.replace``), scheitert aber mit ``EEXIST``, statt
|
||||
eine vorhandene Datei zu ersetzen. ext4, xfs, btrfs und tmpfs können das; NFS
|
||||
und CIFS meist nicht – dann liefert :func:`rename_noreplace` ``False`` und der
|
||||
Aufrufer weicht auf ``link()`` + ``unlink()`` aus.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import errno
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from functools import cache
|
||||
|
||||
AT_FDCWD = -100
|
||||
RENAME_NOREPLACE = 1
|
||||
_UNSUPPORTED = frozenset({errno.EINVAL, errno.ENOSYS, errno.ENOTSUP, errno.EOPNOTSUPP, errno.EPERM})
|
||||
|
||||
|
||||
@cache
|
||||
def _renameat2() -> Callable[..., int] | None:
|
||||
name = ctypes.util.find_library("c")
|
||||
if name is None:
|
||||
return None
|
||||
libc = ctypes.CDLL(name, use_errno=True)
|
||||
fn = getattr(libc, "renameat2", None)
|
||||
if fn is None:
|
||||
return None
|
||||
fn.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint]
|
||||
fn.restype = ctypes.c_int
|
||||
renameat2: Callable[..., int] = fn
|
||||
return renameat2
|
||||
|
||||
|
||||
def rename_noreplace(src: str | os.PathLike[str], dst: str | os.PathLike[str]) -> bool:
|
||||
"""True = umbenannt. False = vom Dateisystem nicht unterstützt (nichts passiert).
|
||||
``FileExistsError``, wenn ``dst`` existiert; andere Fehler als ``OSError``."""
|
||||
fn = _renameat2()
|
||||
if fn is None:
|
||||
return False
|
||||
res = fn(AT_FDCWD, os.fsencode(src), AT_FDCWD, os.fsencode(dst), RENAME_NOREPLACE)
|
||||
if res == 0:
|
||||
return True
|
||||
err = ctypes.get_errno()
|
||||
if err == errno.EEXIST:
|
||||
raise FileExistsError(err, os.strerror(err), os.fsdecode(dst))
|
||||
if err in _UNSUPPORTED:
|
||||
return False
|
||||
raise OSError(err, os.strerror(err), os.fsdecode(src))
|
||||
+58
-6
@@ -240,6 +240,9 @@ def test_source_not_deletable_keeps_target_and_prevents_duplicate(
|
||||
) -> None:
|
||||
if path == "copy":
|
||||
monkeypatch.setattr(LocalDestination, "move_same_device", lambda *a, **k: None)
|
||||
else:
|
||||
# Dateisystem ohne renameat2 → link + unlink, unlink scheitert
|
||||
monkeypatch.setattr("netmover.mover.destinations.rename_noreplace", lambda *a: False)
|
||||
write(src / "scan.pdf")
|
||||
real_unlink = os.unlink
|
||||
|
||||
@@ -637,22 +640,24 @@ def test_local_destination_must_exist(
|
||||
def test_fsync_before_final_rename(
|
||||
engine: Engine, rule_id: int, src: Path, dst: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from netmover.mover import fsops
|
||||
|
||||
events: list[str] = []
|
||||
real_fsync, real_link = os.fsync, os.link
|
||||
real_fsync, real_rename = os.fsync, fsops.rename_noreplace
|
||||
|
||||
def fsync(fd: int) -> None:
|
||||
events.append("fsync")
|
||||
real_fsync(fd)
|
||||
|
||||
def link(a: object, b: object, *args: object, **kw: object) -> None:
|
||||
events.append(f"link:{Path(str(b)).name}")
|
||||
real_link(a, b) # type: ignore[arg-type]
|
||||
def rename(a: object, b: object) -> bool:
|
||||
events.append(f"rename:{Path(str(b)).name}")
|
||||
return real_rename(a, b) # type: ignore[arg-type]
|
||||
|
||||
monkeypatch.setattr("netmover.mover.destinations.os.fsync", fsync)
|
||||
monkeypatch.setattr("netmover.mover.destinations.os.link", link)
|
||||
monkeypatch.setattr("netmover.mover.destinations.rename_noreplace", rename)
|
||||
write(src / "a.pdf")
|
||||
run(engine, snap(rule_id), src, dst)
|
||||
assert events.index("fsync") < events.index("link:a.pdf")
|
||||
assert events.index("fsync") < events.index("rename:a.pdf")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cross_device")
|
||||
@@ -678,3 +683,50 @@ def test_name_taken_between_check_and_link_is_never_overwritten(
|
||||
assert result.items[0].dst == str(dst / "scan-1.pdf")
|
||||
assert (dst / "scan.pdf").read_text() == "fremd"
|
||||
assert (dst / "scan-1.pdf").read_text() == "neu"
|
||||
|
||||
|
||||
# -- renameat2(RENAME_NOREPLACE) ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_rename_noreplace(tmp_path: Path) -> None:
|
||||
from netmover.mover.fsops import rename_noreplace
|
||||
|
||||
a, b = write(tmp_path / "a", "neu"), write(tmp_path / "b", "alt")
|
||||
ino = a.stat().st_ino
|
||||
if not rename_noreplace(a, tmp_path / "c"):
|
||||
pytest.skip("Dateisystem des Testrechners kann kein renameat2")
|
||||
assert (tmp_path / "c").stat().st_ino == ino and not a.exists() # echtes rename
|
||||
with pytest.raises(FileExistsError):
|
||||
rename_noreplace(tmp_path / "c", b)
|
||||
assert b.read_text() == "alt" and (tmp_path / "c").exists()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("cross_device")
|
||||
@pytest.mark.parametrize("supported", [True, False])
|
||||
def test_final_step_is_a_real_rename_when_possible(
|
||||
engine: Engine,
|
||||
rule_id: int,
|
||||
src: Path,
|
||||
dst: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
supported: bool,
|
||||
) -> None:
|
||||
"""Watcher wie Paperless sehen ein rename (IN_MOVED_TO); ohne Unterstützung
|
||||
bleibt es beim link-Rückfallweg – überschrieben wird in keinem Fall."""
|
||||
links: list[object] = []
|
||||
real_link = os.link
|
||||
|
||||
def spy_link(a: object, b: object, *args: object, **kw: object) -> None:
|
||||
links.append(b)
|
||||
real_link(a, b) # type: ignore[arg-type]
|
||||
|
||||
monkeypatch.setattr("netmover.mover.destinations.os.link", spy_link)
|
||||
if not supported:
|
||||
monkeypatch.setattr("netmover.mover.destinations.rename_noreplace", lambda *a: False)
|
||||
write(dst / "a.pdf", "alt")
|
||||
write(src / "a.pdf", "neu")
|
||||
result = run(engine, snap(rule_id), src, dst)
|
||||
assert result.items[0].dst == str(dst / "a-1.pdf")
|
||||
assert (dst / "a.pdf").read_text() == "alt" and (dst / "a-1.pdf").read_text() == "neu"
|
||||
assert bool(links) is not supported
|
||||
assert [p.name for p in dst.iterdir() if p.name.endswith(".part")] == []
|
||||
|
||||
Reference in New Issue
Block a user