Compare commits

..
Author SHA1 Message Date
stumpylog 252652b161 Saves work on a probable fix 2026-10-15 15:32:31 -07:00
4 changed files with 180 additions and 10 deletions

No files matched your search

@@ -3,6 +3,7 @@ from itertools import islice
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from typing import Any from typing import Any
from typing import Final
from allauth.mfa.models import Authenticator from allauth.mfa.models import Authenticator
from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.models import SocialAccount
@@ -70,6 +71,10 @@ from paperless.models import ApplicationConfiguration
from paperless_mail.models import MailAccount from paperless_mail.models import MailAccount
from paperless_mail.models import MailRule from paperless_mail.models import MailRule
# Derived names replace the original's extension with these suffixes
THUMBNAIL_SUFFIX: Final[str] = "-thumbnail.webp"
ARCHIVE_SUFFIX: Final[str] = "-archive.pdf"
def serialize_queryset_batched( def serialize_queryset_batched(
queryset: "QuerySet[Any]", queryset: "QuerySet[Any]",
@@ -512,7 +517,12 @@ class Command(CryptMixin, PaperlessCommand):
counter=filename_counter, counter=filename_counter,
) )
else: else:
base_name = document.get_public_filename(counter=filename_counter) # Leave room for the longer of the derived name suffixes
base_name = document.get_public_filename(
counter=filename_counter,
max_bytes=Document.MAX_PUBLIC_FILENAME_BYTES
- max(len(THUMBNAIL_SUFFIX), len(ARCHIVE_SUFFIX)),
)
# The thumbnail and archive names are derived from the stem, so # The thumbnail and archive names are derived from the stem, so
# uniqueness must hold without the original's extension # uniqueness must hold without the original's extension
@@ -541,7 +551,7 @@ class Command(CryptMixin, PaperlessCommand):
document_dict[EXPORTER_FILE_NAME] = original_arc document_dict[EXPORTER_FILE_NAME] = original_arc
if not self.no_thumbnail: if not self.no_thumbnail:
thumbnail_name = base_name.parent / (base_name.stem + "-thumbnail.webp") thumbnail_name = base_name.parent / (base_name.stem + THUMBNAIL_SUFFIX)
if self.use_folder_prefix: if self.use_folder_prefix:
thumbnail_name = Path("thumbnails") / thumbnail_name thumbnail_name = Path("thumbnails") / thumbnail_name
thumbnail_arc = thumbnail_name.as_posix() thumbnail_arc = thumbnail_name.as_posix()
@@ -550,7 +560,7 @@ class Command(CryptMixin, PaperlessCommand):
thumbnail_arc = None thumbnail_arc = None
if not self.no_archive and document.has_archive_version: if not self.no_archive and document.has_archive_version:
archive_name = base_name.parent / (base_name.stem + "-archive.pdf") archive_name = base_name.parent / (base_name.stem + ARCHIVE_SUFFIX)
if self.use_folder_prefix: if self.use_folder_prefix:
archive_name = Path("archive") / archive_name archive_name = Path("archive") / archive_name
archive_arc = archive_name.as_posix() archive_arc = archive_name.as_posix()
+28 -7
View File
@@ -157,6 +157,8 @@ class StoragePath(MatchingModel):
class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-missing] class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-missing]
MAX_STORED_FILENAME_LENGTH: Final[int] = 1024 MAX_STORED_FILENAME_LENGTH: Final[int] = 1024
# Per-component limit of common filesystems (ext4, XFS, btrfs, ZFS, APFS)
MAX_PUBLIC_FILENAME_BYTES: Final[int] = 255
correspondent = models.ForeignKey( correspondent = models.ForeignKey(
Correspondent, Correspondent,
@@ -484,28 +486,47 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
def archive_file(self): def archive_file(self):
return Path(self.archive_path).open("rb") return Path(self.archive_path).open("rb")
def get_public_filename(self, *, archive=False, counter=0, suffix=None) -> str: def get_public_filename(
self,
*,
archive=False,
counter=0,
suffix=None,
max_bytes: int = MAX_PUBLIC_FILENAME_BYTES,
) -> str:
""" """
Returns a sanitized filename for the document, not including any paths. Returns a sanitized filename for the document, not including any paths.
The name fits in max_bytes (UTF-8). A title that is too long is
shortened, so the counter, suffix and extension are always kept.
""" """
# Root owns metadata for all versions # Root owns metadata for all versions
context_document = ( context_document = (
self.root_document if self.root_document_id is not None else self self.root_document if self.root_document_id is not None else self
) )
result = str(context_document)
tail = ""
if counter: if counter:
result += f"_{counter:02}" tail += f"_{counter:02}"
if suffix: if suffix:
result += suffix tail += suffix
if archive: if archive:
result += ".pdf" tail += ".pdf"
else: else:
result += self.file_type tail += self.file_type
return pathvalidate.sanitize_filename(result, replacement_text="-") # sanitize_filename truncates from the end, which would cut off the
# counter and extension, so shorten the title before appending them
head_bytes = max_bytes - len(tail.encode())
head = str(context_document).encode()[:head_bytes].decode(errors="ignore")
return pathvalidate.sanitize_filename(
head + tail,
replacement_text="-",
max_len=max_bytes,
)
@property @property
def file_type(self): def file_type(self):
@@ -4,6 +4,7 @@ from datetime import date
from pathlib import Path from pathlib import Path
from unittest import mock from unittest import mock
import pytest
from django.test import TestCase from django.test import TestCase
from django.test import override_settings from django.test import override_settings
from faker import Faker from faker import Faker
@@ -257,3 +258,77 @@ def test_suggestion_content() -> None:
) )
doc.content = content_over_limit doc.content = content_over_limit
assert doc.suggestion_content == expected_cropped_content assert doc.suggestion_content == expected_cropped_content
@pytest.mark.django_db
class TestPublicFilenameLength:
@pytest.mark.parametrize(
("title", "archive", "extension"),
[
pytest.param("a" * 300, False, ".jpg", id="ascii-original"),
pytest.param("a" * 300, True, ".pdf", id="ascii-archive"),
pytest.param("ä" * 200, False, ".jpg", id="multibyte-original"),
],
)
def test_long_title_keeps_extension_within_limit(
self,
title: str,
*,
archive: bool,
extension: str,
) -> None:
"""
GIVEN:
- A document whose title alone is longer than 255 bytes
WHEN:
- The public file name is generated
THEN:
- The name fits in 255 bytes and still ends with the extension
"""
doc = Document(mime_type="image/jpeg", title=title, created=date(2020, 12, 25))
name = doc.get_public_filename(archive=archive)
assert len(name.encode()) <= 255
assert name.endswith(extension)
def test_long_title_counters_give_distinct_names(self) -> None:
"""
GIVEN:
- A document whose title alone is longer than 255 bytes
WHEN:
- Public file names are generated with increasing counters
THEN:
- Each counter produces a distinct name that keeps the counter
"""
doc = Document(
mime_type="application/pdf",
title="ä" * 200,
created=date(2020, 12, 25),
)
names = [doc.get_public_filename(counter=counter) for counter in range(3)]
assert len(set(names)) == 3
assert names[1].endswith("_01.pdf")
assert all(len(name.encode()) <= 255 for name in names)
def test_max_bytes_bounds_name(self) -> None:
"""
GIVEN:
- A document with a long title
WHEN:
- The public file name is generated with a smaller byte limit
THEN:
- The name fits in that limit and keeps the counter and extension
"""
doc = Document(
mime_type="application/pdf",
title="a" * 300,
created=date(2020, 12, 25),
)
name = doc.get_public_filename(counter=1, max_bytes=100)
assert len(name.encode()) <= 100
assert name.endswith("_01.pdf")
@@ -1537,3 +1537,67 @@ class TestExportDerivedFileNameCollisions:
assert ( assert (
target / record[EXPORTER_ARCHIVE_NAME] target / record[EXPORTER_ARCHIVE_NAME]
).read_text() == f"archive {pk}" ).read_text() == f"archive {pk}"
def test_long_titles_export_within_file_name_limit(
self,
paperless_dirs: PaperlessDirs,
tmp_path: Path,
) -> None:
"""
GIVEN:
- Two documents with the same created date and the same title
- The title alone is longer than 255 bytes and contains umlauts
WHEN:
- The exporter runs
THEN:
- The export finishes with distinct original, thumbnail and archive names
- Every exported file name fits in 255 bytes
- Each exported file holds its own document's content
"""
for pk in (1, 2):
(paperless_dirs.originals_dir / f"{pk:07}.pdf").write_bytes(
f"original {pk}".encode(),
)
(paperless_dirs.archive_dir / f"{pk:07}.pdf").write_bytes(
f"archive {pk}".encode(),
)
(paperless_dirs.thumbnail_dir / f"{pk:07}.webp").write_bytes(
f"thumbnail {pk}".encode(),
)
DocumentFactory(
pk=pk,
title="Bestellbestätigung " * 15,
created=date(2026, 10, 15),
filename=f"{pk:07}.pdf",
archive_filename=f"{pk:07}.pdf",
mime_type="application/pdf",
)
target = tmp_path / "export"
target.mkdir()
call_command(
"document_exporter",
str(target),
"--no-progress-bar",
skip_checks=True,
)
manifest = json.loads((target / "manifest.json").read_text())
records = {r["pk"]: r for r in manifest if r["model"] == "documents.document"}
for key in (
EXPORTER_FILE_NAME,
EXPORTER_THUMBNAIL_NAME,
EXPORTER_ARCHIVE_NAME,
):
assert records[1][key] != records[2][key]
for pk in (1, 2):
record = records[pk]
for key, content in (
(EXPORTER_FILE_NAME, f"original {pk}"),
(EXPORTER_THUMBNAIL_NAME, f"thumbnail {pk}"),
(EXPORTER_ARCHIVE_NAME, f"archive {pk}"),
):
assert len(Path(record[key]).name.encode()) <= 255
assert (target / record[key]).read_text() == content