mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-16 04:57:13 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
252652b161 |
No files matched your search
@@ -3,6 +3,7 @@ from itertools import islice
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
from allauth.mfa.models import Authenticator
|
from allauth.mfa.models import Authenticator
|
||||||
from allauth.socialaccount.models import SocialAccount
|
from allauth.socialaccount.models import SocialAccount
|
||||||
@@ -70,6 +71,10 @@ from paperless.models import ApplicationConfiguration
|
|||||||
from paperless_mail.models import MailAccount
|
from paperless_mail.models import MailAccount
|
||||||
from paperless_mail.models import MailRule
|
from paperless_mail.models import MailRule
|
||||||
|
|
||||||
|
# Derived names replace the original's extension with these suffixes
|
||||||
|
THUMBNAIL_SUFFIX: Final[str] = "-thumbnail.webp"
|
||||||
|
ARCHIVE_SUFFIX: Final[str] = "-archive.pdf"
|
||||||
|
|
||||||
|
|
||||||
def serialize_queryset_batched(
|
def serialize_queryset_batched(
|
||||||
queryset: "QuerySet[Any]",
|
queryset: "QuerySet[Any]",
|
||||||
@@ -512,7 +517,12 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
counter=filename_counter,
|
counter=filename_counter,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
base_name = document.get_public_filename(counter=filename_counter)
|
# Leave room for the longer of the derived name suffixes
|
||||||
|
base_name = document.get_public_filename(
|
||||||
|
counter=filename_counter,
|
||||||
|
max_bytes=Document.MAX_PUBLIC_FILENAME_BYTES
|
||||||
|
- max(len(THUMBNAIL_SUFFIX), len(ARCHIVE_SUFFIX)),
|
||||||
|
)
|
||||||
|
|
||||||
# The thumbnail and archive names are derived from the stem, so
|
# The thumbnail and archive names are derived from the stem, so
|
||||||
# uniqueness must hold without the original's extension
|
# uniqueness must hold without the original's extension
|
||||||
@@ -541,7 +551,7 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
document_dict[EXPORTER_FILE_NAME] = original_arc
|
document_dict[EXPORTER_FILE_NAME] = original_arc
|
||||||
|
|
||||||
if not self.no_thumbnail:
|
if not self.no_thumbnail:
|
||||||
thumbnail_name = base_name.parent / (base_name.stem + "-thumbnail.webp")
|
thumbnail_name = base_name.parent / (base_name.stem + THUMBNAIL_SUFFIX)
|
||||||
if self.use_folder_prefix:
|
if self.use_folder_prefix:
|
||||||
thumbnail_name = Path("thumbnails") / thumbnail_name
|
thumbnail_name = Path("thumbnails") / thumbnail_name
|
||||||
thumbnail_arc = thumbnail_name.as_posix()
|
thumbnail_arc = thumbnail_name.as_posix()
|
||||||
@@ -550,7 +560,7 @@ class Command(CryptMixin, PaperlessCommand):
|
|||||||
thumbnail_arc = None
|
thumbnail_arc = None
|
||||||
|
|
||||||
if not self.no_archive and document.has_archive_version:
|
if not self.no_archive and document.has_archive_version:
|
||||||
archive_name = base_name.parent / (base_name.stem + "-archive.pdf")
|
archive_name = base_name.parent / (base_name.stem + ARCHIVE_SUFFIX)
|
||||||
if self.use_folder_prefix:
|
if self.use_folder_prefix:
|
||||||
archive_name = Path("archive") / archive_name
|
archive_name = Path("archive") / archive_name
|
||||||
archive_arc = archive_name.as_posix()
|
archive_arc = archive_name.as_posix()
|
||||||
|
|||||||
+28
-7
@@ -157,6 +157,8 @@ class StoragePath(MatchingModel):
|
|||||||
|
|
||||||
class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-missing]
|
class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-missing]
|
||||||
MAX_STORED_FILENAME_LENGTH: Final[int] = 1024
|
MAX_STORED_FILENAME_LENGTH: Final[int] = 1024
|
||||||
|
# Per-component limit of common filesystems (ext4, XFS, btrfs, ZFS, APFS)
|
||||||
|
MAX_PUBLIC_FILENAME_BYTES: Final[int] = 255
|
||||||
|
|
||||||
correspondent = models.ForeignKey(
|
correspondent = models.ForeignKey(
|
||||||
Correspondent,
|
Correspondent,
|
||||||
@@ -484,28 +486,47 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
|||||||
def archive_file(self):
|
def archive_file(self):
|
||||||
return Path(self.archive_path).open("rb")
|
return Path(self.archive_path).open("rb")
|
||||||
|
|
||||||
def get_public_filename(self, *, archive=False, counter=0, suffix=None) -> str:
|
def get_public_filename(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
archive=False,
|
||||||
|
counter=0,
|
||||||
|
suffix=None,
|
||||||
|
max_bytes: int = MAX_PUBLIC_FILENAME_BYTES,
|
||||||
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Returns a sanitized filename for the document, not including any paths.
|
Returns a sanitized filename for the document, not including any paths.
|
||||||
|
|
||||||
|
The name fits in max_bytes (UTF-8). A title that is too long is
|
||||||
|
shortened, so the counter, suffix and extension are always kept.
|
||||||
"""
|
"""
|
||||||
# Root owns metadata for all versions
|
# Root owns metadata for all versions
|
||||||
context_document = (
|
context_document = (
|
||||||
self.root_document if self.root_document_id is not None else self
|
self.root_document if self.root_document_id is not None else self
|
||||||
)
|
)
|
||||||
result = str(context_document)
|
|
||||||
|
|
||||||
|
tail = ""
|
||||||
if counter:
|
if counter:
|
||||||
result += f"_{counter:02}"
|
tail += f"_{counter:02}"
|
||||||
|
|
||||||
if suffix:
|
if suffix:
|
||||||
result += suffix
|
tail += suffix
|
||||||
|
|
||||||
if archive:
|
if archive:
|
||||||
result += ".pdf"
|
tail += ".pdf"
|
||||||
else:
|
else:
|
||||||
result += self.file_type
|
tail += self.file_type
|
||||||
|
|
||||||
return pathvalidate.sanitize_filename(result, replacement_text="-")
|
# sanitize_filename truncates from the end, which would cut off the
|
||||||
|
# counter and extension, so shorten the title before appending them
|
||||||
|
head_bytes = max_bytes - len(tail.encode())
|
||||||
|
head = str(context_document).encode()[:head_bytes].decode(errors="ignore")
|
||||||
|
|
||||||
|
return pathvalidate.sanitize_filename(
|
||||||
|
head + tail,
|
||||||
|
replacement_text="-",
|
||||||
|
max_len=max_bytes,
|
||||||
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def file_type(self):
|
def file_type(self):
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from datetime import date
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
from django.test import TestCase
|
from django.test import TestCase
|
||||||
from django.test import override_settings
|
from django.test import override_settings
|
||||||
from faker import Faker
|
from faker import Faker
|
||||||
@@ -257,3 +258,77 @@ def test_suggestion_content() -> None:
|
|||||||
)
|
)
|
||||||
doc.content = content_over_limit
|
doc.content = content_over_limit
|
||||||
assert doc.suggestion_content == expected_cropped_content
|
assert doc.suggestion_content == expected_cropped_content
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
class TestPublicFilenameLength:
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("title", "archive", "extension"),
|
||||||
|
[
|
||||||
|
pytest.param("a" * 300, False, ".jpg", id="ascii-original"),
|
||||||
|
pytest.param("a" * 300, True, ".pdf", id="ascii-archive"),
|
||||||
|
pytest.param("ä" * 200, False, ".jpg", id="multibyte-original"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_long_title_keeps_extension_within_limit(
|
||||||
|
self,
|
||||||
|
title: str,
|
||||||
|
*,
|
||||||
|
archive: bool,
|
||||||
|
extension: str,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A document whose title alone is longer than 255 bytes
|
||||||
|
WHEN:
|
||||||
|
- The public file name is generated
|
||||||
|
THEN:
|
||||||
|
- The name fits in 255 bytes and still ends with the extension
|
||||||
|
"""
|
||||||
|
doc = Document(mime_type="image/jpeg", title=title, created=date(2020, 12, 25))
|
||||||
|
|
||||||
|
name = doc.get_public_filename(archive=archive)
|
||||||
|
|
||||||
|
assert len(name.encode()) <= 255
|
||||||
|
assert name.endswith(extension)
|
||||||
|
|
||||||
|
def test_long_title_counters_give_distinct_names(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A document whose title alone is longer than 255 bytes
|
||||||
|
WHEN:
|
||||||
|
- Public file names are generated with increasing counters
|
||||||
|
THEN:
|
||||||
|
- Each counter produces a distinct name that keeps the counter
|
||||||
|
"""
|
||||||
|
doc = Document(
|
||||||
|
mime_type="application/pdf",
|
||||||
|
title="ä" * 200,
|
||||||
|
created=date(2020, 12, 25),
|
||||||
|
)
|
||||||
|
|
||||||
|
names = [doc.get_public_filename(counter=counter) for counter in range(3)]
|
||||||
|
|
||||||
|
assert len(set(names)) == 3
|
||||||
|
assert names[1].endswith("_01.pdf")
|
||||||
|
assert all(len(name.encode()) <= 255 for name in names)
|
||||||
|
|
||||||
|
def test_max_bytes_bounds_name(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A document with a long title
|
||||||
|
WHEN:
|
||||||
|
- The public file name is generated with a smaller byte limit
|
||||||
|
THEN:
|
||||||
|
- The name fits in that limit and keeps the counter and extension
|
||||||
|
"""
|
||||||
|
doc = Document(
|
||||||
|
mime_type="application/pdf",
|
||||||
|
title="a" * 300,
|
||||||
|
created=date(2020, 12, 25),
|
||||||
|
)
|
||||||
|
|
||||||
|
name = doc.get_public_filename(counter=1, max_bytes=100)
|
||||||
|
|
||||||
|
assert len(name.encode()) <= 100
|
||||||
|
assert name.endswith("_01.pdf")
|
||||||
@@ -1537,3 +1537,67 @@ class TestExportDerivedFileNameCollisions:
|
|||||||
assert (
|
assert (
|
||||||
target / record[EXPORTER_ARCHIVE_NAME]
|
target / record[EXPORTER_ARCHIVE_NAME]
|
||||||
).read_text() == f"archive {pk}"
|
).read_text() == f"archive {pk}"
|
||||||
|
|
||||||
|
def test_long_titles_export_within_file_name_limit(
|
||||||
|
self,
|
||||||
|
paperless_dirs: PaperlessDirs,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Two documents with the same created date and the same title
|
||||||
|
- The title alone is longer than 255 bytes and contains umlauts
|
||||||
|
WHEN:
|
||||||
|
- The exporter runs
|
||||||
|
THEN:
|
||||||
|
- The export finishes with distinct original, thumbnail and archive names
|
||||||
|
- Every exported file name fits in 255 bytes
|
||||||
|
- Each exported file holds its own document's content
|
||||||
|
"""
|
||||||
|
for pk in (1, 2):
|
||||||
|
(paperless_dirs.originals_dir / f"{pk:07}.pdf").write_bytes(
|
||||||
|
f"original {pk}".encode(),
|
||||||
|
)
|
||||||
|
(paperless_dirs.archive_dir / f"{pk:07}.pdf").write_bytes(
|
||||||
|
f"archive {pk}".encode(),
|
||||||
|
)
|
||||||
|
(paperless_dirs.thumbnail_dir / f"{pk:07}.webp").write_bytes(
|
||||||
|
f"thumbnail {pk}".encode(),
|
||||||
|
)
|
||||||
|
DocumentFactory(
|
||||||
|
pk=pk,
|
||||||
|
title="Bestellbestätigung " * 15,
|
||||||
|
created=date(2026, 10, 15),
|
||||||
|
filename=f"{pk:07}.pdf",
|
||||||
|
archive_filename=f"{pk:07}.pdf",
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
|
||||||
|
target = tmp_path / "export"
|
||||||
|
target.mkdir()
|
||||||
|
call_command(
|
||||||
|
"document_exporter",
|
||||||
|
str(target),
|
||||||
|
"--no-progress-bar",
|
||||||
|
skip_checks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest = json.loads((target / "manifest.json").read_text())
|
||||||
|
records = {r["pk"]: r for r in manifest if r["model"] == "documents.document"}
|
||||||
|
|
||||||
|
for key in (
|
||||||
|
EXPORTER_FILE_NAME,
|
||||||
|
EXPORTER_THUMBNAIL_NAME,
|
||||||
|
EXPORTER_ARCHIVE_NAME,
|
||||||
|
):
|
||||||
|
assert records[1][key] != records[2][key]
|
||||||
|
|
||||||
|
for pk in (1, 2):
|
||||||
|
record = records[pk]
|
||||||
|
for key, content in (
|
||||||
|
(EXPORTER_FILE_NAME, f"original {pk}"),
|
||||||
|
(EXPORTER_THUMBNAIL_NAME, f"thumbnail {pk}"),
|
||||||
|
(EXPORTER_ARCHIVE_NAME, f"archive {pk}"),
|
||||||
|
):
|
||||||
|
assert len(Path(record[key]).name.encode()) <= 255
|
||||||
|
assert (target / record[key]).read_text() == content
|
||||||
Reference in new issue
Block a user