From 1f8cf4cd6e055a094205ac350840d4673d2c902c Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:59:51 -0700 Subject: [PATCH] Fix: consolidate and extend unicode NFC normalization for filenames and matching Consolidates the scattered unicodedata.normalize("NFC", ...) calls introduced by my earlier filename/path normalization work into a single documents.utils.normalize_unicode() helper, and closes several gaps where NFD-normalized filenames could still slip through unnormalized: - Document.get_public_filename() now normalizes, fixing exported filenames built from an NFD title/correspondent name (the default, non-format export path was not covered by the earlier fix). - DocumentViewSet.update_version() now normalizes the uploaded filename, matching PostDocumentView's existing behavior. - ConsumerPlugin normalizes self.filename once at consumption time, covering the title fallback and Document.original_filename for every document source (consume folder, mail, API, barcode splits). - Workflow trigger and mail rule filename/path matching (documents/matching.py, paperless_mail/mail.py) now normalize the document/attachment side before comparing, so an NFD filename matches an NFC-typed filter pattern instead of silently failing to match. - WorkflowTriggerSerializer and MailRuleSerializer normalize filter_filename/ filter_path and the attachment include/exclude patterns once at write time, so the read side isn't re-normalizing an already-canonical value on every match. --- src/documents/consumer.py | 5 +- src/documents/matching.py | 16 ++-- src/documents/models.py | 3 +- src/documents/serialisers.py | 8 ++ src/documents/templating/filepath.py | 24 +++--- .../tests/test_api_update_version_nfc.py | 69 ++++++++++++++++ .../tests/test_get_public_filename_nfc.py | 48 +++++++++++ src/documents/tests/test_matching_nfc.py | 80 +++++++++++++++++++ src/documents/tests/test_normalize_unicode.py | 6 ++ src/documents/utils.py | 20 +++++ src/documents/views.py | 4 +- src/paperless_mail/mail.py | 16 ++-- src/paperless_mail/serialisers.py | 7 ++ 13 files changed, 277 insertions(+), 29 deletions(-) create mode 100644 src/documents/tests/test_api_update_version_nfc.py create mode 100644 src/documents/tests/test_get_public_filename_nfc.py create mode 100644 src/documents/tests/test_matching_nfc.py create mode 100644 src/documents/tests/test_normalize_unicode.py diff --git a/src/documents/consumer.py b/src/documents/consumer.py index edb48c348..3fb90c9e8 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -52,6 +52,7 @@ from documents.templating.workflows import parse_w_workflow_placeholders from documents.utils import compute_checksum from documents.utils import copy_basic_file_stats from documents.utils import copy_file_with_basic_stats +from documents.utils import normalize_unicode from documents.utils import run_subprocess from paperless.config import OcrConfig from paperless.config import RemoteOCRConfig @@ -201,7 +202,9 @@ class ConsumerPluginMixin: self.renew_logging_group() - self.filename = self.metadata.filename or self.input_doc.original_file.name + self.filename = normalize_unicode( + self.metadata.filename or self.input_doc.original_file.name, + ) def _send_progress( self, diff --git a/src/documents/matching.py b/src/documents/matching.py index 65f52d567..58e149e45 100644 --- a/src/documents/matching.py +++ b/src/documents/matching.py @@ -21,6 +21,7 @@ from documents.models import Workflow from documents.models import WorkflowTrigger from documents.permissions import permitted_object_ids from documents.regex import safe_regex_search +from documents.utils import normalize_unicode if TYPE_CHECKING: from django.db.models import QuerySet @@ -311,11 +312,12 @@ def consumable_document_matches_workflow( trigger_matched = False # Document filename vs trigger filename + document_filename = normalize_unicode(document.original_file.name) if ( trigger.filter_filename is not None and len(trigger.filter_filename) > 0 and not fnmatch( - document.original_file.name.lower(), + document_filename.lower(), trigger.filter_filename.lower(), ) ): @@ -328,10 +330,12 @@ def consumable_document_matches_workflow( # Document path vs trigger path # Use the original_path if set, else us the original_file - match_against = ( - document.original_path - if document.original_path is not None - else document.original_file + match_against = normalize_unicode( + str( + document.original_path + if document.original_path is not None + else document.original_file, + ), ) if ( @@ -536,7 +540,7 @@ def existing_document_matches_workflow( and len(trigger.filter_filename) > 0 and document.original_filename is not None and not fnmatch( - document.original_filename.lower(), + normalize_unicode(document.original_filename).lower(), trigger.filter_filename.lower(), ) ): diff --git a/src/documents/models.py b/src/documents/models.py index 0cc4decda..7b4bb37cc 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -27,6 +27,7 @@ from django_softdelete.models import SoftDeleteModel from documents.data_models import DocumentSource from documents.parsers import get_default_file_extension +from documents.utils import normalize_unicode class ModelWithOwner(models.Model): @@ -467,7 +468,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager- context_document = ( self.root_document if self.root_document_id is not None else self ) - result = str(context_document) + result = normalize_unicode(str(context_document)) if counter: result += f"_{counter:02}" diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 94504fcd0..9a0782d7e 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -87,6 +87,7 @@ from documents.regex import validate_regex_pattern from documents.templating.filepath import validate_filepath_template_and_render from documents.templating.utils import convert_format_str_to_template_format from documents.templating.workflows import validate_workflow_template +from documents.utils import normalize_unicode from documents.validators import uri_validator from documents.validators import url_validator from documents.versioning import sort_versions_newest_first @@ -3120,6 +3121,13 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer[WorkflowTrigger]): ): attrs["filter_path"] = None + # Normalize once at write time, since these are matched against many + # documents but edited rarely + if attrs.get("filter_filename") is not None: + attrs["filter_filename"] = normalize_unicode(attrs["filter_filename"]) + if attrs.get("filter_path") is not None: + attrs["filter_path"] = normalize_unicode(attrs["filter_path"]) + if ( "filter_custom_field_query" in attrs and attrs["filter_custom_field_query"] is not None diff --git a/src/documents/templating/filepath.py b/src/documents/templating/filepath.py index a4de6b274..53c026d57 100644 --- a/src/documents/templating/filepath.py +++ b/src/documents/templating/filepath.py @@ -1,7 +1,6 @@ import logging import os import re -import unicodedata from collections.abc import Iterable from pathlib import PurePath @@ -26,6 +25,7 @@ from documents.templating.environment import _template_environment from documents.templating.filters import format_datetime from documents.templating.filters import get_cf_value from documents.templating.filters import localize_date +from documents.utils import normalize_unicode logger = logging.getLogger("paperless.templating") @@ -42,7 +42,7 @@ class FilePathTemplate(Template): 3. Removing extra spaces before and after forward slashes 4. Preserving spaces in other parts of the path """ - value = unicodedata.normalize("NFC", value) + value = normalize_unicode(value) value = value.replace("\n", "").replace("\r", "") value = re.sub(r"\s*/\s*", "/", value) @@ -184,17 +184,17 @@ def get_basic_metadata_context( """ return { "title": pathvalidate.sanitize_filename( - unicodedata.normalize("NFC", document.title), + normalize_unicode(document.title), replacement_text="-", ), "correspondent": pathvalidate.sanitize_filename( - unicodedata.normalize("NFC", document.correspondent.name), + normalize_unicode(document.correspondent.name), replacement_text="-", ) if document.correspondent else no_value_default, "document_type": pathvalidate.sanitize_filename( - unicodedata.normalize("NFC", document.document_type.name), + normalize_unicode(document.document_type.name), replacement_text="-", ) if document.document_type @@ -205,8 +205,7 @@ def get_basic_metadata_context( "owner_username": document.owner.username if document.owner else no_value_default, - "original_name": unicodedata.normalize( - "NFC", + "original_name": normalize_unicode( PurePath(document.original_filename).with_suffix("").name, ) if document.original_filename @@ -275,12 +274,12 @@ def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]: return { "tag_list": pathvalidate.sanitize_filename( ",".join( - sorted(unicodedata.normalize("NFC", tag.name) for tag in tags), + sorted(normalize_unicode(tag.name) for tag in tags), ), replacement_text="-", ), # Assumed to be ordered, but a template could loop through to find what they want - "tag_name_list": [unicodedata.normalize("NFC", x.name) for x in tags], + "tag_name_list": [normalize_unicode(x.name) for x in tags], } @@ -307,7 +306,7 @@ def get_custom_fields_context( CustomField.FieldDataType.LONG_TEXT, }: value = pathvalidate.sanitize_filename( - unicodedata.normalize("NFC", field_instance.value), + normalize_unicode(field_instance.value), replacement_text="-", ) elif ( @@ -316,8 +315,7 @@ def get_custom_fields_context( ): options = field_instance.field.extra_data["select_options"] value = pathvalidate.sanitize_filename( - unicodedata.normalize( - "NFC", + normalize_unicode( next( option["label"] for option in options @@ -330,7 +328,7 @@ def get_custom_fields_context( value = field_instance.value field_data["custom_fields"][ pathvalidate.sanitize_filename( - unicodedata.normalize("NFC", field_instance.field.name), + normalize_unicode(field_instance.field.name), replacement_text="-", ) ] = { diff --git a/src/documents/tests/test_api_update_version_nfc.py b/src/documents/tests/test_api_update_version_nfc.py new file mode 100644 index 000000000..709c5df5f --- /dev/null +++ b/src/documents/tests/test_api_update_version_nfc.py @@ -0,0 +1,69 @@ +import unicodedata +from typing import TYPE_CHECKING +from unittest import mock + +import celery.result +import pytest +from django.core.files.uploadedfile import SimpleUploadedFile + +from documents.models import Document + +if TYPE_CHECKING: + from documents.data_models import ConsumableDocument + + +@pytest.fixture() +def consume_file_mock(): + with mock.patch("documents.tasks.consume_file.apply_async") as m: + m.return_value = celery.result.AsyncResult(id="test-task-id") + yield m + + +@pytest.fixture() +def directories(tmp_path, settings, _media_settings): + scratch = tmp_path / "scratch" + scratch.mkdir() + settings.SCRATCH_DIR = scratch + return scratch + + +@pytest.mark.django_db +class TestUpdateVersionNFCNormalization: + def test_nfd_filename_normalized_to_nfc( + self, + admin_client, + consume_file_mock: mock.MagicMock, + directories, + ): + """Uploaded new-version file with NFD filename must have its temp name stored as NFC.""" + document = Document.objects.create( + title="Test", + content="content", + checksum="checksum", + mime_type="application/pdf", + ) + + nfd = unicodedata.normalize("NFD", "Rechnung März.pdf") + nfc = unicodedata.normalize("NFC", "Rechnung März.pdf") + + assert nfd != nfc + + uploaded = SimpleUploadedFile( + nfd, + b"%PDF-1.4 test", + content_type="application/pdf", + ) + response = admin_client.post( + f"/api/documents/{document.pk}/update_version/", + {"document": uploaded}, + ) + + assert response.status_code == 200 + + task_kwargs = consume_file_mock.call_args.kwargs["kwargs"] + input_doc: ConsumableDocument = task_kwargs["input_doc"] + + assert input_doc.original_file.name == nfc, ( + f"Expected NFC filename {nfc!r}, got {input_doc.original_file.name!r}" + ) + assert unicodedata.is_normalized("NFC", input_doc.original_file.name) diff --git a/src/documents/tests/test_get_public_filename_nfc.py b/src/documents/tests/test_get_public_filename_nfc.py new file mode 100644 index 000000000..7b7a116c7 --- /dev/null +++ b/src/documents/tests/test_get_public_filename_nfc.py @@ -0,0 +1,48 @@ +import unicodedata +from datetime import date + +import pytest + +from documents.models import Correspondent +from documents.models import Document + + +@pytest.mark.django_db +class TestGetPublicFilenameNfc: + def test_normalizes_nfd_title_to_nfc(self) -> None: + nfd_title = unicodedata.normalize("NFD", "Gehaltserhöhung") + assert not unicodedata.is_normalized("NFC", nfd_title) + + doc = Document( + mime_type="application/pdf", + title=nfd_title, + created=date(2025, 10, 17), + ) + + result = doc.get_public_filename() + + assert unicodedata.is_normalized("NFC", result) + assert ( + result + == "2025-10-17 " + + unicodedata.normalize( + "NFC", + nfd_title, + ) + + ".pdf" + ) + + def test_normalizes_nfd_correspondent_name_to_nfc(self) -> None: + nfd_name = unicodedata.normalize("NFD", "Müller GmbH") + correspondent = Correspondent.objects.create(name=nfd_name) + + doc = Document.objects.create( + mime_type="application/pdf", + title="Rechnung", + created=date(2025, 10, 17), + correspondent=correspondent, + ) + + result = doc.get_public_filename() + + assert unicodedata.is_normalized("NFC", result) diff --git a/src/documents/tests/test_matching_nfc.py b/src/documents/tests/test_matching_nfc.py new file mode 100644 index 000000000..22672f011 --- /dev/null +++ b/src/documents/tests/test_matching_nfc.py @@ -0,0 +1,80 @@ +import unicodedata + +import pytest + +from documents.data_models import ConsumableDocument +from documents.data_models import DocumentSource +from documents.matching import consumable_document_matches_workflow +from documents.matching import existing_document_matches_workflow +from documents.models import Document +from documents.models import Workflow +from documents.models import WorkflowTrigger + + +@pytest.mark.django_db +class TestMatchingNfcNormalization: + def test_consumable_document_filename_nfd_matches_nfc_pattern( + self, + tmp_path, + ) -> None: + """ + GIVEN: + - A file on disk whose name is NFD-normalized + - A workflow trigger filename filter typed as NFC + WHEN: + - The consumable document is checked against the trigger + THEN: + - It matches, because both sides are normalized before comparing + """ + nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf") + nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*") + assert nfd_name != unicodedata.normalize("NFC", nfd_name) + + file_path = tmp_path / nfd_name + file_path.write_bytes(b"%PDF-1.4 test") + + document = ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=file_path, + ) + trigger = WorkflowTrigger( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + filter_filename=nfc_pattern, + sources=[], + ) + + matched, reason = consumable_document_matches_workflow(document, trigger) + + assert matched, reason + + def test_existing_document_filename_nfd_matches_nfc_pattern(self) -> None: + """ + GIVEN: + - A Document whose original_filename is NFD-normalized (e.g. from + before normalization was applied at consumption time) + - A workflow trigger filename filter typed as NFC + WHEN: + - The document is checked against the trigger + THEN: + - It matches, because both sides are normalized before comparing + """ + nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf") + nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*") + + document = Document.objects.create( + title="Test", + content="content", + checksum="checksum", + mime_type="application/pdf", + original_filename=nfd_name, + ) + workflow = Workflow.objects.create(name="Test workflow", order=0) + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_filename=nfc_pattern, + ) + workflow.triggers.add(trigger) + + matched, reason = existing_document_matches_workflow(document, trigger) + + assert matched, reason diff --git a/src/documents/tests/test_normalize_unicode.py b/src/documents/tests/test_normalize_unicode.py new file mode 100644 index 000000000..7e3e9ead3 --- /dev/null +++ b/src/documents/tests/test_normalize_unicode.py @@ -0,0 +1,6 @@ +from documents.utils import normalize_unicode + + +class TestNormalizeUnicode: + def test_none_passes_through(self) -> None: + assert normalize_unicode(None) is None diff --git a/src/documents/utils.py b/src/documents/utils.py index d6e505aa4..f5c4002e0 100644 --- a/src/documents/utils.py +++ b/src/documents/utils.py @@ -1,6 +1,7 @@ import hashlib import logging import shutil +import unicodedata from collections.abc import Callable from collections.abc import Iterable from collections.abc import Iterator @@ -31,6 +32,25 @@ def identity(iterable: Iterable[_T]) -> Iterable[_T]: return iterable +def normalize_unicode(value: str | None) -> str | None: + """ + Normalize a string to Unicode NFC form, or return None unchanged. + + This is the single normalization pass for any user- or filesystem-supplied + text that ends up in a filename, path, or is compared/matched against one + (titles, correspondent/tag/type names, uploaded filenames, workflow and + mail rule filename/path filters). Composed (NFC) and decomposed (NFD) + forms of the same visible text are different byte sequences, which breaks + exact comparisons and filesystem lookups even though the text looks + identical. Always normalize through this function rather than calling + unicodedata.normalize() directly, so every call site agrees on the same + form. + """ + if value is None: + return None + return unicodedata.normalize("NFC", value) + + class QuerySetStream(Generic[_M]): """Stream a QuerySet via .iterator(chunk_size=...) instead of materializing it (plus any prefetch caches) all at once, while still diff --git a/src/documents/views.py b/src/documents/views.py index a65c048e3..dc010f8d9 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -231,6 +231,7 @@ from documents.tasks import sanity_check from documents.tasks import train_classifier from documents.tasks import update_document_parent_tags from documents.utils import get_boolean +from documents.utils import normalize_unicode from documents.versioning import VersionResolutionError from documents.versioning import annotate_effective_content from documents.versioning import get_latest_version_for_root @@ -2068,6 +2069,7 @@ class DocumentViewSet( try: doc_name, doc_data = serializer.validated_data.get("document") + doc_name = normalize_unicode(doc_name) version_label = serializer.validated_data.get("version_label") t = int(mktime(datetime.now().timetuple())) @@ -3334,7 +3336,7 @@ class PostDocumentView(GenericAPIView[Any]): serializer.is_valid(raise_exception=True) doc_name, doc_data = serializer.validated_data.get("document") - doc_name = normalize("NFC", doc_name) + doc_name = normalize_unicode(doc_name) correspondent_id = serializer.validated_data.get("correspondent") document_type_id = serializer.validated_data.get("document_type") storage_path_id = serializer.validated_data.get("storage_path") diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 8e9170a6b..6e9dc21b4 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -6,7 +6,6 @@ import socket import ssl import tempfile import traceback -import unicodedata from datetime import date from datetime import timedelta from fnmatch import fnmatch @@ -45,6 +44,7 @@ from documents.models import Correspondent from documents.models import PaperlessTask from documents.parsers import is_mime_type_supported from documents.tasks import consume_file +from documents.utils import normalize_unicode from paperless.network import is_public_ip from paperless.network import resolve_hostname_ips from paperless_mail.models import MailAccount @@ -617,10 +617,10 @@ class MailAccountHandler(LoggingMixin): rule: MailRule, ) -> str | None: if rule.assign_title_from == MailRule.TitleSource.FROM_SUBJECT: - return unicodedata.normalize("NFC", message.subject) + return normalize_unicode(message.subject) elif rule.assign_title_from == MailRule.TitleSource.FROM_FILENAME: - return unicodedata.normalize("NFC", Path(att.filename).stem) + return normalize_unicode(Path(att.filename).stem) elif rule.assign_title_from == MailRule.TitleSource.NONE: return None @@ -1004,6 +1004,8 @@ class MailAccountHandler(LoggingMixin): consume_tasks = [] for att in message.attachments: + attachment_filename = normalize_unicode(att.filename) + if ( att.content_disposition != "attachment" and rule.attachment_type @@ -1018,7 +1020,7 @@ class MailAccountHandler(LoggingMixin): if not self.filename_inclusion_matches( rule.filter_attachment_filename_include, - att.filename, + attachment_filename, ): # Force the filename and pattern to the lowercase # as this is system dependent otherwise @@ -1030,7 +1032,7 @@ class MailAccountHandler(LoggingMixin): continue elif self.filename_exclusion_matches( rule.filter_attachment_filename_exclude, - att.filename, + attachment_filename, ): self.log.debug( f"Rule {rule}: " @@ -1064,7 +1066,7 @@ class MailAccountHandler(LoggingMixin): ) attachment_name = pathvalidate.sanitize_filename( - unicodedata.normalize("NFC", att.filename), + attachment_filename, ) if attachment_name: temp_filename = temp_dir / attachment_name @@ -1175,7 +1177,7 @@ class MailAccountHandler(LoggingMixin): doc_overrides = DocumentMetadataOverrides( title=message.subject, filename=pathvalidate.sanitize_filename( - unicodedata.normalize("NFC", f"{message.subject}.eml"), + normalize_unicode(f"{message.subject}.eml"), ), correspondent_id=correspondent.id if correspondent else None, document_type_id=doc_type.id if doc_type else None, diff --git a/src/paperless_mail/serialisers.py b/src/paperless_mail/serialisers.py index d78c220aa..4b05d7c2e 100644 --- a/src/paperless_mail/serialisers.py +++ b/src/paperless_mail/serialisers.py @@ -8,6 +8,7 @@ from documents.serialisers import CorrespondentField from documents.serialisers import DocumentTypeField from documents.serialisers import OwnedObjectSerializer from documents.serialisers import TagsField +from documents.utils import normalize_unicode from paperless_mail.models import MailAccount from paperless_mail.models import MailRule from paperless_mail.models import ProcessedMail @@ -161,6 +162,12 @@ class MailRuleSerializer(OwnedObjectSerializer): raise serializers.ValidationError("Maximum mail age is unreasonably large.") return value + def validate_filter_attachment_filename_include(self, value): + return normalize_unicode(value) + + def validate_filter_attachment_filename_exclude(self, value): + return normalize_unicode(value) + class ProcessedMailSerializer(OwnedObjectSerializer): class Meta: