mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-11 04:08:00 +00:00
Fix: consolidate and extend unicode NFC normalization for filenames and matching
Consolidates the scattered unicodedata.normalize("NFC", ...) calls introduced
by my earlier filename/path normalization work into a single
documents.utils.normalize_unicode() helper, and closes several gaps where
NFD-normalized filenames could still slip through unnormalized:
- Document.get_public_filename() now normalizes, fixing exported filenames
built from an NFD title/correspondent name (the default, non-format export
path was not covered by the earlier fix).
- DocumentViewSet.update_version() now normalizes the uploaded filename,
matching PostDocumentView's existing behavior.
- ConsumerPlugin normalizes self.filename once at consumption time, covering
the title fallback and Document.original_filename for every document
source (consume folder, mail, API, barcode splits).
- Workflow trigger and mail rule filename/path matching (documents/matching.py,
paperless_mail/mail.py) now normalize the document/attachment side before
comparing, so an NFD filename matches an NFC-typed filter pattern instead of
silently failing to match.
- WorkflowTriggerSerializer and MailRuleSerializer normalize filter_filename/
filter_path and the attachment include/exclude patterns once at write time,
so the read side isn't re-normalizing an already-canonical value on every
match.
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
import unicodedata
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import celery.result
|
||||
import pytest
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.data_models import ConsumableDocument
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def consume_file_mock():
|
||||
with mock.patch("documents.tasks.consume_file.apply_async") as m:
|
||||
m.return_value = celery.result.AsyncResult(id="test-task-id")
|
||||
yield m
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def directories(tmp_path, settings, _media_settings):
|
||||
scratch = tmp_path / "scratch"
|
||||
scratch.mkdir()
|
||||
settings.SCRATCH_DIR = scratch
|
||||
return scratch
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestUpdateVersionNFCNormalization:
|
||||
def test_nfd_filename_normalized_to_nfc(
|
||||
self,
|
||||
admin_client,
|
||||
consume_file_mock: mock.MagicMock,
|
||||
directories,
|
||||
):
|
||||
"""Uploaded new-version file with NFD filename must have its temp name stored as NFC."""
|
||||
document = Document.objects.create(
|
||||
title="Test",
|
||||
content="content",
|
||||
checksum="checksum",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
|
||||
nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
|
||||
|
||||
assert nfd != nfc
|
||||
|
||||
uploaded = SimpleUploadedFile(
|
||||
nfd,
|
||||
b"%PDF-1.4 test",
|
||||
content_type="application/pdf",
|
||||
)
|
||||
response = admin_client.post(
|
||||
f"/api/documents/{document.pk}/update_version/",
|
||||
{"document": uploaded},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]
|
||||
input_doc: ConsumableDocument = task_kwargs["input_doc"]
|
||||
|
||||
assert input_doc.original_file.name == nfc, (
|
||||
f"Expected NFC filename {nfc!r}, got {input_doc.original_file.name!r}"
|
||||
)
|
||||
assert unicodedata.is_normalized("NFC", input_doc.original_file.name)
|
||||
Reference in New Issue
Block a user