mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-09 19:27:59 +00:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
01c120a3cb | ||
|
|
aff0f9cf41 | ||
|
|
bf716ebfd1 | ||
|
|
7d67a10a35 | ||
|
|
8d1bc5dd24 | ||
|
|
43a8d7d412 | ||
|
|
c40922440b |
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import tempfile
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Literal
|
||||
@@ -379,7 +380,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
|
||||
)
|
||||
delete_ids = list({*doc_ids, *version_ids})
|
||||
|
||||
Document.objects.filter(id__in=delete_ids).delete()
|
||||
Document.objects.filter(id__in=delete_ids).delete(transaction_id=uuid.uuid4())
|
||||
|
||||
from documents.search import get_backend
|
||||
|
||||
|
||||
@@ -52,7 +52,6 @@ from documents.templating.workflows import parse_w_workflow_placeholders
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import copy_basic_file_stats
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.utils import normalize_unicode
|
||||
from documents.utils import run_subprocess
|
||||
from paperless.config import OcrConfig
|
||||
from paperless.config import RemoteOCRConfig
|
||||
@@ -202,9 +201,7 @@ class ConsumerPluginMixin:
|
||||
|
||||
self.renew_logging_group()
|
||||
|
||||
self.filename = normalize_unicode(
|
||||
self.metadata.filename or self.input_doc.original_file.name,
|
||||
)
|
||||
self.filename = self.metadata.filename or self.input_doc.original_file.name
|
||||
|
||||
def _send_progress(
|
||||
self,
|
||||
|
||||
@@ -21,7 +21,6 @@ from documents.models import Workflow
|
||||
from documents.models import WorkflowTrigger
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.regex import safe_regex_search
|
||||
from documents.utils import normalize_unicode
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.db.models import QuerySet
|
||||
@@ -312,12 +311,11 @@ def consumable_document_matches_workflow(
|
||||
trigger_matched = False
|
||||
|
||||
# Document filename vs trigger filename
|
||||
document_filename = normalize_unicode(document.original_file.name)
|
||||
if (
|
||||
trigger.filter_filename is not None
|
||||
and len(trigger.filter_filename) > 0
|
||||
and not fnmatch(
|
||||
document_filename.lower(),
|
||||
document.original_file.name.lower(),
|
||||
trigger.filter_filename.lower(),
|
||||
)
|
||||
):
|
||||
@@ -330,12 +328,10 @@ def consumable_document_matches_workflow(
|
||||
# Document path vs trigger path
|
||||
|
||||
# Use the original_path if set, else us the original_file
|
||||
match_against = normalize_unicode(
|
||||
str(
|
||||
document.original_path
|
||||
if document.original_path is not None
|
||||
else document.original_file,
|
||||
),
|
||||
match_against = (
|
||||
document.original_path
|
||||
if document.original_path is not None
|
||||
else document.original_file
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -540,7 +536,7 @@ def existing_document_matches_workflow(
|
||||
and len(trigger.filter_filename) > 0
|
||||
and document.original_filename is not None
|
||||
and not fnmatch(
|
||||
normalize_unicode(document.original_filename).lower(),
|
||||
document.original_filename.lower(),
|
||||
trigger.filter_filename.lower(),
|
||||
)
|
||||
):
|
||||
|
||||
+11
-4
@@ -1,4 +1,5 @@
|
||||
import datetime
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
@@ -27,7 +28,6 @@ from django_softdelete.models import SoftDeleteModel
|
||||
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.parsers import get_default_file_extension
|
||||
from documents.utils import normalize_unicode
|
||||
|
||||
|
||||
class ModelWithOwner(models.Model):
|
||||
@@ -468,7 +468,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
||||
context_document = (
|
||||
self.root_document if self.root_document_id is not None else self
|
||||
)
|
||||
result = normalize_unicode(str(context_document))
|
||||
result = str(context_document)
|
||||
|
||||
if counter:
|
||||
result += f"_{counter:02}"
|
||||
@@ -515,13 +515,20 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
|
||||
def delete(
|
||||
self,
|
||||
*args,
|
||||
transaction_id=None,
|
||||
**kwargs,
|
||||
):
|
||||
# If deleting a root document, move all its versions to trash as well.
|
||||
# Versions must share the root's transaction ID so they are restored
|
||||
# together by django-softdelete.
|
||||
if transaction_id is None:
|
||||
transaction_id = uuid.uuid4()
|
||||
if self.root_document_id is None:
|
||||
Document.objects.filter(root_document=self).delete()
|
||||
Document.objects.filter(root_document=self).delete(
|
||||
transaction_id=transaction_id,
|
||||
)
|
||||
return super().delete(
|
||||
*args,
|
||||
transaction_id=transaction_id,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -87,7 +87,6 @@ from documents.regex import validate_regex_pattern
|
||||
from documents.templating.filepath import validate_filepath_template_and_render
|
||||
from documents.templating.utils import convert_format_str_to_template_format
|
||||
from documents.templating.workflows import validate_workflow_template
|
||||
from documents.utils import normalize_unicode
|
||||
from documents.validators import uri_validator
|
||||
from documents.validators import url_validator
|
||||
from documents.versioning import sort_versions_newest_first
|
||||
@@ -3121,13 +3120,6 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer[WorkflowTrigger]):
|
||||
):
|
||||
attrs["filter_path"] = None
|
||||
|
||||
# Normalize once at write time, since these are matched against many
|
||||
# documents but edited rarely
|
||||
if attrs.get("filter_filename") is not None:
|
||||
attrs["filter_filename"] = normalize_unicode(attrs["filter_filename"])
|
||||
if attrs.get("filter_path") is not None:
|
||||
attrs["filter_path"] = normalize_unicode(attrs["filter_path"])
|
||||
|
||||
if (
|
||||
"filter_custom_field_query" in attrs
|
||||
and attrs["filter_custom_field_query"] is not None
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from collections.abc import Iterable
|
||||
from pathlib import PurePath
|
||||
|
||||
@@ -25,7 +26,6 @@ from documents.templating.environment import _template_environment
|
||||
from documents.templating.filters import format_datetime
|
||||
from documents.templating.filters import get_cf_value
|
||||
from documents.templating.filters import localize_date
|
||||
from documents.utils import normalize_unicode
|
||||
|
||||
logger = logging.getLogger("paperless.templating")
|
||||
|
||||
@@ -42,7 +42,7 @@ class FilePathTemplate(Template):
|
||||
3. Removing extra spaces before and after forward slashes
|
||||
4. Preserving spaces in other parts of the path
|
||||
"""
|
||||
value = normalize_unicode(value)
|
||||
value = unicodedata.normalize("NFC", value)
|
||||
value = value.replace("\n", "").replace("\r", "")
|
||||
value = re.sub(r"\s*/\s*", "/", value)
|
||||
|
||||
@@ -184,17 +184,17 @@ def get_basic_metadata_context(
|
||||
"""
|
||||
return {
|
||||
"title": pathvalidate.sanitize_filename(
|
||||
normalize_unicode(document.title),
|
||||
unicodedata.normalize("NFC", document.title),
|
||||
replacement_text="-",
|
||||
),
|
||||
"correspondent": pathvalidate.sanitize_filename(
|
||||
normalize_unicode(document.correspondent.name),
|
||||
unicodedata.normalize("NFC", document.correspondent.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
if document.correspondent
|
||||
else no_value_default,
|
||||
"document_type": pathvalidate.sanitize_filename(
|
||||
normalize_unicode(document.document_type.name),
|
||||
unicodedata.normalize("NFC", document.document_type.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
if document.document_type
|
||||
@@ -205,7 +205,8 @@ def get_basic_metadata_context(
|
||||
"owner_username": document.owner.username
|
||||
if document.owner
|
||||
else no_value_default,
|
||||
"original_name": normalize_unicode(
|
||||
"original_name": unicodedata.normalize(
|
||||
"NFC",
|
||||
PurePath(document.original_filename).with_suffix("").name,
|
||||
)
|
||||
if document.original_filename
|
||||
@@ -274,12 +275,12 @@ def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]:
|
||||
return {
|
||||
"tag_list": pathvalidate.sanitize_filename(
|
||||
",".join(
|
||||
sorted(normalize_unicode(tag.name) for tag in tags),
|
||||
sorted(unicodedata.normalize("NFC", tag.name) for tag in tags),
|
||||
),
|
||||
replacement_text="-",
|
||||
),
|
||||
# Assumed to be ordered, but a template could loop through to find what they want
|
||||
"tag_name_list": [normalize_unicode(x.name) for x in tags],
|
||||
"tag_name_list": [unicodedata.normalize("NFC", x.name) for x in tags],
|
||||
}
|
||||
|
||||
|
||||
@@ -306,7 +307,7 @@ def get_custom_fields_context(
|
||||
CustomField.FieldDataType.LONG_TEXT,
|
||||
}:
|
||||
value = pathvalidate.sanitize_filename(
|
||||
normalize_unicode(field_instance.value),
|
||||
unicodedata.normalize("NFC", field_instance.value),
|
||||
replacement_text="-",
|
||||
)
|
||||
elif (
|
||||
@@ -315,7 +316,8 @@ def get_custom_fields_context(
|
||||
):
|
||||
options = field_instance.field.extra_data["select_options"]
|
||||
value = pathvalidate.sanitize_filename(
|
||||
normalize_unicode(
|
||||
unicodedata.normalize(
|
||||
"NFC",
|
||||
next(
|
||||
option["label"]
|
||||
for option in options
|
||||
@@ -328,7 +330,7 @@ def get_custom_fields_context(
|
||||
value = field_instance.value
|
||||
field_data["custom_fields"][
|
||||
pathvalidate.sanitize_filename(
|
||||
normalize_unicode(field_instance.field.name),
|
||||
unicodedata.normalize("NFC", field_instance.field.name),
|
||||
replacement_text="-",
|
||||
)
|
||||
] = {
|
||||
|
||||
@@ -207,3 +207,65 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
|
||||
)
|
||||
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn("have not yet been deleted", resp.data["documents"][0])
|
||||
|
||||
def _make_versioned_document(self) -> tuple[Document, list[Document]]:
|
||||
root = Document.objects.create(
|
||||
title="root",
|
||||
content="root-content",
|
||||
checksum="root",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
versions = [
|
||||
Document.objects.create(
|
||||
title=f"v{index}",
|
||||
content=f"v{index}-content",
|
||||
checksum=f"v{index}",
|
||||
mime_type="application/pdf",
|
||||
root_document=root,
|
||||
version_index=index,
|
||||
)
|
||||
for index in range(1, 3)
|
||||
]
|
||||
return root, versions
|
||||
|
||||
def test_api_trash_restore_document_restores_its_versions(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Existing document with two versions
|
||||
WHEN:
|
||||
- API request to delete the document
|
||||
- API request to restore it from the trash
|
||||
THEN:
|
||||
- Only the document itself is listed in the trash
|
||||
- A version cannot be restored without its root
|
||||
- The document is restored together with all of its versions
|
||||
"""
|
||||
root, versions = self._make_versioned_document()
|
||||
|
||||
self.client.force_login(user=self.user)
|
||||
self.client.delete(f"/api/documents/{root.pk}/")
|
||||
self.assertEqual(Document.deleted_objects.count(), 3)
|
||||
|
||||
resp = self.client.get("/api/trash/")
|
||||
self.assertEqual(resp.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(resp.data["count"], 1)
|
||||
self.assertEqual(resp.data["results"][0]["id"], root.pk)
|
||||
|
||||
# A version cannot be restored while its root remains in the trash.
|
||||
resp = self.client.post(
|
||||
"/api/trash/",
|
||||
{"action": "restore", "documents": [versions[0].pk]},
|
||||
)
|
||||
self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn("Restore the root document", resp.data["documents"][0])
|
||||
|
||||
resp = self.client.post(
|
||||
"/api/trash/",
|
||||
{"action": "restore", "documents": [root.pk]},
|
||||
)
|
||||
self.assertEqual(resp.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(Document.deleted_objects.count(), 0)
|
||||
self.assertCountEqual(
|
||||
Document.objects.filter(root_document=root).values_list("id", flat=True),
|
||||
[version.pk for version in versions],
|
||||
)
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
import unicodedata
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import celery.result
|
||||
import pytest
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.data_models import ConsumableDocument
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def consume_file_mock():
|
||||
with mock.patch("documents.tasks.consume_file.apply_async") as m:
|
||||
m.return_value = celery.result.AsyncResult(id="test-task-id")
|
||||
yield m
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def directories(tmp_path, settings, _media_settings):
|
||||
scratch = tmp_path / "scratch"
|
||||
scratch.mkdir()
|
||||
settings.SCRATCH_DIR = scratch
|
||||
return scratch
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestUpdateVersionNFCNormalization:
|
||||
def test_nfd_filename_normalized_to_nfc(
|
||||
self,
|
||||
admin_client,
|
||||
consume_file_mock: mock.MagicMock,
|
||||
directories,
|
||||
):
|
||||
"""Uploaded new-version file with NFD filename must have its temp name stored as NFC."""
|
||||
document = Document.objects.create(
|
||||
title="Test",
|
||||
content="content",
|
||||
checksum="checksum",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
|
||||
nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
|
||||
|
||||
assert nfd != nfc
|
||||
|
||||
uploaded = SimpleUploadedFile(
|
||||
nfd,
|
||||
b"%PDF-1.4 test",
|
||||
content_type="application/pdf",
|
||||
)
|
||||
response = admin_client.post(
|
||||
f"/api/documents/{document.pk}/update_version/",
|
||||
{"document": uploaded},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]
|
||||
input_doc: ConsumableDocument = task_kwargs["input_doc"]
|
||||
|
||||
assert input_doc.original_file.name == nfc, (
|
||||
f"Expected NFC filename {nfc!r}, got {input_doc.original_file.name!r}"
|
||||
)
|
||||
assert unicodedata.is_normalized("NFC", input_doc.original_file.name)
|
||||
@@ -392,6 +392,11 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
|
||||
self.assertFalse(Document.objects.filter(id=self.doc1.id).exists())
|
||||
self.assertFalse(Document.objects.filter(id=version.id).exists())
|
||||
|
||||
Document.deleted_objects.get(id=self.doc1.id).restore(strict=False)
|
||||
|
||||
self.assertTrue(Document.objects.filter(id=self.doc1.id).exists())
|
||||
self.assertTrue(Document.objects.filter(id=version.id).exists())
|
||||
|
||||
def test_delete_version_document_keeps_root(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="A-v1",
|
||||
|
||||
@@ -110,7 +110,7 @@ class TestDocument(TestCase):
|
||||
checksum="checksum",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
Document.objects.create(
|
||||
version = Document.objects.create(
|
||||
root_document=root,
|
||||
correspondent=root.correspondent,
|
||||
title="Version",
|
||||
@@ -124,6 +124,10 @@ class TestDocument(TestCase):
|
||||
self.assertEqual(Document.objects.count(), 0)
|
||||
self.assertEqual(Document.deleted_objects.count(), 2)
|
||||
|
||||
root.restore(strict=False)
|
||||
|
||||
self.assertTrue(Document.objects.filter(pk=version.pk).exists())
|
||||
|
||||
def test_file_name(self) -> None:
|
||||
doc = Document(
|
||||
mime_type="application/pdf",
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
import unicodedata
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestGetPublicFilenameNfc:
|
||||
def test_normalizes_nfd_title_to_nfc(self) -> None:
|
||||
nfd_title = unicodedata.normalize("NFD", "Gehaltserhöhung")
|
||||
assert not unicodedata.is_normalized("NFC", nfd_title)
|
||||
|
||||
doc = Document(
|
||||
mime_type="application/pdf",
|
||||
title=nfd_title,
|
||||
created=date(2025, 10, 17),
|
||||
)
|
||||
|
||||
result = doc.get_public_filename()
|
||||
|
||||
assert unicodedata.is_normalized("NFC", result)
|
||||
assert (
|
||||
result
|
||||
== "2025-10-17 "
|
||||
+ unicodedata.normalize(
|
||||
"NFC",
|
||||
nfd_title,
|
||||
)
|
||||
+ ".pdf"
|
||||
)
|
||||
|
||||
def test_normalizes_nfd_correspondent_name_to_nfc(self) -> None:
|
||||
nfd_name = unicodedata.normalize("NFD", "Müller GmbH")
|
||||
correspondent = Correspondent.objects.create(name=nfd_name)
|
||||
|
||||
doc = Document.objects.create(
|
||||
mime_type="application/pdf",
|
||||
title="Rechnung",
|
||||
created=date(2025, 10, 17),
|
||||
correspondent=correspondent,
|
||||
)
|
||||
|
||||
result = doc.get_public_filename()
|
||||
|
||||
assert unicodedata.is_normalized("NFC", result)
|
||||
@@ -1,80 +0,0 @@
|
||||
import unicodedata
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.data_models import ConsumableDocument
|
||||
from documents.data_models import DocumentSource
|
||||
from documents.matching import consumable_document_matches_workflow
|
||||
from documents.matching import existing_document_matches_workflow
|
||||
from documents.models import Document
|
||||
from documents.models import Workflow
|
||||
from documents.models import WorkflowTrigger
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestMatchingNfcNormalization:
|
||||
def test_consumable_document_filename_nfd_matches_nfc_pattern(
|
||||
self,
|
||||
tmp_path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A file on disk whose name is NFD-normalized
|
||||
- A workflow trigger filename filter typed as NFC
|
||||
WHEN:
|
||||
- The consumable document is checked against the trigger
|
||||
THEN:
|
||||
- It matches, because both sides are normalized before comparing
|
||||
"""
|
||||
nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf")
|
||||
nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*")
|
||||
assert nfd_name != unicodedata.normalize("NFC", nfd_name)
|
||||
|
||||
file_path = tmp_path / nfd_name
|
||||
file_path.write_bytes(b"%PDF-1.4 test")
|
||||
|
||||
document = ConsumableDocument(
|
||||
source=DocumentSource.ConsumeFolder,
|
||||
original_file=file_path,
|
||||
)
|
||||
trigger = WorkflowTrigger(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
|
||||
filter_filename=nfc_pattern,
|
||||
sources=[],
|
||||
)
|
||||
|
||||
matched, reason = consumable_document_matches_workflow(document, trigger)
|
||||
|
||||
assert matched, reason
|
||||
|
||||
def test_existing_document_filename_nfd_matches_nfc_pattern(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A Document whose original_filename is NFD-normalized (e.g. from
|
||||
before normalization was applied at consumption time)
|
||||
- A workflow trigger filename filter typed as NFC
|
||||
WHEN:
|
||||
- The document is checked against the trigger
|
||||
THEN:
|
||||
- It matches, because both sides are normalized before comparing
|
||||
"""
|
||||
nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf")
|
||||
nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*")
|
||||
|
||||
document = Document.objects.create(
|
||||
title="Test",
|
||||
content="content",
|
||||
checksum="checksum",
|
||||
mime_type="application/pdf",
|
||||
original_filename=nfd_name,
|
||||
)
|
||||
workflow = Workflow.objects.create(name="Test workflow", order=0)
|
||||
trigger = WorkflowTrigger.objects.create(
|
||||
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
|
||||
filter_filename=nfc_pattern,
|
||||
)
|
||||
workflow.triggers.add(trigger)
|
||||
|
||||
matched, reason = existing_document_matches_workflow(document, trigger)
|
||||
|
||||
assert matched, reason
|
||||
@@ -1,6 +0,0 @@
|
||||
from documents.utils import normalize_unicode
|
||||
|
||||
|
||||
class TestNormalizeUnicode:
|
||||
def test_none_passes_through(self) -> None:
|
||||
assert normalize_unicode(None) is None
|
||||
@@ -32,6 +32,7 @@ from documents.signals.handlers import update_llm_suggestions_cache
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
from documents.tests.utils import read_streaming_response
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless_ai.exceptions import LLMProviderError
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
|
||||
|
||||
@@ -737,6 +738,38 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="openai-like",
|
||||
)
|
||||
def test_ai_suggestions_with_llm_provider_error(
|
||||
self,
|
||||
mock_get_ai_classification,
|
||||
) -> None:
|
||||
mock_get_ai_classification.side_effect = LLMProviderError(
|
||||
"confidential provider response",
|
||||
)
|
||||
|
||||
self.client.force_login(user=self.user)
|
||||
response = self.client.get(
|
||||
f"/api/documents/{self.document.pk}/ai_suggestions/",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY)
|
||||
self.assertEqual(
|
||||
response.json(),
|
||||
{
|
||||
"ai": [
|
||||
"AI backend rejected the request. Check logs for details.",
|
||||
],
|
||||
},
|
||||
)
|
||||
self.assertNotIn("confidential provider response", response.content.decode())
|
||||
self.assertIsNone(
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import shutil
|
||||
import unicodedata
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterator
|
||||
@@ -32,25 +31,6 @@ def identity(iterable: Iterable[_T]) -> Iterable[_T]:
|
||||
return iterable
|
||||
|
||||
|
||||
def normalize_unicode(value: str | None) -> str | None:
|
||||
"""
|
||||
Normalize a string to Unicode NFC form, or return None unchanged.
|
||||
|
||||
This is the single normalization pass for any user- or filesystem-supplied
|
||||
text that ends up in a filename, path, or is compared/matched against one
|
||||
(titles, correspondent/tag/type names, uploaded filenames, workflow and
|
||||
mail rule filename/path filters). Composed (NFC) and decomposed (NFD)
|
||||
forms of the same visible text are different byte sequences, which breaks
|
||||
exact comparisons and filesystem lookups even though the text looks
|
||||
identical. Always normalize through this function rather than calling
|
||||
unicodedata.normalize() directly, so every call site agrees on the same
|
||||
form.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
return unicodedata.normalize("NFC", value)
|
||||
|
||||
|
||||
class QuerySetStream(Generic[_M]):
|
||||
"""Stream a QuerySet via .iterator(chunk_size=...) instead of
|
||||
materializing it (plus any prefetch caches) all at once, while still
|
||||
|
||||
+31
-5
@@ -231,7 +231,6 @@ from documents.tasks import sanity_check
|
||||
from documents.tasks import train_classifier
|
||||
from documents.tasks import update_document_parent_tags
|
||||
from documents.utils import get_boolean
|
||||
from documents.utils import normalize_unicode
|
||||
from documents.versioning import VersionResolutionError
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.versioning import get_latest_version_for_root
|
||||
@@ -253,6 +252,7 @@ from paperless.views import StandardPagination
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.chat import stream_chat_with_documents
|
||||
from paperless_ai.exceptions import LLMProviderError
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
from paperless_ai.matching import match_correspondents_by_name
|
||||
@@ -1604,6 +1604,22 @@ class DocumentViewSet(
|
||||
{"ai": [_("AI backend request timed out.")]},
|
||||
status=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
except LLMProviderError:
|
||||
logger.exception(
|
||||
"AI backend rejected the request for document %s",
|
||||
doc.pk,
|
||||
)
|
||||
return Response(
|
||||
{
|
||||
"ai": [
|
||||
_(
|
||||
"AI backend rejected the request. "
|
||||
"Check logs for details.",
|
||||
),
|
||||
],
|
||||
},
|
||||
status=status.HTTP_502_BAD_GATEWAY,
|
||||
)
|
||||
set_llm_suggestions_cache(
|
||||
doc.pk,
|
||||
llm_suggestions,
|
||||
@@ -2069,7 +2085,6 @@ class DocumentViewSet(
|
||||
|
||||
try:
|
||||
doc_name, doc_data = serializer.validated_data.get("document")
|
||||
doc_name = normalize_unicode(doc_name)
|
||||
version_label = serializer.validated_data.get("version_label")
|
||||
|
||||
t = int(mktime(datetime.now().timetuple()))
|
||||
@@ -3336,7 +3351,7 @@ class PostDocumentView(GenericAPIView[Any]):
|
||||
serializer.is_valid(raise_exception=True)
|
||||
|
||||
doc_name, doc_data = serializer.validated_data.get("document")
|
||||
doc_name = normalize_unicode(doc_name)
|
||||
doc_name = normalize("NFC", doc_name)
|
||||
correspondent_id = serializer.validated_data.get("correspondent")
|
||||
document_type_id = serializer.validated_data.get("document_type")
|
||||
storage_path_id = serializer.validated_data.get("storage_path")
|
||||
@@ -5439,7 +5454,10 @@ class TrashView(ListModelMixin, PassUserMixin):
|
||||
|
||||
model = Document
|
||||
|
||||
queryset = Document.deleted_objects.all()
|
||||
# A version is listed separately only when its root is not in the trash.
|
||||
queryset = Document.deleted_objects.exclude(
|
||||
root_document_id__in=Document.deleted_objects.values("id"),
|
||||
)
|
||||
|
||||
def get(self, request: Request, format: str | None = None) -> Response:
|
||||
self.serializer_class = DocumentSerializer
|
||||
@@ -5470,7 +5488,15 @@ class TrashView(ListModelMixin, PassUserMixin):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
action = serializer.validated_data.get("action")
|
||||
if action == "restore":
|
||||
restored = list(Document.deleted_objects.filter(id__in=doc_ids))
|
||||
restored = list(self.get_queryset().filter(id__in=doc_ids))
|
||||
if len(restored) != len(doc_ids):
|
||||
raise ValidationError(
|
||||
{
|
||||
"documents": [
|
||||
"Restore the root document instead of one of its versions.",
|
||||
],
|
||||
},
|
||||
)
|
||||
for doc in restored:
|
||||
doc.restore(strict=False)
|
||||
if restored:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user