From d2170f1267f1b3af9c4122efb44d075f8c1a4294 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:23:41 -0700 Subject: [PATCH 01/12] Performance: Loads the classifier through a memory view to reduce memory usage (#14113) --- src/documents/classifier.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/documents/classifier.py b/src/documents/classifier.py index cfed4725c7..3db7c3cfa7 100644 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -158,7 +158,7 @@ class DocumentClassifier: ).hexdigest() @staticmethod - def _compute_hmac(data: bytes) -> bytes: + def _compute_hmac(data: bytes | memoryview) -> bytes: return hmac.new( settings.SECRET_KEY.encode(), data, @@ -173,8 +173,13 @@ class DocumentClassifier: if len(raw) <= self.HMAC_SIZE: raise ClassifierModelCorruptError - signature = raw[: self.HMAC_SIZE] - data = raw[self.HMAC_SIZE :] + # Slice through a memoryview so the (potentially multi-GB) payload is + # not copied; hmac and pickle both accept buffers directly. + # The whole file is still verified from memory before unpickling, rather + # than streamed from disk, so it cannot change between check and load. + view = memoryview(raw) + signature = view[: self.HMAC_SIZE] + data = view[self.HMAC_SIZE :] if not hmac.compare_digest(signature, self._compute_hmac(data)): raise ClassifierModelCorruptError From 970eb55a707a877c1aeaf5eb272ca8f6f694c635 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:43:21 -0700 Subject: [PATCH 02/12] Fix: type edit_pdf operations via a nested serializer (#14116) operations was a ListField with no child, so both EditPdfDocumentsSerializer and BulkEditSerializer._validate_parameters_edit_pdf hand-checked each entry with isinstance(). That accepted booleans (isinstance(True, int) is true) and passed through any extra keys. Adds PdfEditOperationSerializer (page >= 1, doc >= 0, rotate a multiple of 90) and uses it as the ListField child on both paths, dropping the manual type checks and the now-redundant negative/page < 1 bounds checks. Rotations that are not a multiple of 90 previously passed validation and then failed inside the task when QPDF refused them. --- src/documents/serialisers.py | 78 ++++++++++---------- src/documents/tests/test_api_bulk_edit.py | 90 +++++++++++++++++++++-- 2 files changed, 119 insertions(+), 49 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 9cd8d187a7..c13cf23d8c 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1749,8 +1749,23 @@ class MergeDocumentsAsVersionsSerializer(DocumentListSerializer): return attrs +class PdfEditOperationSerializer(serializers.Serializer[dict[str, int]]): + page = serializers.IntegerField(min_value=1) + rotate = serializers.IntegerField(required=False) + doc = serializers.IntegerField(required=False, min_value=0) + + def validate_rotate(self, value: int) -> int: + if value % 90 != 0: + raise serializers.ValidationError("rotate must be a multiple of 90") + return value + + class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin): - operations = serializers.ListField(required=True, allow_empty=False) + operations = serializers.ListField( + child=PdfEditOperationSerializer(), + required=True, + allow_empty=False, + ) delete_original = serializers.BooleanField(required=False, default=False) update_document = serializers.BooleanField(required=False, default=False) include_metadata = serializers.BooleanField(required=False, default=True) @@ -1768,18 +1783,9 @@ class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMix ) operations = attrs["operations"] - if not isinstance(operations, list): - raise serializers.ValidationError("operations must be a list") - for op in operations: - if not isinstance(op, dict): - raise serializers.ValidationError("invalid operation entry") - if "page" not in op or not isinstance(op["page"], int): - raise serializers.ValidationError("page must be an integer") - if "rotate" in op and not isinstance(op["rotate"], int): - raise serializers.ValidationError("rotate must be an integer") - if "doc" in op and not isinstance(op["doc"], int): - raise serializers.ValidationError("doc must be an integer") + if any(op.get("doc", 0) >= len(operations) for op in operations): + raise serializers.ValidationError("doc index is out of bounds") if attrs["update_document"]: max_idx = max(op.get("doc", 0) for op in operations) @@ -1788,16 +1794,10 @@ class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMix "update_document only allowed with a single output document", ) - if any( - op.get("doc", 0) < 0 or op.get("doc", 0) >= len(operations) - for op in operations - ): - raise serializers.ValidationError("doc index is out of bounds") - doc = Document.objects.get(id=documents[0]) if doc.page_count: for op in operations: - if op["page"] < 1 or op["page"] > doc.page_count: + if op["page"] > doc.page_count: raise serializers.ValidationError( f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.", ) @@ -2128,19 +2128,18 @@ class BulkEditSerializer( def _validate_parameters_edit_pdf(self, parameters, document_id) -> None: if "operations" not in parameters: raise serializers.ValidationError("operations not specified") - if not isinstance(parameters["operations"], list): - raise serializers.ValidationError("operations must be a list") - if not parameters["operations"]: - raise serializers.ValidationError("operations must not be empty") - for op in parameters["operations"]: - if not isinstance(op, dict): - raise serializers.ValidationError("invalid operation entry") - if "page" not in op or not isinstance(op["page"], int): - raise serializers.ValidationError("page must be an integer") - if "rotate" in op and not isinstance(op["rotate"], int): - raise serializers.ValidationError("rotate must be an integer") - if "doc" in op and not isinstance(op["doc"], int): - raise serializers.ValidationError("doc must be an integer") + operations_field = serializers.ListField( + child=PdfEditOperationSerializer(), + allow_empty=False, + ) + try: + operations = operations_field.run_validation(parameters["operations"]) + except serializers.ValidationError as e: + # Key the errors under "operations" so they match what the + # dedicated edit_pdf endpoint returns + raise serializers.ValidationError({"operations": e.detail}) from e + parameters["operations"] = operations + if "update_document" in parameters: if not isinstance(parameters["update_document"], bool): raise serializers.ValidationError("update_document must be a boolean") @@ -2152,24 +2151,21 @@ class BulkEditSerializer( else: parameters["include_metadata"] = True + if any(op.get("doc", 0) >= len(operations) for op in operations): + raise serializers.ValidationError("doc index is out of bounds") + if parameters["update_document"]: - max_idx = max(op.get("doc", 0) for op in parameters["operations"]) + max_idx = max(op.get("doc", 0) for op in operations) if max_idx > 0: raise serializers.ValidationError( "update_document only allowed with a single output document", ) - if any( - op.get("doc", 0) < 0 or op.get("doc", 0) >= len(parameters["operations"]) - for op in parameters["operations"] - ): - raise serializers.ValidationError("doc index is out of bounds") - doc = Document.objects.get(id=document_id) # doc existence is already validated if doc.page_count: - for op in parameters["operations"]: - if op["page"] < 1 or op["page"] > doc.page_count: + for op in operations: + if op["page"] > doc.page_count: raise serializers.ValidationError( f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.", ) diff --git a/src/documents/tests/test_api_bulk_edit.py b/src/documents/tests/test_api_bulk_edit.py index ebde4ff1f9..cdced37c64 100644 --- a/src/documents/tests/test_api_bulk_edit.py +++ b/src/documents/tests/test_api_bulk_edit.py @@ -1681,7 +1681,10 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"operations must not be empty", response.content) + self.assertEqual( + response.json(), + {"operations": ["This list may not be empty."]}, + ) @mock.patch("documents.views.bulk_edit.edit_pdf") def test_edit_pdf(self, m) -> None: @@ -1751,7 +1754,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"invalid operation entry", response.content) + self.assertIn(b"Expected a dictionary", response.content) response = self.client.post( "/api/documents/edit_pdf/", @@ -1764,7 +1767,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"page must be an integer", response.content) + self.assertIn(b"valid integer is required", response.content) response = self.client.post( "/api/documents/edit_pdf/", @@ -1777,7 +1780,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"rotate must be an integer", response.content) + self.assertIn(b"valid integer is required", response.content) response = self.client.post( "/api/documents/edit_pdf/", @@ -1790,9 +1793,14 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"doc must be an integer", response.content) + self.assertIn(b"valid integer is required", response.content) - for doc_index in (-1, 2**32): + # A negative index fails the field's min_value before the + # object-level bound against len(operations) is checked + for doc_index, expected_message in ( + (-1, b"greater than or equal to 0"), + (2**32, b"doc index is out of bounds"), + ): with self.subTest(doc_index=doc_index): response = self.client.post( "/api/documents/edit_pdf/", @@ -1805,7 +1813,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"doc index is out of bounds", response.content) + self.assertIn(expected_message, response.content) response = self.client.post( "/api/documents/edit_pdf/", @@ -1813,7 +1821,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): { "documents": [self.doc2.id], "update_document": True, - "operations": [{"page": 1, "doc": 1}, {"page": 2, "doc": 2}], + "operations": [{"page": 1, "doc": 0}, {"page": 2, "doc": 1}], }, ), content_type="application/json", @@ -1838,6 +1846,72 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn(b"Invalid source_mode", response.content) + @mock.patch("documents.views.bulk_edit.edit_pdf") + def test_edit_pdf_rejects_invalid_operation_values(self, m) -> None: + """ + GIVEN: + - An edit_pdf operation with a non-positive page, a boolean in + place of an integer, or a rotation that is not a multiple of 90 + WHEN: + - API to edit the PDF is called + THEN: + - API returns HTTP 400 with the field error + - edit_pdf is not called + """ + self.setup_mock(m, "edit_pdf") + for operation, expected_message in ( + ({"page": 0}, b"greater than or equal to 1"), + ({"page": True}, b"valid integer is required"), + ({"page": 1, "rotate": True}, b"valid integer is required"), + ({"page": 1, "doc": True}, b"valid integer is required"), + ({"page": 1, "rotate": 45}, b"rotate must be a multiple of 90"), + ): + with self.subTest(operation=operation): + response = self.client.post( + "/api/documents/edit_pdf/", + json.dumps( + { + "documents": [self.doc2.id], + "operations": [operation], + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(expected_message, response.content) + m.assert_not_called() + + def test_legacy_bulk_edit_keys_pdf_operation_errors_under_operations( + self, + ) -> None: + """ + GIVEN: + - A legacy bulk_edit edit_pdf request with an invalid operation + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 + - The error is keyed under operations and the operation index, + matching the edit_pdf endpoint + """ + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "edit_pdf", + "parameters": {"operations": [{"page": 1, "rotate": 45}]}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.json(), + {"operations": {"0": {"rotate": ["rotate must be a multiple of 90"]}}}, + ) + @mock.patch("documents.views.bulk_edit.edit_pdf") def test_edit_pdf_page_out_of_bounds(self, m) -> None: self.setup_mock(m, "edit_pdf") From 7354427229582fb50d81ea7a6de9a623396acc93 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:45:22 +0000 Subject: [PATCH 03/12] Auto translate strings --- src/locale/en_US/LC_MESSAGES/django.po | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index cfcf3008e1..5d6622f7ba 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-09-15 14:27+0000\n" +"POT-Creation-Date: 2026-09-15 15:44+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -1632,7 +1632,7 @@ msgid "workflow runs" msgstr "" #: documents/serialisers.py:525 documents/serialisers.py:882 -#: documents/serialisers.py:2868 documents/views.py:343 documents/views.py:2719 +#: documents/serialisers.py:2864 documents/views.py:343 documents/views.py:2719 #: paperless_mail/serialisers.py:156 msgid "Insufficient permissions." msgstr "" @@ -1641,39 +1641,39 @@ msgstr "" msgid "Invalid color." msgstr "" -#: documents/serialisers.py:2341 +#: documents/serialisers.py:2337 #, python-format msgid "File type %(type)s not supported" msgstr "" -#: documents/serialisers.py:2385 +#: documents/serialisers.py:2381 #, python-format msgid "Custom field id must be an integer: %(id)s" msgstr "" -#: documents/serialisers.py:2392 +#: documents/serialisers.py:2388 #, python-format msgid "Custom field with id %(id)s does not exist" msgstr "" -#: documents/serialisers.py:2409 documents/serialisers.py:2419 +#: documents/serialisers.py:2405 documents/serialisers.py:2415 msgid "" "Custom fields must be a list of integers or an object mapping ids to values." msgstr "" -#: documents/serialisers.py:2414 +#: documents/serialisers.py:2410 msgid "Some custom fields don't exist or were specified twice." msgstr "" -#: documents/serialisers.py:2561 +#: documents/serialisers.py:2557 msgid "Invalid variable detected." msgstr "" -#: documents/serialisers.py:2924 +#: documents/serialisers.py:2920 msgid "Duplicate document identifiers are not allowed." msgstr "" -#: documents/serialisers.py:2954 documents/views.py:4742 +#: documents/serialisers.py:2950 documents/views.py:4742 #, python-format msgid "Documents not found: %(ids)s" msgstr "" From 1f374cd65652ff8991f1d7e1efe49354876ac5c2 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:14:01 -0700 Subject: [PATCH 04/12] Fix: reject non-dict user_args/barcode_tag_mapping in config API (#14118) JSONField(binary=True) accepts any JSON value, so a truthy non-dict (bool/int/list/string) silently passed validation. Additionally, require string substitutes in barcode_tag_mapping, a non-string substitute value made the barcode tag substitution fail inside get_tag_ids, and the tag was silently skipped. barcode_tag_mapping values must now be strings (empty still means "use the raw barcode"). --- src/documents/tests/test_api_app_config.py | 64 ++++++++++++++++++++++ src/paperless/serialisers.py | 18 ++++++ 2 files changed, 82 insertions(+) diff --git a/src/documents/tests/test_api_app_config.py b/src/documents/tests/test_api_app_config.py index 0ca6f4412b..8620c3ec3e 100644 --- a/src/documents/tests/test_api_app_config.py +++ b/src/documents/tests/test_api_app_config.py @@ -194,6 +194,70 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): self.assertEqual(config.language, None) self.assertEqual(config.barcode_tag_mapping, None) + def test_api_update_config_json_objects(self) -> None: + """ + GIVEN: + - API request to update app config with JSON objects for the + user_args and barcode_tag_mapping JSONFields + WHEN: + - API is called + THEN: + - Correct HTTP response + - Both objects are stored as sent + """ + user_args = {"unpaper_args": "--pre-rotate 90", "jobs": 2} + barcode_tag_mapping = {"TAG:(.*)": "\\g<1>", "ASN12.*": ""} + response = self.client.patch( + f"{self.ENDPOINT}1/", + json.dumps( + { + "user_args": json.dumps(user_args), + "barcode_tag_mapping": json.dumps(barcode_tag_mapping), + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + config = ApplicationConfiguration.objects.first() + assert config is not None + self.assertEqual(config.user_args, user_args) + self.assertEqual(config.barcode_tag_mapping, barcode_tag_mapping) + + def test_api_update_config_rejects_invalid_json_objects(self) -> None: + """ + GIVEN: + - API request to update app config with a JSON-encoded value that + is not an object for user_args or barcode_tag_mapping, or a + barcode_tag_mapping with a non-string substitute + WHEN: + - API is called + THEN: + - Request is rejected with a 400 naming the problem + - Config is not updated + """ + not_objects = (True, 1, [1, 2, 3], "not a dict") + cases = [ + (field, value, b"must be a JSON object") + for field in ("user_args", "barcode_tag_mapping") + for value in not_objects + ] + cases += [ + ("barcode_tag_mapping", {"TAG:(.*)": 5}, b"values must be strings"), + ("barcode_tag_mapping", {"TAG:(.*)": None}, b"values must be strings"), + ] + for field, value, expected_message in cases: + with self.subTest(field=field, value=value): + response = self.client.patch( + f"{self.ENDPOINT}1/", + json.dumps({field: json.dumps(value)}), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(expected_message, response.content) + config = ApplicationConfiguration.objects.first() + assert config is not None + self.assertIsNone(getattr(config, field)) + def test_api_replace_app_logo(self) -> None: """ GIVEN: diff --git a/src/paperless/serialisers.py b/src/paperless/serialisers.py index f7cfc3d450..7b32230f8f 100644 --- a/src/paperless/serialisers.py +++ b/src/paperless/serialisers.py @@ -235,6 +235,24 @@ class ApplicationConfigurationSerializer( ) -> list[str]: return sorted(name for name in os.environ if name.startswith("PAPERLESS_")) + @staticmethod + def _require_json_object(field: str, value: object) -> None: + if value is not None and not isinstance(value, dict): + raise serializers.ValidationError(f"{field} must be a JSON object.") + + def validate_user_args(self, value): + self._require_json_object("user_args", value) + return value + + def validate_barcode_tag_mapping(self, value): + self._require_json_object("barcode_tag_mapping", value) + # Each value is the regex substitute applied to a matching barcode + if value is not None and not all(isinstance(v, str) for v in value.values()): + raise serializers.ValidationError( + "barcode_tag_mapping values must be strings.", + ) + return value + def run_validation(self, data): # Empty strings treated as None to avoid unexpected behavior if "user_args" in data and data["user_args"] == "": From ff1e61b16232569437c64401661b9dbf42395143 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:12:08 -0700 Subject: [PATCH 05/12] Fix: Use prefetching to reduce query counts during classifier training (#14122) --- src/documents/classifier.py | 20 ++++--- src/documents/tests/test_classifier.py | 72 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/src/documents/classifier.py b/src/documents/classifier.py index 3db7c3cfa7..1cdef4419e 100644 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: from django.conf import settings from django.core.cache import cache from django.core.cache import caches +from django.db.models import Prefetch from documents.caching import CACHE_5_MINUTES from documents.caching import CACHE_50_MINUTES @@ -28,6 +29,7 @@ from documents.caching import CLASSIFIER_VERSION_KEY from documents.caching import StoredLRUCache from documents.models import Document from documents.models import MatchingModel +from documents.models import Tag from paperless.signed_pickle import SignedPickleError from paperless.signed_pickle import signed_pickle_dumps from paperless.signed_pickle import signed_pickle_loads @@ -260,7 +262,17 @@ class DocumentClassifier: tags__is_inbox_tag=True, ) .select_related("document_type", "correspondent", "storage_path") - .prefetch_related("tags") + .prefetch_related( + Prefetch( + "tags", + queryset=Tag.objects.filter( + matching_algorithm=MatchingModel.MATCH_AUTO, + ) + .order_by("pk") + .only("pk"), + to_attr="auto_tags", + ), + ) .order_by("pk") ) @@ -292,11 +304,7 @@ class DocumentClassifier: hasher.update(y.to_bytes(4, "little", signed=True)) labels_correspondent.append(y) - tags: list[int] = list( - doc.tags.filter(matching_algorithm=MatchingModel.MATCH_AUTO) - .order_by("pk") - .values_list("pk", flat=True), - ) + tags: list[int] = [tag.pk for tag in doc.auto_tags] for tag in tags: hasher.update(tag.to_bytes(4, "little", signed=True)) labels_tags.append(tags) diff --git a/src/documents/tests/test_classifier.py b/src/documents/tests/test_classifier.py index 0e09ff0740..9066c4d410 100644 --- a/src/documents/tests/test_classifier.py +++ b/src/documents/tests/test_classifier.py @@ -6,8 +6,11 @@ from unittest import mock import numpy as np import pytest from django.conf import settings +from django.db import connection from django.test import TestCase from django.test import override_settings +from django.test.utils import CaptureQueriesContext +from pytest_mock import MockerFixture from documents.classifier import ClassifierModelCorruptError from documents.classifier import DocumentClassifier @@ -20,6 +23,8 @@ from documents.models import DocumentType from documents.models import MatchingModel from documents.models import StoragePath from documents.models import Tag +from documents.tests.factories import DocumentFactory +from documents.tests.factories import TagFactory from documents.tests.utils import DirectoriesMixin from paperless.signed_pickle import HMAC_SIZE from paperless.signed_pickle import signed_pickle_dumps @@ -1017,3 +1022,70 @@ def test_preprocess_content_nltk_load_fail(mocker) -> None: expected_preprocess_content = f.read().rstrip() result = classifier.preprocess_content(content) assert result == expected_preprocess_content + + +@pytest.mark.django_db +class TestClassifierTrainTagLabels: + @pytest.fixture(autouse=True) + def _simple_preprocess(self, mocker: MockerFixture) -> None: + mocker.patch.object( + DocumentClassifier, + "preprocess_content", + side_effect=dummy_preprocess, + ) + + @pytest.fixture + def auto_tags(self) -> list[Tag]: + return TagFactory.create_batch(2, matching_algorithm=MatchingModel.MATCH_AUTO) + + def test_train_query_count_does_not_scale_with_documents( + self, + auto_tags: list[Tag], + ) -> None: + """ + GIVEN: + - Documents with auto matching tags + WHEN: + - The classifier is trained, then more documents are added and it is + trained again + THEN: + - Both trainings run the same number of queries + """ + for doc in DocumentFactory.create_batch(2): + doc.tags.set(auto_tags) + + with CaptureQueriesContext(connection) as few_documents: + DocumentClassifier().train() + + for doc in DocumentFactory.create_batch(6): + doc.tags.set(auto_tags) + + with CaptureQueriesContext(connection) as more_documents: + DocumentClassifier().train() + + assert len(more_documents) == len(few_documents) + + def test_train_uses_only_auto_tags_as_labels( + self, + auto_tags: list[Tag], + ) -> None: + """ + GIVEN: + - Documents with both auto matching and non auto matching tags + WHEN: + - The classifier is trained + THEN: + - Only the auto matching tags are used as tag labels + """ + manual_tag = TagFactory(matching_algorithm=MatchingModel.MATCH_ANY) + first, second, third = DocumentFactory.create_batch(3) + first.tags.set([auto_tags[0], manual_tag]) + second.tags.set([auto_tags[1], manual_tag]) + third.tags.set([*auto_tags, manual_tag]) + + classifier = DocumentClassifier() + classifier.train() + + assert list(classifier.tags_binarizer.classes_) == sorted( + tag.pk for tag in auto_tags + ) From ef26bc1570a27bc6bd561215293156fb99913a3e Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:46:32 -0700 Subject: [PATCH 06/12] Fix: enforce set_permissions shape via a nested DRF serializer (#14119) SetPermissionsSerializer was a bare DictField, and the legacy bulk_edit set_permissions parameter bypassed even that by hand-calling validate_set_permissions() on an unchecked dict. A bool or a non-list in place of users/groups crashed with a raw TypeError instead of a 400. It is now a nested serializer (view/change, each with users/groups as lists of integers), used for owned-object create/update, the legacy bulk_edit set_permissions parameter, and bulk_edit_objects permissions. Unknown action keys are rejected: previously a typo like "veiw" was silently dropped, leaving an empty permission set that could clear existing grants. An explicit set_permissions null (an owner-only change) remains a no-op, and an empty bulk_edit_objects permissions dict is still rejected. --- src/documents/permissions.py | 18 ++-- src/documents/serialisers.py | 91 +++++++++-------- src/documents/tests/test_api_bulk_edit.py | 73 +++++++++++++- src/documents/tests/test_api_permissions.py | 104 ++++++++++++++++++++ src/documents/tests/test_bulk_edit.py | 11 +-- 5 files changed, 230 insertions(+), 67 deletions(-) diff --git a/src/documents/permissions.py b/src/documents/permissions.py index b7a1d7e0e4..70c48b6ccd 100644 --- a/src/documents/permissions.py +++ b/src/documents/permissions.py @@ -177,13 +177,10 @@ def _resolve_permissions(codenames: set[str], ctype: ContentType) -> list[Permis """ Resolves `codenames` to Permission rows, raising like the single-object assign_perm() this bulk path replaces does (via a `.get()` internally) - if any codename doesn't exist -- e.g. a client-supplied action name that - was never validated (BulkEditObjectsSerializer._validate_permissions - calls validate_set_permissions() only for its side-effecting id checks - and discards the filtered dict it returns, so an unrecognized action key - reaches this function as-is). A plain `.filter()` with no existence - check would otherwise silently build zero rows and no-op instead of - reporting the bad input. + if any codename doesn't exist. SetPermissionsSerializer rejects unknown + action names at the API, but a caller passing one directly would + otherwise get a plain `.filter()` that silently builds zero rows and + no-ops instead of reporting the bad input. """ permission_objs = list( Permission.objects.filter(content_type=ctype, codename__in=codenames), @@ -298,10 +295,9 @@ def set_permissions_for_objects( # Every action is resolved up front, before anything is written, so an # unrecognized action name (see _resolve_permissions) aborts the whole - # call instead of leaving the actions ahead of it already applied -- - # BulkEditObjectsSerializer lets unknown keys through and its view turns - # the exception into a 400, so a half-applied change would otherwise be - # reported to the client as a failure. + # call instead of leaving the actions ahead of it already applied. + # SetPermissionsSerializer rejects unknown actions at the API, so this + # guards any other caller. permissions_by_action: dict[str, list[Permission]] = {} for action, entry in permissions.items(): if "users" not in entry and "groups" not in entry: diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index c13cf23d8c..908ab66e9e 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -210,6 +210,9 @@ class MatchingModelSerializer(serializers.ModelSerializer[Any]): return match +PERMISSION_ACTIONS = ("view", "change") + + class SetPermissionsMixin: def _validate_user_ids(self, user_ids): users = User.objects.none() @@ -232,12 +235,9 @@ class SetPermissionsMixin: return groups def validate_set_permissions(self, set_permissions=None): - permissions_dict = { - "view": {}, - "change": {}, - } + permissions_dict = {action: {} for action in PERMISSION_ACTIONS} if set_permissions is not None: - for action in ["view", "change"]: + for action in PERMISSION_ACTIONS: if action in set_permissions: if "users" in set_permissions[action]: users = set_permissions[action]["users"] @@ -265,41 +265,31 @@ class SerializerWithPerms(serializers.Serializer[dict[str, Any]]): super().__init__(*args, **kwargs) -@extend_schema_field( - field={ - "type": "object", - "properties": { - "view": { - "type": "object", - "properties": { - "users": { - "type": "array", - "items": {"type": "integer"}, - }, - "groups": { - "type": "array", - "items": {"type": "integer"}, - }, - }, - }, - "change": { - "type": "object", - "properties": { - "users": { - "type": "array", - "items": {"type": "integer"}, - }, - "groups": { - "type": "array", - "items": {"type": "integer"}, - }, - }, - }, - }, - }, -) -class SetPermissionsSerializer(serializers.DictField): - pass +class PermissionSetSerializer(serializers.Serializer[dict[str, Any]]): + users = serializers.ListField( + child=serializers.IntegerField(), + required=False, + allow_null=True, + ) + groups = serializers.ListField( + child=serializers.IntegerField(), + required=False, + allow_null=True, + ) + + +class SetPermissionsSerializer(serializers.Serializer[dict[str, Any]]): + view = PermissionSetSerializer(required=False) + change = PermissionSetSerializer(required=False) + + def to_internal_value(self, data): + if isinstance(data, dict): + unknown_keys = set(data) - set(PERMISSION_ACTIONS) + if unknown_keys: + raise serializers.ValidationError( + {key: "Unknown permission action." for key in sorted(unknown_keys)}, + ) + return super().to_internal_value(data) class OwnedObjectSerializer( @@ -470,7 +460,6 @@ class OwnedObjectSerializer( set_permissions = SetPermissionsSerializer( label="Set permissions", - allow_empty=True, required=False, write_only=True, ) @@ -2051,8 +2040,13 @@ class BulkEditSerializer( def _validate_parameters_set_permissions(self, parameters) -> None: if "set_permissions" not in parameters: raise serializers.ValidationError("set_permissions not specified") + set_permissions = parameters["set_permissions"] + if set_permissions is not None: + set_permissions = SetPermissionsSerializer().run_validation( + set_permissions, + ) parameters["set_permissions"] = self.validate_set_permissions( - parameters["set_permissions"], + set_permissions, ) if "owner" in parameters and parameters["owner"] is not None: self._validate_owner(parameters["owner"]) @@ -3011,9 +3005,8 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin): allow_null=True, ) - permissions = serializers.DictField( + permissions = SetPermissionsSerializer( label="Set permissions", - allow_empty=False, required=False, write_only=True, ) @@ -3049,8 +3042,8 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin): ) return objects - def _validate_permissions(self, permissions) -> None: - self.validate_set_permissions( + def _validate_permissions(self, permissions) -> dict: + return self.validate_set_permissions( permissions, ) @@ -3074,7 +3067,11 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin): if operation == "set_permissions": permissions = attrs.get("permissions") if permissions is not None: - self._validate_permissions(permissions) + if not permissions: + raise serializers.ValidationError( + "permissions must not be empty", + ) + attrs["permissions"] = self._validate_permissions(permissions) return attrs diff --git a/src/documents/tests/test_api_bulk_edit.py b/src/documents/tests/test_api_bulk_edit.py index cdced37c64..dafbf4365a 100644 --- a/src/documents/tests/test_api_bulk_edit.py +++ b/src/documents/tests/test_api_bulk_edit.py @@ -1165,6 +1165,77 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.assertIn(b"set_permissions not specified", response.content) m.assert_not_called() + @mock.patch("documents.serialisers.bulk_edit.set_permissions") + def test_set_permissions_rejects_malformed_set_permissions(self, m) -> None: + """ + GIVEN: + - A set_permissions bulk edit where set_permissions is not an + object, has a non-list users value, or has an unknown action + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 describing the problem + - set_permissions is not called + """ + self.setup_mock(m, "set_permissions") + + for set_permissions, expected_message in ( + (False, b"Expected a dictionary"), + ({"view": {"users": False}}, b"Expected a list"), + ({"not_a_real_action": {"users": [1]}}, b"Unknown permission action"), + ): + with self.subTest(set_permissions=set_permissions): + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "set_permissions", + "parameters": {"set_permissions": set_permissions}, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(expected_message, response.content) + m.assert_not_called() + + @mock.patch("documents.serialisers.bulk_edit.set_permissions") + def test_set_permissions_null_is_a_noop(self, m) -> None: + """ + GIVEN: + - A set_permissions bulk edit with set_permissions null and an + owner, i.e. a request that only changes the owner + WHEN: + - API to bulk edit is called + THEN: + - Request succeeds + - set_permissions receives no users or groups for any action + """ + self.setup_mock(m, "set_permissions") + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "set_permissions", + "parameters": { + "set_permissions": None, + "owner": self.user.id, + }, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + self.assertEqual( + m.call_args.kwargs["set_permissions"], + {"view": {}, "change": {}}, + ) + @mock.patch("documents.serialisers.bulk_edit.set_permissions") def test_set_permissions_merge(self, m) -> None: self.setup_mock(m, "set_permissions") @@ -1300,7 +1371,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.client.force_authenticate(user=user1) permissions = { - "owner": user1.id, + "view": {"users": [user1.id]}, } response = self.client.post( diff --git a/src/documents/tests/test_api_permissions.py b/src/documents/tests/test_api_permissions.py index 29eb8f4961..0b2c491a4e 100644 --- a/src/documents/tests/test_api_permissions.py +++ b/src/documents/tests/test_api_permissions.py @@ -1499,6 +1499,110 @@ class TestBulkEditObjectPermissions(APITestCase): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertTrue(Tag.objects.filter(pk=self.t1.id).exists()) + def test_bulk_object_set_permissions_rejects_empty_permissions(self) -> None: + """ + GIVEN: + - Existing objects + WHEN: + - bulk_edit_objects API endpoint is called with set_permissions + operation and an empty permissions dict + THEN: + - Validation fails rather than silently applying a no-op + """ + response = self.client.post( + "/api/bulk_edit_objects/", + json.dumps( + { + "objects": [self.t1.id], + "object_type": "tags", + "operation": "set_permissions", + "permissions": {}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_bulk_object_set_permissions_rejects_non_dict_permissions(self) -> None: + """ + GIVEN: + - Existing objects + WHEN: + - bulk_edit_objects API endpoint is called with set_permissions + operation and a non-dict permissions value + THEN: + - Validation fails rather than crashing + """ + response = self.client.post( + "/api/bulk_edit_objects/", + json.dumps( + { + "objects": [self.t1.id], + "object_type": "tags", + "operation": "set_permissions", + "permissions": False, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_bulk_object_set_permissions_rejects_unknown_action(self) -> None: + """ + GIVEN: + - Existing objects + WHEN: + - bulk_edit_objects API endpoint is called with set_permissions + operation and an unrecognized permission action name + THEN: + - Validation fails rather than silently no-oping + """ + response = self.client.post( + "/api/bulk_edit_objects/", + json.dumps( + { + "objects": [self.t1.id], + "object_type": "tags", + "operation": "set_permissions", + "permissions": {"not_a_real_action": {"users": [self.user1.id]}}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_bulk_object_set_permissions_null_users_clears_users(self) -> None: + """ + GIVEN: + - An object a user has view permission on + WHEN: + - bulk_edit_objects API endpoint is called with set_permissions + operation, merge off, and an explicit null for the view users + THEN: + - Request succeeds and null is treated as an empty user list, + so the existing view permission is removed + """ + assign_perm("view_tag", self.user1, self.t1) + + response = self.client.post( + "/api/bulk_edit_objects/", + json.dumps( + { + "objects": [self.t1.id], + "object_type": "tags", + "operation": "set_permissions", + "permissions": {"view": {"users": None}}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertNotIn(self.user1, get_users_with_perms(self.t1)) + def test_bulk_edit_object_permissions_validation(self) -> None: """ GIVEN: diff --git a/src/documents/tests/test_bulk_edit.py b/src/documents/tests/test_bulk_edit.py index f9d4a5028a..bfd1345163 100644 --- a/src/documents/tests/test_bulk_edit.py +++ b/src/documents/tests/test_bulk_edit.py @@ -636,14 +636,9 @@ class TestBulkEdit(DirectoriesMixin, TestCase): THEN: - Permission.DoesNotExist is raised, not a silent no-op - Regression test: the endpoint that calls this - (BulkEditObjectPermissionsView) never actually validates action - names against the raw client-supplied permissions dict -- - BulkEditObjectsSerializer._validate_permissions calls - validate_set_permissions() only for its side-effecting user/group id - checks and discards the filtered dict it returns -- so a bogus - action key reaches this function as-is. Resolving the Permission via - a bare `.filter()` (which returns empty instead of raising) would + The API rejects unknown action names before they get here, but a + direct caller could still pass one. Resolving the Permission via a + bare `.filter()` (which returns empty instead of raising) would silently drop the grant and report success. """ with self.assertRaises(Permission.DoesNotExist): From 73de6b17bfc581c3661fa142e7acc403bd67944e Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 18:47:52 +0000 Subject: [PATCH 07/12] Auto translate strings --- src/locale/en_US/LC_MESSAGES/django.po | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index 5d6622f7ba..f74b9feaf3 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-09-15 15:44+0000\n" +"POT-Creation-Date: 2026-09-15 18:47+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -1631,49 +1631,49 @@ msgstr "" msgid "workflow runs" msgstr "" -#: documents/serialisers.py:525 documents/serialisers.py:882 -#: documents/serialisers.py:2864 documents/views.py:343 documents/views.py:2719 +#: documents/serialisers.py:514 documents/serialisers.py:871 +#: documents/serialisers.py:2858 documents/views.py:343 documents/views.py:2719 #: paperless_mail/serialisers.py:156 msgid "Insufficient permissions." msgstr "" -#: documents/serialisers.py:718 +#: documents/serialisers.py:707 msgid "Invalid color." msgstr "" -#: documents/serialisers.py:2337 +#: documents/serialisers.py:2331 #, python-format msgid "File type %(type)s not supported" msgstr "" -#: documents/serialisers.py:2381 +#: documents/serialisers.py:2375 #, python-format msgid "Custom field id must be an integer: %(id)s" msgstr "" -#: documents/serialisers.py:2388 +#: documents/serialisers.py:2382 #, python-format msgid "Custom field with id %(id)s does not exist" msgstr "" -#: documents/serialisers.py:2405 documents/serialisers.py:2415 +#: documents/serialisers.py:2399 documents/serialisers.py:2409 msgid "" "Custom fields must be a list of integers or an object mapping ids to values." msgstr "" -#: documents/serialisers.py:2410 +#: documents/serialisers.py:2404 msgid "Some custom fields don't exist or were specified twice." msgstr "" -#: documents/serialisers.py:2557 +#: documents/serialisers.py:2551 msgid "Invalid variable detected." msgstr "" -#: documents/serialisers.py:2920 +#: documents/serialisers.py:2914 msgid "Duplicate document identifiers are not allowed." msgstr "" -#: documents/serialisers.py:2950 documents/views.py:4742 +#: documents/serialisers.py:2944 documents/views.py:4742 #, python-format msgid "Documents not found: %(ids)s" msgstr "" From 683f2ac250d4db33f829fe6bc6d40fa8c4e89a93 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:23:37 -0700 Subject: [PATCH 08/12] Fix: validate legacy bulk_edit owner/rotate/split parameters (#14120) BulkEditSerializer's hand-parsed validators only caught the exceptions their happy paths raised, so wrong-typed input produced a 500 or was passed through to the task: - owner: a nonexistent or wrong-typed id raised an uncaught error, and a boolean was accepted (Django coerces True to pk 1). It is now validated with PrimaryKeyRelatedField and the validated pk is passed on. - rotate: null raised TypeError, while true, "90" and 45 were accepted and failed later in QPDF. Degrees are now an IntegerField plus a multiple-of-90 check, passing an int on. The dedicated rotate endpoint and edit_pdf operations share the same multiple-of-90 check. - split: null raised AttributeError, "0" silently became the last page, "3-1" gave an empty group that crashed the task, and a range like "1-5000000" was expanded into a list during the request with no upper bound. Each range is now checked against 1 <= start <= end <= page_count before it is built. --- src/documents/serialisers.py | 79 ++++---- src/documents/tests/test_api_bulk_edit.py | 211 ++++++++++++++++++++++ 2 files changed, 254 insertions(+), 36 deletions(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 908ab66e9e..10296a392c 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -1669,6 +1669,13 @@ class SourceModeValidationMixin: return source_mode +def _validate_rotation_degrees(degrees: int, field: str = "degrees") -> int: + # QPDF refuses any other angle, which would otherwise fail inside the task + if degrees % 90 != 0: + raise serializers.ValidationError(f"{field} must be a multiple of 90") + return degrees + + class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidationMixin): degrees = serializers.IntegerField(required=True) source_mode = serializers.CharField( @@ -1677,6 +1684,9 @@ class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidatio ) from_webui = serializers.BooleanField(required=False, default=False) + def validate_degrees(self, value: int) -> int: + return _validate_rotation_degrees(value) + class MergeDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin): metadata_document_id = serializers.IntegerField( @@ -1744,9 +1754,7 @@ class PdfEditOperationSerializer(serializers.Serializer[dict[str, int]]): doc = serializers.IntegerField(required=False, min_value=0) def validate_rotate(self, value: int) -> int: - if value % 90 != 0: - raise serializers.ValidationError("rotate must be a multiple of 90") - return value + return _validate_rotation_degrees(value, field="rotate") class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin): @@ -2031,11 +2039,14 @@ class BulkEditSerializer( else: raise serializers.ValidationError("remove_custom_fields not specified") - def _validate_owner(self, owner): - ownerUser = User.objects.get(pk=owner) - if ownerUser is None: - raise serializers.ValidationError("Specified owner cannot be found") - return ownerUser + def _validate_owner(self, owner) -> User: + owner_field = serializers.PrimaryKeyRelatedField(queryset=User.objects.all()) + try: + return owner_field.run_validation(owner) + except serializers.ValidationError as e: + raise serializers.ValidationError( + "Specified owner cannot be found", + ) from e def _validate_parameters_set_permissions(self, parameters) -> None: if "set_permissions" not in parameters: @@ -2049,19 +2060,18 @@ class BulkEditSerializer( set_permissions, ) if "owner" in parameters and parameters["owner"] is not None: - self._validate_owner(parameters["owner"]) + parameters["owner"] = self._validate_owner(parameters["owner"]).pk if "merge" not in parameters: parameters["merge"] = False def _validate_parameters_rotate(self, parameters) -> None: - try: - if ( - "degrees" not in parameters - or not float(parameters["degrees"]).is_integer() - ): - raise serializers.ValidationError("invalid rotation degrees") - except ValueError: + if "degrees" not in parameters: raise serializers.ValidationError("invalid rotation degrees") + try: + degrees = serializers.IntegerField().run_validation(parameters["degrees"]) + except serializers.ValidationError as e: + raise serializers.ValidationError("invalid rotation degrees") from e + parameters["degrees"] = _validate_rotation_degrees(degrees) def _validate_source_mode(self, parameters) -> None: source_mode = parameters.get( @@ -2070,28 +2080,25 @@ class BulkEditSerializer( ) parameters["source_mode"] = self.validate_source_mode(source_mode) - def _validate_parameters_split(self, parameters) -> None: + def _validate_parameters_split(self, parameters, document_id) -> None: if "pages" not in parameters: raise serializers.ValidationError("pages not specified") - try: - pages = [] - docs = parameters["pages"].split(",") - for doc in docs: - if "-" in doc: - pages.append( - [ - x - for x in range( - int(doc.split("-")[0]), - int(doc.split("-")[1]) + 1, - ) - ], - ) - else: - pages.append([int(doc)]) - parameters["pages"] = pages - except ValueError: + if not isinstance(parameters["pages"], str): raise serializers.ValidationError("invalid pages specified") + page_count = Document.objects.get(id=document_id).page_count + pages = [] + for group in parameters["pages"].split(","): + start, is_range, end = group.partition("-") + try: + first = int(start) + last = int(end) if is_range else first + except ValueError as e: + raise serializers.ValidationError("invalid pages specified") from e + # Bound the range before building it, a huge one would exhaust memory + if not 1 <= first <= last or (page_count and last > page_count): + raise serializers.ValidationError("invalid pages specified") + pages.append(list(range(first, last + 1))) + parameters["pages"] = pages if "delete_originals" in parameters: if not isinstance(parameters["delete_originals"], bool): @@ -2218,7 +2225,7 @@ class BulkEditSerializer( raise serializers.ValidationError( "Split method only supports one document", ) - self._validate_parameters_split(parameters) + self._validate_parameters_split(parameters, attrs["documents"][0]) elif method == bulk_edit.delete_pages: if len(attrs["documents"]) > 1: raise serializers.ValidationError( diff --git a/src/documents/tests/test_api_bulk_edit.py b/src/documents/tests/test_api_bulk_edit.py index dafbf4365a..0c13cce406 100644 --- a/src/documents/tests/test_api_bulk_edit.py +++ b/src/documents/tests/test_api_bulk_edit.py @@ -1200,6 +1200,48 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.assertIn(expected_message, response.content) m.assert_not_called() + @mock.patch("documents.serialisers.bulk_edit.set_permissions") + def test_set_permissions_rejects_invalid_owner(self, m) -> None: + """ + GIVEN: + - A set_permissions bulk edit with an owner that is a nonexistent + id, a boolean, a list, a dict, or a non-numeric string + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 + - set_permissions is not called + """ + self.setup_mock(m, "set_permissions") + + for bad_owner in ( + 999999, + True, + ["not", "an", "id"], + {"nested": "dict"}, + "not-a-number", + ): + with self.subTest(owner=bad_owner): + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "set_permissions", + "parameters": { + "set_permissions": { + "view": {"users": [self.user.id]}, + }, + "owner": bad_owner, + }, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(b"Specified owner cannot be found", response.content) + m.assert_not_called() + @mock.patch("documents.serialisers.bulk_edit.set_permissions") def test_set_permissions_null_is_a_noop(self, m) -> None: """ @@ -1236,6 +1278,37 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): {"view": {}, "change": {}}, ) + @mock.patch("documents.serialisers.bulk_edit.set_permissions") + def test_set_permissions_passes_validated_owner_id(self, m) -> None: + """ + GIVEN: + - A set_permissions bulk edit with the owner id given as a string + WHEN: + - API to bulk edit is called + THEN: + - set_permissions receives the owner as an integer id + """ + self.setup_mock(m, "set_permissions") + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "set_permissions", + "parameters": { + "set_permissions": {"view": {"users": [self.user.id]}}, + "owner": str(self.user.id), + }, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + self.assertEqual(m.call_args.kwargs["owner"], self.user.id) + @mock.patch("documents.serialisers.bulk_edit.set_permissions") def test_set_permissions_merge(self, m) -> None: self.setup_mock(m, "set_permissions") @@ -1507,8 +1580,146 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + response = self.client.post( + "/api/documents/rotate/", + json.dumps( + { + "documents": [self.doc2.id, self.doc3.id], + "degrees": 45, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(b"degrees must be a multiple of 90", response.content) m.assert_not_called() + @mock.patch("documents.serialisers.bulk_edit.rotate") + def test_bulk_edit_rotate_rejects_invalid_degrees(self, m) -> None: + """ + GIVEN: + - A legacy rotate bulk edit with degrees that are null, not an + integer, or not a multiple of 90 + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 + - rotate is not called + """ + self.setup_mock(m, "rotate") + + for degrees, expected_message in ( + (None, b"invalid rotation degrees"), + (True, b"invalid rotation degrees"), + ("foo", b"invalid rotation degrees"), + (90.5, b"invalid rotation degrees"), + (45, b"degrees must be a multiple of 90"), + ): + with self.subTest(degrees=degrees): + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "rotate", + "parameters": {"degrees": degrees}, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(expected_message, response.content) + m.assert_not_called() + + @mock.patch("documents.serialisers.bulk_edit.rotate") + def test_bulk_edit_rotate_passes_integer_degrees(self, m) -> None: + """ + GIVEN: + - A legacy rotate bulk edit with degrees given as a string + WHEN: + - API to bulk edit is called + THEN: + - rotate receives the degrees as an integer + """ + self.setup_mock(m, "rotate") + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "rotate", + "parameters": {"degrees": "-90"}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + self.assertEqual(m.call_args.kwargs["degrees"], -90) + + @mock.patch("documents.serialisers.bulk_edit.split") + def test_bulk_edit_split_rejects_invalid_pages(self, m) -> None: + """ + GIVEN: + - A legacy split bulk edit of a 5 page document with pages that + are not a string, not numeric, zero, a reversed range, or past + the last page + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 + - split is not called + """ + self.setup_mock(m, "split") + + for pages in (None, "", "a", "1-2-3", "0", "3-1", "1-6", "1-5000000"): + with self.subTest(pages=pages): + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "split", + "parameters": {"pages": pages}, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(b"invalid pages specified", response.content) + m.assert_not_called() + + @mock.patch("documents.serialisers.bulk_edit.split") + def test_bulk_edit_split_parses_pages(self, m) -> None: + """ + GIVEN: + - A legacy split bulk edit with single pages and ranges + WHEN: + - API to bulk edit is called + THEN: + - split receives one page list per comma separated group + """ + self.setup_mock(m, "split") + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "split", + "parameters": {"pages": "1,2-4,5"}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]]) + @mock.patch("documents.views.bulk_edit.rotate") def test_rotate_insufficient_permissions(self, m) -> None: self.doc1.owner = User.objects.get(username="temp_admin") From 27927fab4dcc076f9a60698ca8973ceb0b1e53a2 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:24:55 +0000 Subject: [PATCH 09/12] Auto translate strings --- src/locale/en_US/LC_MESSAGES/django.po | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index f74b9feaf3..2bbd9b9909 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-09-15 18:47+0000\n" +"POT-Creation-Date: 2026-09-15 19:24+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -1632,7 +1632,7 @@ msgid "workflow runs" msgstr "" #: documents/serialisers.py:514 documents/serialisers.py:871 -#: documents/serialisers.py:2858 documents/views.py:343 documents/views.py:2719 +#: documents/serialisers.py:2865 documents/views.py:343 documents/views.py:2719 #: paperless_mail/serialisers.py:156 msgid "Insufficient permissions." msgstr "" @@ -1641,39 +1641,39 @@ msgstr "" msgid "Invalid color." msgstr "" -#: documents/serialisers.py:2331 +#: documents/serialisers.py:2338 #, python-format msgid "File type %(type)s not supported" msgstr "" -#: documents/serialisers.py:2375 +#: documents/serialisers.py:2382 #, python-format msgid "Custom field id must be an integer: %(id)s" msgstr "" -#: documents/serialisers.py:2382 +#: documents/serialisers.py:2389 #, python-format msgid "Custom field with id %(id)s does not exist" msgstr "" -#: documents/serialisers.py:2399 documents/serialisers.py:2409 +#: documents/serialisers.py:2406 documents/serialisers.py:2416 msgid "" "Custom fields must be a list of integers or an object mapping ids to values." msgstr "" -#: documents/serialisers.py:2404 +#: documents/serialisers.py:2411 msgid "Some custom fields don't exist or were specified twice." msgstr "" -#: documents/serialisers.py:2551 +#: documents/serialisers.py:2558 msgid "Invalid variable detected." msgstr "" -#: documents/serialisers.py:2914 +#: documents/serialisers.py:2921 msgid "Duplicate document identifiers are not allowed." msgstr "" -#: documents/serialisers.py:2944 documents/views.py:4742 +#: documents/serialisers.py:2951 documents/views.py:4742 #, python-format msgid "Documents not found: %(ids)s" msgstr "" From ec4e45ecf40e30caf4f2deeb08d3b22598859b55 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:35:18 -0700 Subject: [PATCH 10/12] Performance: Streams the classifier pickle file during save to file (#14121) --- src/documents/classifier.py | 99 ++++++++++++++++++-------- src/documents/tests/test_classifier.py | 83 +++++++++++++++++++++ 2 files changed, 154 insertions(+), 28 deletions(-) diff --git a/src/documents/classifier.py b/src/documents/classifier.py index 1cdef4419e..bac810cb88 100644 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -13,6 +13,9 @@ if TYPE_CHECKING: from collections.abc import Callable from collections.abc import Iterator from datetime import datetime + from types import TracebackType + from typing import BinaryIO + from typing import Self from numpy import ndarray @@ -68,6 +71,49 @@ RE_DIGIT = re.compile(r"\d") RE_WORD = re.compile(r"\b[\w]+\b") # words that may contain digits +class _SignedFileWriter: + """ + Atomically writes a file made of an HMAC signature followed by the data, + signing the data as it streams to disk rather than holding it in memory. + + The signature is only known once everything is written, so its space is + reserved at the start of the file and filled in on exit. The target is only + replaced once the file is complete; on error the partial file is removed. + """ + + def __init__(self, target: Path, mac: hmac.HMAC) -> None: + self._target = target + self._temp = target.with_name(f"{target.name}.part") + self._mac = mac + self._file: BinaryIO + + def __enter__(self) -> Self: + self._file = self._temp.open("wb") + self._file.write(bytes(self._mac.digest_size)) + return self + + def write(self, data: bytes | memoryview) -> int: + self._mac.update(data) + return self._file.write(data) + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + with self._file: + if exc_type is None: + self._file.seek(0) + self._file.write(self._mac.digest()) + if exc_type is None: + self._temp.rename(self._target) + finally: + # A no-op after a successful rename, otherwise removes the partial file + self._temp.unlink(missing_ok=True) + + class IncompatibleClassifierVersionError(Exception): def __init__(self, message: str, *args: object) -> None: self.message: str = message @@ -159,13 +205,15 @@ class DocumentClassifier: pickle.dumps(self.data_vectorizer), ).hexdigest() + @staticmethod + def _new_hmac() -> hmac.HMAC: + return hmac.new(settings.SECRET_KEY.encode(), digestmod=sha256) + @staticmethod def _compute_hmac(data: bytes | memoryview) -> bytes: - return hmac.new( - settings.SECRET_KEY.encode(), - data, - sha256, - ).digest() + mac = DocumentClassifier._new_hmac() + mac.update(data) + return mac.digest() def load(self) -> None: from sklearn.exceptions import InconsistentVersionWarning @@ -226,29 +274,24 @@ class DocumentClassifier: raise IncompatibleClassifierVersionError("sklearn version update") def save(self) -> None: - target_file: Path = settings.MODEL_FILE - target_file_temp: Path = target_file.with_suffix(".pickle.part") - - data = pickle.dumps( - ( - self.FORMAT_VERSION, - self.last_doc_change_time, - self.last_auto_type_hash, - self.data_vectorizer, - self.tags_binarizer, - self.tags_classifier, - self.correspondent_classifier, - self.document_type_classifier, - self.storage_path_classifier, - ), - ) - - signature = self._compute_hmac(data) - - with target_file_temp.open("wb") as f: - f.write(signature + data) - - target_file_temp.rename(target_file) + # Stream to disk instead of building the payload in memory. Protocol 5+ + # pickles numpy arrays without copying them (the default is 4 before 3.14). + with _SignedFileWriter(settings.MODEL_FILE, self._new_hmac()) as f: + pickle.dump( + ( + self.FORMAT_VERSION, + self.last_doc_change_time, + self.last_auto_type_hash, + self.data_vectorizer, + self.tags_binarizer, + self.tags_classifier, + self.correspondent_classifier, + self.document_type_classifier, + self.storage_path_classifier, + ), + f, + protocol=pickle.HIGHEST_PROTOCOL, + ) def train( self, diff --git a/src/documents/tests/test_classifier.py b/src/documents/tests/test_classifier.py index 9066c4d410..99d7196f3b 100644 --- a/src/documents/tests/test_classifier.py +++ b/src/documents/tests/test_classifier.py @@ -1,5 +1,8 @@ +import pickle import re import warnings +from datetime import UTC +from datetime import datetime from pathlib import Path from unittest import mock @@ -10,6 +13,7 @@ from django.db import connection from django.test import TestCase from django.test import override_settings from django.test.utils import CaptureQueriesContext +from pytest_django.fixtures import Settings from pytest_mock import MockerFixture from documents.classifier import ClassifierModelCorruptError @@ -914,6 +918,85 @@ class TestClassifier(DirectoriesMixin, TestCase): load_classifier(raise_exception=True) +class TestClassifierSave: + @pytest.fixture + def model_file(self, tmp_path: Path, settings: Settings) -> Path: + settings.MODEL_FILE = tmp_path / "classifier.pickle" + return settings.MODEL_FILE + + @pytest.fixture + def classifier(self) -> DocumentClassifier: + classifier = DocumentClassifier() + classifier.last_doc_change_time = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + classifier.last_auto_type_hash = b"\x01" * 32 + return classifier + + def test_save_writes_signed_pickle( + self, + model_file: Path, + classifier: DocumentClassifier, + ) -> None: + """ + GIVEN: + - A classifier with training state + WHEN: + - The classifier is saved + THEN: + - The file is the HMAC of the pickled state followed by that pickle + - The pickle uses the highest protocol + - The saved state loads back into a new classifier + - No temporary file is left behind + """ + classifier.save() + + raw = model_file.read_bytes() + signature = raw[: DocumentClassifier.HMAC_SIZE] + data = raw[DocumentClassifier.HMAC_SIZE :] + assert signature == DocumentClassifier._compute_hmac(data) + # A pickle opens with the PROTO opcode followed by the protocol number + assert data[:2] == bytes([pickle.PROTO[0], pickle.HIGHEST_PROTOCOL]) + assert pickle.loads(data)[:3] == ( + DocumentClassifier.FORMAT_VERSION, + classifier.last_doc_change_time, + classifier.last_auto_type_hash, + ) + + loaded = DocumentClassifier() + loaded.load() + assert loaded.last_doc_change_time == classifier.last_doc_change_time + assert loaded.last_auto_type_hash == classifier.last_auto_type_hash + + assert not model_file.with_name(f"{model_file.name}.part").exists() + + def test_save_failure_removes_partial_file( + self, + model_file: Path, + classifier: DocumentClassifier, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - An existing classifier model file + WHEN: + - Saving a new classifier fails part way through writing + THEN: + - The error is raised + - The partially written temporary file is removed + - The existing model file is left untouched + """ + model_file.write_bytes(b"existing model") + mocker.patch( + "documents.classifier.pickle.dump", + side_effect=RuntimeError("disk full"), + ) + + with pytest.raises(RuntimeError, match="disk full"): + classifier.save() + + assert not model_file.with_name(f"{model_file.name}.part").exists() + assert model_file.read_bytes() == b"existing model" + + class _StubProbaClassifier: """ A fake scikit-learn classifier exposing just enough of the API for From f919c981e10d30d1165e176bdfa43f54bf4cf62a Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:35:44 -0700 Subject: [PATCH 11/12] Peformance: Improves the memory efficency of classifier training (#14124) --- src/documents/classifier.py | 71 +++++++++++++++++--------- src/documents/tests/test_classifier.py | 65 +++++++++++++++++++++++ 2 files changed, 111 insertions(+), 25 deletions(-) diff --git a/src/documents/classifier.py b/src/documents/classifier.py index bac810cb88..1e078c9c93 100644 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -70,6 +70,9 @@ read_cache = caches["read-cache"] RE_DIGIT = re.compile(r"\d") RE_WORD = re.compile(r"\b[\w]+\b") # words that may contain digits +# Documents whose content is fetched per query while training +_CONTENT_CHUNK_SIZE = 1000 + class _SignedFileWriter: """ @@ -300,10 +303,30 @@ class DocumentClassifier: notify = status_callback if status_callback is not None else lambda _: None # Get non-inbox documents - docs_queryset = ( - Document.objects.exclude( - tags__is_inbox_tag=True, - ) + docs_queryset = Document.objects.exclude( + tags__is_inbox_tag=True, + ).order_by("pk") + + # No documents exit to train against + doc_count = docs_queryset.count() + if doc_count == 0: + raise ValueError("No training data available.") + + labels_tags = [] + labels_correspondent = [] + labels_document_type = [] + labels_storage_path = [] + # Content is fetched separately later, for exactly these documents in this + # order, so it never all has to be in memory at once + doc_pks: list[int] = [] + latest_doc_change: datetime | None = None + + # Step 1: Extract and preprocess training data from the database. + logger.debug("Gathering data from database...") + notify(f"Gathering data from {doc_count} document(s)...") + hasher = sha256() + for doc in ( + docs_queryset.defer("content") .select_related("document_type", "correspondent", "storage_path") .prefetch_related( Prefetch( @@ -316,23 +339,12 @@ class DocumentClassifier: to_attr="auto_tags", ), ) - .order_by("pk") - ) + .iterator(chunk_size=2000) + ): + doc_pks.append(doc.pk) + if latest_doc_change is None or doc.modified > latest_doc_change: + latest_doc_change = doc.modified - # No documents exit to train against - if docs_queryset.count() == 0: - raise ValueError("No training data available.") - - labels_tags = [] - labels_correspondent = [] - labels_document_type = [] - labels_storage_path = [] - - # Step 1: Extract and preprocess training data from the database. - logger.debug("Gathering data from database...") - notify(f"Gathering data from {docs_queryset.count()} document(s)...") - hasher = sha256() - for doc in docs_queryset: y = -1 dt = doc.document_type if dt and dt.matching_algorithm == MatchingModel.MATCH_AUTO: @@ -366,7 +378,6 @@ class DocumentClassifier: # Check if retraining is actually required. # A document has been updated since the classifier was trained # New auto tags, types, correspondent, storage paths exist - latest_doc_change = docs_queryset.latest("modified").modified if ( self.last_doc_change_time is not None and self.last_doc_change_time >= latest_doc_change @@ -393,7 +404,7 @@ class DocumentClassifier: num_storage_paths: int = len(set(labels_storage_path) | {-1}) - 1 logger.debug( - f"{docs_queryset.count()} documents, {num_tags} tag(s), {num_correspondents} correspondent(s), " + f"{len(doc_pks)} documents, {num_tags} tag(s), {num_correspondents} correspondent(s), " f"{num_document_types} document type(s). {num_storage_paths} storage path(s)", ) @@ -415,10 +426,20 @@ class DocumentClassifier: def content_generator() -> Iterator[str]: """ - Generates the content for documents, but once at a time + Generates the content for documents, in the same order as the labels, + fetching it a chunk at a time """ - for doc in docs_queryset: - yield self.preprocess_content(doc.content, shared_cache=False) + for start in range(0, len(doc_pks), _CONTENT_CHUNK_SIZE): + chunk = doc_pks[start : start + _CONTENT_CHUNK_SIZE] + docs = Document.objects.only("content").order_by().in_bulk(chunk) + for pk in chunk: + # A document deleted since its labels were gathered still + # needs a row, so labels and content stay aligned + doc = docs.get(pk) + yield self.preprocess_content( + doc.content if doc is not None else "", + shared_cache=False, + ) self.data_vectorizer = CountVectorizer( analyzer="word", diff --git a/src/documents/tests/test_classifier.py b/src/documents/tests/test_classifier.py index 99d7196f3b..673c18fe50 100644 --- a/src/documents/tests/test_classifier.py +++ b/src/documents/tests/test_classifier.py @@ -1172,3 +1172,68 @@ class TestClassifierTrainTagLabels: assert list(classifier.tags_binarizer.classes_) == sorted( tag.pk for tag in auto_tags ) + + +@pytest.mark.django_db +class TestClassifierTrainContent: + def test_train_content_follows_label_order_across_chunks( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - More documents than fit in one content chunk + WHEN: + - The classifier is trained + THEN: + - Every document's content is preprocessed once, in document order + """ + mocker.patch("documents.classifier._CONTENT_CHUNK_SIZE", 2) + docs = DocumentFactory.create_batch(5) + preprocess = mocker.patch.object( + DocumentClassifier, + "preprocess_content", + side_effect=dummy_preprocess, + ) + + DocumentClassifier().train() + + assert [call.args[0] for call in preprocess.call_args_list] == [ + doc.content for doc in sorted(docs, key=lambda doc: doc.pk) + ] + + def test_train_document_deleted_while_training( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - Two documents + WHEN: + - The second document is deleted after its labels were gathered, but + before its content is fetched + THEN: + - Training completes + - The deleted document is trained with empty content, keeping labels + and content aligned + """ + mocker.patch("documents.classifier._CONTENT_CHUNK_SIZE", 1) + first, second = DocumentFactory.create_batch(2) + + def delete_second_then_preprocess(content: str, **kwargs) -> str: + if content == first.content: + second.delete() + return dummy_preprocess(content) + + preprocess = mocker.patch.object( + DocumentClassifier, + "preprocess_content", + side_effect=delete_second_then_preprocess, + ) + + assert DocumentClassifier().train() + + assert [call.args[0] for call in preprocess.call_args_list] == [ + first.content, + "", + ] From e8b7a98d3ec583874e32dc9e5632729878204fea Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:44:12 -0700 Subject: [PATCH 12/12] Fix: better catch email workflow placeholder parsing errors (#14129) --- src/documents/tests/test_workflows.py | 60 +++++++++++++++++++++++ src/documents/workflows/actions.py | 70 +++++++++++++-------------- 2 files changed, 95 insertions(+), 35 deletions(-) diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index 45a3703d2e..8f241e2895 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -3769,6 +3769,66 @@ class TestWorkflows( expected_str = "Error occurred sending email" self.assertIn(expected_str, cm.output[0]) + @override_settings( + EMAIL_ENABLED=True, + PAPERLESS_URL="http://localhost:8000", + ) + @mock.patch("django.core.mail.message.EmailMessage.send") + def test_workflow_email_action_template_error(self, mock_email_send) -> None: + """ + GIVEN: + - Document added workflow with an email action whose body uses an + undefined template variable, followed by an assignment action + WHEN: + - Document consumption finishes + THEN: + - Error is logged, consumption is not aborted + - No email is sent + - Subsequent actions still run + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + email_action = WorkflowActionEmail.objects.create( + subject="Test Notification: {{ doc_title }}", + body="Document Title: {{ title }}", + to="me@example.com", + ) + action = WorkflowAction.objects.create( + type=WorkflowAction.WorkflowActionType.EMAIL, + email=email_action, + order=0, + ) + assignment_action = WorkflowAction.objects.create( + type=WorkflowAction.WorkflowActionType.ASSIGNMENT, + assign_correspondent=self.c2, + order=1, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action, assignment_action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + + with self.assertLogs("paperless.workflows", level="ERROR") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + + self.assertIn("'title' is undefined", cm.output[0]) + mock_email_send.assert_not_called() + doc.refresh_from_db() + self.assertEqual(doc.correspondent, self.c2) + @override_settings( PAPERLESS_EMAIL_HOST="localhost", EMAIL_ENABLED=True, diff --git a/src/documents/workflows/actions.py b/src/documents/workflows/actions.py index be09217475..5a2e0c6be3 100644 --- a/src/documents/workflows/actions.py +++ b/src/documents/workflows/actions.py @@ -102,42 +102,42 @@ def execute_email_action( ) return - subject = ( - parse_w_workflow_placeholders( - action.email.subject, - context["correspondent"], - context["document_type"], - context["owner_username"], - context["added"], - context["filename"], - context["current_filename"], - context["created"], - context["title"], - context["doc_url"], - context["id"], - ) - if action.email.subject - else "" - ) - body = ( - parse_w_workflow_placeholders( - action.email.body, - context["correspondent"], - context["document_type"], - context["owner_username"], - context["added"], - context["filename"], - context["current_filename"], - context["created"], - context["title"], - context["doc_url"], - context["id"], - ) - if action.email.body - else "" - ) - try: + subject = ( + parse_w_workflow_placeholders( + action.email.subject, + context["correspondent"], + context["document_type"], + context["owner_username"], + context["added"], + context["filename"], + context["current_filename"], + context["created"], + context["title"], + context["doc_url"], + context["id"], + ) + if action.email.subject + else "" + ) + body = ( + parse_w_workflow_placeholders( + action.email.body, + context["correspondent"], + context["document_type"], + context["owner_username"], + context["added"], + context["filename"], + context["current_filename"], + context["created"], + context["title"], + context["doc_url"], + context["id"], + ) + if action.email.body + else "" + ) + attachments: list[EmailAttachment] = [] if action.email.include_document: attachment: EmailAttachment | None = None