diff --git a/src/documents/classifier.py b/src/documents/classifier.py index cfed4725c7..1e078c9c93 100644 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -13,12 +13,16 @@ if TYPE_CHECKING: from collections.abc import Callable from collections.abc import Iterator from datetime import datetime + from types import TracebackType + from typing import BinaryIO + from typing import Self from numpy import ndarray from django.conf import settings from django.core.cache import cache from django.core.cache import caches +from django.db.models import Prefetch from documents.caching import CACHE_5_MINUTES from documents.caching import CACHE_50_MINUTES @@ -28,6 +32,7 @@ from documents.caching import CLASSIFIER_VERSION_KEY from documents.caching import StoredLRUCache from documents.models import Document from documents.models import MatchingModel +from documents.models import Tag from paperless.signed_pickle import SignedPickleError from paperless.signed_pickle import signed_pickle_dumps from paperless.signed_pickle import signed_pickle_loads @@ -65,6 +70,52 @@ read_cache = caches["read-cache"] RE_DIGIT = re.compile(r"\d") RE_WORD = re.compile(r"\b[\w]+\b") # words that may contain digits +# Documents whose content is fetched per query while training +_CONTENT_CHUNK_SIZE = 1000 + + +class _SignedFileWriter: + """ + Atomically writes a file made of an HMAC signature followed by the data, + signing the data as it streams to disk rather than holding it in memory. + + The signature is only known once everything is written, so its space is + reserved at the start of the file and filled in on exit. The target is only + replaced once the file is complete; on error the partial file is removed. + """ + + def __init__(self, target: Path, mac: hmac.HMAC) -> None: + self._target = target + self._temp = target.with_name(f"{target.name}.part") + self._mac = mac + self._file: BinaryIO + + def __enter__(self) -> Self: + self._file = self._temp.open("wb") + self._file.write(bytes(self._mac.digest_size)) + return self + + def write(self, data: bytes | memoryview) -> int: + self._mac.update(data) + return self._file.write(data) + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + with self._file: + if exc_type is None: + self._file.seek(0) + self._file.write(self._mac.digest()) + if exc_type is None: + self._temp.rename(self._target) + finally: + # A no-op after a successful rename, otherwise removes the partial file + self._temp.unlink(missing_ok=True) + class IncompatibleClassifierVersionError(Exception): def __init__(self, message: str, *args: object) -> None: @@ -158,12 +209,14 @@ class DocumentClassifier: ).hexdigest() @staticmethod - def _compute_hmac(data: bytes) -> bytes: - return hmac.new( - settings.SECRET_KEY.encode(), - data, - sha256, - ).digest() + def _new_hmac() -> hmac.HMAC: + return hmac.new(settings.SECRET_KEY.encode(), digestmod=sha256) + + @staticmethod + def _compute_hmac(data: bytes | memoryview) -> bytes: + mac = DocumentClassifier._new_hmac() + mac.update(data) + return mac.digest() def load(self) -> None: from sklearn.exceptions import InconsistentVersionWarning @@ -173,8 +226,13 @@ class DocumentClassifier: if len(raw) <= self.HMAC_SIZE: raise ClassifierModelCorruptError - signature = raw[: self.HMAC_SIZE] - data = raw[self.HMAC_SIZE :] + # Slice through a memoryview so the (potentially multi-GB) payload is + # not copied; hmac and pickle both accept buffers directly. + # The whole file is still verified from memory before unpickling, rather + # than streamed from disk, so it cannot change between check and load. + view = memoryview(raw) + signature = view[: self.HMAC_SIZE] + data = view[self.HMAC_SIZE :] if not hmac.compare_digest(signature, self._compute_hmac(data)): raise ClassifierModelCorruptError @@ -219,29 +277,24 @@ class DocumentClassifier: raise IncompatibleClassifierVersionError("sklearn version update") def save(self) -> None: - target_file: Path = settings.MODEL_FILE - target_file_temp: Path = target_file.with_suffix(".pickle.part") - - data = pickle.dumps( - ( - self.FORMAT_VERSION, - self.last_doc_change_time, - self.last_auto_type_hash, - self.data_vectorizer, - self.tags_binarizer, - self.tags_classifier, - self.correspondent_classifier, - self.document_type_classifier, - self.storage_path_classifier, - ), - ) - - signature = self._compute_hmac(data) - - with target_file_temp.open("wb") as f: - f.write(signature + data) - - target_file_temp.rename(target_file) + # Stream to disk instead of building the payload in memory. Protocol 5+ + # pickles numpy arrays without copying them (the default is 4 before 3.14). + with _SignedFileWriter(settings.MODEL_FILE, self._new_hmac()) as f: + pickle.dump( + ( + self.FORMAT_VERSION, + self.last_doc_change_time, + self.last_auto_type_hash, + self.data_vectorizer, + self.tags_binarizer, + self.tags_classifier, + self.correspondent_classifier, + self.document_type_classifier, + self.storage_path_classifier, + ), + f, + protocol=pickle.HIGHEST_PROTOCOL, + ) def train( self, @@ -250,29 +303,48 @@ class DocumentClassifier: notify = status_callback if status_callback is not None else lambda _: None # Get non-inbox documents - docs_queryset = ( - Document.objects.exclude( - tags__is_inbox_tag=True, - ) - .select_related("document_type", "correspondent", "storage_path") - .prefetch_related("tags") - .order_by("pk") - ) + docs_queryset = Document.objects.exclude( + tags__is_inbox_tag=True, + ).order_by("pk") # No documents exit to train against - if docs_queryset.count() == 0: + doc_count = docs_queryset.count() + if doc_count == 0: raise ValueError("No training data available.") labels_tags = [] labels_correspondent = [] labels_document_type = [] labels_storage_path = [] + # Content is fetched separately later, for exactly these documents in this + # order, so it never all has to be in memory at once + doc_pks: list[int] = [] + latest_doc_change: datetime | None = None # Step 1: Extract and preprocess training data from the database. logger.debug("Gathering data from database...") - notify(f"Gathering data from {docs_queryset.count()} document(s)...") + notify(f"Gathering data from {doc_count} document(s)...") hasher = sha256() - for doc in docs_queryset: + for doc in ( + docs_queryset.defer("content") + .select_related("document_type", "correspondent", "storage_path") + .prefetch_related( + Prefetch( + "tags", + queryset=Tag.objects.filter( + matching_algorithm=MatchingModel.MATCH_AUTO, + ) + .order_by("pk") + .only("pk"), + to_attr="auto_tags", + ), + ) + .iterator(chunk_size=2000) + ): + doc_pks.append(doc.pk) + if latest_doc_change is None or doc.modified > latest_doc_change: + latest_doc_change = doc.modified + y = -1 dt = doc.document_type if dt and dt.matching_algorithm == MatchingModel.MATCH_AUTO: @@ -287,11 +359,7 @@ class DocumentClassifier: hasher.update(y.to_bytes(4, "little", signed=True)) labels_correspondent.append(y) - tags: list[int] = list( - doc.tags.filter(matching_algorithm=MatchingModel.MATCH_AUTO) - .order_by("pk") - .values_list("pk", flat=True), - ) + tags: list[int] = [tag.pk for tag in doc.auto_tags] for tag in tags: hasher.update(tag.to_bytes(4, "little", signed=True)) labels_tags.append(tags) @@ -310,7 +378,6 @@ class DocumentClassifier: # Check if retraining is actually required. # A document has been updated since the classifier was trained # New auto tags, types, correspondent, storage paths exist - latest_doc_change = docs_queryset.latest("modified").modified if ( self.last_doc_change_time is not None and self.last_doc_change_time >= latest_doc_change @@ -337,7 +404,7 @@ class DocumentClassifier: num_storage_paths: int = len(set(labels_storage_path) | {-1}) - 1 logger.debug( - f"{docs_queryset.count()} documents, {num_tags} tag(s), {num_correspondents} correspondent(s), " + f"{len(doc_pks)} documents, {num_tags} tag(s), {num_correspondents} correspondent(s), " f"{num_document_types} document type(s). {num_storage_paths} storage path(s)", ) @@ -359,10 +426,20 @@ class DocumentClassifier: def content_generator() -> Iterator[str]: """ - Generates the content for documents, but once at a time + Generates the content for documents, in the same order as the labels, + fetching it a chunk at a time """ - for doc in docs_queryset: - yield self.preprocess_content(doc.content, shared_cache=False) + for start in range(0, len(doc_pks), _CONTENT_CHUNK_SIZE): + chunk = doc_pks[start : start + _CONTENT_CHUNK_SIZE] + docs = Document.objects.only("content").order_by().in_bulk(chunk) + for pk in chunk: + # A document deleted since its labels were gathered still + # needs a row, so labels and content stay aligned + doc = docs.get(pk) + yield self.preprocess_content( + doc.content if doc is not None else "", + shared_cache=False, + ) self.data_vectorizer = CountVectorizer( analyzer="word", diff --git a/src/documents/permissions.py b/src/documents/permissions.py index b7a1d7e0e4..70c48b6ccd 100644 --- a/src/documents/permissions.py +++ b/src/documents/permissions.py @@ -177,13 +177,10 @@ def _resolve_permissions(codenames: set[str], ctype: ContentType) -> list[Permis """ Resolves `codenames` to Permission rows, raising like the single-object assign_perm() this bulk path replaces does (via a `.get()` internally) - if any codename doesn't exist -- e.g. a client-supplied action name that - was never validated (BulkEditObjectsSerializer._validate_permissions - calls validate_set_permissions() only for its side-effecting id checks - and discards the filtered dict it returns, so an unrecognized action key - reaches this function as-is). A plain `.filter()` with no existence - check would otherwise silently build zero rows and no-op instead of - reporting the bad input. + if any codename doesn't exist. SetPermissionsSerializer rejects unknown + action names at the API, but a caller passing one directly would + otherwise get a plain `.filter()` that silently builds zero rows and + no-ops instead of reporting the bad input. """ permission_objs = list( Permission.objects.filter(content_type=ctype, codename__in=codenames), @@ -298,10 +295,9 @@ def set_permissions_for_objects( # Every action is resolved up front, before anything is written, so an # unrecognized action name (see _resolve_permissions) aborts the whole - # call instead of leaving the actions ahead of it already applied -- - # BulkEditObjectsSerializer lets unknown keys through and its view turns - # the exception into a 400, so a half-applied change would otherwise be - # reported to the client as a failure. + # call instead of leaving the actions ahead of it already applied. + # SetPermissionsSerializer rejects unknown actions at the API, so this + # guards any other caller. permissions_by_action: dict[str, list[Permission]] = {} for action, entry in permissions.items(): if "users" not in entry and "groups" not in entry: diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 902fbd21a6..917c68ca46 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -210,6 +210,9 @@ class MatchingModelSerializer(serializers.ModelSerializer[Any]): return match +PERMISSION_ACTIONS = ("view", "change") + + class SetPermissionsMixin: def _validate_user_ids(self, user_ids): users = User.objects.none() @@ -232,12 +235,9 @@ class SetPermissionsMixin: return groups def validate_set_permissions(self, set_permissions=None): - permissions_dict = { - "view": {}, - "change": {}, - } + permissions_dict = {action: {} for action in PERMISSION_ACTIONS} if set_permissions is not None: - for action in ["view", "change"]: + for action in PERMISSION_ACTIONS: if action in set_permissions: if "users" in set_permissions[action]: users = set_permissions[action]["users"] @@ -265,41 +265,31 @@ class SerializerWithPerms(serializers.Serializer[dict[str, Any]]): super().__init__(*args, **kwargs) -@extend_schema_field( - field={ - "type": "object", - "properties": { - "view": { - "type": "object", - "properties": { - "users": { - "type": "array", - "items": {"type": "integer"}, - }, - "groups": { - "type": "array", - "items": {"type": "integer"}, - }, - }, - }, - "change": { - "type": "object", - "properties": { - "users": { - "type": "array", - "items": {"type": "integer"}, - }, - "groups": { - "type": "array", - "items": {"type": "integer"}, - }, - }, - }, - }, - }, -) -class SetPermissionsSerializer(serializers.DictField): - pass +class PermissionSetSerializer(serializers.Serializer[dict[str, Any]]): + users = serializers.ListField( + child=serializers.IntegerField(), + required=False, + allow_null=True, + ) + groups = serializers.ListField( + child=serializers.IntegerField(), + required=False, + allow_null=True, + ) + + +class SetPermissionsSerializer(serializers.Serializer[dict[str, Any]]): + view = PermissionSetSerializer(required=False) + change = PermissionSetSerializer(required=False) + + def to_internal_value(self, data): + if isinstance(data, dict): + unknown_keys = set(data) - set(PERMISSION_ACTIONS) + if unknown_keys: + raise serializers.ValidationError( + {key: "Unknown permission action." for key in sorted(unknown_keys)}, + ) + return super().to_internal_value(data) class OwnedObjectSerializer( @@ -470,7 +460,6 @@ class OwnedObjectSerializer( set_permissions = SetPermissionsSerializer( label="Set permissions", - allow_empty=True, required=False, write_only=True, ) @@ -1680,6 +1669,13 @@ class SourceModeValidationMixin: return source_mode +def _validate_rotation_degrees(degrees: int, field: str = "degrees") -> int: + # QPDF refuses any other angle, which would otherwise fail inside the task + if degrees % 90 != 0: + raise serializers.ValidationError(f"{field} must be a multiple of 90") + return degrees + + class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidationMixin): degrees = serializers.IntegerField(required=True) source_mode = serializers.CharField( @@ -1688,6 +1684,9 @@ class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidatio ) from_webui = serializers.BooleanField(required=False, default=False) + def validate_degrees(self, value: int) -> int: + return _validate_rotation_degrees(value) + class MergeDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin): metadata_document_id = serializers.IntegerField( @@ -1749,8 +1748,21 @@ class MergeDocumentsAsVersionsSerializer(DocumentListSerializer): return attrs +class PdfEditOperationSerializer(serializers.Serializer[dict[str, int]]): + page = serializers.IntegerField(min_value=1) + rotate = serializers.IntegerField(required=False) + doc = serializers.IntegerField(required=False, min_value=0) + + def validate_rotate(self, value: int) -> int: + return _validate_rotation_degrees(value, field="rotate") + + class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin): - operations = serializers.ListField(required=True, allow_empty=False) + operations = serializers.ListField( + child=PdfEditOperationSerializer(), + required=True, + allow_empty=False, + ) delete_original = serializers.BooleanField(required=False, default=False) update_document = serializers.BooleanField(required=False, default=False) include_metadata = serializers.BooleanField(required=False, default=True) @@ -1768,18 +1780,9 @@ class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMix ) operations = attrs["operations"] - if not isinstance(operations, list): - raise serializers.ValidationError("operations must be a list") - for op in operations: - if not isinstance(op, dict): - raise serializers.ValidationError("invalid operation entry") - if "page" not in op or not isinstance(op["page"], int): - raise serializers.ValidationError("page must be an integer") - if "rotate" in op and not isinstance(op["rotate"], int): - raise serializers.ValidationError("rotate must be an integer") - if "doc" in op and not isinstance(op["doc"], int): - raise serializers.ValidationError("doc must be an integer") + if any(op.get("doc", 0) >= len(operations) for op in operations): + raise serializers.ValidationError("doc index is out of bounds") if attrs["update_document"]: max_idx = max(op.get("doc", 0) for op in operations) @@ -1788,16 +1791,10 @@ class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMix "update_document only allowed with a single output document", ) - if any( - op.get("doc", 0) < 0 or op.get("doc", 0) >= len(operations) - for op in operations - ): - raise serializers.ValidationError("doc index is out of bounds") - doc = Document.objects.get(id=documents[0]) if doc.page_count: for op in operations: - if op["page"] < 1 or op["page"] > doc.page_count: + if op["page"] > doc.page_count: raise serializers.ValidationError( f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.", ) @@ -2042,32 +2039,39 @@ class BulkEditSerializer( else: raise serializers.ValidationError("remove_custom_fields not specified") - def _validate_owner(self, owner): - ownerUser = User.objects.get(pk=owner) - if ownerUser is None: - raise serializers.ValidationError("Specified owner cannot be found") - return ownerUser + def _validate_owner(self, owner) -> User: + owner_field = serializers.PrimaryKeyRelatedField(queryset=User.objects.all()) + try: + return owner_field.run_validation(owner) + except serializers.ValidationError as e: + raise serializers.ValidationError( + "Specified owner cannot be found", + ) from e def _validate_parameters_set_permissions(self, parameters) -> None: if "set_permissions" not in parameters: raise serializers.ValidationError("set_permissions not specified") + set_permissions = parameters["set_permissions"] + if set_permissions is not None: + set_permissions = SetPermissionsSerializer().run_validation( + set_permissions, + ) parameters["set_permissions"] = self.validate_set_permissions( - parameters["set_permissions"], + set_permissions, ) if "owner" in parameters and parameters["owner"] is not None: - self._validate_owner(parameters["owner"]) + parameters["owner"] = self._validate_owner(parameters["owner"]).pk if "merge" not in parameters: parameters["merge"] = False def _validate_parameters_rotate(self, parameters) -> None: - try: - if ( - "degrees" not in parameters - or not float(parameters["degrees"]).is_integer() - ): - raise serializers.ValidationError("invalid rotation degrees") - except ValueError: + if "degrees" not in parameters: raise serializers.ValidationError("invalid rotation degrees") + try: + degrees = serializers.IntegerField().run_validation(parameters["degrees"]) + except serializers.ValidationError as e: + raise serializers.ValidationError("invalid rotation degrees") from e + parameters["degrees"] = _validate_rotation_degrees(degrees) def _validate_source_mode(self, parameters) -> None: source_mode = parameters.get( @@ -2076,28 +2080,25 @@ class BulkEditSerializer( ) parameters["source_mode"] = self.validate_source_mode(source_mode) - def _validate_parameters_split(self, parameters) -> None: + def _validate_parameters_split(self, parameters, document_id) -> None: if "pages" not in parameters: raise serializers.ValidationError("pages not specified") - try: - pages = [] - docs = parameters["pages"].split(",") - for doc in docs: - if "-" in doc: - pages.append( - [ - x - for x in range( - int(doc.split("-")[0]), - int(doc.split("-")[1]) + 1, - ) - ], - ) - else: - pages.append([int(doc)]) - parameters["pages"] = pages - except ValueError: + if not isinstance(parameters["pages"], str): raise serializers.ValidationError("invalid pages specified") + page_count = Document.objects.get(id=document_id).page_count + pages = [] + for group in parameters["pages"].split(","): + start, is_range, end = group.partition("-") + try: + first = int(start) + last = int(end) if is_range else first + except ValueError as e: + raise serializers.ValidationError("invalid pages specified") from e + # Bound the range before building it, a huge one would exhaust memory + if not 1 <= first <= last or (page_count and last > page_count): + raise serializers.ValidationError("invalid pages specified") + pages.append(list(range(first, last + 1))) + parameters["pages"] = pages if "delete_originals" in parameters: if not isinstance(parameters["delete_originals"], bool): @@ -2128,19 +2129,18 @@ class BulkEditSerializer( def _validate_parameters_edit_pdf(self, parameters, document_id) -> None: if "operations" not in parameters: raise serializers.ValidationError("operations not specified") - if not isinstance(parameters["operations"], list): - raise serializers.ValidationError("operations must be a list") - if not parameters["operations"]: - raise serializers.ValidationError("operations must not be empty") - for op in parameters["operations"]: - if not isinstance(op, dict): - raise serializers.ValidationError("invalid operation entry") - if "page" not in op or not isinstance(op["page"], int): - raise serializers.ValidationError("page must be an integer") - if "rotate" in op and not isinstance(op["rotate"], int): - raise serializers.ValidationError("rotate must be an integer") - if "doc" in op and not isinstance(op["doc"], int): - raise serializers.ValidationError("doc must be an integer") + operations_field = serializers.ListField( + child=PdfEditOperationSerializer(), + allow_empty=False, + ) + try: + operations = operations_field.run_validation(parameters["operations"]) + except serializers.ValidationError as e: + # Key the errors under "operations" so they match what the + # dedicated edit_pdf endpoint returns + raise serializers.ValidationError({"operations": e.detail}) from e + parameters["operations"] = operations + if "update_document" in parameters: if not isinstance(parameters["update_document"], bool): raise serializers.ValidationError("update_document must be a boolean") @@ -2152,24 +2152,21 @@ class BulkEditSerializer( else: parameters["include_metadata"] = True + if any(op.get("doc", 0) >= len(operations) for op in operations): + raise serializers.ValidationError("doc index is out of bounds") + if parameters["update_document"]: - max_idx = max(op.get("doc", 0) for op in parameters["operations"]) + max_idx = max(op.get("doc", 0) for op in operations) if max_idx > 0: raise serializers.ValidationError( "update_document only allowed with a single output document", ) - if any( - op.get("doc", 0) < 0 or op.get("doc", 0) >= len(parameters["operations"]) - for op in parameters["operations"] - ): - raise serializers.ValidationError("doc index is out of bounds") - doc = Document.objects.get(id=document_id) # doc existence is already validated if doc.page_count: - for op in parameters["operations"]: - if op["page"] < 1 or op["page"] > doc.page_count: + for op in operations: + if op["page"] > doc.page_count: raise serializers.ValidationError( f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.", ) @@ -2228,7 +2225,7 @@ class BulkEditSerializer( raise serializers.ValidationError( "Split method only supports one document", ) - self._validate_parameters_split(parameters) + self._validate_parameters_split(parameters, attrs["documents"][0]) elif method == bulk_edit.delete_pages: if len(attrs["documents"]) > 1: raise serializers.ValidationError( @@ -3021,9 +3018,8 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin): allow_null=True, ) - permissions = serializers.DictField( + permissions = SetPermissionsSerializer( label="Set permissions", - allow_empty=False, required=False, write_only=True, ) @@ -3059,8 +3055,8 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin): ) return objects - def _validate_permissions(self, permissions) -> None: - self.validate_set_permissions( + def _validate_permissions(self, permissions) -> dict: + return self.validate_set_permissions( permissions, ) @@ -3084,7 +3080,11 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin): if operation == "set_permissions": permissions = attrs.get("permissions") if permissions is not None: - self._validate_permissions(permissions) + if not permissions: + raise serializers.ValidationError( + "permissions must not be empty", + ) + attrs["permissions"] = self._validate_permissions(permissions) return attrs diff --git a/src/documents/tests/test_api_app_config.py b/src/documents/tests/test_api_app_config.py index 0ca6f4412b..8620c3ec3e 100644 --- a/src/documents/tests/test_api_app_config.py +++ b/src/documents/tests/test_api_app_config.py @@ -194,6 +194,70 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): self.assertEqual(config.language, None) self.assertEqual(config.barcode_tag_mapping, None) + def test_api_update_config_json_objects(self) -> None: + """ + GIVEN: + - API request to update app config with JSON objects for the + user_args and barcode_tag_mapping JSONFields + WHEN: + - API is called + THEN: + - Correct HTTP response + - Both objects are stored as sent + """ + user_args = {"unpaper_args": "--pre-rotate 90", "jobs": 2} + barcode_tag_mapping = {"TAG:(.*)": "\\g<1>", "ASN12.*": ""} + response = self.client.patch( + f"{self.ENDPOINT}1/", + json.dumps( + { + "user_args": json.dumps(user_args), + "barcode_tag_mapping": json.dumps(barcode_tag_mapping), + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + config = ApplicationConfiguration.objects.first() + assert config is not None + self.assertEqual(config.user_args, user_args) + self.assertEqual(config.barcode_tag_mapping, barcode_tag_mapping) + + def test_api_update_config_rejects_invalid_json_objects(self) -> None: + """ + GIVEN: + - API request to update app config with a JSON-encoded value that + is not an object for user_args or barcode_tag_mapping, or a + barcode_tag_mapping with a non-string substitute + WHEN: + - API is called + THEN: + - Request is rejected with a 400 naming the problem + - Config is not updated + """ + not_objects = (True, 1, [1, 2, 3], "not a dict") + cases = [ + (field, value, b"must be a JSON object") + for field in ("user_args", "barcode_tag_mapping") + for value in not_objects + ] + cases += [ + ("barcode_tag_mapping", {"TAG:(.*)": 5}, b"values must be strings"), + ("barcode_tag_mapping", {"TAG:(.*)": None}, b"values must be strings"), + ] + for field, value, expected_message in cases: + with self.subTest(field=field, value=value): + response = self.client.patch( + f"{self.ENDPOINT}1/", + json.dumps({field: json.dumps(value)}), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(expected_message, response.content) + config = ApplicationConfiguration.objects.first() + assert config is not None + self.assertIsNone(getattr(config, field)) + def test_api_replace_app_logo(self) -> None: """ GIVEN: diff --git a/src/documents/tests/test_api_bulk_edit.py b/src/documents/tests/test_api_bulk_edit.py index ebde4ff1f9..0c13cce406 100644 --- a/src/documents/tests/test_api_bulk_edit.py +++ b/src/documents/tests/test_api_bulk_edit.py @@ -1165,6 +1165,150 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.assertIn(b"set_permissions not specified", response.content) m.assert_not_called() + @mock.patch("documents.serialisers.bulk_edit.set_permissions") + def test_set_permissions_rejects_malformed_set_permissions(self, m) -> None: + """ + GIVEN: + - A set_permissions bulk edit where set_permissions is not an + object, has a non-list users value, or has an unknown action + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 describing the problem + - set_permissions is not called + """ + self.setup_mock(m, "set_permissions") + + for set_permissions, expected_message in ( + (False, b"Expected a dictionary"), + ({"view": {"users": False}}, b"Expected a list"), + ({"not_a_real_action": {"users": [1]}}, b"Unknown permission action"), + ): + with self.subTest(set_permissions=set_permissions): + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "set_permissions", + "parameters": {"set_permissions": set_permissions}, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(expected_message, response.content) + m.assert_not_called() + + @mock.patch("documents.serialisers.bulk_edit.set_permissions") + def test_set_permissions_rejects_invalid_owner(self, m) -> None: + """ + GIVEN: + - A set_permissions bulk edit with an owner that is a nonexistent + id, a boolean, a list, a dict, or a non-numeric string + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 + - set_permissions is not called + """ + self.setup_mock(m, "set_permissions") + + for bad_owner in ( + 999999, + True, + ["not", "an", "id"], + {"nested": "dict"}, + "not-a-number", + ): + with self.subTest(owner=bad_owner): + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "set_permissions", + "parameters": { + "set_permissions": { + "view": {"users": [self.user.id]}, + }, + "owner": bad_owner, + }, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(b"Specified owner cannot be found", response.content) + m.assert_not_called() + + @mock.patch("documents.serialisers.bulk_edit.set_permissions") + def test_set_permissions_null_is_a_noop(self, m) -> None: + """ + GIVEN: + - A set_permissions bulk edit with set_permissions null and an + owner, i.e. a request that only changes the owner + WHEN: + - API to bulk edit is called + THEN: + - Request succeeds + - set_permissions receives no users or groups for any action + """ + self.setup_mock(m, "set_permissions") + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "set_permissions", + "parameters": { + "set_permissions": None, + "owner": self.user.id, + }, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + self.assertEqual( + m.call_args.kwargs["set_permissions"], + {"view": {}, "change": {}}, + ) + + @mock.patch("documents.serialisers.bulk_edit.set_permissions") + def test_set_permissions_passes_validated_owner_id(self, m) -> None: + """ + GIVEN: + - A set_permissions bulk edit with the owner id given as a string + WHEN: + - API to bulk edit is called + THEN: + - set_permissions receives the owner as an integer id + """ + self.setup_mock(m, "set_permissions") + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "set_permissions", + "parameters": { + "set_permissions": {"view": {"users": [self.user.id]}}, + "owner": str(self.user.id), + }, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + self.assertEqual(m.call_args.kwargs["owner"], self.user.id) + @mock.patch("documents.serialisers.bulk_edit.set_permissions") def test_set_permissions_merge(self, m) -> None: self.setup_mock(m, "set_permissions") @@ -1300,7 +1444,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.client.force_authenticate(user=user1) permissions = { - "owner": user1.id, + "view": {"users": [user1.id]}, } response = self.client.post( @@ -1436,8 +1580,146 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + response = self.client.post( + "/api/documents/rotate/", + json.dumps( + { + "documents": [self.doc2.id, self.doc3.id], + "degrees": 45, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(b"degrees must be a multiple of 90", response.content) m.assert_not_called() + @mock.patch("documents.serialisers.bulk_edit.rotate") + def test_bulk_edit_rotate_rejects_invalid_degrees(self, m) -> None: + """ + GIVEN: + - A legacy rotate bulk edit with degrees that are null, not an + integer, or not a multiple of 90 + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 + - rotate is not called + """ + self.setup_mock(m, "rotate") + + for degrees, expected_message in ( + (None, b"invalid rotation degrees"), + (True, b"invalid rotation degrees"), + ("foo", b"invalid rotation degrees"), + (90.5, b"invalid rotation degrees"), + (45, b"degrees must be a multiple of 90"), + ): + with self.subTest(degrees=degrees): + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "rotate", + "parameters": {"degrees": degrees}, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(expected_message, response.content) + m.assert_not_called() + + @mock.patch("documents.serialisers.bulk_edit.rotate") + def test_bulk_edit_rotate_passes_integer_degrees(self, m) -> None: + """ + GIVEN: + - A legacy rotate bulk edit with degrees given as a string + WHEN: + - API to bulk edit is called + THEN: + - rotate receives the degrees as an integer + """ + self.setup_mock(m, "rotate") + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "rotate", + "parameters": {"degrees": "-90"}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + self.assertEqual(m.call_args.kwargs["degrees"], -90) + + @mock.patch("documents.serialisers.bulk_edit.split") + def test_bulk_edit_split_rejects_invalid_pages(self, m) -> None: + """ + GIVEN: + - A legacy split bulk edit of a 5 page document with pages that + are not a string, not numeric, zero, a reversed range, or past + the last page + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 + - split is not called + """ + self.setup_mock(m, "split") + + for pages in (None, "", "a", "1-2-3", "0", "3-1", "1-6", "1-5000000"): + with self.subTest(pages=pages): + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "split", + "parameters": {"pages": pages}, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(b"invalid pages specified", response.content) + m.assert_not_called() + + @mock.patch("documents.serialisers.bulk_edit.split") + def test_bulk_edit_split_parses_pages(self, m) -> None: + """ + GIVEN: + - A legacy split bulk edit with single pages and ranges + WHEN: + - API to bulk edit is called + THEN: + - split receives one page list per comma separated group + """ + self.setup_mock(m, "split") + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "split", + "parameters": {"pages": "1,2-4,5"}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]]) + @mock.patch("documents.views.bulk_edit.rotate") def test_rotate_insufficient_permissions(self, m) -> None: self.doc1.owner = User.objects.get(username="temp_admin") @@ -1681,7 +1963,10 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"operations must not be empty", response.content) + self.assertEqual( + response.json(), + {"operations": ["This list may not be empty."]}, + ) @mock.patch("documents.views.bulk_edit.edit_pdf") def test_edit_pdf(self, m) -> None: @@ -1751,7 +2036,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"invalid operation entry", response.content) + self.assertIn(b"Expected a dictionary", response.content) response = self.client.post( "/api/documents/edit_pdf/", @@ -1764,7 +2049,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"page must be an integer", response.content) + self.assertIn(b"valid integer is required", response.content) response = self.client.post( "/api/documents/edit_pdf/", @@ -1777,7 +2062,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"rotate must be an integer", response.content) + self.assertIn(b"valid integer is required", response.content) response = self.client.post( "/api/documents/edit_pdf/", @@ -1790,9 +2075,14 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"doc must be an integer", response.content) + self.assertIn(b"valid integer is required", response.content) - for doc_index in (-1, 2**32): + # A negative index fails the field's min_value before the + # object-level bound against len(operations) is checked + for doc_index, expected_message in ( + (-1, b"greater than or equal to 0"), + (2**32, b"doc index is out of bounds"), + ): with self.subTest(doc_index=doc_index): response = self.client.post( "/api/documents/edit_pdf/", @@ -1805,7 +2095,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): content_type="application/json", ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertIn(b"doc index is out of bounds", response.content) + self.assertIn(expected_message, response.content) response = self.client.post( "/api/documents/edit_pdf/", @@ -1813,7 +2103,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): { "documents": [self.doc2.id], "update_document": True, - "operations": [{"page": 1, "doc": 1}, {"page": 2, "doc": 2}], + "operations": [{"page": 1, "doc": 0}, {"page": 2, "doc": 1}], }, ), content_type="application/json", @@ -1838,6 +2128,72 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn(b"Invalid source_mode", response.content) + @mock.patch("documents.views.bulk_edit.edit_pdf") + def test_edit_pdf_rejects_invalid_operation_values(self, m) -> None: + """ + GIVEN: + - An edit_pdf operation with a non-positive page, a boolean in + place of an integer, or a rotation that is not a multiple of 90 + WHEN: + - API to edit the PDF is called + THEN: + - API returns HTTP 400 with the field error + - edit_pdf is not called + """ + self.setup_mock(m, "edit_pdf") + for operation, expected_message in ( + ({"page": 0}, b"greater than or equal to 1"), + ({"page": True}, b"valid integer is required"), + ({"page": 1, "rotate": True}, b"valid integer is required"), + ({"page": 1, "doc": True}, b"valid integer is required"), + ({"page": 1, "rotate": 45}, b"rotate must be a multiple of 90"), + ): + with self.subTest(operation=operation): + response = self.client.post( + "/api/documents/edit_pdf/", + json.dumps( + { + "documents": [self.doc2.id], + "operations": [operation], + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(expected_message, response.content) + m.assert_not_called() + + def test_legacy_bulk_edit_keys_pdf_operation_errors_under_operations( + self, + ) -> None: + """ + GIVEN: + - A legacy bulk_edit edit_pdf request with an invalid operation + WHEN: + - API to bulk edit is called + THEN: + - API returns HTTP 400 + - The error is keyed under operations and the operation index, + matching the edit_pdf endpoint + """ + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "documents": [self.doc2.id], + "method": "edit_pdf", + "parameters": {"operations": [{"page": 1, "rotate": 45}]}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.json(), + {"operations": {"0": {"rotate": ["rotate must be a multiple of 90"]}}}, + ) + @mock.patch("documents.views.bulk_edit.edit_pdf") def test_edit_pdf_page_out_of_bounds(self, m) -> None: self.setup_mock(m, "edit_pdf") diff --git a/src/documents/tests/test_api_permissions.py b/src/documents/tests/test_api_permissions.py index 29eb8f4961..0b2c491a4e 100644 --- a/src/documents/tests/test_api_permissions.py +++ b/src/documents/tests/test_api_permissions.py @@ -1499,6 +1499,110 @@ class TestBulkEditObjectPermissions(APITestCase): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertTrue(Tag.objects.filter(pk=self.t1.id).exists()) + def test_bulk_object_set_permissions_rejects_empty_permissions(self) -> None: + """ + GIVEN: + - Existing objects + WHEN: + - bulk_edit_objects API endpoint is called with set_permissions + operation and an empty permissions dict + THEN: + - Validation fails rather than silently applying a no-op + """ + response = self.client.post( + "/api/bulk_edit_objects/", + json.dumps( + { + "objects": [self.t1.id], + "object_type": "tags", + "operation": "set_permissions", + "permissions": {}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_bulk_object_set_permissions_rejects_non_dict_permissions(self) -> None: + """ + GIVEN: + - Existing objects + WHEN: + - bulk_edit_objects API endpoint is called with set_permissions + operation and a non-dict permissions value + THEN: + - Validation fails rather than crashing + """ + response = self.client.post( + "/api/bulk_edit_objects/", + json.dumps( + { + "objects": [self.t1.id], + "object_type": "tags", + "operation": "set_permissions", + "permissions": False, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_bulk_object_set_permissions_rejects_unknown_action(self) -> None: + """ + GIVEN: + - Existing objects + WHEN: + - bulk_edit_objects API endpoint is called with set_permissions + operation and an unrecognized permission action name + THEN: + - Validation fails rather than silently no-oping + """ + response = self.client.post( + "/api/bulk_edit_objects/", + json.dumps( + { + "objects": [self.t1.id], + "object_type": "tags", + "operation": "set_permissions", + "permissions": {"not_a_real_action": {"users": [self.user1.id]}}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_bulk_object_set_permissions_null_users_clears_users(self) -> None: + """ + GIVEN: + - An object a user has view permission on + WHEN: + - bulk_edit_objects API endpoint is called with set_permissions + operation, merge off, and an explicit null for the view users + THEN: + - Request succeeds and null is treated as an empty user list, + so the existing view permission is removed + """ + assign_perm("view_tag", self.user1, self.t1) + + response = self.client.post( + "/api/bulk_edit_objects/", + json.dumps( + { + "objects": [self.t1.id], + "object_type": "tags", + "operation": "set_permissions", + "permissions": {"view": {"users": None}}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertNotIn(self.user1, get_users_with_perms(self.t1)) + def test_bulk_edit_object_permissions_validation(self) -> None: """ GIVEN: diff --git a/src/documents/tests/test_bulk_edit.py b/src/documents/tests/test_bulk_edit.py index f9d4a5028a..bfd1345163 100644 --- a/src/documents/tests/test_bulk_edit.py +++ b/src/documents/tests/test_bulk_edit.py @@ -636,14 +636,9 @@ class TestBulkEdit(DirectoriesMixin, TestCase): THEN: - Permission.DoesNotExist is raised, not a silent no-op - Regression test: the endpoint that calls this - (BulkEditObjectPermissionsView) never actually validates action - names against the raw client-supplied permissions dict -- - BulkEditObjectsSerializer._validate_permissions calls - validate_set_permissions() only for its side-effecting user/group id - checks and discards the filtered dict it returns -- so a bogus - action key reaches this function as-is. Resolving the Permission via - a bare `.filter()` (which returns empty instead of raising) would + The API rejects unknown action names before they get here, but a + direct caller could still pass one. Resolving the Permission via a + bare `.filter()` (which returns empty instead of raising) would silently drop the grant and report success. """ with self.assertRaises(Permission.DoesNotExist): diff --git a/src/documents/tests/test_classifier.py b/src/documents/tests/test_classifier.py index 0e09ff0740..673c18fe50 100644 --- a/src/documents/tests/test_classifier.py +++ b/src/documents/tests/test_classifier.py @@ -1,13 +1,20 @@ +import pickle import re import warnings +from datetime import UTC +from datetime import datetime from pathlib import Path from unittest import mock import numpy as np import pytest from django.conf import settings +from django.db import connection from django.test import TestCase from django.test import override_settings +from django.test.utils import CaptureQueriesContext +from pytest_django.fixtures import Settings +from pytest_mock import MockerFixture from documents.classifier import ClassifierModelCorruptError from documents.classifier import DocumentClassifier @@ -20,6 +27,8 @@ from documents.models import DocumentType from documents.models import MatchingModel from documents.models import StoragePath from documents.models import Tag +from documents.tests.factories import DocumentFactory +from documents.tests.factories import TagFactory from documents.tests.utils import DirectoriesMixin from paperless.signed_pickle import HMAC_SIZE from paperless.signed_pickle import signed_pickle_dumps @@ -909,6 +918,85 @@ class TestClassifier(DirectoriesMixin, TestCase): load_classifier(raise_exception=True) +class TestClassifierSave: + @pytest.fixture + def model_file(self, tmp_path: Path, settings: Settings) -> Path: + settings.MODEL_FILE = tmp_path / "classifier.pickle" + return settings.MODEL_FILE + + @pytest.fixture + def classifier(self) -> DocumentClassifier: + classifier = DocumentClassifier() + classifier.last_doc_change_time = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + classifier.last_auto_type_hash = b"\x01" * 32 + return classifier + + def test_save_writes_signed_pickle( + self, + model_file: Path, + classifier: DocumentClassifier, + ) -> None: + """ + GIVEN: + - A classifier with training state + WHEN: + - The classifier is saved + THEN: + - The file is the HMAC of the pickled state followed by that pickle + - The pickle uses the highest protocol + - The saved state loads back into a new classifier + - No temporary file is left behind + """ + classifier.save() + + raw = model_file.read_bytes() + signature = raw[: DocumentClassifier.HMAC_SIZE] + data = raw[DocumentClassifier.HMAC_SIZE :] + assert signature == DocumentClassifier._compute_hmac(data) + # A pickle opens with the PROTO opcode followed by the protocol number + assert data[:2] == bytes([pickle.PROTO[0], pickle.HIGHEST_PROTOCOL]) + assert pickle.loads(data)[:3] == ( + DocumentClassifier.FORMAT_VERSION, + classifier.last_doc_change_time, + classifier.last_auto_type_hash, + ) + + loaded = DocumentClassifier() + loaded.load() + assert loaded.last_doc_change_time == classifier.last_doc_change_time + assert loaded.last_auto_type_hash == classifier.last_auto_type_hash + + assert not model_file.with_name(f"{model_file.name}.part").exists() + + def test_save_failure_removes_partial_file( + self, + model_file: Path, + classifier: DocumentClassifier, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - An existing classifier model file + WHEN: + - Saving a new classifier fails part way through writing + THEN: + - The error is raised + - The partially written temporary file is removed + - The existing model file is left untouched + """ + model_file.write_bytes(b"existing model") + mocker.patch( + "documents.classifier.pickle.dump", + side_effect=RuntimeError("disk full"), + ) + + with pytest.raises(RuntimeError, match="disk full"): + classifier.save() + + assert not model_file.with_name(f"{model_file.name}.part").exists() + assert model_file.read_bytes() == b"existing model" + + class _StubProbaClassifier: """ A fake scikit-learn classifier exposing just enough of the API for @@ -1017,3 +1105,135 @@ def test_preprocess_content_nltk_load_fail(mocker) -> None: expected_preprocess_content = f.read().rstrip() result = classifier.preprocess_content(content) assert result == expected_preprocess_content + + +@pytest.mark.django_db +class TestClassifierTrainTagLabels: + @pytest.fixture(autouse=True) + def _simple_preprocess(self, mocker: MockerFixture) -> None: + mocker.patch.object( + DocumentClassifier, + "preprocess_content", + side_effect=dummy_preprocess, + ) + + @pytest.fixture + def auto_tags(self) -> list[Tag]: + return TagFactory.create_batch(2, matching_algorithm=MatchingModel.MATCH_AUTO) + + def test_train_query_count_does_not_scale_with_documents( + self, + auto_tags: list[Tag], + ) -> None: + """ + GIVEN: + - Documents with auto matching tags + WHEN: + - The classifier is trained, then more documents are added and it is + trained again + THEN: + - Both trainings run the same number of queries + """ + for doc in DocumentFactory.create_batch(2): + doc.tags.set(auto_tags) + + with CaptureQueriesContext(connection) as few_documents: + DocumentClassifier().train() + + for doc in DocumentFactory.create_batch(6): + doc.tags.set(auto_tags) + + with CaptureQueriesContext(connection) as more_documents: + DocumentClassifier().train() + + assert len(more_documents) == len(few_documents) + + def test_train_uses_only_auto_tags_as_labels( + self, + auto_tags: list[Tag], + ) -> None: + """ + GIVEN: + - Documents with both auto matching and non auto matching tags + WHEN: + - The classifier is trained + THEN: + - Only the auto matching tags are used as tag labels + """ + manual_tag = TagFactory(matching_algorithm=MatchingModel.MATCH_ANY) + first, second, third = DocumentFactory.create_batch(3) + first.tags.set([auto_tags[0], manual_tag]) + second.tags.set([auto_tags[1], manual_tag]) + third.tags.set([*auto_tags, manual_tag]) + + classifier = DocumentClassifier() + classifier.train() + + assert list(classifier.tags_binarizer.classes_) == sorted( + tag.pk for tag in auto_tags + ) + + +@pytest.mark.django_db +class TestClassifierTrainContent: + def test_train_content_follows_label_order_across_chunks( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - More documents than fit in one content chunk + WHEN: + - The classifier is trained + THEN: + - Every document's content is preprocessed once, in document order + """ + mocker.patch("documents.classifier._CONTENT_CHUNK_SIZE", 2) + docs = DocumentFactory.create_batch(5) + preprocess = mocker.patch.object( + DocumentClassifier, + "preprocess_content", + side_effect=dummy_preprocess, + ) + + DocumentClassifier().train() + + assert [call.args[0] for call in preprocess.call_args_list] == [ + doc.content for doc in sorted(docs, key=lambda doc: doc.pk) + ] + + def test_train_document_deleted_while_training( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - Two documents + WHEN: + - The second document is deleted after its labels were gathered, but + before its content is fetched + THEN: + - Training completes + - The deleted document is trained with empty content, keeping labels + and content aligned + """ + mocker.patch("documents.classifier._CONTENT_CHUNK_SIZE", 1) + first, second = DocumentFactory.create_batch(2) + + def delete_second_then_preprocess(content: str, **kwargs) -> str: + if content == first.content: + second.delete() + return dummy_preprocess(content) + + preprocess = mocker.patch.object( + DocumentClassifier, + "preprocess_content", + side_effect=delete_second_then_preprocess, + ) + + assert DocumentClassifier().train() + + assert [call.args[0] for call in preprocess.call_args_list] == [ + first.content, + "", + ] diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index 45a3703d2e..8f241e2895 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -3769,6 +3769,66 @@ class TestWorkflows( expected_str = "Error occurred sending email" self.assertIn(expected_str, cm.output[0]) + @override_settings( + EMAIL_ENABLED=True, + PAPERLESS_URL="http://localhost:8000", + ) + @mock.patch("django.core.mail.message.EmailMessage.send") + def test_workflow_email_action_template_error(self, mock_email_send) -> None: + """ + GIVEN: + - Document added workflow with an email action whose body uses an + undefined template variable, followed by an assignment action + WHEN: + - Document consumption finishes + THEN: + - Error is logged, consumption is not aborted + - No email is sent + - Subsequent actions still run + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + email_action = WorkflowActionEmail.objects.create( + subject="Test Notification: {{ doc_title }}", + body="Document Title: {{ title }}", + to="me@example.com", + ) + action = WorkflowAction.objects.create( + type=WorkflowAction.WorkflowActionType.EMAIL, + email=email_action, + order=0, + ) + assignment_action = WorkflowAction.objects.create( + type=WorkflowAction.WorkflowActionType.ASSIGNMENT, + assign_correspondent=self.c2, + order=1, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action, assignment_action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + + with self.assertLogs("paperless.workflows", level="ERROR") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + + self.assertIn("'title' is undefined", cm.output[0]) + mock_email_send.assert_not_called() + doc.refresh_from_db() + self.assertEqual(doc.correspondent, self.c2) + @override_settings( PAPERLESS_EMAIL_HOST="localhost", EMAIL_ENABLED=True, diff --git a/src/documents/workflows/actions.py b/src/documents/workflows/actions.py index be09217475..5a2e0c6be3 100644 --- a/src/documents/workflows/actions.py +++ b/src/documents/workflows/actions.py @@ -102,42 +102,42 @@ def execute_email_action( ) return - subject = ( - parse_w_workflow_placeholders( - action.email.subject, - context["correspondent"], - context["document_type"], - context["owner_username"], - context["added"], - context["filename"], - context["current_filename"], - context["created"], - context["title"], - context["doc_url"], - context["id"], - ) - if action.email.subject - else "" - ) - body = ( - parse_w_workflow_placeholders( - action.email.body, - context["correspondent"], - context["document_type"], - context["owner_username"], - context["added"], - context["filename"], - context["current_filename"], - context["created"], - context["title"], - context["doc_url"], - context["id"], - ) - if action.email.body - else "" - ) - try: + subject = ( + parse_w_workflow_placeholders( + action.email.subject, + context["correspondent"], + context["document_type"], + context["owner_username"], + context["added"], + context["filename"], + context["current_filename"], + context["created"], + context["title"], + context["doc_url"], + context["id"], + ) + if action.email.subject + else "" + ) + body = ( + parse_w_workflow_placeholders( + action.email.body, + context["correspondent"], + context["document_type"], + context["owner_username"], + context["added"], + context["filename"], + context["current_filename"], + context["created"], + context["title"], + context["doc_url"], + context["id"], + ) + if action.email.body + else "" + ) + attachments: list[EmailAttachment] = [] if action.email.include_document: attachment: EmailAttachment | None = None diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index cfcf3008e1..2bbd9b9909 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-09-15 14:27+0000\n" +"POT-Creation-Date: 2026-09-15 19:24+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -1631,49 +1631,49 @@ msgstr "" msgid "workflow runs" msgstr "" -#: documents/serialisers.py:525 documents/serialisers.py:882 -#: documents/serialisers.py:2868 documents/views.py:343 documents/views.py:2719 +#: documents/serialisers.py:514 documents/serialisers.py:871 +#: documents/serialisers.py:2865 documents/views.py:343 documents/views.py:2719 #: paperless_mail/serialisers.py:156 msgid "Insufficient permissions." msgstr "" -#: documents/serialisers.py:718 +#: documents/serialisers.py:707 msgid "Invalid color." msgstr "" -#: documents/serialisers.py:2341 +#: documents/serialisers.py:2338 #, python-format msgid "File type %(type)s not supported" msgstr "" -#: documents/serialisers.py:2385 +#: documents/serialisers.py:2382 #, python-format msgid "Custom field id must be an integer: %(id)s" msgstr "" -#: documents/serialisers.py:2392 +#: documents/serialisers.py:2389 #, python-format msgid "Custom field with id %(id)s does not exist" msgstr "" -#: documents/serialisers.py:2409 documents/serialisers.py:2419 +#: documents/serialisers.py:2406 documents/serialisers.py:2416 msgid "" "Custom fields must be a list of integers or an object mapping ids to values." msgstr "" -#: documents/serialisers.py:2414 +#: documents/serialisers.py:2411 msgid "Some custom fields don't exist or were specified twice." msgstr "" -#: documents/serialisers.py:2561 +#: documents/serialisers.py:2558 msgid "Invalid variable detected." msgstr "" -#: documents/serialisers.py:2924 +#: documents/serialisers.py:2921 msgid "Duplicate document identifiers are not allowed." msgstr "" -#: documents/serialisers.py:2954 documents/views.py:4742 +#: documents/serialisers.py:2951 documents/views.py:4742 #, python-format msgid "Documents not found: %(ids)s" msgstr "" diff --git a/src/paperless/serialisers.py b/src/paperless/serialisers.py index f7cfc3d450..7b32230f8f 100644 --- a/src/paperless/serialisers.py +++ b/src/paperless/serialisers.py @@ -235,6 +235,24 @@ class ApplicationConfigurationSerializer( ) -> list[str]: return sorted(name for name in os.environ if name.startswith("PAPERLESS_")) + @staticmethod + def _require_json_object(field: str, value: object) -> None: + if value is not None and not isinstance(value, dict): + raise serializers.ValidationError(f"{field} must be a JSON object.") + + def validate_user_args(self, value): + self._require_json_object("user_args", value) + return value + + def validate_barcode_tag_mapping(self, value): + self._require_json_object("barcode_tag_mapping", value) + # Each value is the regex substitute applied to a matching barcode + if value is not None and not all(isinstance(v, str) for v in value.values()): + raise serializers.ValidationError( + "barcode_tag_mapping values must be strings.", + ) + return value + def run_validation(self, data): # Empty strings treated as None to avoid unexpected behavior if "user_args" in data and data["user_args"] == "":