From 4c2df48795e85d7935f09f6fcecf1712a0e6a578 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:04:31 -0800 Subject: [PATCH] Chore: give OCR templates their own matching fields Templates are never assigned to documents, so the classifier-based Automatic matching inherited from MatchingModel could never work. Declare the matching fields with the workflow trigger choices instead. Also keep zones when their custom field is deleted, allow longer OCR language lists, fix help texts, and regenerate the unreleased migration. --- src/documents/matching.py | 8 +++- .../0031_ocrtemplate_ocrtemplatezone.py | 11 +++-- src/documents/models.py | 41 ++++++++++++++++--- src/documents/serialisers.py | 37 ++++++++--------- src/documents/tests/test_api_ocr_templates.py | 29 ++++++++++++- src/paperless/settings/__init__.py | 1 + 6 files changed, 91 insertions(+), 36 deletions(-) diff --git a/src/documents/matching.py b/src/documents/matching.py index 8bc39d219b..e139f5e1fd 100644 --- a/src/documents/matching.py +++ b/src/documents/matching.py @@ -14,6 +14,7 @@ from documents.models import Correspondent from documents.models import Document from documents.models import DocumentType from documents.models import MatchingModel +from documents.models import OcrTemplate from documents.models import StoragePath from documents.models import Tag from documents.models import Workflow @@ -30,7 +31,7 @@ logger = logging.getLogger("paperless.matching") def log_reason( - matching_model: MatchingModel | WorkflowTrigger, + matching_model: MatchingModel | WorkflowTrigger | OcrTemplate, document: Document, reason: str, ) -> None: @@ -161,7 +162,10 @@ def match_storage_paths(document: Document, classifier: DocumentClassifier, user ) -def matches(matching_model: MatchingModel, document: Document): +def matches( + matching_model: MatchingModel | WorkflowTrigger | OcrTemplate, + document: Document, +): search_flags = 0 document_content = document.get_effective_content() or "" diff --git a/src/documents/migrations/0031_ocrtemplate_ocrtemplatezone.py b/src/documents/migrations/0031_ocrtemplate_ocrtemplatezone.py index c1c6f0063e..4f2ed61e3d 100644 --- a/src/documents/migrations/0031_ocrtemplate_ocrtemplatezone.py +++ b/src/documents/migrations/0031_ocrtemplate_ocrtemplatezone.py @@ -1,4 +1,4 @@ -# Generated by Django 5.2.17 on 2026-10-20 20:29 +# Generated by Django 5.2.18 on 2026-11-19 17:29 import django.core.validators import django.db.models.deletion @@ -42,7 +42,6 @@ class Migration(migrations.Migration): (3, "Exact match"), (4, "Regular expression"), (5, "Fuzzy word"), - (6, "Automatic"), ], default=1, verbose_name="matching algorithm", @@ -161,7 +160,7 @@ class Migration(migrations.Migration): "page", models.IntegerField( blank=True, - help_text="Page (1 = first, -1 = last; blank uses the template default)", + help_text="Page (1 = first, -1 = last; blank = first page)", null=True, verbose_name="page", ), @@ -219,7 +218,7 @@ class Migration(migrations.Migration): models.CharField( default="eng", help_text="Tesseract language code(s), e.g. 'deu+eng'", - max_length=20, + max_length=32, verbose_name="OCR language", ), ), @@ -259,7 +258,7 @@ class Migration(migrations.Migration): models.CharField( blank=True, default="", - help_text="Optional regex pattern — extracted text is only accepted if it matches", + help_text="Optional regex pattern. Extracted text is only accepted if it fully matches.", max_length=256, verbose_name="validation regex", ), @@ -271,7 +270,7 @@ class Migration(migrations.Migration): blank=True, help_text="Target custom field (only used when target is 'custom_field')", null=True, - on_delete=django.db.models.deletion.CASCADE, + on_delete=django.db.models.deletion.SET_NULL, related_name="ocr_zones", to="documents.customfield", verbose_name="custom field", diff --git a/src/documents/models.py b/src/documents/models.py index 3d6ea8359b..a7ba0eb4db 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -2218,9 +2218,22 @@ class WorkflowRun(SoftDeleteModel): return f"WorkflowRun of {self.workflow} at {self.run_at} on {self.document}" -class OcrTemplate(MatchingModel): +class OcrTemplate(ModelWithOwner): """Matching rules and extraction zones for documents with a common layout.""" + name = models.CharField(_("name"), max_length=128) + + match = models.CharField(_("match"), max_length=256, blank=True) + + # Templates are never assigned by the classifier, so there is no auto matching. + matching_algorithm = models.PositiveSmallIntegerField( + _("matching algorithm"), + choices=WorkflowTrigger.WorkflowTriggerMatching.choices, + default=MatchingModel.MATCH_ANY, + ) + + is_insensitive = models.BooleanField(_("is insensitive"), default=True) + source_width = models.PositiveIntegerField( _("source width"), validators=[MinValueValidator(1)], @@ -2268,9 +2281,25 @@ class OcrTemplate(MatchingModel): auto_now=True, ) - class Meta(MatchingModel.Meta): + class Meta(ModelWithOwner.Meta): + abstract = False verbose_name = _("OCR template") verbose_name_plural = _("OCR templates") + ordering = ("name",) + constraints = [ + models.UniqueConstraint( + fields=["name", "owner"], + name="%(app_label)s_%(class)s_unique_name_owner", + ), + models.UniqueConstraint( + name="%(app_label)s_%(class)s_name_uniq", + fields=["name"], + condition=models.Q(owner__isnull=True), + ), + ] + + def __str__(self) -> str: + return self.name class OcrTemplateZone(models.Model): @@ -2308,7 +2337,7 @@ class OcrTemplateZone(models.Model): custom_field = models.ForeignKey( "documents.CustomField", - on_delete=models.CASCADE, + on_delete=models.SET_NULL, related_name="ocr_zones", verbose_name=_("custom field"), null=True, @@ -2320,7 +2349,7 @@ class OcrTemplateZone(models.Model): _("page"), null=True, blank=True, - help_text=_("Page (1 = first, -1 = last; blank uses the template default)"), + help_text=_("Page (1 = first, -1 = last; blank = first page)"), ) x = models.PositiveIntegerField(_("x"), help_text=_("Left edge (px)")) @@ -2358,7 +2387,7 @@ class OcrTemplateZone(models.Model): ocr_language = models.CharField( _("OCR language"), - max_length=20, + max_length=32, default="eng", help_text=_("Tesseract language code(s), e.g. 'deu+eng'"), ) @@ -2400,7 +2429,7 @@ class OcrTemplateZone(models.Model): blank=True, default="", help_text=_( - "Optional regex pattern — extracted text is only accepted if it matches", + "Optional regex pattern. Extracted text is only accepted if it fully matches.", ), ) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index abf2cc3ff8..20cb5c20cc 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -171,14 +171,10 @@ class DocumentUpdateFieldsModelSerializer(DynamicFieldsModelSerializer): return instance -class MatchingModelSerializer(serializers.ModelSerializer[Any]): - document_count = serializers.IntegerField(read_only=True) - - def get_slug(self, obj) -> str: - return slugify(obj.name) - - slug = SerializerMethodField() - +# Name and owner uniqueness and match regex validation for models with a name, +# an owner and match rules. A comment, not a docstring, so it stays out of the +# API schema descriptions of every serializer built on it. +class MatchRuleSerializer(serializers.ModelSerializer[Any]): def validate(self, data): # TODO: remove pending https://github.com/encode/django-rest-framework/issues/7173 name = data.get( @@ -217,6 +213,15 @@ class MatchingModelSerializer(serializers.ModelSerializer[Any]): return match +class MatchingModelSerializer(MatchRuleSerializer): + document_count = serializers.IntegerField(read_only=True) + + def get_slug(self, obj) -> str: + return slugify(obj.name) + + slug = SerializerMethodField() + + class SetPermissionsMixin: def _validate_user_ids(self, user_ids): users = User.objects.none() @@ -3827,7 +3832,8 @@ class OcrTemplateZoneSerializer(serializers.ModelSerializer): {name: field.error_messages["required"]}, ) # A custom field target may be left unassigned, as the editor creates new - # zones; built-in targets never use a custom field. + # zones, and a zone is left unassigned when its custom field is deleted; + # built-in targets never use a custom field. if ( data.get("target", OcrTemplateZone.TargetType.CUSTOM_FIELD) != OcrTemplateZone.TargetType.CUSTOM_FIELD @@ -3897,17 +3903,13 @@ def _zone_field_key(zone: dict[str, Any]) -> str: return target -class OcrTemplateSerializer(MatchingModelSerializer, OwnedObjectSerializer): +class OcrTemplateSerializer(MatchRuleSerializer, OwnedObjectSerializer): zones = OcrTemplateZoneSerializer(many=True, required=False) combine_formats = serializers.DictField( child=serializers.CharField(allow_blank=True, trim_whitespace=False), required=False, ) - # Templates are never assigned to documents - slug = None - document_count = None - def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) # Writes accept documents the user can view, plus the sample document @@ -3947,13 +3949,6 @@ class OcrTemplateSerializer(MatchingModelSerializer, OwnedObjectSerializer): ] read_only_fields = ["created", "updated"] - def validate_matching_algorithm(self, value): - if value == MatchingModel.MATCH_AUTO: - raise serializers.ValidationError( - "Automatic matching is not supported for OCR templates.", - ) - return value - def validate(self, data): data = super().validate(data) if "zones" in data: diff --git a/src/documents/tests/test_api_ocr_templates.py b/src/documents/tests/test_api_ocr_templates.py index 6afa5c8837..959e22abc7 100644 --- a/src/documents/tests/test_api_ocr_templates.py +++ b/src/documents/tests/test_api_ocr_templates.py @@ -1680,6 +1680,33 @@ class TestOcrTemplateDelete: assert OcrTemplate.objects.count() == 0 assert OcrTemplateZone.objects.count() == 0 + def test_deleting_a_custom_field_keeps_its_zones( + self, + text_field: CustomField, + ) -> None: + """ + GIVEN: + - A template with a zone targeting a custom field + WHEN: + - The custom field is deleted + THEN: + - The zone is kept with no custom field assigned + """ + template = OcrTemplateFactory(name="Keeps Zones", match="Northwind") + zone = OcrTemplateZoneFactory( + template=template, + name="Zone", + target=OcrTemplateZone.TargetType.CUSTOM_FIELD, + custom_field=text_field, + height=100, + ) + + text_field.delete() + + zone.refresh_from_db() + assert zone.custom_field is None + assert OcrTemplate.objects.filter(pk=template.pk).exists() + def test_nonexistent_returns_404(self, admin_client: APIClient) -> None: """ GIVEN: @@ -2585,7 +2612,7 @@ class TestZonePreview: pytest.param({"page": 0}, id="page-zero"), pytest.param({"page": -2}, id="page-below-last"), pytest.param({"transform": "invalid"}, id="unknown-transform"), - pytest.param({"ocr_language": "x" * 21}, id="language-too-long"), + pytest.param({"ocr_language": "x" * 33}, id="language-too-long"), pytest.param( {"zone_source_width": "invalid"}, id="non-numeric-source-width", diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py index ed01604c56..f6059750a8 100644 --- a/src/paperless/settings/__init__.py +++ b/src/paperless/settings/__init__.py @@ -227,6 +227,7 @@ SPECTACULAR_SETTINGS = { }, "ENUM_NAME_OVERRIDES": { "MatchingAlgorithm": "documents.models.MatchingModel.MATCHING_ALGORITHMS", + "WorkflowTriggerMatchingAlgorithmEnum": "documents.models.WorkflowTrigger.WorkflowTriggerMatching", "BarcodeFormatEnum": "documents.models.DocumentBarcode.Format", "NoteFormatEnum": "documents.models.Note.Format", },