diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 20cb5c20cc..ef13a0845e 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -98,6 +98,7 @@ from documents.validators import uri_validator from documents.validators import url_validator from documents.versioning import has_prefetched_effective_content from documents.versioning import sort_versions_newest_first +from documents.zone_ocr import ZoneOcrReason if TYPE_CHECKING: from collections.abc import Iterable @@ -1826,6 +1827,28 @@ class RunZoneOcrSerializer(serializers.Serializer): ).prefetch_related("zones", "zones__custom_field") +class ZoneOcrResultSerializer(serializers.Serializer): + """The outcome of one zone, mirroring the ZoneOcrResult dataclass.""" + + template = serializers.CharField(read_only=True) + zone = serializers.CharField(read_only=True) + custom_field = serializers.CharField(read_only=True) + value = serializers.JSONField( + read_only=True, + allow_null=True, + help_text=( + "The value written to the target. A string, number, boolean or ISO " + "date string, depending on the target, or null when nothing was written." + ), + ) + reason = serializers.ChoiceField( + choices=[reason.value for reason in ZoneOcrReason], + read_only=True, + allow_null=True, + help_text="Why no value was written, or null when one was.", + ) + + class RunZoneOcrDocumentsSerializer(DocumentSelectionSerializer, RunZoneOcrSerializer): from_webui = serializers.BooleanField(required=False, default=False) diff --git a/src/documents/tests/test_api_ocr_templates.py b/src/documents/tests/test_api_ocr_templates.py index 959e22abc7..256bf0d2e0 100644 --- a/src/documents/tests/test_api_ocr_templates.py +++ b/src/documents/tests/test_api_ocr_templates.py @@ -24,6 +24,8 @@ from documents.parsers import ParseError from documents.search import get_backend from documents.signals import document_consumption_finished from documents.tasks import extract_zones +from documents.zone_ocr import ZoneExtraction +from documents.zone_ocr import ZoneOcrReason from paperless_testing.factories import CustomFieldFactory from paperless_testing.factories import DocumentFactory from paperless_testing.factories import OcrTemplateFactory @@ -2028,7 +2030,7 @@ class TestRunZoneOcr: response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk)) assert response.status_code == status.HTTP_200_OK - assert response.json()["results"][0]["reason"] == "no_value" + assert response.json()["results"][0]["reason"] == ZoneOcrReason.NO_VALUE assert response.json()["results"][0]["value"] is None document.refresh_from_db() assert document.title == "Existing title" @@ -2524,7 +2526,7 @@ class TestZonePreview: - The result includes the regex match - No zone is saved """ - mock_preview.return_value = {"raw_text": " 123 ", "value": "123"} + mock_preview.return_value = ZoneExtraction(raw_text=" 123 ", value="123") response = preview_zone( preview_client, @@ -2588,7 +2590,7 @@ class TestZonePreview: - regex_match reflects the outcome - Source dimensions default to 0 """ - mock_preview.return_value = {"raw_text": value, "value": value} + mock_preview.return_value = ZoneExtraction(raw_text=value, value=value) response = preview_zone( preview_client, @@ -2776,7 +2778,7 @@ class TestOcrTemplatePreviewPermissions: grant_global(regular_user, "view_document", *template_perms) document = DocumentFactory(owner=None, mime_type="image/png") document.source_path.write_bytes(b"document contents") - mock_preview.return_value = {"raw_text": None, "value": None} + mock_preview.return_value = ZoneExtraction() if action == "document-page-image": response = user_client.get(preview_url(document)) @@ -2823,7 +2825,7 @@ class TestOcrTemplatePreviewPermissions: mock_preview.assert_not_called() grant_object(preview_user, root, "view_document") - mock_preview.return_value = {"raw_text": None, "value": None} + mock_preview.return_value = ZoneExtraction() assert ( preview_client.get(preview_url(version)).status_code == status.HTTP_200_OK ) diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index c1e9617ab8..fa162234e5 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -5,6 +5,7 @@ from __future__ import annotations import shutil import subprocess from collections import Counter +from dataclasses import asdict from datetime import date from decimal import Decimal from pathlib import Path @@ -15,6 +16,7 @@ import pytest from babel import Locale from django.contrib.auth.models import Group from PIL import Image +from rest_framework.renderers import JSONRenderer from documents.models import CustomField from documents.models import CustomFieldInstance @@ -23,7 +25,11 @@ from documents.models import MatchingModel from documents.models import OcrTemplate from documents.models import OcrTemplateZone from documents.parsers import ParseError +from documents.serialisers import ZoneOcrResultSerializer from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH +from documents.zone_ocr import ZoneExtraction +from documents.zone_ocr import ZoneOcrReason +from documents.zone_ocr import ZoneOcrResult from documents.zone_ocr import _apply_transform from documents.zone_ocr import _combine_field_value from documents.zone_ocr import _convert_value @@ -378,10 +384,7 @@ class TestZoneExtraction: - An empty result is returned and the reader is never run """ setattr(zone, attribute, value) - assert extract_zone_preview(page, zone, 100, 50, 1) == { - "raw_text": None, - "value": None, - } + assert extract_zone_preview(page, zone, 100, 50, 1) == ZoneExtraction() mock_zone_ocr_run.assert_not_called() def test_date_transform_uses_zone_format( @@ -485,10 +488,10 @@ class TestZoneExtraction: - The raw text and transformed value are returned """ mock_zone_ocr_run.return_value.stdout = "account abc" - assert extract_zone_preview(page, zone, 0, 0, 1) == { - "raw_text": "account abc", - "value": "ACCOUNT ABC", - } + assert extract_zone_preview(page, zone, 0, 0, 1) == ZoneExtraction( + raw_text="account abc", + value="ACCOUNT ABC", + ) @pytest.mark.django_db @@ -582,8 +585,8 @@ class TestSavedZoneExtraction: ) assert [(result.zone, result.reason) for result in results] == [ - ("Date", "transform_failed"), - ("Ref", "incomplete"), + ("Date", ZoneOcrReason.TRANSFORM_FAILED), + ("Ref", ZoneOcrReason.INCOMPLETE), ] document.refresh_from_db() assert document.title == "Original" @@ -887,7 +890,7 @@ class TestSavedZoneExtraction: paperless_dirs.scratch_dir, ) - assert results[0].reason == "missing_page" + assert results[0].reason == ZoneOcrReason.MISSING_PAGE assert results[0].value is None def test_missing_document_file_returns_skipped_results( @@ -908,7 +911,7 @@ class TestSavedZoneExtraction: results = run_zone_extraction(document, None, templates=[template]) assert len(results) == 1 - assert results[0].reason == "missing_file" + assert results[0].reason == ZoneOcrReason.MISSING_FILE assert results[0].value is None def test_empty_combination_cannot_clear_a_field( @@ -939,7 +942,7 @@ class TestSavedZoneExtraction: paperless_dirs.scratch_dir, ) - assert results[0].reason == "invalid_value" + assert results[0].reason == ZoneOcrReason.INVALID_VALUE assert results[0].value is None assert document.title == "Original" @@ -980,10 +983,10 @@ class TestSavedZoneExtraction: ) assert [result.reason for result in results] == [ - "no_value", - "regex_mismatch", - "invalid_value", - "unassigned", + ZoneOcrReason.NO_VALUE, + ZoneOcrReason.REGEX_MISMATCH, + ZoneOcrReason.INVALID_VALUE, + ZoneOcrReason.UNASSIGNED, ] assert [result.value for result in results] == [None] * 4 assert document.title == "Original" @@ -1021,8 +1024,8 @@ class TestSavedZoneExtraction: results = run_zone_extraction(document, page, templates=[template, second]) assert [result.reason for result in results] == [ - "template_failed", - "template_failed", + ZoneOcrReason.TEMPLATE_FAILED, + ZoneOcrReason.TEMPLATE_FAILED, None, ] assert [result.value for result in results] == [None, None, "Applied"] @@ -1073,7 +1076,7 @@ class TestSavedZoneExtraction: assert [(result.zone, result.value, result.reason) for result in results] == [ ("Title", "New title", None), - ("ASN", None, "duplicate_asn"), + ("ASN", None, ZoneOcrReason.DUPLICATE_ASN), ] document.refresh_from_db() assert document.title == "New title" @@ -1140,7 +1143,7 @@ class TestSavedZoneExtraction: paperless_dirs.scratch_dir, ) - assert results[0].reason == "no_value" + assert results[0].reason == ZoneOcrReason.NO_VALUE assert results[0].value is None assert document.title == "Original" @@ -1172,8 +1175,8 @@ class TestSavedZoneExtraction: def preview_and_save(raw_text: str) -> None: mock_zone_ocr_run.return_value.stdout = raw_text preview = extract_zone_preview(page, zone, 100, 50, 1) - assert preview["raw_text"] == raw_text - assert preview["value"] == raw_text.upper() + assert preview.raw_text == raw_text + assert preview.value == raw_text.upper() _process_template( document, {0: page}, @@ -1193,10 +1196,14 @@ class TestSavedZoneExtraction: [ pytest.param( {"validation_regex": r"\d{2}"}, - "regex_mismatch", + ZoneOcrReason.REGEX_MISMATCH, id="regex-mismatch", ), - pytest.param({"page": 2}, "missing_page", id="missing-page"), + pytest.param( + {"page": 2}, + ZoneOcrReason.MISSING_PAGE, + id="missing-page", + ), ], ) def test_failed_zone_blocks_combined_field( @@ -1243,8 +1250,8 @@ class TestSavedZoneExtraction: assert [result.reason for result in results] == [ expected_failure, - "incomplete", - "incomplete", + ZoneOcrReason.INCOMPLETE, + ZoneOcrReason.INCOMPLETE, ] assert [result.value for result in results] == [None] * 3 document.refresh_from_db() @@ -1279,7 +1286,7 @@ class TestSavedZoneExtraction: paperless_dirs.scratch_dir, ) - assert [result.reason for result in results] == ["no_value", None] + assert [result.reason for result in results] == [ZoneOcrReason.NO_VALUE, None] assert results[1].value == "two" document.refresh_from_db() assert document.title == "two" @@ -1315,7 +1322,10 @@ class TestSavedZoneExtraction: paperless_dirs.scratch_dir, ) - assert [result.reason for result in results] == [None, "regex_mismatch"] + assert [result.reason for result in results] == [ + None, + ZoneOcrReason.REGEX_MISMATCH, + ] document.refresh_from_db() assert document.title == "New title" @@ -3602,7 +3612,7 @@ class TestRunZoneExtraction: results = run_zone_extraction(document, source) assert Counter((r.value, r.reason) for r in results) == Counter( - [(None, "template_failed"), ("extracted", None)], + [(None, ZoneOcrReason.TEMPLATE_FAILED), ("extracted", None)], ) assert {r.template for r in results} == {"First", "Second"} @@ -3627,7 +3637,7 @@ class TestRunZoneExtraction: results = run_zone_extraction(document, None) assert [(r.template, r.zone, r.reason) for r in results] == [ - ("Matching", zone.name, "missing_file"), + ("Matching", zone.name, ZoneOcrReason.MISSING_FILE), ] mock_zone_ocr_run.assert_not_called() @@ -3741,3 +3751,41 @@ class TestRealZoneOcr: assert [result.reason for result in results] == [None] document.refresh_from_db() assert "This is a test document" in document.title + + +class TestZoneOcrResultSerializer: + @pytest.mark.parametrize( + "value", + [ + pytest.param("text", id="string"), + pytest.param(42, id="int"), + pytest.param(1.5, id="float"), + pytest.param(True, id="bool"), + pytest.param(date(2025, 12, 25), id="date"), + pytest.param(Decimal("12.50"), id="decimal"), + pytest.param(None, id="none"), + ], + ) + @pytest.mark.parametrize( + "reason", + [pytest.param(None, id="no-reason")] + + [pytest.param(reason, id=reason.value) for reason in ZoneOcrReason], + ) + def test_wire_format_matches_dataclass( + self, + value: object, + reason: ZoneOcrReason | None, + ) -> None: + """ + GIVEN: + - A zone result with any value type and any reason + WHEN: + - It is serialized with the result serializer and rendered as JSON + THEN: + - The JSON equals the rendering of the dataclass as a dict + """ + result = ZoneOcrResult("Template", "Zone", "Title", value, reason) + + assert JSONRenderer().render( + ZoneOcrResultSerializer(result).data, + ) == JSONRenderer().render(asdict(result)) diff --git a/src/documents/views.py b/src/documents/views.py index 37b1800ab6..a5a2de2be9 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -10,7 +10,6 @@ from collections import deque from collections.abc import AsyncIterator from collections.abc import Callable from collections.abc import Generator -from dataclasses import asdict from datetime import datetime from datetime import timedelta from http import HTTPStatus @@ -244,6 +243,7 @@ from documents.serialisers import UiSettingsViewSerializer from documents.serialisers import WorkflowActionSerializer from documents.serialisers import WorkflowSerializer from documents.serialisers import WorkflowTriggerSerializer +from documents.serialisers import ZoneOcrResultSerializer from documents.signals import document_updated from documents.tasks import build_share_link_bundle from documents.tasks import consume_file @@ -2290,7 +2290,7 @@ class DocumentViewSet( choices=["no_template"], required=False, ), - "results": serializers.ListField(child=serializers.DictField()), + "results": ZoneOcrResultSerializer(many=True), }, ), ) @@ -2350,7 +2350,10 @@ class DocumentViewSet( results = run_zone_extraction(file_document, templates=templates, reindex=True) return Response( - {"status": "completed", "results": [asdict(result) for result in results]}, + { + "status": "completed", + "results": ZoneOcrResultSerializer(results, many=True).data, + }, ) @extend_schema( @@ -5923,15 +5926,15 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem ) regex_match = None - if zone.validation_regex and result.get("value") is not None: + if zone.validation_regex and result.value is not None: regex_match = ( - safe_regex_fullmatch(zone.validation_regex, result["value"]) is not None + safe_regex_fullmatch(zone.validation_regex, result.value) is not None ) return Response( { - "raw_text": result.get("raw_text"), - "value": result.get("value"), + "raw_text": result.raw_text, + "value": result.value, "regex": zone.validation_regex, "regex_match": regex_match, }, diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index c8e5f8d8d2..d38cbb1f54 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -12,6 +12,7 @@ from datetime import date from datetime import datetime from decimal import Decimal from decimal import InvalidOperation +from enum import StrEnum from functools import lru_cache from functools import partial from pathlib import Path @@ -98,13 +99,36 @@ class FieldUpdate: custom_field: CustomField | None = None +class ZoneOcrReason(StrEnum): + """Why a zone produced no value for its target.""" + + MISSING_FILE = "missing_file" + MISSING_PAGE = "missing_page" + NO_VALUE = "no_value" + TRANSFORM_FAILED = "transform_failed" + REGEX_MISMATCH = "regex_mismatch" + INCOMPLETE = "incomplete" + DUPLICATE_ASN = "duplicate_asn" + UNASSIGNED = "unassigned" + INVALID_VALUE = "invalid_value" + TEMPLATE_FAILED = "template_failed" + + @dataclass class ZoneOcrResult: template: str zone: str custom_field: str value: object = None - reason: str | None = None + reason: ZoneOcrReason | None = None + + +# Result labels for zones that target a built-in field. +BUILTIN_TARGET_LABELS: dict[OcrTemplateZone.TargetType, str] = { + OcrTemplateZone.TargetType.TITLE: "Title", + OcrTemplateZone.TargetType.ASN: "ASN", + OcrTemplateZone.TargetType.CREATED: "Created", +} def _zone_result( @@ -112,12 +136,12 @@ def _zone_result( zone: OcrTemplateZone, *, value: object = None, - reason: str | None = None, + reason: ZoneOcrReason | None = None, ) -> ZoneOcrResult: if zone.target == OcrTemplateZone.TargetType.CUSTOM_FIELD: label = zone.custom_field.name if zone.custom_field_id else "(no field)" else: - label = {"title": "Title", "asn": "ASN", "created": "Created"}[zone.target] + label = BUILTIN_TARGET_LABELS[zone.target] return ZoneOcrResult(template.name, zone.name, label, value, reason) @@ -185,7 +209,7 @@ def run_zone_extraction( document.pk, ) return [ - _zone_result(template, zone, reason="missing_file") + _zone_result(template, zone, reason=ZoneOcrReason.MISSING_FILE) for template in templates for zone in template.zones.all() ] @@ -215,7 +239,7 @@ def run_zone_extraction( document.pk, ) results.extend( - _zone_result(template, zone, reason="template_failed") + _zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED) for zone in zones ) if reindex and any(result.reason is None for result in results): @@ -269,7 +293,7 @@ def _process_template( for zone in zones: page_idx = _resolve_page_idx(zone.page, document.page_count) extracted = None - reason = "missing_page" + reason: ZoneOcrReason | None = ZoneOcrReason.MISSING_PAGE if page_idx in page_images: extraction = _extract_zone( page_images[page_idx], @@ -283,15 +307,15 @@ def _process_template( reason = None elif extraction.raw_text: # Text was read but the transform found nothing usable in it. - reason = "transform_failed" + reason = ZoneOcrReason.TRANSFORM_FAILED else: - reason = "no_value" + reason = ZoneOcrReason.NO_VALUE if ( extracted is not None and zone.validation_regex and not safe_regex_fullmatch(zone.validation_regex, extracted) ): - reason = "regex_mismatch" + reason = ZoneOcrReason.REGEX_MISMATCH result = _zone_result( template, zone, @@ -309,10 +333,13 @@ def _process_template( if not accepted: continue # An empty zone is an optional token; any other failure blocks the group. - if any(result.reason not in (None, "no_value") for _, result in field_zones): + if any( + result.reason not in (None, ZoneOcrReason.NO_VALUE) + for _, result in field_zones + ): for result in accepted: result.value = None - result.reason = "incomplete" + result.reason = ZoneOcrReason.INCOMPLETE continue value = _combine_field_value( combine_formats.get(key, ""), @@ -325,7 +352,7 @@ def _process_template( # write of this template, so only this field is dropped. for result in accepted: result.value = None - result.reason = "duplicate_asn" + result.reason = ZoneOcrReason.DUPLICATE_ASN continue if update is not None: updates.append(update) @@ -333,7 +360,9 @@ def _process_template( result.value = update.value if update else None if update is None: result.reason = ( - "unassigned" if key == "custom_field" else "invalid_value" + ZoneOcrReason.UNASSIGNED + if key == "custom_field" + else ZoneOcrReason.INVALID_VALUE ) _apply_field_updates(root_document, updates) return results @@ -581,23 +610,22 @@ def extract_zone_preview( source_width: int, source_height: int, page_count: int | None, -) -> dict: +) -> ZoneExtraction: """Extract a zone without writing its value to the document.""" page_idx = _resolve_page_idx(zone.page, page_count) with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: tmp_path = Path(tmp_dir) page_images = _render_pages(doc_path, {page_idx}, tmp_path, page_count) if page_idx not in page_images: - return {"raw_text": None, "value": None} + return ZoneExtraction() - result = _extract_zone( + return _extract_zone( page_images[page_idx], zone, source_width, source_height, tmp_path, ) - return {"raw_text": result.raw_text, "value": result.value} @dataclass(frozen=True)