Chore: type zone OCR results and reasons

Reasons become a StrEnum, the run response has a serializer instead of an
untyped dict list, and the zone preview returns its dataclass. The JSON is
unchanged.
This commit is contained in:
stumpylog committed 2026-11-19 10:15:43 -08:00
1 parent 4c2df48795
commit a490d9d348
5 files changed
+164 -60

No files matched your search

+23
View File
@@ -98,6 +98,7 @@ from documents.validators import uri_validator
from documents.validators import url_validator
from documents.versioning import has_prefetched_effective_content
from documents.versioning import sort_versions_newest_first
from documents.zone_ocr import ZoneOcrReason
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -1826,6 +1827,28 @@ class RunZoneOcrSerializer(serializers.Serializer):
).prefetch_related("zones", "zones__custom_field")
class ZoneOcrResultSerializer(serializers.Serializer):
"""The outcome of one zone, mirroring the ZoneOcrResult dataclass."""
template = serializers.CharField(read_only=True)
zone = serializers.CharField(read_only=True)
custom_field = serializers.CharField(read_only=True)
value = serializers.JSONField(
read_only=True,
allow_null=True,
help_text=(
"The value written to the target. A string, number, boolean or ISO "
"date string, depending on the target, or null when nothing was written."
),
)
reason = serializers.ChoiceField(
choices=[reason.value for reason in ZoneOcrReason],
read_only=True,
allow_null=True,
help_text="Why no value was written, or null when one was.",
)
class RunZoneOcrDocumentsSerializer(DocumentSelectionSerializer, RunZoneOcrSerializer):
from_webui = serializers.BooleanField(required=False, default=False)
@@ -24,6 +24,8 @@ from documents.parsers import ParseError
from documents.search import get_backend
from documents.signals import document_consumption_finished
from documents.tasks import extract_zones
from documents.zone_ocr import ZoneExtraction
from documents.zone_ocr import ZoneOcrReason
from paperless_testing.factories import CustomFieldFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
@@ -2028,7 +2030,7 @@ class TestRunZoneOcr:
response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk))
assert response.status_code == status.HTTP_200_OK
assert response.json()["results"][0]["reason"] == "no_value"
assert response.json()["results"][0]["reason"] == ZoneOcrReason.NO_VALUE
assert response.json()["results"][0]["value"] is None
document.refresh_from_db()
assert document.title == "Existing title"
@@ -2524,7 +2526,7 @@ class TestZonePreview:
- The result includes the regex match
- No zone is saved
"""
mock_preview.return_value = {"raw_text": " 123 ", "value": "123"}
mock_preview.return_value = ZoneExtraction(raw_text=" 123 ", value="123")
response = preview_zone(
preview_client,
@@ -2588,7 +2590,7 @@ class TestZonePreview:
- regex_match reflects the outcome
- Source dimensions default to 0
"""
mock_preview.return_value = {"raw_text": value, "value": value}
mock_preview.return_value = ZoneExtraction(raw_text=value, value=value)
response = preview_zone(
preview_client,
@@ -2776,7 +2778,7 @@ class TestOcrTemplatePreviewPermissions:
grant_global(regular_user, "view_document", *template_perms)
document = DocumentFactory(owner=None, mime_type="image/png")
document.source_path.write_bytes(b"document contents")
mock_preview.return_value = {"raw_text": None, "value": None}
mock_preview.return_value = ZoneExtraction()
if action == "document-page-image":
response = user_client.get(preview_url(document))
@@ -2823,7 +2825,7 @@ class TestOcrTemplatePreviewPermissions:
mock_preview.assert_not_called()
grant_object(preview_user, root, "view_document")
mock_preview.return_value = {"raw_text": None, "value": None}
mock_preview.return_value = ZoneExtraction()
assert (
preview_client.get(preview_url(version)).status_code == status.HTTP_200_OK
)
+79 -31
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import shutil
import subprocess
from collections import Counter
from dataclasses import asdict
from datetime import date
from decimal import Decimal
from pathlib import Path
@@ -15,6 +16,7 @@ import pytest
from babel import Locale
from django.contrib.auth.models import Group
from PIL import Image
from rest_framework.renderers import JSONRenderer
from documents.models import CustomField
from documents.models import CustomFieldInstance
@@ -23,7 +25,11 @@ from documents.models import MatchingModel
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.parsers import ParseError
from documents.serialisers import ZoneOcrResultSerializer
from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH
from documents.zone_ocr import ZoneExtraction
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from documents.zone_ocr import _apply_transform
from documents.zone_ocr import _combine_field_value
from documents.zone_ocr import _convert_value
@@ -378,10 +384,7 @@ class TestZoneExtraction:
- An empty result is returned and the reader is never run
"""
setattr(zone, attribute, value)
assert extract_zone_preview(page, zone, 100, 50, 1) == {
"raw_text": None,
"value": None,
}
assert extract_zone_preview(page, zone, 100, 50, 1) == ZoneExtraction()
mock_zone_ocr_run.assert_not_called()
def test_date_transform_uses_zone_format(
@@ -485,10 +488,10 @@ class TestZoneExtraction:
- The raw text and transformed value are returned
"""
mock_zone_ocr_run.return_value.stdout = "account abc"
assert extract_zone_preview(page, zone, 0, 0, 1) == {
"raw_text": "account abc",
"value": "ACCOUNT ABC",
}
assert extract_zone_preview(page, zone, 0, 0, 1) == ZoneExtraction(
raw_text="account abc",
value="ACCOUNT ABC",
)
@pytest.mark.django_db
@@ -582,8 +585,8 @@ class TestSavedZoneExtraction:
)
assert [(result.zone, result.reason) for result in results] == [
("Date", "transform_failed"),
("Ref", "incomplete"),
("Date", ZoneOcrReason.TRANSFORM_FAILED),
("Ref", ZoneOcrReason.INCOMPLETE),
]
document.refresh_from_db()
assert document.title == "Original"
@@ -887,7 +890,7 @@ class TestSavedZoneExtraction:
paperless_dirs.scratch_dir,
)
assert results[0].reason == "missing_page"
assert results[0].reason == ZoneOcrReason.MISSING_PAGE
assert results[0].value is None
def test_missing_document_file_returns_skipped_results(
@@ -908,7 +911,7 @@ class TestSavedZoneExtraction:
results = run_zone_extraction(document, None, templates=[template])
assert len(results) == 1
assert results[0].reason == "missing_file"
assert results[0].reason == ZoneOcrReason.MISSING_FILE
assert results[0].value is None
def test_empty_combination_cannot_clear_a_field(
@@ -939,7 +942,7 @@ class TestSavedZoneExtraction:
paperless_dirs.scratch_dir,
)
assert results[0].reason == "invalid_value"
assert results[0].reason == ZoneOcrReason.INVALID_VALUE
assert results[0].value is None
assert document.title == "Original"
@@ -980,10 +983,10 @@ class TestSavedZoneExtraction:
)
assert [result.reason for result in results] == [
"no_value",
"regex_mismatch",
"invalid_value",
"unassigned",
ZoneOcrReason.NO_VALUE,
ZoneOcrReason.REGEX_MISMATCH,
ZoneOcrReason.INVALID_VALUE,
ZoneOcrReason.UNASSIGNED,
]
assert [result.value for result in results] == [None] * 4
assert document.title == "Original"
@@ -1021,8 +1024,8 @@ class TestSavedZoneExtraction:
results = run_zone_extraction(document, page, templates=[template, second])
assert [result.reason for result in results] == [
"template_failed",
"template_failed",
ZoneOcrReason.TEMPLATE_FAILED,
ZoneOcrReason.TEMPLATE_FAILED,
None,
]
assert [result.value for result in results] == [None, None, "Applied"]
@@ -1073,7 +1076,7 @@ class TestSavedZoneExtraction:
assert [(result.zone, result.value, result.reason) for result in results] == [
("Title", "New title", None),
("ASN", None, "duplicate_asn"),
("ASN", None, ZoneOcrReason.DUPLICATE_ASN),
]
document.refresh_from_db()
assert document.title == "New title"
@@ -1140,7 +1143,7 @@ class TestSavedZoneExtraction:
paperless_dirs.scratch_dir,
)
assert results[0].reason == "no_value"
assert results[0].reason == ZoneOcrReason.NO_VALUE
assert results[0].value is None
assert document.title == "Original"
@@ -1172,8 +1175,8 @@ class TestSavedZoneExtraction:
def preview_and_save(raw_text: str) -> None:
mock_zone_ocr_run.return_value.stdout = raw_text
preview = extract_zone_preview(page, zone, 100, 50, 1)
assert preview["raw_text"] == raw_text
assert preview["value"] == raw_text.upper()
assert preview.raw_text == raw_text
assert preview.value == raw_text.upper()
_process_template(
document,
{0: page},
@@ -1193,10 +1196,14 @@ class TestSavedZoneExtraction:
[
pytest.param(
{"validation_regex": r"\d{2}"},
"regex_mismatch",
ZoneOcrReason.REGEX_MISMATCH,
id="regex-mismatch",
),
pytest.param({"page": 2}, "missing_page", id="missing-page"),
pytest.param(
{"page": 2},
ZoneOcrReason.MISSING_PAGE,
id="missing-page",
),
],
)
def test_failed_zone_blocks_combined_field(
@@ -1243,8 +1250,8 @@ class TestSavedZoneExtraction:
assert [result.reason for result in results] == [
expected_failure,
"incomplete",
"incomplete",
ZoneOcrReason.INCOMPLETE,
ZoneOcrReason.INCOMPLETE,
]
assert [result.value for result in results] == [None] * 3
document.refresh_from_db()
@@ -1279,7 +1286,7 @@ class TestSavedZoneExtraction:
paperless_dirs.scratch_dir,
)
assert [result.reason for result in results] == ["no_value", None]
assert [result.reason for result in results] == [ZoneOcrReason.NO_VALUE, None]
assert results[1].value == "two"
document.refresh_from_db()
assert document.title == "two"
@@ -1315,7 +1322,10 @@ class TestSavedZoneExtraction:
paperless_dirs.scratch_dir,
)
assert [result.reason for result in results] == [None, "regex_mismatch"]
assert [result.reason for result in results] == [
None,
ZoneOcrReason.REGEX_MISMATCH,
]
document.refresh_from_db()
assert document.title == "New title"
@@ -3602,7 +3612,7 @@ class TestRunZoneExtraction:
results = run_zone_extraction(document, source)
assert Counter((r.value, r.reason) for r in results) == Counter(
[(None, "template_failed"), ("extracted", None)],
[(None, ZoneOcrReason.TEMPLATE_FAILED), ("extracted", None)],
)
assert {r.template for r in results} == {"First", "Second"}
@@ -3627,7 +3637,7 @@ class TestRunZoneExtraction:
results = run_zone_extraction(document, None)
assert [(r.template, r.zone, r.reason) for r in results] == [
("Matching", zone.name, "missing_file"),
("Matching", zone.name, ZoneOcrReason.MISSING_FILE),
]
mock_zone_ocr_run.assert_not_called()
@@ -3741,3 +3751,41 @@ class TestRealZoneOcr:
assert [result.reason for result in results] == [None]
document.refresh_from_db()
assert "This is a test document" in document.title
class TestZoneOcrResultSerializer:
@pytest.mark.parametrize(
"value",
[
pytest.param("text", id="string"),
pytest.param(42, id="int"),
pytest.param(1.5, id="float"),
pytest.param(True, id="bool"),
pytest.param(date(2025, 12, 25), id="date"),
pytest.param(Decimal("12.50"), id="decimal"),
pytest.param(None, id="none"),
],
)
@pytest.mark.parametrize(
"reason",
[pytest.param(None, id="no-reason")]
+ [pytest.param(reason, id=reason.value) for reason in ZoneOcrReason],
)
def test_wire_format_matches_dataclass(
self,
value: object,
reason: ZoneOcrReason | None,
) -> None:
"""
GIVEN:
- A zone result with any value type and any reason
WHEN:
- It is serialized with the result serializer and rendered as JSON
THEN:
- The JSON equals the rendering of the dataclass as a dict
"""
result = ZoneOcrResult("Template", "Zone", "Title", value, reason)
assert JSONRenderer().render(
ZoneOcrResultSerializer(result).data,
) == JSONRenderer().render(asdict(result))
+10 -7
View File
@@ -10,7 +10,6 @@ from collections import deque
from collections.abc import AsyncIterator
from collections.abc import Callable
from collections.abc import Generator
from dataclasses import asdict
from datetime import datetime
from datetime import timedelta
from http import HTTPStatus
@@ -244,6 +243,7 @@ from documents.serialisers import UiSettingsViewSerializer
from documents.serialisers import WorkflowActionSerializer
from documents.serialisers import WorkflowSerializer
from documents.serialisers import WorkflowTriggerSerializer
from documents.serialisers import ZoneOcrResultSerializer
from documents.signals import document_updated
from documents.tasks import build_share_link_bundle
from documents.tasks import consume_file
@@ -2290,7 +2290,7 @@ class DocumentViewSet(
choices=["no_template"],
required=False,
),
"results": serializers.ListField(child=serializers.DictField()),
"results": ZoneOcrResultSerializer(many=True),
},
),
)
@@ -2350,7 +2350,10 @@ class DocumentViewSet(
results = run_zone_extraction(file_document, templates=templates, reindex=True)
return Response(
{"status": "completed", "results": [asdict(result) for result in results]},
{
"status": "completed",
"results": ZoneOcrResultSerializer(results, many=True).data,
},
)
@extend_schema(
@@ -5923,15 +5926,15 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem
)
regex_match = None
if zone.validation_regex and result.get("value") is not None:
if zone.validation_regex and result.value is not None:
regex_match = (
safe_regex_fullmatch(zone.validation_regex, result["value"]) is not None
safe_regex_fullmatch(zone.validation_regex, result.value) is not None
)
return Response(
{
"raw_text": result.get("raw_text"),
"value": result.get("value"),
"raw_text": result.raw_text,
"value": result.value,
"regex": zone.validation_regex,
"regex_match": regex_match,
},
+45 -17
View File
@@ -12,6 +12,7 @@ from datetime import date
from datetime import datetime
from decimal import Decimal
from decimal import InvalidOperation
from enum import StrEnum
from functools import lru_cache
from functools import partial
from pathlib import Path
@@ -98,13 +99,36 @@ class FieldUpdate:
custom_field: CustomField | None = None
class ZoneOcrReason(StrEnum):
"""Why a zone produced no value for its target."""
MISSING_FILE = "missing_file"
MISSING_PAGE = "missing_page"
NO_VALUE = "no_value"
TRANSFORM_FAILED = "transform_failed"
REGEX_MISMATCH = "regex_mismatch"
INCOMPLETE = "incomplete"
DUPLICATE_ASN = "duplicate_asn"
UNASSIGNED = "unassigned"
INVALID_VALUE = "invalid_value"
TEMPLATE_FAILED = "template_failed"
@dataclass
class ZoneOcrResult:
template: str
zone: str
custom_field: str
value: object = None
reason: str | None = None
reason: ZoneOcrReason | None = None
# Result labels for zones that target a built-in field.
BUILTIN_TARGET_LABELS: dict[OcrTemplateZone.TargetType, str] = {
OcrTemplateZone.TargetType.TITLE: "Title",
OcrTemplateZone.TargetType.ASN: "ASN",
OcrTemplateZone.TargetType.CREATED: "Created",
}
def _zone_result(
@@ -112,12 +136,12 @@ def _zone_result(
zone: OcrTemplateZone,
*,
value: object = None,
reason: str | None = None,
reason: ZoneOcrReason | None = None,
) -> ZoneOcrResult:
if zone.target == OcrTemplateZone.TargetType.CUSTOM_FIELD:
label = zone.custom_field.name if zone.custom_field_id else "(no field)"
else:
label = {"title": "Title", "asn": "ASN", "created": "Created"}[zone.target]
label = BUILTIN_TARGET_LABELS[zone.target]
return ZoneOcrResult(template.name, zone.name, label, value, reason)
@@ -185,7 +209,7 @@ def run_zone_extraction(
document.pk,
)
return [
_zone_result(template, zone, reason="missing_file")
_zone_result(template, zone, reason=ZoneOcrReason.MISSING_FILE)
for template in templates
for zone in template.zones.all()
]
@@ -215,7 +239,7 @@ def run_zone_extraction(
document.pk,
)
results.extend(
_zone_result(template, zone, reason="template_failed")
_zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED)
for zone in zones
)
if reindex and any(result.reason is None for result in results):
@@ -269,7 +293,7 @@ def _process_template(
for zone in zones:
page_idx = _resolve_page_idx(zone.page, document.page_count)
extracted = None
reason = "missing_page"
reason: ZoneOcrReason | None = ZoneOcrReason.MISSING_PAGE
if page_idx in page_images:
extraction = _extract_zone(
page_images[page_idx],
@@ -283,15 +307,15 @@ def _process_template(
reason = None
elif extraction.raw_text:
# Text was read but the transform found nothing usable in it.
reason = "transform_failed"
reason = ZoneOcrReason.TRANSFORM_FAILED
else:
reason = "no_value"
reason = ZoneOcrReason.NO_VALUE
if (
extracted is not None
and zone.validation_regex
and not safe_regex_fullmatch(zone.validation_regex, extracted)
):
reason = "regex_mismatch"
reason = ZoneOcrReason.REGEX_MISMATCH
result = _zone_result(
template,
zone,
@@ -309,10 +333,13 @@ def _process_template(
if not accepted:
continue
# An empty zone is an optional token; any other failure blocks the group.
if any(result.reason not in (None, "no_value") for _, result in field_zones):
if any(
result.reason not in (None, ZoneOcrReason.NO_VALUE)
for _, result in field_zones
):
for result in accepted:
result.value = None
result.reason = "incomplete"
result.reason = ZoneOcrReason.INCOMPLETE
continue
value = _combine_field_value(
combine_formats.get(key, ""),
@@ -325,7 +352,7 @@ def _process_template(
# write of this template, so only this field is dropped.
for result in accepted:
result.value = None
result.reason = "duplicate_asn"
result.reason = ZoneOcrReason.DUPLICATE_ASN
continue
if update is not None:
updates.append(update)
@@ -333,7 +360,9 @@ def _process_template(
result.value = update.value if update else None
if update is None:
result.reason = (
"unassigned" if key == "custom_field" else "invalid_value"
ZoneOcrReason.UNASSIGNED
if key == "custom_field"
else ZoneOcrReason.INVALID_VALUE
)
_apply_field_updates(root_document, updates)
return results
@@ -581,23 +610,22 @@ def extract_zone_preview(
source_width: int,
source_height: int,
page_count: int | None,
) -> dict:
) -> ZoneExtraction:
"""Extract a zone without writing its value to the document."""
page_idx = _resolve_page_idx(zone.page, page_count)
with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir:
tmp_path = Path(tmp_dir)
page_images = _render_pages(doc_path, {page_idx}, tmp_path, page_count)
if page_idx not in page_images:
return {"raw_text": None, "value": None}
return ZoneExtraction()
result = _extract_zone(
return _extract_zone(
page_images[page_idx],
zone,
source_width,
source_height,
tmp_path,
)
return {"raw_text": result.raw_text, "value": result.value}
@dataclass(frozen=True)