From 17d5cb88554f6f603b6fcf842c3e742d4db8623e Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:30:31 -0800 Subject: [PATCH] Fix: let the first matching OCR template win a field Later templates no longer overwrite it. The skipped zones report superseded and a warning is logged. --- docs/usage.md | 3 +- src/documents/tests/test_zone_ocr.py | 186 +++++++++++++++++++++++++-- src/documents/zone_ocr.py | 46 +++++-- 3 files changed, 217 insertions(+), 18 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 988ae91392..26ff879305 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -845,7 +845,8 @@ decides which documents it automatically applies to (or not), based on their con Zones are positioned relative to the sample document's page, so templates work best when the documents they match have an identical layout. Keep the matching specific enough that it only catches documents with that -layout. +layout. When more than one template applies to a document, the first one by name fills a field and later +templates do not overwrite it. ### Zone settings diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index f26a2b8a85..9a5bdfa0dc 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging import shutil import subprocess from collections import Counter @@ -52,6 +53,7 @@ from documents.zone_ocr import get_templates_for_content from documents.zone_ocr import get_templates_for_document from documents.zone_ocr import resolve_zone_source from documents.zone_ocr import run_zone_extraction +from documents.zone_ocr import zone_results_written from paperless_testing.factories import CustomFieldFactory from paperless_testing.factories import DocumentFactory from paperless_testing.factories import OcrTemplateFactory @@ -846,7 +848,7 @@ class TestSavedZoneExtraction: assert document.title == "Original" assert not document.custom_fields.exists() - def test_last_successful_template_wins( + def test_first_template_with_an_accepted_value_wins( self, mock_zone_ocr_run: MagicMock, page: Path, @@ -855,21 +857,21 @@ class TestSavedZoneExtraction: ) -> None: """ GIVEN: - - Three templates writing the title, the last of which rejects its value + - Three templates writing the title, the first of which rejects its value WHEN: - Zone extraction runs over all of them THEN: - - The title comes from the last template that produced a value + - The title comes from the first template that produced a value """ - make_zone(template, "First") + make_zone(template, "First", validation_regex="accepted") second = make_template("Second") make_zone(second, "Second") third = make_template("Third") - make_zone(third, "Third", validation_regex="accepted") + make_zone(third, "Third") templates = list(OcrTemplate.objects.order_by("pk")) mock_zone_ocr_run.side_effect = [ subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") - for value in ["First", "Second", "rejected"] + for value in ["rejected", "Second", "Third"] ] run_zone_extraction(document, templates=templates) @@ -1019,7 +1021,11 @@ class TestSavedZoneExtraction: results = run_zone_extraction(document, templates=[template, second]) assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2] - assert [result.reason for result in results] == [None, None, None] + assert [result.reason for result in results] == [ + None, + None, + ZoneOcrReason.SUPERSEDED, + ] assert not any(path.exists() for path in rendered) def test_zone_dimensions_override_template_dimensions( @@ -1263,6 +1269,166 @@ class TestSavedZoneExtraction: document.refresh_from_db() assert document.title == "Applied" + def test_first_template_wins_a_field_written_by_both( + self, + caplog: pytest.LogCaptureFixture, + mock_zone_ocr_run: MagicMock, + paperless_dirs: PaperlessDirs, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - Two matching templates whose title zones both read a value + WHEN: + - Zone extraction runs + THEN: + - The first template by name sets the title, the second template's + zone is reported as superseded with a warning, and the run still + counts as having written + """ + make_zone(template, "First title") + second = make_template("Second") + make_zone(second, "Second title") + mock_zone_ocr_run.side_effect = ocr_outputs("From first", "From second") + + with caplog.at_level(logging.WARNING, logger="paperless.zone_ocr"): + results = run_zone_extraction(document, templates=[template, second]) + + assert [(r.template, r.value, r.reason) for r in results] == [ + ("Account", "From first", None), + ("Second", None, ZoneOcrReason.SUPERSEDED), + ] + assert zone_results_written(results) + assert any( + "Second" in message and "title" in message for message in caplog.messages + ) + document.refresh_from_db() + assert document.title == "From first" + + def test_single_template_logs_no_superseded_warning( + self, + caplog: pytest.LogCaptureFixture, + mock_zone_ocr_run: MagicMock, + paperless_dirs: PaperlessDirs, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - One matching template with a title zone + WHEN: + - Zone extraction runs + THEN: + - The title is written and no warning is logged + """ + make_zone(template, "Title") + mock_zone_ocr_run.side_effect = ocr_outputs("Only") + + with caplog.at_level(logging.WARNING, logger="paperless.zone_ocr"): + run_zone_extraction(document, templates=[template]) + + assert caplog.messages == [] + + def test_field_dropped_after_planning_stays_available_to_later_templates( + self, + mock_zone_ocr_run: MagicMock, + paperless_dirs: PaperlessDirs, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - Two matching templates with ASN zones, where the first template's + ASN is the ASN of another document + WHEN: + - Zone extraction runs + THEN: + - The first template's ASN is reported as duplicate_asn and the + second template's ASN is written + """ + DocumentFactory(archive_serial_number=42) + make_zone(template, "First ASN", "asn") + second = make_template("Second") + make_zone(second, "Second ASN", "asn") + mock_zone_ocr_run.side_effect = ocr_outputs("42", "43") + + results = run_zone_extraction(document, templates=[template, second]) + + assert [r.reason for r in results] == [ZoneOcrReason.DUPLICATE_ASN, None] + document.refresh_from_db() + assert document.archive_serial_number == 43 + + def test_field_the_first_template_did_not_write_stays_available( + self, + mock_zone_ocr_run: MagicMock, + paperless_dirs: PaperlessDirs, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - Two matching templates, where the first template's title zone + reads nothing + WHEN: + - Zone extraction runs + THEN: + - The second template's title is written + """ + make_zone(template, "First title") + second = make_template("Second") + make_zone(second, "Second title") + mock_zone_ocr_run.side_effect = ocr_outputs("", "From second") + + results = run_zone_extraction(document, templates=[template, second]) + + assert [r.reason for r in results] == [ZoneOcrReason.NO_VALUE, None] + document.refresh_from_db() + assert document.title == "From second" + + def test_later_template_writes_only_the_fields_still_free( + self, + mock_zone_ocr_run: MagicMock, + paperless_dirs: PaperlessDirs, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - A first template writing the title and a second template with a + title zone and an ASN zone + WHEN: + - Zone extraction runs + THEN: + - Only the second template's ASN is written, its title zone is + reported as superseded + """ + make_zone(template, "First title") + second = make_template("Second") + make_zone(second, "Second title") + make_zone(second, "Second ASN", "asn") + mock_zone_ocr_run.side_effect = ocr_outputs( + "From first", + "From second", + "17", + ) + + results = run_zone_extraction(document, templates=[template, second]) + + assert [(r.zone, r.reason) for r in results] == [ + ("First title", None), + ("Second title", ZoneOcrReason.SUPERSEDED), + ("Second ASN", None), + ] + document.refresh_from_db() + assert document.title == "From first" + assert document.archive_serial_number == 17 + @pytest.mark.parametrize( "deleted", [ @@ -4110,7 +4276,8 @@ class TestRunZoneExtraction: WHEN: - Zone extraction runs THEN: - - Each template reports exactly one successful result + - Each template reports exactly one result, and only the first + template by name writes the shared title """ for name in template_names: template_with_zone(name, enabled=True) @@ -4118,7 +4285,8 @@ class TestRunZoneExtraction: results = run_zone_extraction(document) assert sorted((r.template, r.reason) for r in results) == [ - (name, None) for name in sorted(template_names) + (name, None if index == 0 else ZoneOcrReason.SUPERSEDED) + for index, name in enumerate(sorted(template_names)) ] diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 8885280312..dcf8fac41d 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -134,7 +134,10 @@ def resolve_zone_source(document: Document) -> ZoneSourceFile | None: class ZoneOcrReason(StrEnum): - """Why a zone produced no value for its target.""" + """ + Why a zone produced no value for its target. SUPERSEDED means an earlier + template already wrote the field. + """ MISSING_FILE = "missing_file" MISSING_PAGE = "missing_page" @@ -146,6 +149,7 @@ class ZoneOcrReason(StrEnum): UNASSIGNED = "unassigned" INVALID_VALUE = "invalid_value" TEMPLATE_FAILED = "template_failed" + SUPERSEDED = "superseded" class ZoneOcrSkipReason(StrEnum): @@ -382,9 +386,11 @@ def apply_zone_plans( in its own atomic block, and return the zone results. Fields whose key is in protected are not written, and the results of every zone targeting them are left out. A template whose writes fail gets TEMPLATE_FAILED results and the - others still apply. + others still apply. A field an earlier template wrote is not written by a + later one, and the zones targeting it get SUPERSEDED results. """ results: list[ZoneOcrResult] = [] + written: set[FieldKey] = set() for plan in plans: for planned in plan.updates: if planned.key in protected: @@ -394,11 +400,31 @@ def apply_zone_plans( planned.update.field_name, label, ) - updates = [planned for planned in plan.updates if planned.key not in protected] + superseded: set[FieldKey] = set() + for planned in plan.updates: + if planned.key in written: + superseded.add(planned.key) + logger.warning( + "Zone OCR: template '%s' did not write %s of %s, an earlier " + "template already did", + plan.template.name, + planned.update.field_name, + label, + ) + updates = [ + planned + for planned in plan.updates + if planned.key not in protected and planned.key not in superseded + ] try: if updates: _apply_template_updates(root_document, updates) plan_results = plan.results + written.update( + planned.key + for planned in updates + if all(result.reason is None for result in planned.results) + ) except Exception: logger.exception( "Zone OCR: error processing template '%s' for %s", @@ -410,12 +436,16 @@ def apply_zone_plans( plan.zones, ZoneOcrReason.TEMPLATE_FAILED, ).results + superseded = set() # Each zone has exactly one result, in zone order. - results.extend( - result - for zone, result in zip(plan.zones, plan_results, strict=True) - if _zone_key(zone) not in protected - ) + for zone, result in zip(plan.zones, plan_results, strict=True): + key = _zone_key(zone) + if key in protected: + continue + if key in superseded: + result.value = None + result.reason = ZoneOcrReason.SUPERSEDED + results.append(result) return results