Fix: let the first matching OCR template win a field

Later templates no longer overwrite it. The skipped zones report superseded and a warning is logged.
This commit is contained in:
stumpylog committed 2026-11-20 11:30:31 -08:00
1 parent a2c18123fa
commit 17d5cb8855
3 files changed
+217 -18

No files matched your search

+177 -9
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
import shutil
import subprocess
from collections import Counter
@@ -52,6 +53,7 @@ from documents.zone_ocr import get_templates_for_content
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import resolve_zone_source
from documents.zone_ocr import run_zone_extraction
from documents.zone_ocr import zone_results_written
from paperless_testing.factories import CustomFieldFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
@@ -846,7 +848,7 @@ class TestSavedZoneExtraction:
assert document.title == "Original"
assert not document.custom_fields.exists()
def test_last_successful_template_wins(
def test_first_template_with_an_accepted_value_wins(
self,
mock_zone_ocr_run: MagicMock,
page: Path,
@@ -855,21 +857,21 @@ class TestSavedZoneExtraction:
) -> None:
"""
GIVEN:
- Three templates writing the title, the last of which rejects its value
- Three templates writing the title, the first of which rejects its value
WHEN:
- Zone extraction runs over all of them
THEN:
- The title comes from the last template that produced a value
- The title comes from the first template that produced a value
"""
make_zone(template, "First")
make_zone(template, "First", validation_regex="accepted")
second = make_template("Second")
make_zone(second, "Second")
third = make_template("Third")
make_zone(third, "Third", validation_regex="accepted")
make_zone(third, "Third")
templates = list(OcrTemplate.objects.order_by("pk"))
mock_zone_ocr_run.side_effect = [
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
for value in ["First", "Second", "rejected"]
for value in ["rejected", "Second", "Third"]
]
run_zone_extraction(document, templates=templates)
@@ -1019,7 +1021,11 @@ class TestSavedZoneExtraction:
results = run_zone_extraction(document, templates=[template, second])
assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2]
assert [result.reason for result in results] == [None, None, None]
assert [result.reason for result in results] == [
None,
None,
ZoneOcrReason.SUPERSEDED,
]
assert not any(path.exists() for path in rendered)
def test_zone_dimensions_override_template_dimensions(
@@ -1263,6 +1269,166 @@ class TestSavedZoneExtraction:
document.refresh_from_db()
assert document.title == "Applied"
def test_first_template_wins_a_field_written_by_both(
self,
caplog: pytest.LogCaptureFixture,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- Two matching templates whose title zones both read a value
WHEN:
- Zone extraction runs
THEN:
- The first template by name sets the title, the second template's
zone is reported as superseded with a warning, and the run still
counts as having written
"""
make_zone(template, "First title")
second = make_template("Second")
make_zone(second, "Second title")
mock_zone_ocr_run.side_effect = ocr_outputs("From first", "From second")
with caplog.at_level(logging.WARNING, logger="paperless.zone_ocr"):
results = run_zone_extraction(document, templates=[template, second])
assert [(r.template, r.value, r.reason) for r in results] == [
("Account", "From first", None),
("Second", None, ZoneOcrReason.SUPERSEDED),
]
assert zone_results_written(results)
assert any(
"Second" in message and "title" in message for message in caplog.messages
)
document.refresh_from_db()
assert document.title == "From first"
def test_single_template_logs_no_superseded_warning(
self,
caplog: pytest.LogCaptureFixture,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- One matching template with a title zone
WHEN:
- Zone extraction runs
THEN:
- The title is written and no warning is logged
"""
make_zone(template, "Title")
mock_zone_ocr_run.side_effect = ocr_outputs("Only")
with caplog.at_level(logging.WARNING, logger="paperless.zone_ocr"):
run_zone_extraction(document, templates=[template])
assert caplog.messages == []
def test_field_dropped_after_planning_stays_available_to_later_templates(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- Two matching templates with ASN zones, where the first template's
ASN is the ASN of another document
WHEN:
- Zone extraction runs
THEN:
- The first template's ASN is reported as duplicate_asn and the
second template's ASN is written
"""
DocumentFactory(archive_serial_number=42)
make_zone(template, "First ASN", "asn")
second = make_template("Second")
make_zone(second, "Second ASN", "asn")
mock_zone_ocr_run.side_effect = ocr_outputs("42", "43")
results = run_zone_extraction(document, templates=[template, second])
assert [r.reason for r in results] == [ZoneOcrReason.DUPLICATE_ASN, None]
document.refresh_from_db()
assert document.archive_serial_number == 43
def test_field_the_first_template_did_not_write_stays_available(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- Two matching templates, where the first template's title zone
reads nothing
WHEN:
- Zone extraction runs
THEN:
- The second template's title is written
"""
make_zone(template, "First title")
second = make_template("Second")
make_zone(second, "Second title")
mock_zone_ocr_run.side_effect = ocr_outputs("", "From second")
results = run_zone_extraction(document, templates=[template, second])
assert [r.reason for r in results] == [ZoneOcrReason.NO_VALUE, None]
document.refresh_from_db()
assert document.title == "From second"
def test_later_template_writes_only_the_fields_still_free(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- A first template writing the title and a second template with a
title zone and an ASN zone
WHEN:
- Zone extraction runs
THEN:
- Only the second template's ASN is written, its title zone is
reported as superseded
"""
make_zone(template, "First title")
second = make_template("Second")
make_zone(second, "Second title")
make_zone(second, "Second ASN", "asn")
mock_zone_ocr_run.side_effect = ocr_outputs(
"From first",
"From second",
"17",
)
results = run_zone_extraction(document, templates=[template, second])
assert [(r.zone, r.reason) for r in results] == [
("First title", None),
("Second title", ZoneOcrReason.SUPERSEDED),
("Second ASN", None),
]
document.refresh_from_db()
assert document.title == "From first"
assert document.archive_serial_number == 17
@pytest.mark.parametrize(
"deleted",
[
@@ -4110,7 +4276,8 @@ class TestRunZoneExtraction:
WHEN:
- Zone extraction runs
THEN:
- Each template reports exactly one successful result
- Each template reports exactly one result, and only the first
template by name writes the shared title
"""
for name in template_names:
template_with_zone(name, enabled=True)
@@ -4118,7 +4285,8 @@ class TestRunZoneExtraction:
results = run_zone_extraction(document)
assert sorted((r.template, r.reason) for r in results) == [
(name, None) for name in sorted(template_names)
(name, None if index == 0 else ZoneOcrReason.SUPERSEDED)
for index, name in enumerate(sorted(template_names))
]
+38 -8
View File
@@ -134,7 +134,10 @@ def resolve_zone_source(document: Document) -> ZoneSourceFile | None:
class ZoneOcrReason(StrEnum):
"""Why a zone produced no value for its target."""
"""
Why a zone produced no value for its target. SUPERSEDED means an earlier
template already wrote the field.
"""
MISSING_FILE = "missing_file"
MISSING_PAGE = "missing_page"
@@ -146,6 +149,7 @@ class ZoneOcrReason(StrEnum):
UNASSIGNED = "unassigned"
INVALID_VALUE = "invalid_value"
TEMPLATE_FAILED = "template_failed"
SUPERSEDED = "superseded"
class ZoneOcrSkipReason(StrEnum):
@@ -382,9 +386,11 @@ def apply_zone_plans(
in its own atomic block, and return the zone results. Fields whose key is in
protected are not written, and the results of every zone targeting them are
left out. A template whose writes fail gets TEMPLATE_FAILED results and the
others still apply.
others still apply. A field an earlier template wrote is not written by a
later one, and the zones targeting it get SUPERSEDED results.
"""
results: list[ZoneOcrResult] = []
written: set[FieldKey] = set()
for plan in plans:
for planned in plan.updates:
if planned.key in protected:
@@ -394,11 +400,31 @@ def apply_zone_plans(
planned.update.field_name,
label,
)
updates = [planned for planned in plan.updates if planned.key not in protected]
superseded: set[FieldKey] = set()
for planned in plan.updates:
if planned.key in written:
superseded.add(planned.key)
logger.warning(
"Zone OCR: template '%s' did not write %s of %s, an earlier "
"template already did",
plan.template.name,
planned.update.field_name,
label,
)
updates = [
planned
for planned in plan.updates
if planned.key not in protected and planned.key not in superseded
]
try:
if updates:
_apply_template_updates(root_document, updates)
plan_results = plan.results
written.update(
planned.key
for planned in updates
if all(result.reason is None for result in planned.results)
)
except Exception:
logger.exception(
"Zone OCR: error processing template '%s' for %s",
@@ -410,12 +436,16 @@ def apply_zone_plans(
plan.zones,
ZoneOcrReason.TEMPLATE_FAILED,
).results
superseded = set()
# Each zone has exactly one result, in zone order.
results.extend(
result
for zone, result in zip(plan.zones, plan_results, strict=True)
if _zone_key(zone) not in protected
)
for zone, result in zip(plan.zones, plan_results, strict=True):
key = _zone_key(zone)
if key in protected:
continue
if key in superseded:
result.value = None
result.reason = ZoneOcrReason.SUPERSEDED
results.append(result)
return results