Fix: let the first matching OCR template win a field

Later templates no longer overwrite it. The skipped zones report superseded and a warning is logged.
This commit is contained in:
stumpylog committed 2026-11-20 11:30:31 -08:00
1 parent a2c18123fa
commit 17d5cb8855
3 files changed
+217 -18

No files matched your search

+2 -1
View File
@@ -845,7 +845,8 @@ decides which documents it automatically applies to (or not), based on their con
Zones are positioned relative to the sample document's page, so templates work best when the documents they
match have an identical layout. Keep the matching specific enough that it only catches documents with that
layout.
layout. When more than one template applies to a document, the first one by name fills a field and later
templates do not overwrite it.
### Zone settings
+177 -9
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
import shutil
import subprocess
from collections import Counter
@@ -52,6 +53,7 @@ from documents.zone_ocr import get_templates_for_content
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import resolve_zone_source
from documents.zone_ocr import run_zone_extraction
from documents.zone_ocr import zone_results_written
from paperless_testing.factories import CustomFieldFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
@@ -846,7 +848,7 @@ class TestSavedZoneExtraction:
assert document.title == "Original"
assert not document.custom_fields.exists()
def test_last_successful_template_wins(
def test_first_template_with_an_accepted_value_wins(
self,
mock_zone_ocr_run: MagicMock,
page: Path,
@@ -855,21 +857,21 @@ class TestSavedZoneExtraction:
) -> None:
"""
GIVEN:
- Three templates writing the title, the last of which rejects its value
- Three templates writing the title, the first of which rejects its value
WHEN:
- Zone extraction runs over all of them
THEN:
- The title comes from the last template that produced a value
- The title comes from the first template that produced a value
"""
make_zone(template, "First")
make_zone(template, "First", validation_regex="accepted")
second = make_template("Second")
make_zone(second, "Second")
third = make_template("Third")
make_zone(third, "Third", validation_regex="accepted")
make_zone(third, "Third")
templates = list(OcrTemplate.objects.order_by("pk"))
mock_zone_ocr_run.side_effect = [
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
for value in ["First", "Second", "rejected"]
for value in ["rejected", "Second", "Third"]
]
run_zone_extraction(document, templates=templates)
@@ -1019,7 +1021,11 @@ class TestSavedZoneExtraction:
results = run_zone_extraction(document, templates=[template, second])
assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2]
assert [result.reason for result in results] == [None, None, None]
assert [result.reason for result in results] == [
None,
None,
ZoneOcrReason.SUPERSEDED,
]
assert not any(path.exists() for path in rendered)
def test_zone_dimensions_override_template_dimensions(
@@ -1263,6 +1269,166 @@ class TestSavedZoneExtraction:
document.refresh_from_db()
assert document.title == "Applied"
def test_first_template_wins_a_field_written_by_both(
self,
caplog: pytest.LogCaptureFixture,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- Two matching templates whose title zones both read a value
WHEN:
- Zone extraction runs
THEN:
- The first template by name sets the title, the second template's
zone is reported as superseded with a warning, and the run still
counts as having written
"""
make_zone(template, "First title")
second = make_template("Second")
make_zone(second, "Second title")
mock_zone_ocr_run.side_effect = ocr_outputs("From first", "From second")
with caplog.at_level(logging.WARNING, logger="paperless.zone_ocr"):
results = run_zone_extraction(document, templates=[template, second])
assert [(r.template, r.value, r.reason) for r in results] == [
("Account", "From first", None),
("Second", None, ZoneOcrReason.SUPERSEDED),
]
assert zone_results_written(results)
assert any(
"Second" in message and "title" in message for message in caplog.messages
)
document.refresh_from_db()
assert document.title == "From first"
def test_single_template_logs_no_superseded_warning(
self,
caplog: pytest.LogCaptureFixture,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- One matching template with a title zone
WHEN:
- Zone extraction runs
THEN:
- The title is written and no warning is logged
"""
make_zone(template, "Title")
mock_zone_ocr_run.side_effect = ocr_outputs("Only")
with caplog.at_level(logging.WARNING, logger="paperless.zone_ocr"):
run_zone_extraction(document, templates=[template])
assert caplog.messages == []
def test_field_dropped_after_planning_stays_available_to_later_templates(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- Two matching templates with ASN zones, where the first template's
ASN is the ASN of another document
WHEN:
- Zone extraction runs
THEN:
- The first template's ASN is reported as duplicate_asn and the
second template's ASN is written
"""
DocumentFactory(archive_serial_number=42)
make_zone(template, "First ASN", "asn")
second = make_template("Second")
make_zone(second, "Second ASN", "asn")
mock_zone_ocr_run.side_effect = ocr_outputs("42", "43")
results = run_zone_extraction(document, templates=[template, second])
assert [r.reason for r in results] == [ZoneOcrReason.DUPLICATE_ASN, None]
document.refresh_from_db()
assert document.archive_serial_number == 43
def test_field_the_first_template_did_not_write_stays_available(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- Two matching templates, where the first template's title zone
reads nothing
WHEN:
- Zone extraction runs
THEN:
- The second template's title is written
"""
make_zone(template, "First title")
second = make_template("Second")
make_zone(second, "Second title")
mock_zone_ocr_run.side_effect = ocr_outputs("", "From second")
results = run_zone_extraction(document, templates=[template, second])
assert [r.reason for r in results] == [ZoneOcrReason.NO_VALUE, None]
document.refresh_from_db()
assert document.title == "From second"
def test_later_template_writes_only_the_fields_still_free(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- A first template writing the title and a second template with a
title zone and an ASN zone
WHEN:
- Zone extraction runs
THEN:
- Only the second template's ASN is written, its title zone is
reported as superseded
"""
make_zone(template, "First title")
second = make_template("Second")
make_zone(second, "Second title")
make_zone(second, "Second ASN", "asn")
mock_zone_ocr_run.side_effect = ocr_outputs(
"From first",
"From second",
"17",
)
results = run_zone_extraction(document, templates=[template, second])
assert [(r.zone, r.reason) for r in results] == [
("First title", None),
("Second title", ZoneOcrReason.SUPERSEDED),
("Second ASN", None),
]
document.refresh_from_db()
assert document.title == "From first"
assert document.archive_serial_number == 17
@pytest.mark.parametrize(
"deleted",
[
@@ -4110,7 +4276,8 @@ class TestRunZoneExtraction:
WHEN:
- Zone extraction runs
THEN:
- Each template reports exactly one successful result
- Each template reports exactly one result, and only the first
template by name writes the shared title
"""
for name in template_names:
template_with_zone(name, enabled=True)
@@ -4118,7 +4285,8 @@ class TestRunZoneExtraction:
results = run_zone_extraction(document)
assert sorted((r.template, r.reason) for r in results) == [
(name, None) for name in sorted(template_names)
(name, None if index == 0 else ZoneOcrReason.SUPERSEDED)
for index, name in enumerate(sorted(template_names))
]
+38 -8
View File
@@ -134,7 +134,10 @@ def resolve_zone_source(document: Document) -> ZoneSourceFile | None:
class ZoneOcrReason(StrEnum):
"""Why a zone produced no value for its target."""
"""
Why a zone produced no value for its target. SUPERSEDED means an earlier
template already wrote the field.
"""
MISSING_FILE = "missing_file"
MISSING_PAGE = "missing_page"
@@ -146,6 +149,7 @@ class ZoneOcrReason(StrEnum):
UNASSIGNED = "unassigned"
INVALID_VALUE = "invalid_value"
TEMPLATE_FAILED = "template_failed"
SUPERSEDED = "superseded"
class ZoneOcrSkipReason(StrEnum):
@@ -382,9 +386,11 @@ def apply_zone_plans(
in its own atomic block, and return the zone results. Fields whose key is in
protected are not written, and the results of every zone targeting them are
left out. A template whose writes fail gets TEMPLATE_FAILED results and the
others still apply.
others still apply. A field an earlier template wrote is not written by a
later one, and the zones targeting it get SUPERSEDED results.
"""
results: list[ZoneOcrResult] = []
written: set[FieldKey] = set()
for plan in plans:
for planned in plan.updates:
if planned.key in protected:
@@ -394,11 +400,31 @@ def apply_zone_plans(
planned.update.field_name,
label,
)
updates = [planned for planned in plan.updates if planned.key not in protected]
superseded: set[FieldKey] = set()
for planned in plan.updates:
if planned.key in written:
superseded.add(planned.key)
logger.warning(
"Zone OCR: template '%s' did not write %s of %s, an earlier "
"template already did",
plan.template.name,
planned.update.field_name,
label,
)
updates = [
planned
for planned in plan.updates
if planned.key not in protected and planned.key not in superseded
]
try:
if updates:
_apply_template_updates(root_document, updates)
plan_results = plan.results
written.update(
planned.key
for planned in updates
if all(result.reason is None for result in planned.results)
)
except Exception:
logger.exception(
"Zone OCR: error processing template '%s' for %s",
@@ -410,12 +436,16 @@ def apply_zone_plans(
plan.zones,
ZoneOcrReason.TEMPLATE_FAILED,
).results
superseded = set()
# Each zone has exactly one result, in zone order.
results.extend(
result
for zone, result in zip(plan.zones, plan_results, strict=True)
if _zone_key(zone) not in protected
)
for zone, result in zip(plan.zones, plan_results, strict=True):
key = _zone_key(zone)
if key in protected:
continue
if key in superseded:
result.value = None
result.reason = ZoneOcrReason.SUPERSEDED
results.append(result)
return results