From c1bb327f1fd48bcbade95123cb9d8472738d7cf8 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:20:11 -0800 Subject: [PATCH] Fix: keep OCR values intact when combining zones into one field Separator cleanup ran over the extracted values too, so a negative amount lost its sign. A zone failing validation also no longer lets a partial value through. --- src/documents/tests/test_zone_ocr.py | 229 +++++++++++++++++++++++++++ src/documents/zone_ocr.py | 67 ++++++-- 2 files changed, 285 insertions(+), 11 deletions(-) diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index 90afc58bcb..92090ec73d 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -85,6 +85,14 @@ def make_zone( ) +def ocr_outputs(*values: str) -> list[subprocess.CompletedProcess[str]]: + """Build one successful fake OCR process result per extracted text.""" + return [ + subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") + for value in values + ] + + def make_visible_template(name: str, **kwargs: Any) -> OcrTemplate: """Build a 100x100 template matching "Northwind".""" kwargs.setdefault("match", "Northwind") @@ -1023,6 +1031,137 @@ class TestSavedZoneExtraction: preview_and_save("invalid") assert document.title == "ACCOUNT ABC" + @pytest.mark.parametrize( + ("failing_zone", "expected_failure"), + [ + pytest.param( + {"validation_regex": r"\d{2}"}, + "regex_mismatch", + id="regex-mismatch", + ), + pytest.param({"page": 2}, "missing_page", id="missing-page"), + ], + ) + def test_failed_zone_blocks_combined_field( + self, + mock_zone_ocr_run: MagicMock, + paperless_dirs: PaperlessDirs, + page: Path, + document: Document, + template: OcrTemplate, + failing_zone: dict[str, Any], + expected_failure: str, + ) -> None: + """ + GIVEN: + - Three title zones combined as a date, where the first one fails + WHEN: + - The template is processed + THEN: + - Nothing is written, the failing zone reports its own reason and + the accepted zones report incomplete + """ + zones = [ + make_zone(template, "Day", **failing_zone), + make_zone(template, "Month"), + make_zone(template, "Year"), + ] + template.combine_formats = {"title": "{Day}.{Month}.{Year}"} + # A zone on a missing page is never read. + mock_zone_ocr_run.side_effect = ocr_outputs( + *( + ["x", "05", "2024"] + if "validation_regex" in failing_zone + else ["05", "2024"] + ), + ) + + results = _process_template( + document, + {0: page}, + template, + zones, + paperless_dirs.scratch_dir, + ) + + assert [result.reason for result in results] == [ + expected_failure, + "incomplete", + "incomplete", + ] + assert [result.value for result in results] == [None] * 3 + document.refresh_from_db() + assert document.title == "Original" + + def test_zone_with_no_value_is_an_optional_token( + self, + mock_zone_ocr_run: MagicMock, + paperless_dirs: PaperlessDirs, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - Two title zones combined with a separator, where the first reads + nothing + WHEN: + - The template is processed + THEN: + - The remaining value is still written and reported + """ + zones = [make_zone(template, "First"), make_zone(template, "Second")] + template.combine_formats = {"title": "{First} / {Second}"} + mock_zone_ocr_run.side_effect = ocr_outputs("", "two") + + results = _process_template( + document, + {0: page}, + template, + zones, + paperless_dirs.scratch_dir, + ) + + assert [result.reason for result in results] == ["no_value", None] + assert results[1].value == "two" + document.refresh_from_db() + assert document.title == "two" + + def test_failed_single_zone_field_does_not_affect_other_fields( + self, + mock_zone_ocr_run: MagicMock, + paperless_dirs: PaperlessDirs, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - A title zone and a separate ASN zone, where the ASN fails its + validation regex + WHEN: + - The template is processed + THEN: + - The title is written and the failing zone keeps regex_mismatch + """ + zones = [ + make_zone(template, "Title"), + make_zone(template, "ASN", "asn", validation_regex=r"\d+"), + ] + mock_zone_ocr_run.side_effect = ocr_outputs("New title", "abc") + + results = _process_template( + document, + {0: page}, + template, + zones, + paperless_dirs.scratch_dir, + ) + + assert [result.reason for result in results] == [None, "regex_mismatch"] + document.refresh_from_db() + assert document.title == "New title" + class TestCombineFieldValue: """Tests for combining zone values into one field value.""" @@ -1042,6 +1181,96 @@ class TestCombineFieldValue: "one", id="format-preserves-existing-missing-token-cleanup", ), + pytest.param( + "{Amount}", + [("Amount", "-12.50")], + "-12.50", + id="negative-amount-sign-kept", + ), + pytest.param( + "{Url}", + [("Url", "http://x.com/a//b")], + "http://x.com/a//b", + id="url-double-slash-kept-verbatim", + ), + pytest.param( + "{Value}", + [("Value", "A--B")], + "A--B", + id="repeated-punctuation-in-value-kept", + ), + pytest.param( + "{Name}", + [("Name", "Dr. Smith Jr.")], + "Dr. Smith Jr.", + id="trailing-period-in-value-kept", + ), + pytest.param( + "{Value}", + [("Value", "wait...")], + "wait...", + id="trailing-ellipsis-in-value-kept", + ), + pytest.param( + "{A} / {B} / {C}", + [("A", "a"), ("B", None), ("C", "c")], + "a / c", + id="middle-empty-token-drops-one-separator", + ), + pytest.param( + "{A} / {B} / {C}", + [("A", None), ("B", "b"), ("C", "c")], + "b / c", + id="leading-empty-token-drops-leading-separator", + ), + pytest.param( + "{A} / {B} / {C}", + [("A", "a"), ("B", "b"), ("C", None)], + "a / b", + id="trailing-empty-token-drops-trailing-separator", + ), + pytest.param( + "{A} / {B} / {C}", + [("A", "a"), ("B", None), ("C", None)], + "a", + id="trailing-run-of-empty-tokens-drops-nearest-separator", + ), + pytest.param( + "Total: {A} / {B}", + [("A", None), ("B", "b")], + "Total: b", + id="leading-literal-kept-before-empty-first-token", + ), + pytest.param( + "- {A} {B}", + [("A", None), ("B", "b")], + "b", + id="leading-separator-dropped-before-empty-first-token", + ), + pytest.param( + "{Day}.{Month}.{Year}", + [("Day", "12"), ("Month", "05"), ("Year", "2024")], + "12.05.2024", + id="all-tokens-present", + ), + pytest.param( + "INV-{Number}", + [("Number", "123")], + "INV-123", + id="literal-prefix-kept-with-value", + ), + pytest.param( + "INV-{Number}", + [("Number", None)], + "", + id="literal-prefix-dropped-when-every-token-empty", + ), + pytest.param( + "{A} / {B}", + [("A", None), ("B", " ")], + "", + id="every-token-empty", + ), ], ) def test_combine_field_value( diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index a11e99dd8e..214abcac2c 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -68,6 +68,9 @@ DATE_TOKEN_RE = re.compile( r"|(? str: - return values.get(match.group(1).strip(), "") + values = {name.strip(): (value or "").strip() for name, value in field_zones} - combined = re.sub(r"\{([^{}]+)\}", _replace, fmt) - # Tidy up separators an empty token may have left behind. - combined = re.sub(r"\s{2,}", " ", combined) - combined = re.sub(r"([^\w\s])\s*\1+", r"\1", combined) - return combined.strip().strip("-/.,;:| \t") + # Pairs of (is_token, text); tokens carry their value. + parts: list[tuple[bool, str]] = [] + for index, piece in enumerate(re.split(r"(\{[^{}]+\})", fmt)): + if not piece: + continue + if index % 2: + parts.append((True, values.get(piece[1:-1].strip(), ""))) + else: + parts.append((False, piece)) + + if not any(is_token and text for is_token, text in parts): + return "" + + def _is_separator(part: tuple[bool, str]) -> bool: + return not part[0] and SEPARATOR_ONLY_RE.fullmatch(part[1]) is not None + + kept: list[tuple[bool, str]] = [] + drop_next_separator = False + for position, part in enumerate(parts): + if drop_next_separator: + drop_next_separator = False + continue + is_token, text = part + if not is_token or text: + kept.append(part) + continue + following = parts[position + 1] if position + 1 < len(parts) else None + if following is not None: + drop_next_separator = _is_separator(following) + elif kept and _is_separator(kept[-1]): + kept.pop() + # A leading separator left in front of an empty first token. + if len(kept) == 1 and _is_separator(kept[0]): + kept.pop() + return "".join(text for _, text in kept).strip() def _render_pages(