Fix: keep OCR values intact when combining zones into one field

Separator cleanup ran over the extracted values too, so a negative amount lost its sign. A zone failing validation also no longer lets a partial value through.
This commit is contained in:
stumpylog committed 2026-11-18 14:20:11 -08:00
1 parent 0e6d06c255
commit c1bb327f1f
2 files changed
+285 -11

No files matched your search

+229
View File
@@ -85,6 +85,14 @@ def make_zone(
)
def ocr_outputs(*values: str) -> list[subprocess.CompletedProcess[str]]:
"""Build one successful fake OCR process result per extracted text."""
return [
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
for value in values
]
def make_visible_template(name: str, **kwargs: Any) -> OcrTemplate:
"""Build a 100x100 template matching "Northwind"."""
kwargs.setdefault("match", "Northwind")
@@ -1023,6 +1031,137 @@ class TestSavedZoneExtraction:
preview_and_save("invalid")
assert document.title == "ACCOUNT ABC"
@pytest.mark.parametrize(
("failing_zone", "expected_failure"),
[
pytest.param(
{"validation_regex": r"\d{2}"},
"regex_mismatch",
id="regex-mismatch",
),
pytest.param({"page": 2}, "missing_page", id="missing-page"),
],
)
def test_failed_zone_blocks_combined_field(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
failing_zone: dict[str, Any],
expected_failure: str,
) -> None:
"""
GIVEN:
- Three title zones combined as a date, where the first one fails
WHEN:
- The template is processed
THEN:
- Nothing is written, the failing zone reports its own reason and
the accepted zones report incomplete
"""
zones = [
make_zone(template, "Day", **failing_zone),
make_zone(template, "Month"),
make_zone(template, "Year"),
]
template.combine_formats = {"title": "{Day}.{Month}.{Year}"}
# A zone on a missing page is never read.
mock_zone_ocr_run.side_effect = ocr_outputs(
*(
["x", "05", "2024"]
if "validation_regex" in failing_zone
else ["05", "2024"]
),
)
results = _process_template(
document,
{0: page},
template,
zones,
paperless_dirs.scratch_dir,
)
assert [result.reason for result in results] == [
expected_failure,
"incomplete",
"incomplete",
]
assert [result.value for result in results] == [None] * 3
document.refresh_from_db()
assert document.title == "Original"
def test_zone_with_no_value_is_an_optional_token(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- Two title zones combined with a separator, where the first reads
nothing
WHEN:
- The template is processed
THEN:
- The remaining value is still written and reported
"""
zones = [make_zone(template, "First"), make_zone(template, "Second")]
template.combine_formats = {"title": "{First} / {Second}"}
mock_zone_ocr_run.side_effect = ocr_outputs("", "two")
results = _process_template(
document,
{0: page},
template,
zones,
paperless_dirs.scratch_dir,
)
assert [result.reason for result in results] == ["no_value", None]
assert results[1].value == "two"
document.refresh_from_db()
assert document.title == "two"
def test_failed_single_zone_field_does_not_affect_other_fields(
self,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- A title zone and a separate ASN zone, where the ASN fails its
validation regex
WHEN:
- The template is processed
THEN:
- The title is written and the failing zone keeps regex_mismatch
"""
zones = [
make_zone(template, "Title"),
make_zone(template, "ASN", "asn", validation_regex=r"\d+"),
]
mock_zone_ocr_run.side_effect = ocr_outputs("New title", "abc")
results = _process_template(
document,
{0: page},
template,
zones,
paperless_dirs.scratch_dir,
)
assert [result.reason for result in results] == [None, "regex_mismatch"]
document.refresh_from_db()
assert document.title == "New title"
class TestCombineFieldValue:
"""Tests for combining zone values into one field value."""
@@ -1042,6 +1181,96 @@ class TestCombineFieldValue:
"one",
id="format-preserves-existing-missing-token-cleanup",
),
pytest.param(
"{Amount}",
[("Amount", "-12.50")],
"-12.50",
id="negative-amount-sign-kept",
),
pytest.param(
"{Url}",
[("Url", "http://x.com/a//b")],
"http://x.com/a//b",
id="url-double-slash-kept-verbatim",
),
pytest.param(
"{Value}",
[("Value", "A--B")],
"A--B",
id="repeated-punctuation-in-value-kept",
),
pytest.param(
"{Name}",
[("Name", "Dr. Smith Jr.")],
"Dr. Smith Jr.",
id="trailing-period-in-value-kept",
),
pytest.param(
"{Value}",
[("Value", "wait...")],
"wait...",
id="trailing-ellipsis-in-value-kept",
),
pytest.param(
"{A} / {B} / {C}",
[("A", "a"), ("B", None), ("C", "c")],
"a / c",
id="middle-empty-token-drops-one-separator",
),
pytest.param(
"{A} / {B} / {C}",
[("A", None), ("B", "b"), ("C", "c")],
"b / c",
id="leading-empty-token-drops-leading-separator",
),
pytest.param(
"{A} / {B} / {C}",
[("A", "a"), ("B", "b"), ("C", None)],
"a / b",
id="trailing-empty-token-drops-trailing-separator",
),
pytest.param(
"{A} / {B} / {C}",
[("A", "a"), ("B", None), ("C", None)],
"a",
id="trailing-run-of-empty-tokens-drops-nearest-separator",
),
pytest.param(
"Total: {A} / {B}",
[("A", None), ("B", "b")],
"Total: b",
id="leading-literal-kept-before-empty-first-token",
),
pytest.param(
"- {A} {B}",
[("A", None), ("B", "b")],
"b",
id="leading-separator-dropped-before-empty-first-token",
),
pytest.param(
"{Day}.{Month}.{Year}",
[("Day", "12"), ("Month", "05"), ("Year", "2024")],
"12.05.2024",
id="all-tokens-present",
),
pytest.param(
"INV-{Number}",
[("Number", "123")],
"INV-123",
id="literal-prefix-kept-with-value",
),
pytest.param(
"INV-{Number}",
[("Number", None)],
"",
id="literal-prefix-dropped-when-every-token-empty",
),
pytest.param(
"{A} / {B}",
[("A", None), ("B", " ")],
"",
id="every-token-empty",
),
],
)
def test_combine_field_value(
+56 -11
View File
@@ -68,6 +68,9 @@ DATE_TOKEN_RE = re.compile(
r"|(?<!\d)\d{1,2}\.?\s+[^\W\d_]{3,}\.?\s+\d{4}(?!\d)",
)
# Literal text made only of whitespace and separator punctuation.
SEPARATOR_ONLY_RE = re.compile(r"[\s\-/.,;:|]*")
# Earlier dates are far more likely OCR noise than a real document date.
MIN_ZONE_DATE_YEAR = 1900
@@ -285,6 +288,12 @@ def _process_template(
accepted = [result for _, result in field_zones if result.reason is None]
if not accepted:
continue
# An empty zone is an optional token; any other failure blocks the group.
if any(result.reason not in (None, "no_value") for _, result in field_zones):
for result in accepted:
result.value = None
result.reason = "incomplete"
continue
value = _combine_field_value(
combine_formats.get(key, ""),
[(zone.name, result.value) for zone, result in field_zones],
@@ -318,22 +327,58 @@ def _combine_field_value(
"""Combine the OCR values of all zones targeting one field.
With a format string, `{Zone Name}` tokens are replaced by that zone's value
and literal text is kept; separators left dangling by an empty token are
cleaned up. Without a format, the zone values are joined in order by a space.
and literal text is kept. Values are inserted verbatim (only surrounding
whitespace is removed), so punctuation inside an OCR value is never altered.
Unknown token names count as empty. Tidying only touches the format's own
literal text: a separator-only literal following an empty token is dropped
(for a trailing empty token, the nearest preceding kept separator is
dropped instead), as is a leading separator-only literal in front of an
empty first token. Other literal text, such as brackets around an empty
token ("{A}, {B} ({C})" gives "a, b ()"), is kept as long as at least one
token has a value; when every token is empty the result is an empty
string. Without a format, the non-empty zone values are joined in order by
a space.
"""
values = {name: value or "" for name, value in field_zones}
if not fmt:
return " ".join(value for _, value in field_zones if value).strip()
def _replace(match: re.Match) -> str:
return values.get(match.group(1).strip(), "")
values = {name.strip(): (value or "").strip() for name, value in field_zones}
combined = re.sub(r"\{([^{}]+)\}", _replace, fmt)
# Tidy up separators an empty token may have left behind.
combined = re.sub(r"\s{2,}", " ", combined)
combined = re.sub(r"([^\w\s])\s*\1+", r"\1", combined)
return combined.strip().strip("-/.,;:| \t")
# Pairs of (is_token, text); tokens carry their value.
parts: list[tuple[bool, str]] = []
for index, piece in enumerate(re.split(r"(\{[^{}]+\})", fmt)):
if not piece:
continue
if index % 2:
parts.append((True, values.get(piece[1:-1].strip(), "")))
else:
parts.append((False, piece))
if not any(is_token and text for is_token, text in parts):
return ""
def _is_separator(part: tuple[bool, str]) -> bool:
return not part[0] and SEPARATOR_ONLY_RE.fullmatch(part[1]) is not None
kept: list[tuple[bool, str]] = []
drop_next_separator = False
for position, part in enumerate(parts):
if drop_next_separator:
drop_next_separator = False
continue
is_token, text = part
if not is_token or text:
kept.append(part)
continue
following = parts[position + 1] if position + 1 < len(parts) else None
if following is not None:
drop_next_separator = _is_separator(following)
elif kept and _is_separator(kept[-1]):
kept.pop()
# A leading separator left in front of an empty first token.
if len(kept) == 1 and _is_separator(kept[0]):
kept.pop()
return "".join(text for _, text in kept).strip()
def _render_pages(