mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-19 20:25:21 +00:00
Fix: keep OCR values intact when combining zones into one field
Separator cleanup ran over the extracted values too, so a negative amount lost its sign. A zone failing validation also no longer lets a partial value through.
This commit is contained in:
1 parent
0e6d06c255
commit
c1bb327f1f
2 files changed
+285
-11
No files matched your search
@@ -85,6 +85,14 @@ def make_zone(
|
||||
)
|
||||
|
||||
|
||||
def ocr_outputs(*values: str) -> list[subprocess.CompletedProcess[str]]:
|
||||
"""Build one successful fake OCR process result per extracted text."""
|
||||
return [
|
||||
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
|
||||
for value in values
|
||||
]
|
||||
|
||||
|
||||
def make_visible_template(name: str, **kwargs: Any) -> OcrTemplate:
|
||||
"""Build a 100x100 template matching "Northwind"."""
|
||||
kwargs.setdefault("match", "Northwind")
|
||||
@@ -1023,6 +1031,137 @@ class TestSavedZoneExtraction:
|
||||
preview_and_save("invalid")
|
||||
assert document.title == "ACCOUNT ABC"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("failing_zone", "expected_failure"),
|
||||
[
|
||||
pytest.param(
|
||||
{"validation_regex": r"\d{2}"},
|
||||
"regex_mismatch",
|
||||
id="regex-mismatch",
|
||||
),
|
||||
pytest.param({"page": 2}, "missing_page", id="missing-page"),
|
||||
],
|
||||
)
|
||||
def test_failed_zone_blocks_combined_field(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
paperless_dirs: PaperlessDirs,
|
||||
page: Path,
|
||||
document: Document,
|
||||
template: OcrTemplate,
|
||||
failing_zone: dict[str, Any],
|
||||
expected_failure: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Three title zones combined as a date, where the first one fails
|
||||
WHEN:
|
||||
- The template is processed
|
||||
THEN:
|
||||
- Nothing is written, the failing zone reports its own reason and
|
||||
the accepted zones report incomplete
|
||||
"""
|
||||
zones = [
|
||||
make_zone(template, "Day", **failing_zone),
|
||||
make_zone(template, "Month"),
|
||||
make_zone(template, "Year"),
|
||||
]
|
||||
template.combine_formats = {"title": "{Day}.{Month}.{Year}"}
|
||||
# A zone on a missing page is never read.
|
||||
mock_zone_ocr_run.side_effect = ocr_outputs(
|
||||
*(
|
||||
["x", "05", "2024"]
|
||||
if "validation_regex" in failing_zone
|
||||
else ["05", "2024"]
|
||||
),
|
||||
)
|
||||
|
||||
results = _process_template(
|
||||
document,
|
||||
{0: page},
|
||||
template,
|
||||
zones,
|
||||
paperless_dirs.scratch_dir,
|
||||
)
|
||||
|
||||
assert [result.reason for result in results] == [
|
||||
expected_failure,
|
||||
"incomplete",
|
||||
"incomplete",
|
||||
]
|
||||
assert [result.value for result in results] == [None] * 3
|
||||
document.refresh_from_db()
|
||||
assert document.title == "Original"
|
||||
|
||||
def test_zone_with_no_value_is_an_optional_token(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
paperless_dirs: PaperlessDirs,
|
||||
page: Path,
|
||||
document: Document,
|
||||
template: OcrTemplate,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Two title zones combined with a separator, where the first reads
|
||||
nothing
|
||||
WHEN:
|
||||
- The template is processed
|
||||
THEN:
|
||||
- The remaining value is still written and reported
|
||||
"""
|
||||
zones = [make_zone(template, "First"), make_zone(template, "Second")]
|
||||
template.combine_formats = {"title": "{First} / {Second}"}
|
||||
mock_zone_ocr_run.side_effect = ocr_outputs("", "two")
|
||||
|
||||
results = _process_template(
|
||||
document,
|
||||
{0: page},
|
||||
template,
|
||||
zones,
|
||||
paperless_dirs.scratch_dir,
|
||||
)
|
||||
|
||||
assert [result.reason for result in results] == ["no_value", None]
|
||||
assert results[1].value == "two"
|
||||
document.refresh_from_db()
|
||||
assert document.title == "two"
|
||||
|
||||
def test_failed_single_zone_field_does_not_affect_other_fields(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
paperless_dirs: PaperlessDirs,
|
||||
page: Path,
|
||||
document: Document,
|
||||
template: OcrTemplate,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A title zone and a separate ASN zone, where the ASN fails its
|
||||
validation regex
|
||||
WHEN:
|
||||
- The template is processed
|
||||
THEN:
|
||||
- The title is written and the failing zone keeps regex_mismatch
|
||||
"""
|
||||
zones = [
|
||||
make_zone(template, "Title"),
|
||||
make_zone(template, "ASN", "asn", validation_regex=r"\d+"),
|
||||
]
|
||||
mock_zone_ocr_run.side_effect = ocr_outputs("New title", "abc")
|
||||
|
||||
results = _process_template(
|
||||
document,
|
||||
{0: page},
|
||||
template,
|
||||
zones,
|
||||
paperless_dirs.scratch_dir,
|
||||
)
|
||||
|
||||
assert [result.reason for result in results] == [None, "regex_mismatch"]
|
||||
document.refresh_from_db()
|
||||
assert document.title == "New title"
|
||||
|
||||
|
||||
class TestCombineFieldValue:
|
||||
"""Tests for combining zone values into one field value."""
|
||||
@@ -1042,6 +1181,96 @@ class TestCombineFieldValue:
|
||||
"one",
|
||||
id="format-preserves-existing-missing-token-cleanup",
|
||||
),
|
||||
pytest.param(
|
||||
"{Amount}",
|
||||
[("Amount", "-12.50")],
|
||||
"-12.50",
|
||||
id="negative-amount-sign-kept",
|
||||
),
|
||||
pytest.param(
|
||||
"{Url}",
|
||||
[("Url", "http://x.com/a//b")],
|
||||
"http://x.com/a//b",
|
||||
id="url-double-slash-kept-verbatim",
|
||||
),
|
||||
pytest.param(
|
||||
"{Value}",
|
||||
[("Value", "A--B")],
|
||||
"A--B",
|
||||
id="repeated-punctuation-in-value-kept",
|
||||
),
|
||||
pytest.param(
|
||||
"{Name}",
|
||||
[("Name", "Dr. Smith Jr.")],
|
||||
"Dr. Smith Jr.",
|
||||
id="trailing-period-in-value-kept",
|
||||
),
|
||||
pytest.param(
|
||||
"{Value}",
|
||||
[("Value", "wait...")],
|
||||
"wait...",
|
||||
id="trailing-ellipsis-in-value-kept",
|
||||
),
|
||||
pytest.param(
|
||||
"{A} / {B} / {C}",
|
||||
[("A", "a"), ("B", None), ("C", "c")],
|
||||
"a / c",
|
||||
id="middle-empty-token-drops-one-separator",
|
||||
),
|
||||
pytest.param(
|
||||
"{A} / {B} / {C}",
|
||||
[("A", None), ("B", "b"), ("C", "c")],
|
||||
"b / c",
|
||||
id="leading-empty-token-drops-leading-separator",
|
||||
),
|
||||
pytest.param(
|
||||
"{A} / {B} / {C}",
|
||||
[("A", "a"), ("B", "b"), ("C", None)],
|
||||
"a / b",
|
||||
id="trailing-empty-token-drops-trailing-separator",
|
||||
),
|
||||
pytest.param(
|
||||
"{A} / {B} / {C}",
|
||||
[("A", "a"), ("B", None), ("C", None)],
|
||||
"a",
|
||||
id="trailing-run-of-empty-tokens-drops-nearest-separator",
|
||||
),
|
||||
pytest.param(
|
||||
"Total: {A} / {B}",
|
||||
[("A", None), ("B", "b")],
|
||||
"Total: b",
|
||||
id="leading-literal-kept-before-empty-first-token",
|
||||
),
|
||||
pytest.param(
|
||||
"- {A} {B}",
|
||||
[("A", None), ("B", "b")],
|
||||
"b",
|
||||
id="leading-separator-dropped-before-empty-first-token",
|
||||
),
|
||||
pytest.param(
|
||||
"{Day}.{Month}.{Year}",
|
||||
[("Day", "12"), ("Month", "05"), ("Year", "2024")],
|
||||
"12.05.2024",
|
||||
id="all-tokens-present",
|
||||
),
|
||||
pytest.param(
|
||||
"INV-{Number}",
|
||||
[("Number", "123")],
|
||||
"INV-123",
|
||||
id="literal-prefix-kept-with-value",
|
||||
),
|
||||
pytest.param(
|
||||
"INV-{Number}",
|
||||
[("Number", None)],
|
||||
"",
|
||||
id="literal-prefix-dropped-when-every-token-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"{A} / {B}",
|
||||
[("A", None), ("B", " ")],
|
||||
"",
|
||||
id="every-token-empty",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_combine_field_value(
|
||||
|
||||
+56
-11
@@ -68,6 +68,9 @@ DATE_TOKEN_RE = re.compile(
|
||||
r"|(?<!\d)\d{1,2}\.?\s+[^\W\d_]{3,}\.?\s+\d{4}(?!\d)",
|
||||
)
|
||||
|
||||
# Literal text made only of whitespace and separator punctuation.
|
||||
SEPARATOR_ONLY_RE = re.compile(r"[\s\-/.,;:|]*")
|
||||
|
||||
# Earlier dates are far more likely OCR noise than a real document date.
|
||||
MIN_ZONE_DATE_YEAR = 1900
|
||||
|
||||
@@ -285,6 +288,12 @@ def _process_template(
|
||||
accepted = [result for _, result in field_zones if result.reason is None]
|
||||
if not accepted:
|
||||
continue
|
||||
# An empty zone is an optional token; any other failure blocks the group.
|
||||
if any(result.reason not in (None, "no_value") for _, result in field_zones):
|
||||
for result in accepted:
|
||||
result.value = None
|
||||
result.reason = "incomplete"
|
||||
continue
|
||||
value = _combine_field_value(
|
||||
combine_formats.get(key, ""),
|
||||
[(zone.name, result.value) for zone, result in field_zones],
|
||||
@@ -318,22 +327,58 @@ def _combine_field_value(
|
||||
"""Combine the OCR values of all zones targeting one field.
|
||||
|
||||
With a format string, `{Zone Name}` tokens are replaced by that zone's value
|
||||
and literal text is kept; separators left dangling by an empty token are
|
||||
cleaned up. Without a format, the zone values are joined in order by a space.
|
||||
and literal text is kept. Values are inserted verbatim (only surrounding
|
||||
whitespace is removed), so punctuation inside an OCR value is never altered.
|
||||
Unknown token names count as empty. Tidying only touches the format's own
|
||||
literal text: a separator-only literal following an empty token is dropped
|
||||
(for a trailing empty token, the nearest preceding kept separator is
|
||||
dropped instead), as is a leading separator-only literal in front of an
|
||||
empty first token. Other literal text, such as brackets around an empty
|
||||
token ("{A}, {B} ({C})" gives "a, b ()"), is kept as long as at least one
|
||||
token has a value; when every token is empty the result is an empty
|
||||
string. Without a format, the non-empty zone values are joined in order by
|
||||
a space.
|
||||
"""
|
||||
values = {name: value or "" for name, value in field_zones}
|
||||
|
||||
if not fmt:
|
||||
return " ".join(value for _, value in field_zones if value).strip()
|
||||
|
||||
def _replace(match: re.Match) -> str:
|
||||
return values.get(match.group(1).strip(), "")
|
||||
values = {name.strip(): (value or "").strip() for name, value in field_zones}
|
||||
|
||||
combined = re.sub(r"\{([^{}]+)\}", _replace, fmt)
|
||||
# Tidy up separators an empty token may have left behind.
|
||||
combined = re.sub(r"\s{2,}", " ", combined)
|
||||
combined = re.sub(r"([^\w\s])\s*\1+", r"\1", combined)
|
||||
return combined.strip().strip("-/.,;:| \t")
|
||||
# Pairs of (is_token, text); tokens carry their value.
|
||||
parts: list[tuple[bool, str]] = []
|
||||
for index, piece in enumerate(re.split(r"(\{[^{}]+\})", fmt)):
|
||||
if not piece:
|
||||
continue
|
||||
if index % 2:
|
||||
parts.append((True, values.get(piece[1:-1].strip(), "")))
|
||||
else:
|
||||
parts.append((False, piece))
|
||||
|
||||
if not any(is_token and text for is_token, text in parts):
|
||||
return ""
|
||||
|
||||
def _is_separator(part: tuple[bool, str]) -> bool:
|
||||
return not part[0] and SEPARATOR_ONLY_RE.fullmatch(part[1]) is not None
|
||||
|
||||
kept: list[tuple[bool, str]] = []
|
||||
drop_next_separator = False
|
||||
for position, part in enumerate(parts):
|
||||
if drop_next_separator:
|
||||
drop_next_separator = False
|
||||
continue
|
||||
is_token, text = part
|
||||
if not is_token or text:
|
||||
kept.append(part)
|
||||
continue
|
||||
following = parts[position + 1] if position + 1 < len(parts) else None
|
||||
if following is not None:
|
||||
drop_next_separator = _is_separator(following)
|
||||
elif kept and _is_separator(kept[-1]):
|
||||
kept.pop()
|
||||
# A leading separator left in front of an empty first token.
|
||||
if len(kept) == 1 and _is_separator(kept[0]):
|
||||
kept.pop()
|
||||
return "".join(text for _, text in kept).strip()
|
||||
|
||||
|
||||
def _render_pages(
|
||||
|
||||
Reference in new issue
Block a user