mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-20 04:26:53 +00:00
Fix: don't pass raw text on when a zone transform fails
A date or numeric transform that found nothing returned the OCR text, so the field got junk. Return nothing and report transform_failed, which also blocks a combined field.
This commit is contained in:
1 parent
fcc465ec2e
commit
0e7cbda3d9
2 files changed
+105
-21
No files matched your search
@@ -406,6 +406,36 @@ class TestZoneExtraction:
|
||||
assert result.raw_text == "25/12/2025"
|
||||
assert result.value == "2025-12-25"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("transform", "text"),
|
||||
[
|
||||
pytest.param("date", "not a date", id="date"),
|
||||
pytest.param("numeric", "abc", id="numeric"),
|
||||
],
|
||||
)
|
||||
def test_failed_transform_yields_no_value(
|
||||
self,
|
||||
page: Path,
|
||||
zone: OcrTemplateZone,
|
||||
tmp_path: Path,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
transform: str,
|
||||
text: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A date or numeric zone reading text with no date or number
|
||||
WHEN:
|
||||
- The zone is extracted
|
||||
THEN:
|
||||
- The raw text is retained for the preview and there is no value
|
||||
"""
|
||||
zone.transform = transform
|
||||
mock_zone_ocr_run.return_value.stdout = text
|
||||
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
||||
assert result.raw_text == text
|
||||
assert result.value is None
|
||||
|
||||
def test_barcode_extraction_bypasses_tesseract(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
@@ -518,6 +548,46 @@ class TestSavedZoneExtraction:
|
||||
assert document.title == "two / one"
|
||||
save.assert_called_once_with(update_fields=["title"])
|
||||
|
||||
def test_failed_transform_blocks_the_combined_field(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
paperless_dirs: PaperlessDirs,
|
||||
page: Path,
|
||||
document: Document,
|
||||
template: OcrTemplate,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Two title zones combined as "{Date} {Ref}", where the date zone
|
||||
reads text that is not a date
|
||||
WHEN:
|
||||
- The template is processed
|
||||
THEN:
|
||||
- The date zone is reported as transform_failed, the other zone as
|
||||
incomplete, and the title is not written
|
||||
"""
|
||||
zones = [
|
||||
make_zone(template, "Date", transform="date"),
|
||||
make_zone(template, "Ref"),
|
||||
]
|
||||
template.combine_formats = {"title": "{Date} {Ref}"}
|
||||
mock_zone_ocr_run.side_effect = ocr_outputs("not a date", "R-1")
|
||||
|
||||
results = _process_template(
|
||||
document,
|
||||
{0: page},
|
||||
template,
|
||||
zones,
|
||||
paperless_dirs.scratch_dir,
|
||||
)
|
||||
|
||||
assert [(result.zone, result.reason) for result in results] == [
|
||||
("Date", "transform_failed"),
|
||||
("Ref", "incomplete"),
|
||||
]
|
||||
document.refresh_from_db()
|
||||
assert document.title == "Original"
|
||||
|
||||
def test_converts_custom_and_builtin_fields_before_saving(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
@@ -1676,8 +1746,15 @@ class TestApplyTransform:
|
||||
"abc",
|
||||
"numeric",
|
||||
"",
|
||||
"abc",
|
||||
id="numeric-empty-result-falls-back",
|
||||
"",
|
||||
id="numeric-without-digits-is-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"a-b.c",
|
||||
"numeric",
|
||||
"",
|
||||
"",
|
||||
id="numeric-punctuation-only-is-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"13.04.2026",
|
||||
@@ -1732,22 +1809,22 @@ class TestApplyTransform:
|
||||
"32.13.2026",
|
||||
"date",
|
||||
"%d.%m.%Y",
|
||||
"32.13.2026",
|
||||
id="date-invalid-with-format-returns-text",
|
||||
"",
|
||||
id="date-invalid-with-format-is-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"2026-13-32",
|
||||
"date",
|
||||
"",
|
||||
"2026-13-32",
|
||||
id="date-invalid-auto-detect-returns-text",
|
||||
"",
|
||||
id="date-invalid-auto-detect-is-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"not a date",
|
||||
"date",
|
||||
"",
|
||||
"not a date",
|
||||
id="date-not-a-date-returns-text",
|
||||
"",
|
||||
id="date-not-a-date-is-empty",
|
||||
),
|
||||
pytest.param("", "strip", "", "", id="empty-string"),
|
||||
pytest.param(" ", "strip", "", "", id="whitespace-only"),
|
||||
@@ -1786,19 +1863,17 @@ class TestApplyTransform:
|
||||
rules = _date_rules("eng", date_format)
|
||||
assert _apply_transform(text, transform, rules) == expected
|
||||
|
||||
def test_unparseable_date_preserves_text(self, mocker: MockerFixture) -> None:
|
||||
def test_dateparser_error_gives_empty_text(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- dateparser raises OverflowError for the input
|
||||
WHEN:
|
||||
- A date transform is applied
|
||||
THEN:
|
||||
- The original text is returned unchanged
|
||||
- An empty string is returned
|
||||
"""
|
||||
mocker.patch("documents.zone_ocr.dateparser.parse", side_effect=OverflowError)
|
||||
assert (
|
||||
_apply_transform("unparsable", "date", _date_rules("eng")) == "unparsable"
|
||||
)
|
||||
assert _apply_transform("unparsable", "date", _date_rules("eng")) == ""
|
||||
|
||||
|
||||
class TestZoneLanguage:
|
||||
|
||||
@@ -276,7 +276,13 @@ def _process_template(
|
||||
tmp_path,
|
||||
)
|
||||
extracted = extraction.value
|
||||
reason = None if extracted else "no_value"
|
||||
if extracted:
|
||||
reason = None
|
||||
elif extraction.raw_text:
|
||||
# Text was read but the transform found nothing usable in it.
|
||||
reason = "transform_failed"
|
||||
else:
|
||||
reason = "no_value"
|
||||
if (
|
||||
extracted is not None
|
||||
and zone.validation_regex
|
||||
@@ -558,12 +564,11 @@ def _extract_zone(
|
||||
return ZoneExtraction(
|
||||
raw_text=raw_text,
|
||||
value=_apply_transform(
|
||||
raw_text,
|
||||
raw_text or "",
|
||||
zone.transform,
|
||||
_date_rules(zone.ocr_language, zone.date_format),
|
||||
)
|
||||
if raw_text
|
||||
else None,
|
||||
or None,
|
||||
)
|
||||
|
||||
|
||||
@@ -687,7 +692,11 @@ def _apply_transform(
|
||||
transform: str,
|
||||
dates: DateRules,
|
||||
) -> str:
|
||||
"""Apply post-processing transform to extracted text."""
|
||||
"""Apply post-processing transform to extracted text.
|
||||
|
||||
A date or numeric transform that finds no date or digit returns an empty
|
||||
string instead of passing the raw text on.
|
||||
"""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return text
|
||||
@@ -696,14 +705,14 @@ def _apply_transform(
|
||||
return text
|
||||
elif transform == "date":
|
||||
parsed = _parse_date(text, dates)
|
||||
return parsed.isoformat() if parsed else text
|
||||
return parsed.isoformat() if parsed else ""
|
||||
elif transform == "uppercase":
|
||||
return text.upper()
|
||||
elif transform == "lowercase":
|
||||
return text.lower()
|
||||
elif transform == "numeric":
|
||||
result = re.sub(r"[^\d.,\-]", "", text)
|
||||
return result if result else text
|
||||
numeric = re.sub(r"[^\d.,\-]", "", text)
|
||||
return numeric if re.search(r"\d", numeric) else ""
|
||||
elif transform == "strip_punctuation":
|
||||
return text.strip(string.punctuation + " \t\r\n")
|
||||
return text
|
||||
|
||||
Reference in new issue
Block a user