mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-20 12:36:53 +00:00
Chore: dispatch zone OCR on the target, transform and data type enums
The string if/elif chains ended in catch-alls. Match on the enums with assert_never instead. A stored value outside the choices now raises, failing only that template.
This commit is contained in:
1 parent
f35a2a4343
commit
ff6b5f3820
2 files changed
+208
-120
No files matched your search
@@ -1034,6 +1034,37 @@ class TestSavedZoneExtraction:
|
||||
assert document.title == "Applied"
|
||||
assert document.archive_serial_number is None
|
||||
|
||||
def test_unknown_stored_transform_fails_only_its_template(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
page: Path,
|
||||
document: Document,
|
||||
template: OcrTemplate,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A template whose zone has a transform value outside the choices
|
||||
- A second template that would succeed
|
||||
WHEN:
|
||||
- Zone extraction runs over both templates
|
||||
THEN:
|
||||
- The first template is reported as failed and the second template
|
||||
is still applied
|
||||
"""
|
||||
make_zone(template, "Bogus", transform="bogus")
|
||||
second = make_template("Second")
|
||||
make_zone(second, "Second")
|
||||
mock_zone_ocr_run.side_effect = ocr_outputs("Ignored", "Applied")
|
||||
|
||||
results = run_zone_extraction(document, page, templates=[template, second])
|
||||
|
||||
assert [result.reason for result in results] == [
|
||||
ZoneOcrReason.TEMPLATE_FAILED,
|
||||
None,
|
||||
]
|
||||
document.refresh_from_db()
|
||||
assert document.title == "Applied"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deleted",
|
||||
[
|
||||
@@ -1734,132 +1765,156 @@ class TestApplyTransform:
|
||||
@pytest.mark.parametrize(
|
||||
("text", "transform", "date_format", "expected"),
|
||||
[
|
||||
pytest.param(" hello ", "strip", "", "hello", id="strip"),
|
||||
pytest.param(" hello ", "none", "", "hello", id="none-transform"),
|
||||
pytest.param(
|
||||
" hello ",
|
||||
OcrTemplateZone.TransformType.STRIP,
|
||||
"",
|
||||
"hello",
|
||||
id="strip",
|
||||
),
|
||||
pytest.param(
|
||||
" hello ",
|
||||
OcrTemplateZone.TransformType.NONE,
|
||||
"",
|
||||
"hello",
|
||||
id="none-transform",
|
||||
),
|
||||
pytest.param(
|
||||
"hello world",
|
||||
"uppercase",
|
||||
OcrTemplateZone.TransformType.UPPERCASE,
|
||||
"",
|
||||
"HELLO WORLD",
|
||||
id="uppercase",
|
||||
),
|
||||
pytest.param(
|
||||
"HELLO WORLD",
|
||||
"lowercase",
|
||||
OcrTemplateZone.TransformType.LOWERCASE,
|
||||
"",
|
||||
"hello world",
|
||||
id="lowercase",
|
||||
),
|
||||
pytest.param(
|
||||
"Invoice 2026-001",
|
||||
"numeric",
|
||||
OcrTemplateZone.TransformType.NUMERIC,
|
||||
"",
|
||||
"2026-001",
|
||||
id="numeric-basic",
|
||||
),
|
||||
pytest.param(
|
||||
"\u20ac1,234.56",
|
||||
"numeric",
|
||||
OcrTemplateZone.TransformType.NUMERIC,
|
||||
"",
|
||||
"1,234.56",
|
||||
id="numeric-with-currency",
|
||||
),
|
||||
pytest.param(
|
||||
"abc",
|
||||
"numeric",
|
||||
OcrTemplateZone.TransformType.NUMERIC,
|
||||
"",
|
||||
"",
|
||||
id="numeric-without-digits-is-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"a-b.c",
|
||||
"numeric",
|
||||
OcrTemplateZone.TransformType.NUMERIC,
|
||||
"",
|
||||
"",
|
||||
id="numeric-punctuation-only-is-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"13.04.2026",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"%d.%m.%Y",
|
||||
"2026-04-13",
|
||||
id="date-with-format-dotted",
|
||||
),
|
||||
pytest.param(
|
||||
"01/12/2025",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"%d/%m/%Y",
|
||||
"2025-12-01",
|
||||
id="date-with-format-slashed",
|
||||
),
|
||||
pytest.param(
|
||||
"13.04.26",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"%d.%m.%y",
|
||||
"2026-04-13",
|
||||
id="date-two-digit-year",
|
||||
),
|
||||
pytest.param(
|
||||
"2026-04-13",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"",
|
||||
"2026-04-13",
|
||||
id="date-auto-detect-iso",
|
||||
),
|
||||
pytest.param(
|
||||
"2026/04/13",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"",
|
||||
"2026-04-13",
|
||||
id="date-auto-detect-slashed",
|
||||
),
|
||||
pytest.param(
|
||||
"13 April 2026",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"",
|
||||
"2026-04-13",
|
||||
id="date-auto-detect-month-name",
|
||||
),
|
||||
pytest.param(
|
||||
"2026-04-13",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"%d.%m.%Y",
|
||||
"2026-04-13",
|
||||
id="date-format-mismatch-falls-back-to-auto-detect",
|
||||
),
|
||||
pytest.param(
|
||||
"32.13.2026",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"%d.%m.%Y",
|
||||
"",
|
||||
id="date-invalid-with-format-is-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"2026-13-32",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"",
|
||||
"",
|
||||
id="date-invalid-auto-detect-is-empty",
|
||||
),
|
||||
pytest.param(
|
||||
"not a date",
|
||||
"date",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
"",
|
||||
"",
|
||||
id="date-not-a-date-is-empty",
|
||||
),
|
||||
pytest.param("", "strip", "", "", id="empty-string"),
|
||||
pytest.param(" ", "strip", "", "", id="whitespace-only"),
|
||||
pytest.param(
|
||||
"",
|
||||
OcrTemplateZone.TransformType.STRIP,
|
||||
"",
|
||||
"",
|
||||
id="empty-string",
|
||||
),
|
||||
pytest.param(
|
||||
" ",
|
||||
OcrTemplateZone.TransformType.STRIP,
|
||||
"",
|
||||
"",
|
||||
id="whitespace-only",
|
||||
),
|
||||
pytest.param(
|
||||
" hello ",
|
||||
"unknown",
|
||||
OcrTemplateZone.TransformType.QR_CODE,
|
||||
"",
|
||||
"hello",
|
||||
id="unknown-transform-strips",
|
||||
id="qr-code-strips",
|
||||
),
|
||||
pytest.param(
|
||||
" -- Account #123. -- ",
|
||||
"strip_punctuation",
|
||||
OcrTemplateZone.TransformType.STRIP_PUNCTUATION,
|
||||
"",
|
||||
"Account #123",
|
||||
id="strip-punctuation",
|
||||
@@ -1869,7 +1924,7 @@ class TestApplyTransform:
|
||||
def test_apply_transform(
|
||||
self,
|
||||
text: str,
|
||||
transform: str,
|
||||
transform: OcrTemplateZone.TransformType,
|
||||
date_format: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
@@ -1895,7 +1950,14 @@ class TestApplyTransform:
|
||||
- An empty string is returned
|
||||
"""
|
||||
mocker.patch("documents.zone_ocr.dateparser.parse", side_effect=OverflowError)
|
||||
assert _apply_transform("unparsable", "date", _date_rules("eng")) == ""
|
||||
assert (
|
||||
_apply_transform(
|
||||
"unparsable",
|
||||
OcrTemplateZone.TransformType.DATE,
|
||||
_date_rules("eng"),
|
||||
)
|
||||
== ""
|
||||
)
|
||||
|
||||
|
||||
class TestZoneLanguage:
|
||||
@@ -2326,16 +2388,17 @@ class TestParseDate:
|
||||
class TestPrepareFieldUpdate:
|
||||
"""Tests for converting extracted values into field updates."""
|
||||
|
||||
def test_unknown_target_cannot_produce_a_field_update(self) -> None:
|
||||
def test_unknown_target_raises(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A zone whose target is not a known target
|
||||
WHEN:
|
||||
- A field update is prepared for it
|
||||
THEN:
|
||||
- No field update is produced
|
||||
- A ValueError is raised
|
||||
"""
|
||||
assert _prepare_field_update(OcrTemplateZone(target="unknown"), "value") is None
|
||||
with pytest.raises(ValueError, match="unknown"):
|
||||
_prepare_field_update(OcrTemplateZone(target="unknown"), "value")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ocr_language", "expected"),
|
||||
@@ -2406,7 +2469,7 @@ class TestPrepareFieldUpdate:
|
||||
)
|
||||
def test_text_values_are_kept_whole_or_rejected(
|
||||
self,
|
||||
data_type: str,
|
||||
data_type: CustomField.FieldDataType,
|
||||
text: str,
|
||||
expected: str | None,
|
||||
) -> None:
|
||||
@@ -2961,7 +3024,7 @@ class TestConvertValue:
|
||||
def test_converts_to_exact_value(
|
||||
self,
|
||||
value: str,
|
||||
data_type: str,
|
||||
data_type: CustomField.FieldDataType,
|
||||
expected: object,
|
||||
locale: Locale,
|
||||
) -> None:
|
||||
@@ -3161,7 +3224,12 @@ class TestConvertValue:
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_returns_none(self, value: str, data_type: str, locale: Locale) -> None:
|
||||
def test_returns_none(
|
||||
self,
|
||||
value: str,
|
||||
data_type: CustomField.FieldDataType,
|
||||
locale: Locale,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A value that cannot be converted to the given data type in a locale
|
||||
@@ -3193,7 +3261,7 @@ class TestConvertValue:
|
||||
)
|
||||
def test_numeric_types_reject_malformed_numbers(
|
||||
self,
|
||||
data_type: str,
|
||||
data_type: CustomField.FieldDataType,
|
||||
value: str,
|
||||
) -> None:
|
||||
"""
|
||||
|
||||
+108
-88
@@ -18,6 +18,7 @@ from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TypeAlias
|
||||
from typing import assert_never
|
||||
|
||||
import dateparser
|
||||
import magic
|
||||
@@ -143,10 +144,12 @@ def _zone_result(
|
||||
*,
|
||||
reason: ZoneOcrReason | None = None,
|
||||
) -> ZoneOcrResult:
|
||||
if zone.target == OcrTemplateZone.TargetType.CUSTOM_FIELD:
|
||||
label = zone.custom_field.name if zone.custom_field_id else "(no field)"
|
||||
target = OcrTemplateZone.TargetType(zone.target)
|
||||
if target == OcrTemplateZone.TargetType.CUSTOM_FIELD:
|
||||
custom_field = zone.custom_field
|
||||
label = custom_field.name if custom_field is not None else "(no field)"
|
||||
else:
|
||||
label = BUILTIN_TARGET_LABELS[zone.target]
|
||||
label = BUILTIN_TARGET_LABELS[target]
|
||||
return ZoneOcrResult(
|
||||
template=template.name,
|
||||
zone=zone.name,
|
||||
@@ -606,7 +609,7 @@ def _extract_zone(
|
||||
raw_text=raw_text,
|
||||
value=_apply_transform(
|
||||
raw_text or "",
|
||||
zone.transform,
|
||||
OcrTemplateZone.TransformType(zone.transform),
|
||||
_date_rules(zone.ocr_language, zone.date_format),
|
||||
)
|
||||
or None,
|
||||
@@ -729,7 +732,7 @@ def _parse_date(text: str, rules: DateRules) -> date | None:
|
||||
|
||||
def _apply_transform(
|
||||
text: str,
|
||||
transform: str,
|
||||
transform: OcrTemplateZone.TransformType,
|
||||
dates: DateRules,
|
||||
) -> str:
|
||||
"""Apply post-processing transform to extracted text.
|
||||
@@ -741,21 +744,27 @@ def _apply_transform(
|
||||
if not text:
|
||||
return text
|
||||
|
||||
if transform in ("strip", "none"):
|
||||
return text
|
||||
elif transform == "date":
|
||||
parsed = _parse_date(text, dates)
|
||||
return parsed.isoformat() if parsed else ""
|
||||
elif transform == "uppercase":
|
||||
return text.upper()
|
||||
elif transform == "lowercase":
|
||||
return text.lower()
|
||||
elif transform == "numeric":
|
||||
numeric = re.sub(r"[^\d.,\-]", "", text)
|
||||
return numeric if re.search(r"\d", numeric) else ""
|
||||
elif transform == "strip_punctuation":
|
||||
return text.strip(string.punctuation + " \t\r\n")
|
||||
return text
|
||||
match transform:
|
||||
case (
|
||||
OcrTemplateZone.TransformType.NONE
|
||||
| OcrTemplateZone.TransformType.STRIP
|
||||
| OcrTemplateZone.TransformType.QR_CODE
|
||||
):
|
||||
return text
|
||||
case OcrTemplateZone.TransformType.DATE:
|
||||
parsed = _parse_date(text, dates)
|
||||
return parsed.isoformat() if parsed else ""
|
||||
case OcrTemplateZone.TransformType.UPPERCASE:
|
||||
return text.upper()
|
||||
case OcrTemplateZone.TransformType.LOWERCASE:
|
||||
return text.lower()
|
||||
case OcrTemplateZone.TransformType.NUMERIC:
|
||||
numeric = re.sub(r"[^\d.,\-]", "", text)
|
||||
return numeric if re.search(r"\d", numeric) else ""
|
||||
case OcrTemplateZone.TransformType.STRIP_PUNCTUATION:
|
||||
return text.strip(string.punctuation + " \t\r\n")
|
||||
case unreachable:
|
||||
assert_never(unreachable)
|
||||
|
||||
|
||||
def _prepare_field_update(
|
||||
@@ -765,37 +774,41 @@ def _prepare_field_update(
|
||||
"""Convert extracted text and enforce the destination field's constraints."""
|
||||
custom_field = None
|
||||
typed_value: ZoneValue | None
|
||||
if zone.target == OcrTemplateZone.TargetType.CUSTOM_FIELD:
|
||||
if not zone.custom_field_id:
|
||||
return None
|
||||
custom_field = zone.custom_field
|
||||
field_name = CustomFieldInstance.get_value_field_name(custom_field.data_type)
|
||||
typed_value = _convert_value(
|
||||
value,
|
||||
custom_field.data_type,
|
||||
_number_locale(zone.ocr_language),
|
||||
_date_rules(zone.ocr_language, zone.date_format),
|
||||
)
|
||||
model = CustomFieldInstance
|
||||
elif zone.target == OcrTemplateZone.TargetType.TITLE:
|
||||
field_name, typed_value = "title", value[:128]
|
||||
model = Document
|
||||
elif zone.target == OcrTemplateZone.TargetType.ASN:
|
||||
digits = re.sub(r"[^\d]", "", value)
|
||||
field_name = "archive_serial_number"
|
||||
typed_value = int(digits) if digits else None
|
||||
model = Document
|
||||
elif zone.target == OcrTemplateZone.TargetType.CREATED:
|
||||
field_name, typed_value = (
|
||||
"created",
|
||||
_parse_date(
|
||||
model: type[Document | CustomFieldInstance]
|
||||
target = OcrTemplateZone.TargetType(zone.target)
|
||||
match target:
|
||||
case OcrTemplateZone.TargetType.CUSTOM_FIELD:
|
||||
custom_field = zone.custom_field
|
||||
if custom_field is None:
|
||||
return None
|
||||
data_type = CustomField.FieldDataType(custom_field.data_type)
|
||||
field_name = CustomFieldInstance.get_value_field_name(data_type)
|
||||
typed_value = _convert_value(
|
||||
value,
|
||||
data_type,
|
||||
_number_locale(zone.ocr_language),
|
||||
_date_rules(zone.ocr_language, zone.date_format),
|
||||
),
|
||||
)
|
||||
model = Document
|
||||
else:
|
||||
return None
|
||||
)
|
||||
model = CustomFieldInstance
|
||||
case OcrTemplateZone.TargetType.TITLE:
|
||||
field_name, typed_value = "title", value[:128]
|
||||
model = Document
|
||||
case OcrTemplateZone.TargetType.ASN:
|
||||
digits = re.sub(r"[^\d]", "", value)
|
||||
field_name = "archive_serial_number"
|
||||
typed_value = int(digits) if digits else None
|
||||
model = Document
|
||||
case OcrTemplateZone.TargetType.CREATED:
|
||||
field_name, typed_value = (
|
||||
"created",
|
||||
_parse_date(
|
||||
value,
|
||||
_date_rules(zone.ocr_language, zone.date_format),
|
||||
),
|
||||
)
|
||||
model = Document
|
||||
case unreachable:
|
||||
assert_never(unreachable)
|
||||
if typed_value is None:
|
||||
return None
|
||||
field = model._meta.get_field(field_name)
|
||||
@@ -1027,7 +1040,7 @@ def _parse_monetary(value: str, locale: Locale) -> str | None:
|
||||
|
||||
def _convert_value(
|
||||
value: str,
|
||||
data_type: str,
|
||||
data_type: CustomField.FieldDataType,
|
||||
locale: Locale,
|
||||
dates: DateRules,
|
||||
) -> ZoneValue | None:
|
||||
@@ -1040,49 +1053,56 @@ def _convert_value(
|
||||
return None
|
||||
|
||||
try:
|
||||
if data_type in (
|
||||
CustomField.FieldDataType.STRING,
|
||||
CustomField.FieldDataType.URL,
|
||||
CustomField.FieldDataType.LONG_TEXT,
|
||||
):
|
||||
# Returned whole; _prepare_field_update validates length and URL
|
||||
# format, so a value is rejected rather than truncated.
|
||||
return value
|
||||
match data_type:
|
||||
case (
|
||||
CustomField.FieldDataType.STRING
|
||||
| CustomField.FieldDataType.URL
|
||||
| CustomField.FieldDataType.LONG_TEXT
|
||||
):
|
||||
# Returned whole; _prepare_field_update validates length and URL
|
||||
# format, so a value is rejected rather than truncated.
|
||||
return value
|
||||
|
||||
elif data_type == CustomField.FieldDataType.INT:
|
||||
integer = _parse_number(value, locale)
|
||||
if integer is None or integer != integer.to_integral_value():
|
||||
case CustomField.FieldDataType.INT:
|
||||
integer = _parse_number(value, locale)
|
||||
if integer is None or integer != integer.to_integral_value():
|
||||
return None
|
||||
return int(integer)
|
||||
|
||||
case CustomField.FieldDataType.FLOAT:
|
||||
decimal = _parse_number(value, locale)
|
||||
if decimal is None:
|
||||
return None
|
||||
number = float(decimal)
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
case CustomField.FieldDataType.DATE:
|
||||
parsed = _parse_date(value, dates)
|
||||
return parsed.isoformat() if parsed else None
|
||||
|
||||
case CustomField.FieldDataType.MONETARY:
|
||||
return _parse_monetary(value, locale)
|
||||
|
||||
case CustomField.FieldDataType.BOOL:
|
||||
lower = value.lower().strip()
|
||||
if lower in ("true", "yes", "1", "ja", "oui", "si", "x"):
|
||||
return True
|
||||
elif lower in ("false", "no", "0", "nein", "non"):
|
||||
return False
|
||||
return None
|
||||
return int(integer)
|
||||
|
||||
elif data_type == CustomField.FieldDataType.FLOAT:
|
||||
decimal = _parse_number(value, locale)
|
||||
if decimal is None:
|
||||
case (
|
||||
CustomField.FieldDataType.SELECT
|
||||
| CustomField.FieldDataType.DOCUMENTLINK
|
||||
):
|
||||
logger.debug(
|
||||
"Zone OCR: unsupported custom field type %s for OCR extraction",
|
||||
data_type,
|
||||
)
|
||||
return None
|
||||
number = float(decimal)
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
elif data_type == CustomField.FieldDataType.DATE:
|
||||
parsed = _parse_date(value, dates)
|
||||
return parsed.isoformat() if parsed else None
|
||||
|
||||
elif data_type == CustomField.FieldDataType.MONETARY:
|
||||
return _parse_monetary(value, locale)
|
||||
|
||||
elif data_type == CustomField.FieldDataType.BOOL:
|
||||
lower = value.lower().strip()
|
||||
if lower in ("true", "yes", "1", "ja", "oui", "si", "x"):
|
||||
return True
|
||||
elif lower in ("false", "no", "0", "nein", "non"):
|
||||
return False
|
||||
return None
|
||||
|
||||
else:
|
||||
logger.debug(
|
||||
"Zone OCR: unsupported custom field type %s for OCR extraction",
|
||||
data_type,
|
||||
)
|
||||
return None
|
||||
case unreachable:
|
||||
assert_never(unreachable)
|
||||
|
||||
except (ValueError, InvalidOperation):
|
||||
logger.info("Zone OCR: could not convert value to %s", data_type)
|
||||
|
||||
Reference in new issue
Block a user