diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index c32ed18b57..a86ca49362 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -1034,6 +1034,37 @@ class TestSavedZoneExtraction: assert document.title == "Applied" assert document.archive_serial_number is None + def test_unknown_stored_transform_fails_only_its_template( + self, + mock_zone_ocr_run: MagicMock, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - A template whose zone has a transform value outside the choices + - A second template that would succeed + WHEN: + - Zone extraction runs over both templates + THEN: + - The first template is reported as failed and the second template + is still applied + """ + make_zone(template, "Bogus", transform="bogus") + second = make_template("Second") + make_zone(second, "Second") + mock_zone_ocr_run.side_effect = ocr_outputs("Ignored", "Applied") + + results = run_zone_extraction(document, page, templates=[template, second]) + + assert [result.reason for result in results] == [ + ZoneOcrReason.TEMPLATE_FAILED, + None, + ] + document.refresh_from_db() + assert document.title == "Applied" + @pytest.mark.parametrize( "deleted", [ @@ -1734,132 +1765,156 @@ class TestApplyTransform: @pytest.mark.parametrize( ("text", "transform", "date_format", "expected"), [ - pytest.param(" hello ", "strip", "", "hello", id="strip"), - pytest.param(" hello ", "none", "", "hello", id="none-transform"), + pytest.param( + " hello ", + OcrTemplateZone.TransformType.STRIP, + "", + "hello", + id="strip", + ), + pytest.param( + " hello ", + OcrTemplateZone.TransformType.NONE, + "", + "hello", + id="none-transform", + ), pytest.param( "hello world", - "uppercase", + OcrTemplateZone.TransformType.UPPERCASE, "", "HELLO WORLD", id="uppercase", ), pytest.param( "HELLO WORLD", - "lowercase", + OcrTemplateZone.TransformType.LOWERCASE, "", "hello world", id="lowercase", ), pytest.param( "Invoice 2026-001", - "numeric", + OcrTemplateZone.TransformType.NUMERIC, "", "2026-001", id="numeric-basic", ), pytest.param( "\u20ac1,234.56", - "numeric", + OcrTemplateZone.TransformType.NUMERIC, "", "1,234.56", id="numeric-with-currency", ), pytest.param( "abc", - "numeric", + OcrTemplateZone.TransformType.NUMERIC, "", "", id="numeric-without-digits-is-empty", ), pytest.param( "a-b.c", - "numeric", + OcrTemplateZone.TransformType.NUMERIC, "", "", id="numeric-punctuation-only-is-empty", ), pytest.param( "13.04.2026", - "date", + OcrTemplateZone.TransformType.DATE, "%d.%m.%Y", "2026-04-13", id="date-with-format-dotted", ), pytest.param( "01/12/2025", - "date", + OcrTemplateZone.TransformType.DATE, "%d/%m/%Y", "2025-12-01", id="date-with-format-slashed", ), pytest.param( "13.04.26", - "date", + OcrTemplateZone.TransformType.DATE, "%d.%m.%y", "2026-04-13", id="date-two-digit-year", ), pytest.param( "2026-04-13", - "date", + OcrTemplateZone.TransformType.DATE, "", "2026-04-13", id="date-auto-detect-iso", ), pytest.param( "2026/04/13", - "date", + OcrTemplateZone.TransformType.DATE, "", "2026-04-13", id="date-auto-detect-slashed", ), pytest.param( "13 April 2026", - "date", + OcrTemplateZone.TransformType.DATE, "", "2026-04-13", id="date-auto-detect-month-name", ), pytest.param( "2026-04-13", - "date", + OcrTemplateZone.TransformType.DATE, "%d.%m.%Y", "2026-04-13", id="date-format-mismatch-falls-back-to-auto-detect", ), pytest.param( "32.13.2026", - "date", + OcrTemplateZone.TransformType.DATE, "%d.%m.%Y", "", id="date-invalid-with-format-is-empty", ), pytest.param( "2026-13-32", - "date", + OcrTemplateZone.TransformType.DATE, "", "", id="date-invalid-auto-detect-is-empty", ), pytest.param( "not a date", - "date", + OcrTemplateZone.TransformType.DATE, "", "", id="date-not-a-date-is-empty", ), - pytest.param("", "strip", "", "", id="empty-string"), - pytest.param(" ", "strip", "", "", id="whitespace-only"), + pytest.param( + "", + OcrTemplateZone.TransformType.STRIP, + "", + "", + id="empty-string", + ), + pytest.param( + " ", + OcrTemplateZone.TransformType.STRIP, + "", + "", + id="whitespace-only", + ), pytest.param( " hello ", - "unknown", + OcrTemplateZone.TransformType.QR_CODE, "", "hello", - id="unknown-transform-strips", + id="qr-code-strips", ), pytest.param( " -- Account #123. -- ", - "strip_punctuation", + OcrTemplateZone.TransformType.STRIP_PUNCTUATION, "", "Account #123", id="strip-punctuation", @@ -1869,7 +1924,7 @@ class TestApplyTransform: def test_apply_transform( self, text: str, - transform: str, + transform: OcrTemplateZone.TransformType, date_format: str, expected: str, ) -> None: @@ -1895,7 +1950,14 @@ class TestApplyTransform: - An empty string is returned """ mocker.patch("documents.zone_ocr.dateparser.parse", side_effect=OverflowError) - assert _apply_transform("unparsable", "date", _date_rules("eng")) == "" + assert ( + _apply_transform( + "unparsable", + OcrTemplateZone.TransformType.DATE, + _date_rules("eng"), + ) + == "" + ) class TestZoneLanguage: @@ -2326,16 +2388,17 @@ class TestParseDate: class TestPrepareFieldUpdate: """Tests for converting extracted values into field updates.""" - def test_unknown_target_cannot_produce_a_field_update(self) -> None: + def test_unknown_target_raises(self) -> None: """ GIVEN: - A zone whose target is not a known target WHEN: - A field update is prepared for it THEN: - - No field update is produced + - A ValueError is raised """ - assert _prepare_field_update(OcrTemplateZone(target="unknown"), "value") is None + with pytest.raises(ValueError, match="unknown"): + _prepare_field_update(OcrTemplateZone(target="unknown"), "value") @pytest.mark.parametrize( ("ocr_language", "expected"), @@ -2406,7 +2469,7 @@ class TestPrepareFieldUpdate: ) def test_text_values_are_kept_whole_or_rejected( self, - data_type: str, + data_type: CustomField.FieldDataType, text: str, expected: str | None, ) -> None: @@ -2961,7 +3024,7 @@ class TestConvertValue: def test_converts_to_exact_value( self, value: str, - data_type: str, + data_type: CustomField.FieldDataType, expected: object, locale: Locale, ) -> None: @@ -3161,7 +3224,12 @@ class TestConvertValue: ), ], ) - def test_returns_none(self, value: str, data_type: str, locale: Locale) -> None: + def test_returns_none( + self, + value: str, + data_type: CustomField.FieldDataType, + locale: Locale, + ) -> None: """ GIVEN: - A value that cannot be converted to the given data type in a locale @@ -3193,7 +3261,7 @@ class TestConvertValue: ) def test_numeric_types_reject_malformed_numbers( self, - data_type: str, + data_type: CustomField.FieldDataType, value: str, ) -> None: """ diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 625313326f..9714d75b1b 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -18,6 +18,7 @@ from functools import partial from pathlib import Path from typing import TYPE_CHECKING from typing import TypeAlias +from typing import assert_never import dateparser import magic @@ -143,10 +144,12 @@ def _zone_result( *, reason: ZoneOcrReason | None = None, ) -> ZoneOcrResult: - if zone.target == OcrTemplateZone.TargetType.CUSTOM_FIELD: - label = zone.custom_field.name if zone.custom_field_id else "(no field)" + target = OcrTemplateZone.TargetType(zone.target) + if target == OcrTemplateZone.TargetType.CUSTOM_FIELD: + custom_field = zone.custom_field + label = custom_field.name if custom_field is not None else "(no field)" else: - label = BUILTIN_TARGET_LABELS[zone.target] + label = BUILTIN_TARGET_LABELS[target] return ZoneOcrResult( template=template.name, zone=zone.name, @@ -606,7 +609,7 @@ def _extract_zone( raw_text=raw_text, value=_apply_transform( raw_text or "", - zone.transform, + OcrTemplateZone.TransformType(zone.transform), _date_rules(zone.ocr_language, zone.date_format), ) or None, @@ -729,7 +732,7 @@ def _parse_date(text: str, rules: DateRules) -> date | None: def _apply_transform( text: str, - transform: str, + transform: OcrTemplateZone.TransformType, dates: DateRules, ) -> str: """Apply post-processing transform to extracted text. @@ -741,21 +744,27 @@ def _apply_transform( if not text: return text - if transform in ("strip", "none"): - return text - elif transform == "date": - parsed = _parse_date(text, dates) - return parsed.isoformat() if parsed else "" - elif transform == "uppercase": - return text.upper() - elif transform == "lowercase": - return text.lower() - elif transform == "numeric": - numeric = re.sub(r"[^\d.,\-]", "", text) - return numeric if re.search(r"\d", numeric) else "" - elif transform == "strip_punctuation": - return text.strip(string.punctuation + " \t\r\n") - return text + match transform: + case ( + OcrTemplateZone.TransformType.NONE + | OcrTemplateZone.TransformType.STRIP + | OcrTemplateZone.TransformType.QR_CODE + ): + return text + case OcrTemplateZone.TransformType.DATE: + parsed = _parse_date(text, dates) + return parsed.isoformat() if parsed else "" + case OcrTemplateZone.TransformType.UPPERCASE: + return text.upper() + case OcrTemplateZone.TransformType.LOWERCASE: + return text.lower() + case OcrTemplateZone.TransformType.NUMERIC: + numeric = re.sub(r"[^\d.,\-]", "", text) + return numeric if re.search(r"\d", numeric) else "" + case OcrTemplateZone.TransformType.STRIP_PUNCTUATION: + return text.strip(string.punctuation + " \t\r\n") + case unreachable: + assert_never(unreachable) def _prepare_field_update( @@ -765,37 +774,41 @@ def _prepare_field_update( """Convert extracted text and enforce the destination field's constraints.""" custom_field = None typed_value: ZoneValue | None - if zone.target == OcrTemplateZone.TargetType.CUSTOM_FIELD: - if not zone.custom_field_id: - return None - custom_field = zone.custom_field - field_name = CustomFieldInstance.get_value_field_name(custom_field.data_type) - typed_value = _convert_value( - value, - custom_field.data_type, - _number_locale(zone.ocr_language), - _date_rules(zone.ocr_language, zone.date_format), - ) - model = CustomFieldInstance - elif zone.target == OcrTemplateZone.TargetType.TITLE: - field_name, typed_value = "title", value[:128] - model = Document - elif zone.target == OcrTemplateZone.TargetType.ASN: - digits = re.sub(r"[^\d]", "", value) - field_name = "archive_serial_number" - typed_value = int(digits) if digits else None - model = Document - elif zone.target == OcrTemplateZone.TargetType.CREATED: - field_name, typed_value = ( - "created", - _parse_date( + model: type[Document | CustomFieldInstance] + target = OcrTemplateZone.TargetType(zone.target) + match target: + case OcrTemplateZone.TargetType.CUSTOM_FIELD: + custom_field = zone.custom_field + if custom_field is None: + return None + data_type = CustomField.FieldDataType(custom_field.data_type) + field_name = CustomFieldInstance.get_value_field_name(data_type) + typed_value = _convert_value( value, + data_type, + _number_locale(zone.ocr_language), _date_rules(zone.ocr_language, zone.date_format), - ), - ) - model = Document - else: - return None + ) + model = CustomFieldInstance + case OcrTemplateZone.TargetType.TITLE: + field_name, typed_value = "title", value[:128] + model = Document + case OcrTemplateZone.TargetType.ASN: + digits = re.sub(r"[^\d]", "", value) + field_name = "archive_serial_number" + typed_value = int(digits) if digits else None + model = Document + case OcrTemplateZone.TargetType.CREATED: + field_name, typed_value = ( + "created", + _parse_date( + value, + _date_rules(zone.ocr_language, zone.date_format), + ), + ) + model = Document + case unreachable: + assert_never(unreachable) if typed_value is None: return None field = model._meta.get_field(field_name) @@ -1027,7 +1040,7 @@ def _parse_monetary(value: str, locale: Locale) -> str | None: def _convert_value( value: str, - data_type: str, + data_type: CustomField.FieldDataType, locale: Locale, dates: DateRules, ) -> ZoneValue | None: @@ -1040,49 +1053,56 @@ def _convert_value( return None try: - if data_type in ( - CustomField.FieldDataType.STRING, - CustomField.FieldDataType.URL, - CustomField.FieldDataType.LONG_TEXT, - ): - # Returned whole; _prepare_field_update validates length and URL - # format, so a value is rejected rather than truncated. - return value + match data_type: + case ( + CustomField.FieldDataType.STRING + | CustomField.FieldDataType.URL + | CustomField.FieldDataType.LONG_TEXT + ): + # Returned whole; _prepare_field_update validates length and URL + # format, so a value is rejected rather than truncated. + return value - elif data_type == CustomField.FieldDataType.INT: - integer = _parse_number(value, locale) - if integer is None or integer != integer.to_integral_value(): + case CustomField.FieldDataType.INT: + integer = _parse_number(value, locale) + if integer is None or integer != integer.to_integral_value(): + return None + return int(integer) + + case CustomField.FieldDataType.FLOAT: + decimal = _parse_number(value, locale) + if decimal is None: + return None + number = float(decimal) + return number if math.isfinite(number) else None + + case CustomField.FieldDataType.DATE: + parsed = _parse_date(value, dates) + return parsed.isoformat() if parsed else None + + case CustomField.FieldDataType.MONETARY: + return _parse_monetary(value, locale) + + case CustomField.FieldDataType.BOOL: + lower = value.lower().strip() + if lower in ("true", "yes", "1", "ja", "oui", "si", "x"): + return True + elif lower in ("false", "no", "0", "nein", "non"): + return False return None - return int(integer) - elif data_type == CustomField.FieldDataType.FLOAT: - decimal = _parse_number(value, locale) - if decimal is None: + case ( + CustomField.FieldDataType.SELECT + | CustomField.FieldDataType.DOCUMENTLINK + ): + logger.debug( + "Zone OCR: unsupported custom field type %s for OCR extraction", + data_type, + ) return None - number = float(decimal) - return number if math.isfinite(number) else None - elif data_type == CustomField.FieldDataType.DATE: - parsed = _parse_date(value, dates) - return parsed.isoformat() if parsed else None - - elif data_type == CustomField.FieldDataType.MONETARY: - return _parse_monetary(value, locale) - - elif data_type == CustomField.FieldDataType.BOOL: - lower = value.lower().strip() - if lower in ("true", "yes", "1", "ja", "oui", "si", "x"): - return True - elif lower in ("false", "no", "0", "nein", "non"): - return False - return None - - else: - logger.debug( - "Zone OCR: unsupported custom field type %s for OCR extraction", - data_type, - ) - return None + case unreachable: + assert_never(unreachable) except (ValueError, InvalidOperation): logger.info("Zone OCR: could not convert value to %s", data_type)