diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index 2a5295f4ea..36939e565f 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -24,13 +24,16 @@ from documents.zone_ocr import _apply_transform from documents.zone_ocr import _combine_field_value from documents.zone_ocr import _convert_value from documents.zone_ocr import _crop_zone +from documents.zone_ocr import _date_rules from documents.zone_ocr import _detect_mime from documents.zone_ocr import _extract_zone from documents.zone_ocr import _number_locale +from documents.zone_ocr import _parse_date from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages from documents.zone_ocr import _resolve_doc_path +from documents.zone_ocr import _zone_language from documents.zone_ocr import extract_zone_preview from documents.zone_ocr import get_templates_for_document from documents.zone_ocr import run_zone_extraction @@ -54,6 +57,7 @@ if TYPE_CHECKING: EN = Locale.parse("en") DE = Locale.parse("de") +ENG_DATES = _date_rules("eng") FR = Locale.parse("fr") @@ -1449,12 +1453,14 @@ class TestApplyTransform: """ GIVEN: - Extracted text, a transform name and an optional date format + - A zone reading English WHEN: - The transform is applied THEN: - The transformed text matches the expected result """ - assert _apply_transform(text, transform, date_format) == expected + rules = _date_rules("eng", date_format) + assert _apply_transform(text, transform, rules) == expected def test_unparseable_date_preserves_text(self, mocker: MockerFixture) -> None: """ @@ -1466,7 +1472,278 @@ class TestApplyTransform: - The original text is returned unchanged """ mocker.patch("documents.zone_ocr.dateparser.parse", side_effect=OverflowError) - assert _apply_transform("unparsable", "date") == "unparsable" + assert ( + _apply_transform("unparsable", "date", _date_rules("eng")) == "unparsable" + ) + + +class TestZoneLanguage: + """Tests for mapping the OCR language to a dateparser language.""" + + @pytest.mark.parametrize( + ("ocr_language", "expected"), + [ + pytest.param("deu", "de", id="single-language"), + pytest.param("deu+eng", "de", id="first-of-several-languages"), + pytest.param("aze_Cyrl", "az", id="script-suffix-is-dropped"), + pytest.param("xyz", None, id="unmapped-language-is-none"), + ], + ) + def test_zone_language(self, ocr_language: str, expected: str | None) -> None: + """ + GIVEN: + - A tesseract language code, possibly several or with a script + WHEN: + - The zone language is derived + THEN: + - The first language is mapped, or None when it is unknown + """ + assert _zone_language(ocr_language) == expected + + +class TestParseDate: + """Tests for the zone date parser.""" + + @pytest.mark.parametrize( + ("text", "ocr_language", "expected"), + [ + pytest.param("12.05.2024", "deu", date(2024, 5, 12), id="german-day-first"), + pytest.param( + "05/12/2024", + "eng", + date(2024, 5, 12), + id="english-month-first", + ), + pytest.param( + "12. Mai 2024", + "deu", + date(2024, 5, 12), + id="german-month-name", + ), + pytest.param( + "13 April 2026", + "eng", + date(2026, 4, 13), + id="english-month-name", + ), + pytest.param( + "2026/04/13", + "eng", + date(2026, 4, 13), + id="year-first-slashed", + ), + pytest.param( + "01.01.2999", + "deu", + date(2999, 1, 1), + id="future-date-allowed", + ), + ], + ) + def test_locale_decides_the_order( + self, + text: str, + ocr_language: str, + expected: date, + ) -> None: + """ + GIVEN: + - A date written in the convention of the zone's language + WHEN: + - The date is parsed + THEN: + - The locale's day and month order is used + """ + assert _parse_date(text, _date_rules(ocr_language)) == expected + + @pytest.mark.parametrize( + ("date_order", "expected"), + [ + pytest.param("DMY", date(2024, 5, 12), id="day-first"), + pytest.param("MDY", date(2024, 12, 5), id="month-first"), + ], + ) + def test_unmapped_language_uses_the_configured_date_order( + self, + settings: SettingsWrapper, + date_order: str, + expected: date, + ) -> None: + """ + GIVEN: + - A zone whose language has no dateparser mapping + - A configured PAPERLESS_DATE_ORDER + WHEN: + - An ambiguous numeric date is parsed + THEN: + - The configured date order decides the result + """ + settings.DATE_ORDER = date_order + assert _parse_date("12.05.2024", _date_rules("xyz")) == expected + + @pytest.mark.parametrize( + "text", + [ + pytest.param("now", id="now"), + pytest.param("yesterday", id="yesterday"), + pytest.param("today", id="today"), + pytest.param("in 2 days", id="relative-offset"), + pytest.param("Mon", id="weekday"), + pytest.param("March", id="month-only"), + pytest.param("2024", id="year-only"), + pytest.param("5", id="day-only"), + pytest.param("12", id="two-digits"), + pytest.param("not a date", id="words"), + ], + ) + @pytest.mark.parametrize( + "ocr_language", + [ + pytest.param("eng", id="mapped-language"), + pytest.param("xyz", id="unmapped-language"), + ], + ) + def test_non_dates_are_rejected(self, text: str, ocr_language: str) -> None: + """ + GIVEN: + - Text that is not a complete calendar date + WHEN: + - The date is parsed + THEN: + - No date is returned + """ + assert _parse_date(text, _date_rules(ocr_language)) is None + + @pytest.mark.parametrize( + ("text", "date_format", "expected"), + [ + pytest.param( + "12.05.2024", + "%m.%d.%Y", + date(2024, 12, 5), + id="format-overrides-locale-order", + ), + pytest.param( + "12.05.2024", + "%Y/%m/%d", + date(2024, 5, 12), + id="invalid-format-falls-back", + ), + ], + ) + def test_date_format_is_tried_first( + self, + text: str, + date_format: str, + expected: date, + ) -> None: + """ + GIVEN: + - A German zone with a date format + WHEN: + - A date is parsed + THEN: + - A matching format is used, otherwise parsing falls back to the locale + """ + assert _parse_date(text, _date_rules("deu", date_format)) == expected + + @pytest.mark.parametrize( + ("text", "expected"), + [ + pytest.param("2024-05-06", date(2024, 5, 6), id="plain-iso"), + pytest.param("Due 2024-05-06.", date(2024, 5, 6), id="iso-inside-text"), + pytest.param("No. 12345-06-07", None, id="digit-before-is-not-iso"), + pytest.param("2024-05-061", None, id="digit-after-is-not-iso"), + pytest.param( + "2024-13-45 and 2024-05-06", + date(2024, 5, 6), + id="invalid-match-is-skipped", + ), + pytest.param("2024-13-45", None, id="only-invalid-match"), + ], + ) + def test_iso_dates_are_anchored(self, text: str, expected: date | None) -> None: + """ + GIVEN: + - Text containing ISO-like dates, some invalid or part of longer numbers + WHEN: + - The date is parsed + THEN: + - Only whole, valid ISO dates are returned, trying every match + """ + assert _parse_date(text, _date_rules("eng")) == expected + + @pytest.mark.parametrize( + "text", + [ + pytest.param("2026/04/13", id="slashed"), + pytest.param("2026.04.13", id="dotted"), + pytest.param("2026-04-13", id="dashed"), + ], + ) + @pytest.mark.parametrize( + ("ocr_language", "date_order"), + [ + pytest.param("xyz", "DMY", id="unmapped-language-day-first"), + pytest.param("xyz", "MDY", id="unmapped-language-month-first"), + pytest.param("deu", "DMY", id="mapped-language"), + ], + ) + def test_year_first_dates_ignore_the_date_order( + self, + settings: SettingsWrapper, + text: str, + ocr_language: str, + date_order: str, + ) -> None: + """ + GIVEN: + - A year-first date separated by a dash, slash or dot + - A mapped or unmapped zone language and a configured date order + WHEN: + - The date is parsed + THEN: + - The date is read as year, month, day regardless of the order + """ + settings.DATE_ORDER = date_order + assert _parse_date(text, _date_rules(ocr_language)) == date(2026, 4, 13) + + @pytest.mark.parametrize( + "text", + [ + pytest.param("1.2026/04/133", id="digit-after"), + pytest.param("12026/04/13", id="digit-before"), + pytest.param("2026/13/45", id="invalid-month"), + ], + ) + def test_year_first_dates_are_anchored_and_validated(self, text: str) -> None: + """ + GIVEN: + - Year-first looking text that is part of longer digits or invalid + WHEN: + - The date is parsed + THEN: + - No date is returned + """ + assert _parse_date(text, _date_rules("xyz")) is None + + @pytest.mark.parametrize( + "text", + [ + pytest.param("1899-12-31", id="iso"), + pytest.param("31.12.1899", id="locale-numeric"), + ], + ) + def test_years_before_1900_are_rejected(self, text: str) -> None: + """ + GIVEN: + - A valid calendar date before the year 1900 + WHEN: + - The date is parsed + THEN: + - No date is returned + """ + assert _parse_date(text, _date_rules("deu")) is None @pytest.mark.django_db @@ -1516,6 +1793,82 @@ class TestPrepareFieldUpdate: assert (update.value if update else None) == expected + @pytest.mark.parametrize( + "target", + [ + pytest.param(OcrTemplateZone.TargetType.CREATED, id="created"), + pytest.param(OcrTemplateZone.TargetType.CUSTOM_FIELD, id="date-field"), + ], + ) + @pytest.mark.parametrize( + ("ocr_language", "date_format", "text", "expected"), + [ + pytest.param("deu", "", "12.05.2024", date(2024, 5, 12), id="german-order"), + pytest.param( + "eng", + "", + "05/12/2024", + date(2024, 5, 12), + id="english-order", + ), + pytest.param( + "deu", + "", + "12. Mai 2024", + date(2024, 5, 12), + id="german-month-name", + ), + pytest.param( + "deu", + "%m.%d.%Y", + "12.05.2024", + date(2024, 12, 5), + id="zone-date-format", + ), + pytest.param( + "deu", + "%Y/%m/%d", + "12.05.2024", + date(2024, 5, 12), + id="invalid-format-falls-back", + ), + pytest.param("eng", "", "2024", None, id="year-only-is-rejected"), + ], + ) + def test_date_targets_parse_alike( + self, + target: str, + ocr_language: str, + date_format: str, + text: str, + expected: date | None, + ) -> None: + """ + GIVEN: + - A zone targeting the created date or a date custom field + - An OCR language and optionally a date format + WHEN: + - A field update is prepared for non-ISO date text + THEN: + - Both targets read the text the same way, honoring the zone's + language and date format + """ + field = ( + CustomFieldFactory(data_type=CustomField.FieldDataType.DATE) + if target == OcrTemplateZone.TargetType.CUSTOM_FIELD + else None + ) + zone = OcrTemplateZoneFactory( + target=target, + custom_field=field, + ocr_language=ocr_language, + date_format=date_format, + ) + + update = _prepare_field_update(zone, text) + + assert (update.value if update else None) == expected + class TestNumberLocale: """Tests for deriving a number locale from the OCR language.""" @@ -1775,7 +2128,7 @@ class TestConvertValue: THEN: - The converted value equals the expected value exactly """ - assert _convert_value(value, data_type, locale) == expected + assert _convert_value(value, data_type, locale, ENG_DATES) == expected def test_string_truncation(self) -> None: """ @@ -1786,7 +2139,12 @@ class TestConvertValue: THEN: - The result is truncated to 128 characters """ - result = _convert_value("x" * 200, CustomField.FieldDataType.STRING, EN) + result = _convert_value( + "x" * 200, + CustomField.FieldDataType.STRING, + EN, + ENG_DATES, + ) assert isinstance(result, str) assert len(result) == 128 @@ -1829,7 +2187,8 @@ class TestConvertValue: - The result is the expected float """ assert ( - _convert_value(value, CustomField.FieldDataType.FLOAT, locale) == expected + _convert_value(value, CustomField.FieldDataType.FLOAT, locale, ENG_DATES) + == expected ) @pytest.mark.parametrize( @@ -1984,7 +2343,7 @@ class TestConvertValue: THEN: - None is returned """ - assert _convert_value(value, data_type, locale) is None + assert _convert_value(value, data_type, locale, ENG_DATES) is None @pytest.mark.parametrize( "data_type", @@ -2018,7 +2377,7 @@ class TestConvertValue: THEN: - None is returned """ - assert _convert_value(value, data_type, EN) is None + assert _convert_value(value, data_type, EN, ENG_DATES) is None @pytest.mark.parametrize( "value", @@ -2041,7 +2400,9 @@ class TestConvertValue: THEN: - The result is True """ - assert _convert_value(value, CustomField.FieldDataType.BOOL, EN) is True + assert ( + _convert_value(value, CustomField.FieldDataType.BOOL, EN, ENG_DATES) is True + ) @pytest.mark.parametrize( "value", @@ -2062,7 +2423,10 @@ class TestConvertValue: THEN: - The result is False """ - assert _convert_value(value, CustomField.FieldDataType.BOOL, EN) is False + assert ( + _convert_value(value, CustomField.FieldDataType.BOOL, EN, ENG_DATES) + is False + ) class TestRenderPages: diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index b487bf8604..4fa943e71d 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -44,6 +44,9 @@ from documents.versioning import get_root_document from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any + from django.contrib.auth.models import User logger = logging.getLogger("paperless.zone_ocr") @@ -51,6 +54,15 @@ logger = logging.getLogger("paperless.zone_ocr") # Unicode general category for decimal digits in any script. UNICODE_DECIMAL_DIGIT = "Nd" # codespell:ignore +# Year-first dates (ISO and its slashed or dotted variants), never a slice of a +# longer digit run. They are unambiguous, so no date order is needed. +YEAR_FIRST_DATE_RE = re.compile( + r"(? date | None: - match = re.search(r"\d{4}-\d{2}-\d{2}", text) - return date.fromisoformat(match[0]) if match else None +@dataclass(frozen=True) +class DateRules: + """How a zone reads dates: its language and optional strptime format.""" + + language: str | None = None + date_format: str = "" -def _parse_date(text: str, fmt: str = "") -> date | None: - """Try the zone's date format, ISO dates, then automatic date detection.""" - if fmt: +def _date_rules(ocr_language: str, date_format: str = "") -> DateRules: + return DateRules(_zone_language(ocr_language), date_format) + + +def _candidate_dates(text: str, rules: DateRules) -> Iterator[date]: + """Yield the dates the text may hold, most specific reading first.""" + if rules.date_format: try: - return datetime.strptime(text, fmt).date() + yield datetime.strptime(text, rules.date_format).date() except ValueError: pass - try: - iso_date = _parse_iso_date(text) - except ValueError: - iso_date = None - if iso_date is not None: - return iso_date + for match in YEAR_FIRST_DATE_RE.finditer(text): + year, _, month, day = match.groups() + try: + yield date(int(year), int(month), int(day)) + except ValueError: + continue + # Only absolute dates: dateparser would also read "now" or "yesterday" + date_settings: dict[str, Any] = { + "PARSERS": ["custom-formats", "absolute-time"], + "REQUIRE_PARTS": ["day", "month", "year"], + "RETURN_AS_TIMEZONE_AWARE": False, + } + if rules.language is None: + date_settings["DATE_ORDER"] = settings.DATE_ORDER try: parsed = dateparser.parse( text, - settings={ - "PREFER_DAY_OF_MONTH": "first", - "RETURN_AS_TIMEZONE_AWARE": False, - }, + languages=None if rules.language is None else [rules.language], + settings=date_settings, ) - if parsed: - return parsed.date() except (ValueError, OverflowError): logger.debug("Zone OCR: date parsing failed") - return None + return + if parsed: + yield parsed.date() -def _apply_transform(text: str, transform: str, date_format: str = "") -> str: +def _parse_date(text: str, rules: DateRules) -> date | None: + """Read a calendar date from OCR text. + + Tries the zone's date format, year-first dates, then dateparser in the zone's + language (or the configured date order when the language is unknown). + Dates before MIN_ZONE_DATE_YEAR are treated as OCR noise. + """ + return next( + ( + candidate + for candidate in _candidate_dates(text, rules) + if candidate.year >= MIN_ZONE_DATE_YEAR + ), + None, + ) + + +def _apply_transform( + text: str, + transform: str, + dates: DateRules, +) -> str: """Apply post-processing transform to extracted text.""" text = text.strip() if not text: @@ -555,7 +605,7 @@ def _apply_transform(text: str, transform: str, date_format: str = "") -> str: if transform in ("strip", "none"): return text elif transform == "date": - parsed = _parse_date(text, date_format) + parsed = _parse_date(text, dates) return parsed.isoformat() if parsed else text elif transform == "uppercase": return text.upper() @@ -584,6 +634,7 @@ def _prepare_field_update( value, custom_field.data_type, _number_locale(zone.ocr_language), + _date_rules(zone.ocr_language, zone.date_format), ) model = CustomFieldInstance elif zone.target == OcrTemplateZone.TargetType.TITLE: @@ -595,7 +646,13 @@ def _prepare_field_update( typed_value = int(digits) if digits else None model = Document elif zone.target == OcrTemplateZone.TargetType.CREATED: - field_name, typed_value = "created", _parse_date(value) + field_name, typed_value = ( + "created", + _parse_date( + value, + _date_rules(zone.ocr_language, zone.date_format), + ), + ) model = Document else: return None @@ -636,14 +693,22 @@ def _apply_field_updates(document: Document, updates: list[FieldUpdate]) -> None raise +def _zone_language(ocr_language: str) -> str | None: + """Map the first tesseract language of a zone to a dateparser/Babel code. + + Only the first language counts, so "deu+eng" is German. Returns None when + the language is not mapped. + """ + tesseract_code = ocr_language.split("+", maxsplit=1)[0].split("_", maxsplit=1)[0] + return OCR_TO_DATEPARSER_LANGUAGES.get(tesseract_code) + + def _number_locale(ocr_language: str) -> Locale: """Derive the locale used to read numbers from a tesseract language code. - Only the first language counts, so "deu+eng" reads numbers the German way. Languages Babel does not know fall back to English. """ - tesseract_code = ocr_language.split("+", maxsplit=1)[0].split("_", maxsplit=1)[0] - code = OCR_TO_DATEPARSER_LANGUAGES.get(tesseract_code) + code = _zone_language(ocr_language) if code is not None: try: return Locale.parse(code) @@ -700,10 +765,12 @@ def _convert_value( value: str, data_type: str, locale: Locale, + dates: DateRules, ) -> object | None: """Convert an extracted OCR string to the appropriate type for the custom field. - Numeric types are read with the number format of the given locale. + Numeric types are read with the number format of the given locale and + dates with the given date rules. """ if not value: return None @@ -732,7 +799,7 @@ def _convert_value( return number if math.isfinite(number) else None elif data_type == CustomField.FieldDataType.DATE: - parsed = _parse_iso_date(value) + parsed = _parse_date(value, dates) return parsed.isoformat() if parsed else None elif data_type == CustomField.FieldDataType.MONETARY: