diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index d7ec706822..bba1bebfbe 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -6,6 +6,7 @@ import shutil import subprocess from collections import Counter from datetime import date +from decimal import Decimal from pathlib import Path from typing import TYPE_CHECKING from typing import Any @@ -22,6 +23,7 @@ from documents.models import MatchingModel from documents.models import OcrTemplate from documents.models import OcrTemplateZone from documents.parsers import ParseError +from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH from documents.zone_ocr import _apply_transform from documents.zone_ocr import _combine_field_value from documents.zone_ocr import _convert_value @@ -32,6 +34,7 @@ from documents.zone_ocr import _extract_zone from documents.zone_ocr import _number_locale from documents.zone_ocr import _parse_date from documents.zone_ocr import _parse_monetary +from documents.zone_ocr import _parse_number from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages @@ -2371,6 +2374,135 @@ class TestParseMonetary: assert _parse_monetary(value, locale) == expected +class TestParsedTextLength: + """Tests for the length cap on text read as a number, amount or date.""" + + @pytest.mark.parametrize( + ("parse", "value", "expected"), + [ + pytest.param( + lambda text: _parse_number(text, EN), + "12.50", + Decimal("12.50"), + id="number", + ), + pytest.param( + lambda text: _parse_monetary(text, EN), + "USD 12.50", + "USD12.50", + id="monetary", + ), + ], + ) + def test_text_longer_than_the_cap_is_not_read( + self, + parse: Callable[[str], object], + value: str, + expected: object, + ) -> None: + """ + GIVEN: + - A valid number or amount padded with spaces to exactly the length + cap, and the same value padded one character past it + WHEN: + - Each text is parsed + THEN: + - The text at the cap is read and the longer text is not + """ + at_cap = value.rjust(MAX_PARSED_TEXT_LENGTH) + + assert parse(at_cap) == expected + assert parse(f" {at_cap}") is None + + @pytest.mark.parametrize( + ("text", "expected"), + [ + pytest.param( + "Datum: 13.04.2026 " + "x" * 300, + date(2026, 4, 13), + id="date-near-the-start", + ), + pytest.param( + "x" * 300 + " Datum: 13.04.2026", + None, + id="date-past-the-cap", + ), + pytest.param( + "x " * 118 + "xxxx Date 2026-04-131 more", + None, + id="token-split-by-the-cut", + ), + pytest.param( + "x" * 246 + "2026-04-13" + "1" * 50, + None, + id="cut-inside-text-without-spaces", + ), + pytest.param( + "x " * 118 + "xxxx Date 2026-04-13 more", + date(2026, 4, 13), + id="cut-on-a-space-keeps-the-last-word", + ), + pytest.param(" " * 300, None, id="whitespace-only"), + ], + ) + def test_long_text_is_searched_for_dates_up_to_the_cap( + self, + text: str, + expected: date | None, + ) -> None: + """ + GIVEN: + - Text longer than the cap with a date before it, after it, or with a + non-date token that the cut would turn into a date + WHEN: + - The text is parsed as a date + THEN: + - Only whole words within the cap are searched + """ + assert _parse_date(text, ENG_DATES) == expected + + @pytest.mark.parametrize( + ("parse", "value"), + [ + pytest.param( + lambda text: _parse_number(text, EN), + "- " * 128, + id="number-dashes-without-digit", + ), + pytest.param( + lambda text: _parse_number(text, EN), + "- " * 127 + "1", + id="number-dashes-before-digit", + ), + pytest.param( + lambda text: _parse_monetary(text, EN), + "- " * 128, + id="monetary-dashes-without-digit", + ), + pytest.param( + lambda text: _parse_monetary(text, EN), + "(" + "1 " * 127, + id="monetary-unclosed-parenthesis", + ), + ], + ) + def test_malformed_text_at_the_cap_is_rejected( + self, + parse: Callable[[str], object], + value: str, + ) -> None: + """ + GIVEN: + - Malformed text at the length cap, made of dash runs or an + unclosed parenthesis + WHEN: + - The text is parsed + THEN: + - No value is read + """ + assert parse(value) is None + + class TestConvertValue: """Tests for the _convert_value function.""" diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 76bc70d6d4..b542216aff 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -79,6 +79,11 @@ SEPARATOR_ONLY_RE = re.compile(r"[\s\-/.,;:|]*") # Earlier dates are far more likely OCR noise than a real document date. MIN_ZONE_DATE_YEAR = 1900 +# Longest OCR text read as a number, amount or date. Longer numbers and amounts +# are not parsed, and dates are only searched for in the leading text, which +# bounds parsing time when a zone covers a large region. +MAX_PARSED_TEXT_LENGTH = 256 + @dataclass(frozen=True) class ZoneExtraction: @@ -633,8 +638,19 @@ def _parse_date(text: str, rules: DateRules) -> date | None: month name and year), and the first token dateparser accepts is used. Bare numbers and two-digit years in surrounding text are never read as dates, so version or article numbers like "1.2.34" are ignored. Dates before - MIN_ZONE_DATE_YEAR are treated as OCR noise. + MIN_ZONE_DATE_YEAR are treated as OCR noise. Only the first + MAX_PARSED_TEXT_LENGTH characters are read, without a word the cut splits. """ + if len(text) > MAX_PARSED_TEXT_LENGTH: + head = text[:MAX_PARSED_TEXT_LENGTH] + if not text[MAX_PARSED_TEXT_LENGTH].isspace(): + # Drop the word split by the cut, so a truncated token cannot match + # as a different date. + kept = re.match(r".*\s", head, re.DOTALL) + head = kept.group() if kept else "" + text = head + if not text.strip(): + return None return next( ( candidate @@ -779,8 +795,11 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None: when it comes before the first digit and does not directly follow a letter or digit, so "-5", "EUR -5" and "$-5" are negative while "INV-123" is just 123. Any other dash makes the text invalid. Returns None when the - text is not a well-formed number in the locale. + text is not a well-formed number in the locale or is longer than + MAX_PARSED_TEXT_LENGTH. """ + if len(value) > MAX_PARSED_TEXT_LENGTH: + return None text = "".join( str(unicodedata.digit(char)) if unicodedata.category(char) == UNICODE_DECIMAL_DIGIT @@ -793,8 +812,15 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None: .replace("\N{ARABIC DECIMAL SEPARATOR}", ".") .replace("\N{ARABIC THOUSANDS SEPARATOR}", ",") ) - sign = re.search(r"(? str | None: - The currency is an ISO 4217 code written as an uppercase word, or else a currency symbol of the locale that belongs to a single currency. Text naming two different currencies is rejected, and text naming none gives - an amount without a code. Zone crops are expected to be short, but an - uppercase word that happens to be an ISO code (e.g. "ALL" or "TOP") is - still read as a currency. + an amount without a code. Any uppercase word that is an ISO code + (e.g. "ALL" or "TOP") is read as a currency. - A number enclosed in parentheses is negative, "(12.50)". Digits outside the parentheses, as in "100 (19% VAT)", or a minus sign combined with them are rejected. @@ -877,9 +902,19 @@ def _parse_monetary(value: str, locale: Locale) -> str | None: most 12 digits, with or without a currency code. - The result must pass the same validation as values sent through the API. - Returns None when the text is not a valid monetary value. + Returns None when the text is not a valid monetary value or is longer than + MAX_PARSED_TEXT_LENGTH. """ - group = re.search(r"\([^()]*\d[^()]*\)", value) + if len(value) > MAX_PARSED_TEXT_LENGTH: + return None + group = next( + ( + match + for match in re.finditer(r"\([^()]*\)", value) + if re.search(r"\d", match.group()) + ), + None, + ) parenthesised = group is not None text = value if group is not None: