Fix: keep zone number parsing linear and cap its input

Two patterns backtracked quadratically on long OCR text, taking seconds on a
large zone. Rewrite both, skip numbers and amounts longer than 256 characters,
and search only the first 256 characters for a date.
This commit is contained in:
stumpylog committed 2026-11-19 08:02:51 -08:00
1 parent 6fc4526947
commit 3504ad07c6
2 files changed
+176 -9

No files matched your search

+132
View File
@@ -6,6 +6,7 @@ import shutil
import subprocess
from collections import Counter
from datetime import date
from decimal import Decimal
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
@@ -22,6 +23,7 @@ from documents.models import MatchingModel
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.parsers import ParseError
from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH
from documents.zone_ocr import _apply_transform
from documents.zone_ocr import _combine_field_value
from documents.zone_ocr import _convert_value
@@ -32,6 +34,7 @@ from documents.zone_ocr import _extract_zone
from documents.zone_ocr import _number_locale
from documents.zone_ocr import _parse_date
from documents.zone_ocr import _parse_monetary
from documents.zone_ocr import _parse_number
from documents.zone_ocr import _prepare_field_update
from documents.zone_ocr import _process_template
from documents.zone_ocr import _render_pages
@@ -2371,6 +2374,135 @@ class TestParseMonetary:
assert _parse_monetary(value, locale) == expected
class TestParsedTextLength:
"""Tests for the length cap on text read as a number, amount or date."""
@pytest.mark.parametrize(
("parse", "value", "expected"),
[
pytest.param(
lambda text: _parse_number(text, EN),
"12.50",
Decimal("12.50"),
id="number",
),
pytest.param(
lambda text: _parse_monetary(text, EN),
"USD 12.50",
"USD12.50",
id="monetary",
),
],
)
def test_text_longer_than_the_cap_is_not_read(
self,
parse: Callable[[str], object],
value: str,
expected: object,
) -> None:
"""
GIVEN:
- A valid number or amount padded with spaces to exactly the length
cap, and the same value padded one character past it
WHEN:
- Each text is parsed
THEN:
- The text at the cap is read and the longer text is not
"""
at_cap = value.rjust(MAX_PARSED_TEXT_LENGTH)
assert parse(at_cap) == expected
assert parse(f" {at_cap}") is None
@pytest.mark.parametrize(
("text", "expected"),
[
pytest.param(
"Datum: 13.04.2026 " + "x" * 300,
date(2026, 4, 13),
id="date-near-the-start",
),
pytest.param(
"x" * 300 + " Datum: 13.04.2026",
None,
id="date-past-the-cap",
),
pytest.param(
"x " * 118 + "xxxx Date 2026-04-131 more",
None,
id="token-split-by-the-cut",
),
pytest.param(
"x" * 246 + "2026-04-13" + "1" * 50,
None,
id="cut-inside-text-without-spaces",
),
pytest.param(
"x " * 118 + "xxxx Date 2026-04-13 more",
date(2026, 4, 13),
id="cut-on-a-space-keeps-the-last-word",
),
pytest.param(" " * 300, None, id="whitespace-only"),
],
)
def test_long_text_is_searched_for_dates_up_to_the_cap(
self,
text: str,
expected: date | None,
) -> None:
"""
GIVEN:
- Text longer than the cap with a date before it, after it, or with a
non-date token that the cut would turn into a date
WHEN:
- The text is parsed as a date
THEN:
- Only whole words within the cap are searched
"""
assert _parse_date(text, ENG_DATES) == expected
@pytest.mark.parametrize(
("parse", "value"),
[
pytest.param(
lambda text: _parse_number(text, EN),
"- " * 128,
id="number-dashes-without-digit",
),
pytest.param(
lambda text: _parse_number(text, EN),
"- " * 127 + "1",
id="number-dashes-before-digit",
),
pytest.param(
lambda text: _parse_monetary(text, EN),
"- " * 128,
id="monetary-dashes-without-digit",
),
pytest.param(
lambda text: _parse_monetary(text, EN),
"(" + "1 " * 127,
id="monetary-unclosed-parenthesis",
),
],
)
def test_malformed_text_at_the_cap_is_rejected(
self,
parse: Callable[[str], object],
value: str,
) -> None:
"""
GIVEN:
- Malformed text at the length cap, made of dash runs or an
unclosed parenthesis
WHEN:
- The text is parsed
THEN:
- No value is read
"""
assert parse(value) is None
class TestConvertValue:
"""Tests for the _convert_value function."""
+44 -9
View File
@@ -79,6 +79,11 @@ SEPARATOR_ONLY_RE = re.compile(r"[\s\-/.,;:|]*")
# Earlier dates are far more likely OCR noise than a real document date.
MIN_ZONE_DATE_YEAR = 1900
# Longest OCR text read as a number, amount or date. Longer numbers and amounts
# are not parsed, and dates are only searched for in the leading text, which
# bounds parsing time when a zone covers a large region.
MAX_PARSED_TEXT_LENGTH = 256
@dataclass(frozen=True)
class ZoneExtraction:
@@ -633,8 +638,19 @@ def _parse_date(text: str, rules: DateRules) -> date | None:
month name and year), and the first token dateparser accepts is used. Bare
numbers and two-digit years in surrounding text are never read as dates, so
version or article numbers like "1.2.34" are ignored. Dates before
MIN_ZONE_DATE_YEAR are treated as OCR noise.
MIN_ZONE_DATE_YEAR are treated as OCR noise. Only the first
MAX_PARSED_TEXT_LENGTH characters are read, without a word the cut splits.
"""
if len(text) > MAX_PARSED_TEXT_LENGTH:
head = text[:MAX_PARSED_TEXT_LENGTH]
if not text[MAX_PARSED_TEXT_LENGTH].isspace():
# Drop the word split by the cut, so a truncated token cannot match
# as a different date.
kept = re.match(r".*\s", head, re.DOTALL)
head = kept.group() if kept else ""
text = head
if not text.strip():
return None
return next(
(
candidate
@@ -779,8 +795,11 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None:
when it comes before the first digit and does not directly follow a letter
or digit, so "-5", "EUR -5" and "$-5" are negative while "INV-123" is
just 123. Any other dash makes the text invalid. Returns None when the
text is not a well-formed number in the locale.
text is not a well-formed number in the locale or is longer than
MAX_PARSED_TEXT_LENGTH.
"""
if len(value) > MAX_PARSED_TEXT_LENGTH:
return None
text = "".join(
str(unicodedata.digit(char))
if unicodedata.category(char) == UNICODE_DECIMAL_DIGIT
@@ -793,8 +812,15 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None:
.replace("\N{ARABIC DECIMAL SEPARATOR}", ".")
.replace("\N{ARABIC THOUSANDS SEPARATOR}", ",")
)
sign = re.search(r"(?<!\w)-(?=\D*\d)", text)
negative = sign is not None and not re.search(r"\d", text[: sign.start()])
# Only a dash before the first digit can be a sign. Searching that prefix
# (instead of a lookahead for a later digit) keeps this linear.
first_digit = re.search(r"\d", text)
sign = (
re.search(r"(?<!\w)-", text[: first_digit.start()])
if first_digit is not None
else None
)
negative = sign is not None
if negative:
text = text[: sign.start()] + text[sign.end() :]
if re.search(r"\d\D*-|(?<!\w)-", text):
@@ -866,9 +892,8 @@ def _parse_monetary(value: str, locale: Locale) -> str | None:
- The currency is an ISO 4217 code written as an uppercase word, or else a
currency symbol of the locale that belongs to a single currency. Text
naming two different currencies is rejected, and text naming none gives
an amount without a code. Zone crops are expected to be short, but an
uppercase word that happens to be an ISO code (e.g. "ALL" or "TOP") is
still read as a currency.
an amount without a code. Any uppercase word that is an ISO code
(e.g. "ALL" or "TOP") is read as a currency.
- A number enclosed in parentheses is negative, "(12.50)". Digits outside
the parentheses, as in "100 (19% VAT)", or a minus sign combined with
them are rejected.
@@ -877,9 +902,19 @@ def _parse_monetary(value: str, locale: Locale) -> str | None:
most 12 digits, with or without a currency code.
- The result must pass the same validation as values sent through the API.
Returns None when the text is not a valid monetary value.
Returns None when the text is not a valid monetary value or is longer than
MAX_PARSED_TEXT_LENGTH.
"""
group = re.search(r"\([^()]*\d[^()]*\)", value)
if len(value) > MAX_PARSED_TEXT_LENGTH:
return None
group = next(
(
match
for match in re.finditer(r"\([^()]*\)", value)
if re.search(r"\d", match.group())
),
None,
)
parenthesised = group is not None
text = value
if group is not None: