mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-20 04:26:53 +00:00
Fix: keep zone number parsing linear and cap its input
Two patterns backtracked quadratically on long OCR text, taking seconds on a large zone. Rewrite both, skip numbers and amounts longer than 256 characters, and search only the first 256 characters for a date.
This commit is contained in:
1 parent
6fc4526947
commit
3504ad07c6
2 files changed
+176
-9
No files matched your search
@@ -6,6 +6,7 @@ import shutil
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
@@ -22,6 +23,7 @@ from documents.models import MatchingModel
|
||||
from documents.models import OcrTemplate
|
||||
from documents.models import OcrTemplateZone
|
||||
from documents.parsers import ParseError
|
||||
from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH
|
||||
from documents.zone_ocr import _apply_transform
|
||||
from documents.zone_ocr import _combine_field_value
|
||||
from documents.zone_ocr import _convert_value
|
||||
@@ -32,6 +34,7 @@ from documents.zone_ocr import _extract_zone
|
||||
from documents.zone_ocr import _number_locale
|
||||
from documents.zone_ocr import _parse_date
|
||||
from documents.zone_ocr import _parse_monetary
|
||||
from documents.zone_ocr import _parse_number
|
||||
from documents.zone_ocr import _prepare_field_update
|
||||
from documents.zone_ocr import _process_template
|
||||
from documents.zone_ocr import _render_pages
|
||||
@@ -2371,6 +2374,135 @@ class TestParseMonetary:
|
||||
assert _parse_monetary(value, locale) == expected
|
||||
|
||||
|
||||
class TestParsedTextLength:
|
||||
"""Tests for the length cap on text read as a number, amount or date."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("parse", "value", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
lambda text: _parse_number(text, EN),
|
||||
"12.50",
|
||||
Decimal("12.50"),
|
||||
id="number",
|
||||
),
|
||||
pytest.param(
|
||||
lambda text: _parse_monetary(text, EN),
|
||||
"USD 12.50",
|
||||
"USD12.50",
|
||||
id="monetary",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_text_longer_than_the_cap_is_not_read(
|
||||
self,
|
||||
parse: Callable[[str], object],
|
||||
value: str,
|
||||
expected: object,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A valid number or amount padded with spaces to exactly the length
|
||||
cap, and the same value padded one character past it
|
||||
WHEN:
|
||||
- Each text is parsed
|
||||
THEN:
|
||||
- The text at the cap is read and the longer text is not
|
||||
"""
|
||||
at_cap = value.rjust(MAX_PARSED_TEXT_LENGTH)
|
||||
|
||||
assert parse(at_cap) == expected
|
||||
assert parse(f" {at_cap}") is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"Datum: 13.04.2026 " + "x" * 300,
|
||||
date(2026, 4, 13),
|
||||
id="date-near-the-start",
|
||||
),
|
||||
pytest.param(
|
||||
"x" * 300 + " Datum: 13.04.2026",
|
||||
None,
|
||||
id="date-past-the-cap",
|
||||
),
|
||||
pytest.param(
|
||||
"x " * 118 + "xxxx Date 2026-04-131 more",
|
||||
None,
|
||||
id="token-split-by-the-cut",
|
||||
),
|
||||
pytest.param(
|
||||
"x" * 246 + "2026-04-13" + "1" * 50,
|
||||
None,
|
||||
id="cut-inside-text-without-spaces",
|
||||
),
|
||||
pytest.param(
|
||||
"x " * 118 + "xxxx Date 2026-04-13 more",
|
||||
date(2026, 4, 13),
|
||||
id="cut-on-a-space-keeps-the-last-word",
|
||||
),
|
||||
pytest.param(" " * 300, None, id="whitespace-only"),
|
||||
],
|
||||
)
|
||||
def test_long_text_is_searched_for_dates_up_to_the_cap(
|
||||
self,
|
||||
text: str,
|
||||
expected: date | None,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Text longer than the cap with a date before it, after it, or with a
|
||||
non-date token that the cut would turn into a date
|
||||
WHEN:
|
||||
- The text is parsed as a date
|
||||
THEN:
|
||||
- Only whole words within the cap are searched
|
||||
"""
|
||||
assert _parse_date(text, ENG_DATES) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("parse", "value"),
|
||||
[
|
||||
pytest.param(
|
||||
lambda text: _parse_number(text, EN),
|
||||
"- " * 128,
|
||||
id="number-dashes-without-digit",
|
||||
),
|
||||
pytest.param(
|
||||
lambda text: _parse_number(text, EN),
|
||||
"- " * 127 + "1",
|
||||
id="number-dashes-before-digit",
|
||||
),
|
||||
pytest.param(
|
||||
lambda text: _parse_monetary(text, EN),
|
||||
"- " * 128,
|
||||
id="monetary-dashes-without-digit",
|
||||
),
|
||||
pytest.param(
|
||||
lambda text: _parse_monetary(text, EN),
|
||||
"(" + "1 " * 127,
|
||||
id="monetary-unclosed-parenthesis",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_malformed_text_at_the_cap_is_rejected(
|
||||
self,
|
||||
parse: Callable[[str], object],
|
||||
value: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Malformed text at the length cap, made of dash runs or an
|
||||
unclosed parenthesis
|
||||
WHEN:
|
||||
- The text is parsed
|
||||
THEN:
|
||||
- No value is read
|
||||
"""
|
||||
assert parse(value) is None
|
||||
|
||||
|
||||
class TestConvertValue:
|
||||
"""Tests for the _convert_value function."""
|
||||
|
||||
|
||||
@@ -79,6 +79,11 @@ SEPARATOR_ONLY_RE = re.compile(r"[\s\-/.,;:|]*")
|
||||
# Earlier dates are far more likely OCR noise than a real document date.
|
||||
MIN_ZONE_DATE_YEAR = 1900
|
||||
|
||||
# Longest OCR text read as a number, amount or date. Longer numbers and amounts
|
||||
# are not parsed, and dates are only searched for in the leading text, which
|
||||
# bounds parsing time when a zone covers a large region.
|
||||
MAX_PARSED_TEXT_LENGTH = 256
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ZoneExtraction:
|
||||
@@ -633,8 +638,19 @@ def _parse_date(text: str, rules: DateRules) -> date | None:
|
||||
month name and year), and the first token dateparser accepts is used. Bare
|
||||
numbers and two-digit years in surrounding text are never read as dates, so
|
||||
version or article numbers like "1.2.34" are ignored. Dates before
|
||||
MIN_ZONE_DATE_YEAR are treated as OCR noise.
|
||||
MIN_ZONE_DATE_YEAR are treated as OCR noise. Only the first
|
||||
MAX_PARSED_TEXT_LENGTH characters are read, without a word the cut splits.
|
||||
"""
|
||||
if len(text) > MAX_PARSED_TEXT_LENGTH:
|
||||
head = text[:MAX_PARSED_TEXT_LENGTH]
|
||||
if not text[MAX_PARSED_TEXT_LENGTH].isspace():
|
||||
# Drop the word split by the cut, so a truncated token cannot match
|
||||
# as a different date.
|
||||
kept = re.match(r".*\s", head, re.DOTALL)
|
||||
head = kept.group() if kept else ""
|
||||
text = head
|
||||
if not text.strip():
|
||||
return None
|
||||
return next(
|
||||
(
|
||||
candidate
|
||||
@@ -779,8 +795,11 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None:
|
||||
when it comes before the first digit and does not directly follow a letter
|
||||
or digit, so "-5", "EUR -5" and "$-5" are negative while "INV-123" is
|
||||
just 123. Any other dash makes the text invalid. Returns None when the
|
||||
text is not a well-formed number in the locale.
|
||||
text is not a well-formed number in the locale or is longer than
|
||||
MAX_PARSED_TEXT_LENGTH.
|
||||
"""
|
||||
if len(value) > MAX_PARSED_TEXT_LENGTH:
|
||||
return None
|
||||
text = "".join(
|
||||
str(unicodedata.digit(char))
|
||||
if unicodedata.category(char) == UNICODE_DECIMAL_DIGIT
|
||||
@@ -793,8 +812,15 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None:
|
||||
.replace("\N{ARABIC DECIMAL SEPARATOR}", ".")
|
||||
.replace("\N{ARABIC THOUSANDS SEPARATOR}", ",")
|
||||
)
|
||||
sign = re.search(r"(?<!\w)-(?=\D*\d)", text)
|
||||
negative = sign is not None and not re.search(r"\d", text[: sign.start()])
|
||||
# Only a dash before the first digit can be a sign. Searching that prefix
|
||||
# (instead of a lookahead for a later digit) keeps this linear.
|
||||
first_digit = re.search(r"\d", text)
|
||||
sign = (
|
||||
re.search(r"(?<!\w)-", text[: first_digit.start()])
|
||||
if first_digit is not None
|
||||
else None
|
||||
)
|
||||
negative = sign is not None
|
||||
if negative:
|
||||
text = text[: sign.start()] + text[sign.end() :]
|
||||
if re.search(r"\d\D*-|(?<!\w)-", text):
|
||||
@@ -866,9 +892,8 @@ def _parse_monetary(value: str, locale: Locale) -> str | None:
|
||||
- The currency is an ISO 4217 code written as an uppercase word, or else a
|
||||
currency symbol of the locale that belongs to a single currency. Text
|
||||
naming two different currencies is rejected, and text naming none gives
|
||||
an amount without a code. Zone crops are expected to be short, but an
|
||||
uppercase word that happens to be an ISO code (e.g. "ALL" or "TOP") is
|
||||
still read as a currency.
|
||||
an amount without a code. Any uppercase word that is an ISO code
|
||||
(e.g. "ALL" or "TOP") is read as a currency.
|
||||
- A number enclosed in parentheses is negative, "(12.50)". Digits outside
|
||||
the parentheses, as in "100 (19% VAT)", or a minus sign combined with
|
||||
them are rejected.
|
||||
@@ -877,9 +902,19 @@ def _parse_monetary(value: str, locale: Locale) -> str | None:
|
||||
most 12 digits, with or without a currency code.
|
||||
- The result must pass the same validation as values sent through the API.
|
||||
|
||||
Returns None when the text is not a valid monetary value.
|
||||
Returns None when the text is not a valid monetary value or is longer than
|
||||
MAX_PARSED_TEXT_LENGTH.
|
||||
"""
|
||||
group = re.search(r"\([^()]*\d[^()]*\)", value)
|
||||
if len(value) > MAX_PARSED_TEXT_LENGTH:
|
||||
return None
|
||||
group = next(
|
||||
(
|
||||
match
|
||||
for match in re.finditer(r"\([^()]*\)", value)
|
||||
if re.search(r"\d", match.group())
|
||||
),
|
||||
None,
|
||||
)
|
||||
parenthesised = group is not None
|
||||
text = value
|
||||
if group is not None:
|
||||
|
||||
Reference in new issue
Block a user