mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-19 20:25:21 +00:00
Fix: parse OCR zone dates strictly using the zone's language
Dates ignored the document's day/month order, and words like "now" or a bare "5" were accepted as dates.
This commit is contained in:
1 parent
63a87dd11b
commit
acf7ba346c
2 files changed
+469
-38
No files matched your search
@@ -24,13 +24,16 @@ from documents.zone_ocr import _apply_transform
|
||||
from documents.zone_ocr import _combine_field_value
|
||||
from documents.zone_ocr import _convert_value
|
||||
from documents.zone_ocr import _crop_zone
|
||||
from documents.zone_ocr import _date_rules
|
||||
from documents.zone_ocr import _detect_mime
|
||||
from documents.zone_ocr import _extract_zone
|
||||
from documents.zone_ocr import _number_locale
|
||||
from documents.zone_ocr import _parse_date
|
||||
from documents.zone_ocr import _prepare_field_update
|
||||
from documents.zone_ocr import _process_template
|
||||
from documents.zone_ocr import _render_pages
|
||||
from documents.zone_ocr import _resolve_doc_path
|
||||
from documents.zone_ocr import _zone_language
|
||||
from documents.zone_ocr import extract_zone_preview
|
||||
from documents.zone_ocr import get_templates_for_document
|
||||
from documents.zone_ocr import run_zone_extraction
|
||||
@@ -54,6 +57,7 @@ if TYPE_CHECKING:
|
||||
|
||||
EN = Locale.parse("en")
|
||||
DE = Locale.parse("de")
|
||||
ENG_DATES = _date_rules("eng")
|
||||
FR = Locale.parse("fr")
|
||||
|
||||
|
||||
@@ -1449,12 +1453,14 @@ class TestApplyTransform:
|
||||
"""
|
||||
GIVEN:
|
||||
- Extracted text, a transform name and an optional date format
|
||||
- A zone reading English
|
||||
WHEN:
|
||||
- The transform is applied
|
||||
THEN:
|
||||
- The transformed text matches the expected result
|
||||
"""
|
||||
assert _apply_transform(text, transform, date_format) == expected
|
||||
rules = _date_rules("eng", date_format)
|
||||
assert _apply_transform(text, transform, rules) == expected
|
||||
|
||||
def test_unparseable_date_preserves_text(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
@@ -1466,7 +1472,278 @@ class TestApplyTransform:
|
||||
- The original text is returned unchanged
|
||||
"""
|
||||
mocker.patch("documents.zone_ocr.dateparser.parse", side_effect=OverflowError)
|
||||
assert _apply_transform("unparsable", "date") == "unparsable"
|
||||
assert (
|
||||
_apply_transform("unparsable", "date", _date_rules("eng")) == "unparsable"
|
||||
)
|
||||
|
||||
|
||||
class TestZoneLanguage:
|
||||
"""Tests for mapping the OCR language to a dateparser language."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ocr_language", "expected"),
|
||||
[
|
||||
pytest.param("deu", "de", id="single-language"),
|
||||
pytest.param("deu+eng", "de", id="first-of-several-languages"),
|
||||
pytest.param("aze_Cyrl", "az", id="script-suffix-is-dropped"),
|
||||
pytest.param("xyz", None, id="unmapped-language-is-none"),
|
||||
],
|
||||
)
|
||||
def test_zone_language(self, ocr_language: str, expected: str | None) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A tesseract language code, possibly several or with a script
|
||||
WHEN:
|
||||
- The zone language is derived
|
||||
THEN:
|
||||
- The first language is mapped, or None when it is unknown
|
||||
"""
|
||||
assert _zone_language(ocr_language) == expected
|
||||
|
||||
|
||||
class TestParseDate:
|
||||
"""Tests for the zone date parser."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "ocr_language", "expected"),
|
||||
[
|
||||
pytest.param("12.05.2024", "deu", date(2024, 5, 12), id="german-day-first"),
|
||||
pytest.param(
|
||||
"05/12/2024",
|
||||
"eng",
|
||||
date(2024, 5, 12),
|
||||
id="english-month-first",
|
||||
),
|
||||
pytest.param(
|
||||
"12. Mai 2024",
|
||||
"deu",
|
||||
date(2024, 5, 12),
|
||||
id="german-month-name",
|
||||
),
|
||||
pytest.param(
|
||||
"13 April 2026",
|
||||
"eng",
|
||||
date(2026, 4, 13),
|
||||
id="english-month-name",
|
||||
),
|
||||
pytest.param(
|
||||
"2026/04/13",
|
||||
"eng",
|
||||
date(2026, 4, 13),
|
||||
id="year-first-slashed",
|
||||
),
|
||||
pytest.param(
|
||||
"01.01.2999",
|
||||
"deu",
|
||||
date(2999, 1, 1),
|
||||
id="future-date-allowed",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_locale_decides_the_order(
|
||||
self,
|
||||
text: str,
|
||||
ocr_language: str,
|
||||
expected: date,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A date written in the convention of the zone's language
|
||||
WHEN:
|
||||
- The date is parsed
|
||||
THEN:
|
||||
- The locale's day and month order is used
|
||||
"""
|
||||
assert _parse_date(text, _date_rules(ocr_language)) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("date_order", "expected"),
|
||||
[
|
||||
pytest.param("DMY", date(2024, 5, 12), id="day-first"),
|
||||
pytest.param("MDY", date(2024, 12, 5), id="month-first"),
|
||||
],
|
||||
)
|
||||
def test_unmapped_language_uses_the_configured_date_order(
|
||||
self,
|
||||
settings: SettingsWrapper,
|
||||
date_order: str,
|
||||
expected: date,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A zone whose language has no dateparser mapping
|
||||
- A configured PAPERLESS_DATE_ORDER
|
||||
WHEN:
|
||||
- An ambiguous numeric date is parsed
|
||||
THEN:
|
||||
- The configured date order decides the result
|
||||
"""
|
||||
settings.DATE_ORDER = date_order
|
||||
assert _parse_date("12.05.2024", _date_rules("xyz")) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
pytest.param("now", id="now"),
|
||||
pytest.param("yesterday", id="yesterday"),
|
||||
pytest.param("today", id="today"),
|
||||
pytest.param("in 2 days", id="relative-offset"),
|
||||
pytest.param("Mon", id="weekday"),
|
||||
pytest.param("March", id="month-only"),
|
||||
pytest.param("2024", id="year-only"),
|
||||
pytest.param("5", id="day-only"),
|
||||
pytest.param("12", id="two-digits"),
|
||||
pytest.param("not a date", id="words"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"ocr_language",
|
||||
[
|
||||
pytest.param("eng", id="mapped-language"),
|
||||
pytest.param("xyz", id="unmapped-language"),
|
||||
],
|
||||
)
|
||||
def test_non_dates_are_rejected(self, text: str, ocr_language: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Text that is not a complete calendar date
|
||||
WHEN:
|
||||
- The date is parsed
|
||||
THEN:
|
||||
- No date is returned
|
||||
"""
|
||||
assert _parse_date(text, _date_rules(ocr_language)) is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "date_format", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"12.05.2024",
|
||||
"%m.%d.%Y",
|
||||
date(2024, 12, 5),
|
||||
id="format-overrides-locale-order",
|
||||
),
|
||||
pytest.param(
|
||||
"12.05.2024",
|
||||
"%Y/%m/%d",
|
||||
date(2024, 5, 12),
|
||||
id="invalid-format-falls-back",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_date_format_is_tried_first(
|
||||
self,
|
||||
text: str,
|
||||
date_format: str,
|
||||
expected: date,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A German zone with a date format
|
||||
WHEN:
|
||||
- A date is parsed
|
||||
THEN:
|
||||
- A matching format is used, otherwise parsing falls back to the locale
|
||||
"""
|
||||
assert _parse_date(text, _date_rules("deu", date_format)) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "expected"),
|
||||
[
|
||||
pytest.param("2024-05-06", date(2024, 5, 6), id="plain-iso"),
|
||||
pytest.param("Due 2024-05-06.", date(2024, 5, 6), id="iso-inside-text"),
|
||||
pytest.param("No. 12345-06-07", None, id="digit-before-is-not-iso"),
|
||||
pytest.param("2024-05-061", None, id="digit-after-is-not-iso"),
|
||||
pytest.param(
|
||||
"2024-13-45 and 2024-05-06",
|
||||
date(2024, 5, 6),
|
||||
id="invalid-match-is-skipped",
|
||||
),
|
||||
pytest.param("2024-13-45", None, id="only-invalid-match"),
|
||||
],
|
||||
)
|
||||
def test_iso_dates_are_anchored(self, text: str, expected: date | None) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Text containing ISO-like dates, some invalid or part of longer numbers
|
||||
WHEN:
|
||||
- The date is parsed
|
||||
THEN:
|
||||
- Only whole, valid ISO dates are returned, trying every match
|
||||
"""
|
||||
assert _parse_date(text, _date_rules("eng")) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
pytest.param("2026/04/13", id="slashed"),
|
||||
pytest.param("2026.04.13", id="dotted"),
|
||||
pytest.param("2026-04-13", id="dashed"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("ocr_language", "date_order"),
|
||||
[
|
||||
pytest.param("xyz", "DMY", id="unmapped-language-day-first"),
|
||||
pytest.param("xyz", "MDY", id="unmapped-language-month-first"),
|
||||
pytest.param("deu", "DMY", id="mapped-language"),
|
||||
],
|
||||
)
|
||||
def test_year_first_dates_ignore_the_date_order(
|
||||
self,
|
||||
settings: SettingsWrapper,
|
||||
text: str,
|
||||
ocr_language: str,
|
||||
date_order: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A year-first date separated by a dash, slash or dot
|
||||
- A mapped or unmapped zone language and a configured date order
|
||||
WHEN:
|
||||
- The date is parsed
|
||||
THEN:
|
||||
- The date is read as year, month, day regardless of the order
|
||||
"""
|
||||
settings.DATE_ORDER = date_order
|
||||
assert _parse_date(text, _date_rules(ocr_language)) == date(2026, 4, 13)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
pytest.param("1.2026/04/133", id="digit-after"),
|
||||
pytest.param("12026/04/13", id="digit-before"),
|
||||
pytest.param("2026/13/45", id="invalid-month"),
|
||||
],
|
||||
)
|
||||
def test_year_first_dates_are_anchored_and_validated(self, text: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Year-first looking text that is part of longer digits or invalid
|
||||
WHEN:
|
||||
- The date is parsed
|
||||
THEN:
|
||||
- No date is returned
|
||||
"""
|
||||
assert _parse_date(text, _date_rules("xyz")) is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
pytest.param("1899-12-31", id="iso"),
|
||||
pytest.param("31.12.1899", id="locale-numeric"),
|
||||
],
|
||||
)
|
||||
def test_years_before_1900_are_rejected(self, text: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A valid calendar date before the year 1900
|
||||
WHEN:
|
||||
- The date is parsed
|
||||
THEN:
|
||||
- No date is returned
|
||||
"""
|
||||
assert _parse_date(text, _date_rules("deu")) is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -1516,6 +1793,82 @@ class TestPrepareFieldUpdate:
|
||||
|
||||
assert (update.value if update else None) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
pytest.param(OcrTemplateZone.TargetType.CREATED, id="created"),
|
||||
pytest.param(OcrTemplateZone.TargetType.CUSTOM_FIELD, id="date-field"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("ocr_language", "date_format", "text", "expected"),
|
||||
[
|
||||
pytest.param("deu", "", "12.05.2024", date(2024, 5, 12), id="german-order"),
|
||||
pytest.param(
|
||||
"eng",
|
||||
"",
|
||||
"05/12/2024",
|
||||
date(2024, 5, 12),
|
||||
id="english-order",
|
||||
),
|
||||
pytest.param(
|
||||
"deu",
|
||||
"",
|
||||
"12. Mai 2024",
|
||||
date(2024, 5, 12),
|
||||
id="german-month-name",
|
||||
),
|
||||
pytest.param(
|
||||
"deu",
|
||||
"%m.%d.%Y",
|
||||
"12.05.2024",
|
||||
date(2024, 12, 5),
|
||||
id="zone-date-format",
|
||||
),
|
||||
pytest.param(
|
||||
"deu",
|
||||
"%Y/%m/%d",
|
||||
"12.05.2024",
|
||||
date(2024, 5, 12),
|
||||
id="invalid-format-falls-back",
|
||||
),
|
||||
pytest.param("eng", "", "2024", None, id="year-only-is-rejected"),
|
||||
],
|
||||
)
|
||||
def test_date_targets_parse_alike(
|
||||
self,
|
||||
target: str,
|
||||
ocr_language: str,
|
||||
date_format: str,
|
||||
text: str,
|
||||
expected: date | None,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A zone targeting the created date or a date custom field
|
||||
- An OCR language and optionally a date format
|
||||
WHEN:
|
||||
- A field update is prepared for non-ISO date text
|
||||
THEN:
|
||||
- Both targets read the text the same way, honoring the zone's
|
||||
language and date format
|
||||
"""
|
||||
field = (
|
||||
CustomFieldFactory(data_type=CustomField.FieldDataType.DATE)
|
||||
if target == OcrTemplateZone.TargetType.CUSTOM_FIELD
|
||||
else None
|
||||
)
|
||||
zone = OcrTemplateZoneFactory(
|
||||
target=target,
|
||||
custom_field=field,
|
||||
ocr_language=ocr_language,
|
||||
date_format=date_format,
|
||||
)
|
||||
|
||||
update = _prepare_field_update(zone, text)
|
||||
|
||||
assert (update.value if update else None) == expected
|
||||
|
||||
|
||||
class TestNumberLocale:
|
||||
"""Tests for deriving a number locale from the OCR language."""
|
||||
@@ -1775,7 +2128,7 @@ class TestConvertValue:
|
||||
THEN:
|
||||
- The converted value equals the expected value exactly
|
||||
"""
|
||||
assert _convert_value(value, data_type, locale) == expected
|
||||
assert _convert_value(value, data_type, locale, ENG_DATES) == expected
|
||||
|
||||
def test_string_truncation(self) -> None:
|
||||
"""
|
||||
@@ -1786,7 +2139,12 @@ class TestConvertValue:
|
||||
THEN:
|
||||
- The result is truncated to 128 characters
|
||||
"""
|
||||
result = _convert_value("x" * 200, CustomField.FieldDataType.STRING, EN)
|
||||
result = _convert_value(
|
||||
"x" * 200,
|
||||
CustomField.FieldDataType.STRING,
|
||||
EN,
|
||||
ENG_DATES,
|
||||
)
|
||||
assert isinstance(result, str)
|
||||
assert len(result) == 128
|
||||
|
||||
@@ -1829,7 +2187,8 @@ class TestConvertValue:
|
||||
- The result is the expected float
|
||||
"""
|
||||
assert (
|
||||
_convert_value(value, CustomField.FieldDataType.FLOAT, locale) == expected
|
||||
_convert_value(value, CustomField.FieldDataType.FLOAT, locale, ENG_DATES)
|
||||
== expected
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -1984,7 +2343,7 @@ class TestConvertValue:
|
||||
THEN:
|
||||
- None is returned
|
||||
"""
|
||||
assert _convert_value(value, data_type, locale) is None
|
||||
assert _convert_value(value, data_type, locale, ENG_DATES) is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data_type",
|
||||
@@ -2018,7 +2377,7 @@ class TestConvertValue:
|
||||
THEN:
|
||||
- None is returned
|
||||
"""
|
||||
assert _convert_value(value, data_type, EN) is None
|
||||
assert _convert_value(value, data_type, EN, ENG_DATES) is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
@@ -2041,7 +2400,9 @@ class TestConvertValue:
|
||||
THEN:
|
||||
- The result is True
|
||||
"""
|
||||
assert _convert_value(value, CustomField.FieldDataType.BOOL, EN) is True
|
||||
assert (
|
||||
_convert_value(value, CustomField.FieldDataType.BOOL, EN, ENG_DATES) is True
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
@@ -2062,7 +2423,10 @@ class TestConvertValue:
|
||||
THEN:
|
||||
- The result is False
|
||||
"""
|
||||
assert _convert_value(value, CustomField.FieldDataType.BOOL, EN) is False
|
||||
assert (
|
||||
_convert_value(value, CustomField.FieldDataType.BOOL, EN, ENG_DATES)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
class TestRenderPages:
|
||||
|
||||
+96
-29
@@ -44,6 +44,9 @@ from documents.versioning import get_root_document
|
||||
from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
logger = logging.getLogger("paperless.zone_ocr")
|
||||
@@ -51,6 +54,15 @@ logger = logging.getLogger("paperless.zone_ocr")
|
||||
# Unicode general category for decimal digits in any script.
|
||||
UNICODE_DECIMAL_DIGIT = "Nd" # codespell:ignore
|
||||
|
||||
# Year-first dates (ISO and its slashed or dotted variants), never a slice of a
|
||||
# longer digit run. They are unambiguous, so no date order is needed.
|
||||
YEAR_FIRST_DATE_RE = re.compile(
|
||||
r"(?<!\d)(\d{4})([-/.])(\d{2})\2(\d{2})(?!\d)",
|
||||
)
|
||||
|
||||
# Earlier dates are far more likely OCR noise than a real document date.
|
||||
MIN_ZONE_DATE_YEAR = 1900
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ZoneExtraction:
|
||||
@@ -482,7 +494,11 @@ def _extract_zone(
|
||||
cropped.close()
|
||||
return ZoneExtraction(
|
||||
raw_text=raw_text,
|
||||
value=_apply_transform(raw_text, zone.transform, zone.date_format)
|
||||
value=_apply_transform(
|
||||
raw_text,
|
||||
zone.transform,
|
||||
_date_rules(zone.ocr_language, zone.date_format),
|
||||
)
|
||||
if raw_text
|
||||
else None,
|
||||
)
|
||||
@@ -513,40 +529,74 @@ def extract_zone_preview(
|
||||
return {"raw_text": result.raw_text, "value": result.value}
|
||||
|
||||
|
||||
def _parse_iso_date(text: str) -> date | None:
|
||||
match = re.search(r"\d{4}-\d{2}-\d{2}", text)
|
||||
return date.fromisoformat(match[0]) if match else None
|
||||
@dataclass(frozen=True)
|
||||
class DateRules:
|
||||
"""How a zone reads dates: its language and optional strptime format."""
|
||||
|
||||
language: str | None = None
|
||||
date_format: str = ""
|
||||
|
||||
|
||||
def _parse_date(text: str, fmt: str = "") -> date | None:
|
||||
"""Try the zone's date format, ISO dates, then automatic date detection."""
|
||||
if fmt:
|
||||
def _date_rules(ocr_language: str, date_format: str = "") -> DateRules:
|
||||
return DateRules(_zone_language(ocr_language), date_format)
|
||||
|
||||
|
||||
def _candidate_dates(text: str, rules: DateRules) -> Iterator[date]:
|
||||
"""Yield the dates the text may hold, most specific reading first."""
|
||||
if rules.date_format:
|
||||
try:
|
||||
return datetime.strptime(text, fmt).date()
|
||||
yield datetime.strptime(text, rules.date_format).date()
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
iso_date = _parse_iso_date(text)
|
||||
except ValueError:
|
||||
iso_date = None
|
||||
if iso_date is not None:
|
||||
return iso_date
|
||||
for match in YEAR_FIRST_DATE_RE.finditer(text):
|
||||
year, _, month, day = match.groups()
|
||||
try:
|
||||
yield date(int(year), int(month), int(day))
|
||||
except ValueError:
|
||||
continue
|
||||
# Only absolute dates: dateparser would also read "now" or "yesterday"
|
||||
date_settings: dict[str, Any] = {
|
||||
"PARSERS": ["custom-formats", "absolute-time"],
|
||||
"REQUIRE_PARTS": ["day", "month", "year"],
|
||||
"RETURN_AS_TIMEZONE_AWARE": False,
|
||||
}
|
||||
if rules.language is None:
|
||||
date_settings["DATE_ORDER"] = settings.DATE_ORDER
|
||||
try:
|
||||
parsed = dateparser.parse(
|
||||
text,
|
||||
settings={
|
||||
"PREFER_DAY_OF_MONTH": "first",
|
||||
"RETURN_AS_TIMEZONE_AWARE": False,
|
||||
},
|
||||
languages=None if rules.language is None else [rules.language],
|
||||
settings=date_settings,
|
||||
)
|
||||
if parsed:
|
||||
return parsed.date()
|
||||
except (ValueError, OverflowError):
|
||||
logger.debug("Zone OCR: date parsing failed")
|
||||
return None
|
||||
return
|
||||
if parsed:
|
||||
yield parsed.date()
|
||||
|
||||
|
||||
def _apply_transform(text: str, transform: str, date_format: str = "") -> str:
|
||||
def _parse_date(text: str, rules: DateRules) -> date | None:
|
||||
"""Read a calendar date from OCR text.
|
||||
|
||||
Tries the zone's date format, year-first dates, then dateparser in the zone's
|
||||
language (or the configured date order when the language is unknown).
|
||||
Dates before MIN_ZONE_DATE_YEAR are treated as OCR noise.
|
||||
"""
|
||||
return next(
|
||||
(
|
||||
candidate
|
||||
for candidate in _candidate_dates(text, rules)
|
||||
if candidate.year >= MIN_ZONE_DATE_YEAR
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _apply_transform(
|
||||
text: str,
|
||||
transform: str,
|
||||
dates: DateRules,
|
||||
) -> str:
|
||||
"""Apply post-processing transform to extracted text."""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
@@ -555,7 +605,7 @@ def _apply_transform(text: str, transform: str, date_format: str = "") -> str:
|
||||
if transform in ("strip", "none"):
|
||||
return text
|
||||
elif transform == "date":
|
||||
parsed = _parse_date(text, date_format)
|
||||
parsed = _parse_date(text, dates)
|
||||
return parsed.isoformat() if parsed else text
|
||||
elif transform == "uppercase":
|
||||
return text.upper()
|
||||
@@ -584,6 +634,7 @@ def _prepare_field_update(
|
||||
value,
|
||||
custom_field.data_type,
|
||||
_number_locale(zone.ocr_language),
|
||||
_date_rules(zone.ocr_language, zone.date_format),
|
||||
)
|
||||
model = CustomFieldInstance
|
||||
elif zone.target == OcrTemplateZone.TargetType.TITLE:
|
||||
@@ -595,7 +646,13 @@ def _prepare_field_update(
|
||||
typed_value = int(digits) if digits else None
|
||||
model = Document
|
||||
elif zone.target == OcrTemplateZone.TargetType.CREATED:
|
||||
field_name, typed_value = "created", _parse_date(value)
|
||||
field_name, typed_value = (
|
||||
"created",
|
||||
_parse_date(
|
||||
value,
|
||||
_date_rules(zone.ocr_language, zone.date_format),
|
||||
),
|
||||
)
|
||||
model = Document
|
||||
else:
|
||||
return None
|
||||
@@ -636,14 +693,22 @@ def _apply_field_updates(document: Document, updates: list[FieldUpdate]) -> None
|
||||
raise
|
||||
|
||||
|
||||
def _zone_language(ocr_language: str) -> str | None:
|
||||
"""Map the first tesseract language of a zone to a dateparser/Babel code.
|
||||
|
||||
Only the first language counts, so "deu+eng" is German. Returns None when
|
||||
the language is not mapped.
|
||||
"""
|
||||
tesseract_code = ocr_language.split("+", maxsplit=1)[0].split("_", maxsplit=1)[0]
|
||||
return OCR_TO_DATEPARSER_LANGUAGES.get(tesseract_code)
|
||||
|
||||
|
||||
def _number_locale(ocr_language: str) -> Locale:
|
||||
"""Derive the locale used to read numbers from a tesseract language code.
|
||||
|
||||
Only the first language counts, so "deu+eng" reads numbers the German way.
|
||||
Languages Babel does not know fall back to English.
|
||||
"""
|
||||
tesseract_code = ocr_language.split("+", maxsplit=1)[0].split("_", maxsplit=1)[0]
|
||||
code = OCR_TO_DATEPARSER_LANGUAGES.get(tesseract_code)
|
||||
code = _zone_language(ocr_language)
|
||||
if code is not None:
|
||||
try:
|
||||
return Locale.parse(code)
|
||||
@@ -700,10 +765,12 @@ def _convert_value(
|
||||
value: str,
|
||||
data_type: str,
|
||||
locale: Locale,
|
||||
dates: DateRules,
|
||||
) -> object | None:
|
||||
"""Convert an extracted OCR string to the appropriate type for the custom field.
|
||||
|
||||
Numeric types are read with the number format of the given locale.
|
||||
Numeric types are read with the number format of the given locale and
|
||||
dates with the given date rules.
|
||||
"""
|
||||
if not value:
|
||||
return None
|
||||
@@ -732,7 +799,7 @@ def _convert_value(
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
elif data_type == CustomField.FieldDataType.DATE:
|
||||
parsed = _parse_iso_date(value)
|
||||
parsed = _parse_date(value, dates)
|
||||
return parsed.isoformat() if parsed else None
|
||||
|
||||
elif data_type == CustomField.FieldDataType.MONETARY:
|
||||
|
||||
Reference in new issue
Block a user