diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index f50e5871f2..2a5295f4ea 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING from typing import Any import pytest +from babel import Locale from django.contrib.auth.models import Group from PIL import Image @@ -25,6 +26,7 @@ from documents.zone_ocr import _convert_value from documents.zone_ocr import _crop_zone from documents.zone_ocr import _detect_mime from documents.zone_ocr import _extract_zone +from documents.zone_ocr import _number_locale from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages @@ -50,6 +52,11 @@ if TYPE_CHECKING: from paperless_testing.dirs import PaperlessDirs +EN = Locale.parse("en") +DE = Locale.parse("de") +FR = Locale.parse("fr") + + def make_template(name: str, **kwargs: Any) -> OcrTemplate: """Build a saved template, 100x50 by default.""" kwargs.setdefault("source_width", 100) @@ -1462,6 +1469,7 @@ class TestApplyTransform: assert _apply_transform("unparsable", "date") == "unparsable" +@pytest.mark.django_db class TestPrepareFieldUpdate: """Tests for converting extracted values into field updates.""" @@ -1476,69 +1484,280 @@ class TestPrepareFieldUpdate: """ assert _prepare_field_update(OcrTemplateZone(target="unknown"), "value") is None + @pytest.mark.parametrize( + ("ocr_language", "expected"), + [ + pytest.param("deu", 1234, id="german-reads-dot-as-thousands"), + pytest.param("eng", None, id="english-reads-dot-as-fraction"), + ], + ) + def test_int_field_uses_the_zone_language_for_numbers( + self, + ocr_language: str, + expected: int | None, + ) -> None: + """ + GIVEN: + - A zone targeting an integer custom field with a given OCR language + WHEN: + - A field update is prepared for the text "1.234" + THEN: + - A German zone reads 1234, while an English zone reads a fraction + and produces no update + """ + field = CustomFieldFactory(data_type=CustomField.FieldDataType.INT) + zone = OcrTemplateZoneFactory( + target=OcrTemplateZone.TargetType.CUSTOM_FIELD, + custom_field=field, + ocr_language=ocr_language, + ) + + update = _prepare_field_update(zone, "1.234") + + assert (update.value if update else None) == expected + + +class TestNumberLocale: + """Tests for deriving a number locale from the OCR language.""" + + @pytest.mark.parametrize( + ("ocr_language", "expected"), + [ + pytest.param("deu", "de", id="single-language"), + pytest.param("deu+eng", "de", id="first-of-several-languages"), + pytest.param("aze_Cyrl", "az", id="script-suffix-is-dropped"), + pytest.param("xyz", "en", id="unknown-language-falls-back-to-english"), + ], + ) + def test_number_locale(self, ocr_language: str, expected: str) -> None: + """ + GIVEN: + - A tesseract language code, possibly several or with a script + WHEN: + - The number locale is derived + THEN: + - The first language decides the locale, falling back to English + """ + assert _number_locale(ocr_language) == Locale.parse(expected) + class TestConvertValue: """Tests for the _convert_value function.""" @pytest.mark.parametrize( - ("value", "data_type", "expected"), + ("value", "data_type", "expected", "locale"), [ pytest.param( "Hello", CustomField.FieldDataType.STRING, "Hello", + EN, id="string", ), pytest.param( "https://example.com", CustomField.FieldDataType.URL, "https://example.com", + EN, id="url", ), pytest.param( "x" * 500, CustomField.FieldDataType.LONG_TEXT, "x" * 500, + EN, id="long-text", ), - pytest.param("42", CustomField.FieldDataType.INT, 42, id="int-simple"), + pytest.param("42", CustomField.FieldDataType.INT, 42, EN, id="int-simple"), pytest.param( "INV-123", CustomField.FieldDataType.INT, 123, + EN, id="int-with-noise", ), - pytest.param("-42", CustomField.FieldDataType.INT, -42, id="int-negative"), + pytest.param( + "-42", + CustomField.FieldDataType.INT, + -42, + EN, + id="int-negative", + ), + pytest.param( + "\N{MINUS SIGN}5", + CustomField.FieldDataType.INT, + -5, + EN, + id="int-unicode-minus", + ), + pytest.param( + "1,234", + CustomField.FieldDataType.INT, + 1234, + EN, + id="int-comma-thousands", + ), + pytest.param( + "1.234.567", + CustomField.FieldDataType.INT, + 1234567, + DE, + id="int-dot-thousands", + ), + pytest.param( + "12.00", + CustomField.FieldDataType.INT, + 12, + EN, + id="int-zero-fraction", + ), + pytest.param( + "1.234,00", + CustomField.FieldDataType.INT, + 1234, + DE, + id="int-thousands-and-zero-fraction", + ), + pytest.param( + "1234.00", + CustomField.FieldDataType.INT, + 1234, + EN, + id="int-ungrouped-with-zero-fraction", + ), + pytest.param( + "1.234", + CustomField.FieldDataType.INT, + 1234, + DE, + id="int-dot-is-thousands-in-german", + ), + pytest.param( + "1\N{NARROW NO-BREAK SPACE}234", + CustomField.FieldDataType.INT, + 1234, + FR, + id="int-narrow-space-thousands-in-french", + ), pytest.param( "2026-04-13", CustomField.FieldDataType.DATE, "2026-04-13", + EN, id="date-iso", ), pytest.param( "123.45", CustomField.FieldDataType.MONETARY, "123.45", + EN, id="monetary-simple", ), pytest.param( "1.234,56", CustomField.FieldDataType.MONETARY, "1234.56", + DE, id="monetary-european", ), pytest.param( "\u20ac1,234.56", CustomField.FieldDataType.MONETARY, "1234.56", + EN, id="monetary-with-currency-symbol", ), pytest.param( "12345678901234567890,12345678901234567890", CustomField.FieldDataType.MONETARY, "12345678901234567890.12345678901234567890", + DE, id="monetary-preserves-decimal-precision", ), + pytest.param( + "-12345678901234567890,12345678901234567890", + CustomField.FieldDataType.MONETARY, + "-12345678901234567890.12345678901234567890", + DE, + id="monetary-negative-preserves-decimal-precision", + ), + pytest.param( + "\u20ac-12.50", + CustomField.FieldDataType.MONETARY, + "-12.50", + EN, + id="monetary-sign-after-currency-symbol", + ), + pytest.param( + "-\u20ac12.50", + CustomField.FieldDataType.MONETARY, + "-12.50", + EN, + id="monetary-sign-before-currency-symbol", + ), + pytest.param( + "EUR -12.50", + CustomField.FieldDataType.MONETARY, + "-12.50", + EN, + id="monetary-sign-after-currency-code", + ), + pytest.param( + "Total: -5", + CustomField.FieldDataType.INT, + -5, + EN, + id="int-sign-after-label", + ), + pytest.param( + "\u20ac-12.50", + CustomField.FieldDataType.FLOAT, + -12.5, + EN, + id="float-sign-after-currency-symbol", + ), + pytest.param( + "\u0661\u0662\u0663", + CustomField.FieldDataType.INT, + 123, + Locale.parse("ar"), + id="int-arabic-indic-digits", + ), + pytest.param( + "\u0661\u066c\u0662\u0663\u0664\u066b\u0665", + CustomField.FieldDataType.FLOAT, + 1234.5, + Locale.parse("ar"), + id="float-arabic-separators", + ), + pytest.param( + "\u06f1\u06f2\u06f3", + CustomField.FieldDataType.INT, + 123, + Locale.parse("fa"), + id="int-persian-digits", + ), + pytest.param( + "\u0967\u0968\u0969", + CustomField.FieldDataType.INT, + 123, + Locale.parse("hi"), + id="int-devanagari-digits", + ), + pytest.param( + "1 234,56", + CustomField.FieldDataType.FLOAT, + 1234.56, + Locale.parse("ru"), + id="float-plain-space-in-russian", + ), + pytest.param( + "1\N{RIGHT SINGLE QUOTATION MARK}234.56", + CustomField.FieldDataType.FLOAT, + 1234.56, + Locale.parse("de_CH"), + id="float-apostrophe-group-in-swiss-german", + ), ], ) def test_converts_to_exact_value( @@ -1546,16 +1765,17 @@ class TestConvertValue: value: str, data_type: str, expected: object, + locale: Locale, ) -> None: """ GIVEN: - - An extracted string and a custom field data type + - An extracted string, a custom field data type and a number locale WHEN: - The value is converted THEN: - The converted value equals the expected value exactly """ - assert _convert_value(value, data_type) == expected + assert _convert_value(value, data_type, locale) == expected def test_string_truncation(self) -> None: """ @@ -1566,105 +1786,205 @@ class TestConvertValue: THEN: - The result is truncated to 128 characters """ - result = _convert_value("x" * 200, CustomField.FieldDataType.STRING) + result = _convert_value("x" * 200, CustomField.FieldDataType.STRING, EN) assert isinstance(result, str) assert len(result) == 128 @pytest.mark.parametrize( - ("value", "expected"), + ("value", "locale", "expected"), [ - pytest.param("1234.56", 1234.56, id="float-simple"), - pytest.param("1.234,56", 1234.56, id="float-european-format"), - pytest.param("1,234.56", 1234.56, id="float-us-format"), - pytest.param("1234,56", 1234.56, id="float-comma-only"), + pytest.param("1234.56", EN, 1234.56, id="float-simple"), + pytest.param("1.234,56", DE, 1234.56, id="float-european-format"), + pytest.param("1,234.56", EN, 1234.56, id="float-us-format"), + pytest.param("1234,56", DE, 1234.56, id="float-comma-only"), + pytest.param("1,234", DE, 1.234, id="float-comma-is-decimal-in-german"), + pytest.param("1,234", EN, 1234.0, id="float-comma-is-thousands-in-english"), + pytest.param("1 234,56", FR, 1234.56, id="float-plain-space-in-french"), + pytest.param( + "1\N{NARROW NO-BREAK SPACE}234,56", + FR, + 1234.56, + id="float-narrow-space-in-french", + ), + pytest.param( + "\N{MINUS SIGN}1.5", + EN, + -1.5, + id="float-unicode-minus", + ), ], ) - def test_float_conversion(self, value: str, expected: float) -> None: + def test_float_conversion( + self, + value: str, + locale: Locale, + expected: float, + ) -> None: """ GIVEN: - - A numeric string in one of several decimal formats + - A numeric string in one of several decimal formats and a locale WHEN: - The value is converted to a FLOAT field value THEN: - The result is the expected float """ - assert _convert_value(value, CustomField.FieldDataType.FLOAT) == expected + assert ( + _convert_value(value, CustomField.FieldDataType.FLOAT, locale) == expected + ) @pytest.mark.parametrize( - ("value", "data_type"), + ("value", "data_type", "locale"), [ pytest.param( "abc", CustomField.FieldDataType.INT, + EN, id="int-empty-returns-none", ), pytest.param( "-", CustomField.FieldDataType.INT, + EN, id="int-only-dash-returns-none", ), + pytest.param( + "12.50", + CustomField.FieldDataType.INT, + EN, + id="int-fraction-returns-none", + ), + pytest.param( + "1.5", + CustomField.FieldDataType.INT, + EN, + id="int-short-fraction-returns-none", + ), + pytest.param( + "0.500", + CustomField.FieldDataType.INT, + EN, + id="int-leading-zero-fraction-returns-none", + ), + pytest.param( + "1..2", + CustomField.FieldDataType.INT, + EN, + id="int-empty-group-returns-none", + ), + pytest.param( + "5-3", + CustomField.FieldDataType.INT, + EN, + id="int-inner-dash-returns-none", + ), + pytest.param( + "1,234.567", + CustomField.FieldDataType.INT, + EN, + id="int-thousands-and-fraction-returns-none", + ), + pytest.param( + "12.3,00", + CustomField.FieldDataType.INT, + EN, + id="int-bad-grouping-before-fraction-returns-none", + ), + pytest.param( + "1,2345.00", + CustomField.FieldDataType.INT, + EN, + id="int-long-group-before-fraction-returns-none", + ), pytest.param( "abc", CustomField.FieldDataType.FLOAT, + EN, id="float-empty-returns-none", ), pytest.param( ",", CustomField.FieldDataType.FLOAT, + EN, id="float-only-separator-returns-none", ), pytest.param( "not a date", CustomField.FieldDataType.DATE, + EN, id="date-invalid-returns-none", ), pytest.param( "2026-13-32", CustomField.FieldDataType.DATE, + EN, id="date-invalid-values-returns-none", ), pytest.param( "CHF", CustomField.FieldDataType.MONETARY, + EN, id="monetary-empty-returns-none", ), pytest.param( "maybe", CustomField.FieldDataType.BOOL, + EN, id="bool-unknown-returns-none", ), pytest.param( "test", CustomField.FieldDataType.DOCUMENTLINK, + EN, id="unsupported-type-documentlink-returns-none", ), pytest.param( "test", CustomField.FieldDataType.SELECT, + EN, id="unsupported-type-select-returns-none", ), pytest.param( "", CustomField.FieldDataType.STRING, + EN, id="empty-string-returns-none", ), pytest.param( "9" * 400, CustomField.FieldDataType.FLOAT, + EN, id="float-overflow-is-not-stored-as-infinity", ), + pytest.param( + "1.234", + CustomField.FieldDataType.INT, + EN, + id="int-german-thousands-is-a-fraction-in-english", + ), + pytest.param( + "12.50", + CustomField.FieldDataType.INT, + DE, + id="int-english-decimal-is-malformed-in-german", + ), + pytest.param( + "1.234.567", + CustomField.FieldDataType.FLOAT, + EN, + id="float-multiple-dots-in-english", + ), ], ) - def test_returns_none(self, value: str, data_type: str) -> None: + def test_returns_none(self, value: str, data_type: str, locale: Locale) -> None: """ GIVEN: - - A value that cannot be converted to the given data type + - A value that cannot be converted to the given data type in a locale WHEN: - The value is converted THEN: - None is returned """ - assert _convert_value(value, data_type) is None + assert _convert_value(value, data_type, locale) is None @pytest.mark.parametrize( "data_type", @@ -1698,7 +2018,7 @@ class TestConvertValue: THEN: - None is returned """ - assert _convert_value(value, data_type) is None + assert _convert_value(value, data_type, EN) is None @pytest.mark.parametrize( "value", @@ -1721,7 +2041,7 @@ class TestConvertValue: THEN: - The result is True """ - assert _convert_value(value, CustomField.FieldDataType.BOOL) is True + assert _convert_value(value, CustomField.FieldDataType.BOOL, EN) is True @pytest.mark.parametrize( "value", @@ -1742,7 +2062,7 @@ class TestConvertValue: THEN: - The result is False """ - assert _convert_value(value, CustomField.FieldDataType.BOOL) is False + assert _convert_value(value, CustomField.FieldDataType.BOOL, EN) is False class TestRenderPages: diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index a594fca3aa..b487bf8604 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -6,6 +6,7 @@ import re import string import subprocess import tempfile +import unicodedata from dataclasses import dataclass from datetime import date from datetime import datetime @@ -18,6 +19,12 @@ from typing import TYPE_CHECKING import dateparser import magic import zxingcpp +from babel import Locale +from babel import UnknownLocaleError +from babel.numbers import NumberFormatError +from babel.numbers import get_decimal_symbol +from babel.numbers import get_group_symbol +from babel.numbers import parse_decimal from django.conf import settings from django.core.exceptions import ValidationError from django.db import transaction @@ -34,12 +41,16 @@ from documents.permissions import permitted_object_ids from documents.regex import safe_regex_fullmatch from documents.signals.handlers import add_to_index from documents.versioning import get_root_document +from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES if TYPE_CHECKING: from django.contrib.auth.models import User logger = logging.getLogger("paperless.zone_ocr") +# Unicode general category for decimal digits in any script. +UNICODE_DECIMAL_DIGIT = "Nd" # codespell:ignore + @dataclass(frozen=True) class ZoneExtraction: @@ -569,7 +580,11 @@ def _prepare_field_update( return None custom_field = zone.custom_field field_name = CustomFieldInstance.get_value_field_name(custom_field.data_type) - typed_value = _convert_value(value, custom_field.data_type) + typed_value = _convert_value( + value, + custom_field.data_type, + _number_locale(zone.ocr_language), + ) model = CustomFieldInstance elif zone.target == OcrTemplateZone.TargetType.TITLE: field_name, typed_value = "title", value[:128] @@ -621,22 +636,75 @@ def _apply_field_updates(document: Document, updates: list[FieldUpdate]) -> None raise -def _normalize_number(value: str) -> str: - cleaned = re.sub(r"[^\d.,\-]", "", value) - # When both separators occur, the rightmost one is the decimal separator. - if "," in cleaned and "." in cleaned: - if cleaned.rindex(",") > cleaned.rindex("."): - cleaned = cleaned.replace(".", "").replace(",", ".") - else: - cleaned = cleaned.replace(",", "") - else: - cleaned = cleaned.replace(",", ".") - Decimal(cleaned) - return cleaned +def _number_locale(ocr_language: str) -> Locale: + """Derive the locale used to read numbers from a tesseract language code. + + Only the first language counts, so "deu+eng" reads numbers the German way. + Languages Babel does not know fall back to English. + """ + tesseract_code = ocr_language.split("+", maxsplit=1)[0].split("_", maxsplit=1)[0] + code = OCR_TO_DATEPARSER_LANGUAGES.get(tesseract_code) + if code is not None: + try: + return Locale.parse(code) + except (UnknownLocaleError, ValueError): + pass + return Locale.parse("en") -def _convert_value(value: str, data_type: str) -> object | None: - """Convert an extracted OCR string to the appropriate type for the custom field.""" +def _parse_number(value: str, locale: Locale) -> Decimal | None: + """Parse OCR text as a decimal number using the locale's number format. + + Digits of any script are read as their ASCII equivalents, and characters + that are not part of a number, such as letters or currency symbols, are + dropped. A dash (also a unicode minus or en dash) makes the number negative + when it comes before the first digit and does not directly follow a letter + or digit, so "-5", "EUR -5" and "$-5" are negative while "INV-123" is + just 123. Any other dash makes the text invalid. Returns None when the + text is not a well-formed number in the locale. + """ + text = "".join( + str(unicodedata.digit(char)) + if unicodedata.category(char) == UNICODE_DECIMAL_DIGIT + else char + for char in value.strip() + ) + text = ( + text.replace("\N{MINUS SIGN}", "-") + .replace("\N{EN DASH}", "-") + .replace("\N{ARABIC DECIMAL SEPARATOR}", ".") + .replace("\N{ARABIC THOUSANDS SEPARATOR}", ",") + ) + sign = re.search(r"(? object | None: + """Convert an extracted OCR string to the appropriate type for the custom field. + + Numeric types are read with the number format of the given locale. + """ if not value: return None @@ -651,16 +719,16 @@ def _convert_value(value: str, data_type: str) -> object | None: return value elif data_type == CustomField.FieldDataType.INT: - digits = re.sub(r"[^\d\-]", "", value) - digits = digits.lstrip("-") - if not digits: + integer = _parse_number(value, locale) + if integer is None or integer != integer.to_integral_value(): return None - if value.strip().startswith("-"): - digits = "-" + digits - return int(digits) + return int(integer) elif data_type == CustomField.FieldDataType.FLOAT: - number = float(_normalize_number(value)) + decimal = _parse_number(value, locale) + if decimal is None: + return None + number = float(decimal) return number if math.isfinite(number) else None elif data_type == CustomField.FieldDataType.DATE: @@ -668,7 +736,8 @@ def _convert_value(value: str, data_type: str) -> object | None: return parsed.isoformat() if parsed else None elif data_type == CustomField.FieldDataType.MONETARY: - return _normalize_number(value) + monetary = _parse_number(value, locale) + return None if monetary is None else format(monetary, "f") elif data_type == CustomField.FieldDataType.BOOL: lower = value.lower().strip()