From 63a87dd11b2e7570f40573a44c94142928d2937c Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:34:26 -0800 Subject: [PATCH] Fix: parse OCR numbers with Babel using the zone's language Separators were guessed from the text, so "12.50" became 1250 for integer fields and the thousands and decimal separators were often swapped. --- src/documents/tests/test_zone_ocr.py | 362 +++++++++++++++++++++++++-- src/documents/zone_ocr.py | 115 +++++++-- 2 files changed, 433 insertions(+), 44 deletions(-) diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index f50e5871f2..2a5295f4ea 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING from typing import Any import pytest +from babel import Locale from django.contrib.auth.models import Group from PIL import Image @@ -25,6 +26,7 @@ from documents.zone_ocr import _convert_value from documents.zone_ocr import _crop_zone from documents.zone_ocr import _detect_mime from documents.zone_ocr import _extract_zone +from documents.zone_ocr import _number_locale from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages @@ -50,6 +52,11 @@ if TYPE_CHECKING: from paperless_testing.dirs import PaperlessDirs +EN = Locale.parse("en") +DE = Locale.parse("de") +FR = Locale.parse("fr") + + def make_template(name: str, **kwargs: Any) -> OcrTemplate: """Build a saved template, 100x50 by default.""" kwargs.setdefault("source_width", 100) @@ -1462,6 +1469,7 @@ class TestApplyTransform: assert _apply_transform("unparsable", "date") == "unparsable" +@pytest.mark.django_db class TestPrepareFieldUpdate: """Tests for converting extracted values into field updates.""" @@ -1476,69 +1484,280 @@ class TestPrepareFieldUpdate: """ assert _prepare_field_update(OcrTemplateZone(target="unknown"), "value") is None + @pytest.mark.parametrize( + ("ocr_language", "expected"), + [ + pytest.param("deu", 1234, id="german-reads-dot-as-thousands"), + pytest.param("eng", None, id="english-reads-dot-as-fraction"), + ], + ) + def test_int_field_uses_the_zone_language_for_numbers( + self, + ocr_language: str, + expected: int | None, + ) -> None: + """ + GIVEN: + - A zone targeting an integer custom field with a given OCR language + WHEN: + - A field update is prepared for the text "1.234" + THEN: + - A German zone reads 1234, while an English zone reads a fraction + and produces no update + """ + field = CustomFieldFactory(data_type=CustomField.FieldDataType.INT) + zone = OcrTemplateZoneFactory( + target=OcrTemplateZone.TargetType.CUSTOM_FIELD, + custom_field=field, + ocr_language=ocr_language, + ) + + update = _prepare_field_update(zone, "1.234") + + assert (update.value if update else None) == expected + + +class TestNumberLocale: + """Tests for deriving a number locale from the OCR language.""" + + @pytest.mark.parametrize( + ("ocr_language", "expected"), + [ + pytest.param("deu", "de", id="single-language"), + pytest.param("deu+eng", "de", id="first-of-several-languages"), + pytest.param("aze_Cyrl", "az", id="script-suffix-is-dropped"), + pytest.param("xyz", "en", id="unknown-language-falls-back-to-english"), + ], + ) + def test_number_locale(self, ocr_language: str, expected: str) -> None: + """ + GIVEN: + - A tesseract language code, possibly several or with a script + WHEN: + - The number locale is derived + THEN: + - The first language decides the locale, falling back to English + """ + assert _number_locale(ocr_language) == Locale.parse(expected) + class TestConvertValue: """Tests for the _convert_value function.""" @pytest.mark.parametrize( - ("value", "data_type", "expected"), + ("value", "data_type", "expected", "locale"), [ pytest.param( "Hello", CustomField.FieldDataType.STRING, "Hello", + EN, id="string", ), pytest.param( "https://example.com", CustomField.FieldDataType.URL, "https://example.com", + EN, id="url", ), pytest.param( "x" * 500, CustomField.FieldDataType.LONG_TEXT, "x" * 500, + EN, id="long-text", ), - pytest.param("42", CustomField.FieldDataType.INT, 42, id="int-simple"), + pytest.param("42", CustomField.FieldDataType.INT, 42, EN, id="int-simple"), pytest.param( "INV-123", CustomField.FieldDataType.INT, 123, + EN, id="int-with-noise", ), - pytest.param("-42", CustomField.FieldDataType.INT, -42, id="int-negative"), + pytest.param( + "-42", + CustomField.FieldDataType.INT, + -42, + EN, + id="int-negative", + ), + pytest.param( + "\N{MINUS SIGN}5", + CustomField.FieldDataType.INT, + -5, + EN, + id="int-unicode-minus", + ), + pytest.param( + "1,234", + CustomField.FieldDataType.INT, + 1234, + EN, + id="int-comma-thousands", + ), + pytest.param( + "1.234.567", + CustomField.FieldDataType.INT, + 1234567, + DE, + id="int-dot-thousands", + ), + pytest.param( + "12.00", + CustomField.FieldDataType.INT, + 12, + EN, + id="int-zero-fraction", + ), + pytest.param( + "1.234,00", + CustomField.FieldDataType.INT, + 1234, + DE, + id="int-thousands-and-zero-fraction", + ), + pytest.param( + "1234.00", + CustomField.FieldDataType.INT, + 1234, + EN, + id="int-ungrouped-with-zero-fraction", + ), + pytest.param( + "1.234", + CustomField.FieldDataType.INT, + 1234, + DE, + id="int-dot-is-thousands-in-german", + ), + pytest.param( + "1\N{NARROW NO-BREAK SPACE}234", + CustomField.FieldDataType.INT, + 1234, + FR, + id="int-narrow-space-thousands-in-french", + ), pytest.param( "2026-04-13", CustomField.FieldDataType.DATE, "2026-04-13", + EN, id="date-iso", ), pytest.param( "123.45", CustomField.FieldDataType.MONETARY, "123.45", + EN, id="monetary-simple", ), pytest.param( "1.234,56", CustomField.FieldDataType.MONETARY, "1234.56", + DE, id="monetary-european", ), pytest.param( "\u20ac1,234.56", CustomField.FieldDataType.MONETARY, "1234.56", + EN, id="monetary-with-currency-symbol", ), pytest.param( "12345678901234567890,12345678901234567890", CustomField.FieldDataType.MONETARY, "12345678901234567890.12345678901234567890", + DE, id="monetary-preserves-decimal-precision", ), + pytest.param( + "-12345678901234567890,12345678901234567890", + CustomField.FieldDataType.MONETARY, + "-12345678901234567890.12345678901234567890", + DE, + id="monetary-negative-preserves-decimal-precision", + ), + pytest.param( + "\u20ac-12.50", + CustomField.FieldDataType.MONETARY, + "-12.50", + EN, + id="monetary-sign-after-currency-symbol", + ), + pytest.param( + "-\u20ac12.50", + CustomField.FieldDataType.MONETARY, + "-12.50", + EN, + id="monetary-sign-before-currency-symbol", + ), + pytest.param( + "EUR -12.50", + CustomField.FieldDataType.MONETARY, + "-12.50", + EN, + id="monetary-sign-after-currency-code", + ), + pytest.param( + "Total: -5", + CustomField.FieldDataType.INT, + -5, + EN, + id="int-sign-after-label", + ), + pytest.param( + "\u20ac-12.50", + CustomField.FieldDataType.FLOAT, + -12.5, + EN, + id="float-sign-after-currency-symbol", + ), + pytest.param( + "\u0661\u0662\u0663", + CustomField.FieldDataType.INT, + 123, + Locale.parse("ar"), + id="int-arabic-indic-digits", + ), + pytest.param( + "\u0661\u066c\u0662\u0663\u0664\u066b\u0665", + CustomField.FieldDataType.FLOAT, + 1234.5, + Locale.parse("ar"), + id="float-arabic-separators", + ), + pytest.param( + "\u06f1\u06f2\u06f3", + CustomField.FieldDataType.INT, + 123, + Locale.parse("fa"), + id="int-persian-digits", + ), + pytest.param( + "\u0967\u0968\u0969", + CustomField.FieldDataType.INT, + 123, + Locale.parse("hi"), + id="int-devanagari-digits", + ), + pytest.param( + "1 234,56", + CustomField.FieldDataType.FLOAT, + 1234.56, + Locale.parse("ru"), + id="float-plain-space-in-russian", + ), + pytest.param( + "1\N{RIGHT SINGLE QUOTATION MARK}234.56", + CustomField.FieldDataType.FLOAT, + 1234.56, + Locale.parse("de_CH"), + id="float-apostrophe-group-in-swiss-german", + ), ], ) def test_converts_to_exact_value( @@ -1546,16 +1765,17 @@ class TestConvertValue: value: str, data_type: str, expected: object, + locale: Locale, ) -> None: """ GIVEN: - - An extracted string and a custom field data type + - An extracted string, a custom field data type and a number locale WHEN: - The value is converted THEN: - The converted value equals the expected value exactly """ - assert _convert_value(value, data_type) == expected + assert _convert_value(value, data_type, locale) == expected def test_string_truncation(self) -> None: """ @@ -1566,105 +1786,205 @@ class TestConvertValue: THEN: - The result is truncated to 128 characters """ - result = _convert_value("x" * 200, CustomField.FieldDataType.STRING) + result = _convert_value("x" * 200, CustomField.FieldDataType.STRING, EN) assert isinstance(result, str) assert len(result) == 128 @pytest.mark.parametrize( - ("value", "expected"), + ("value", "locale", "expected"), [ - pytest.param("1234.56", 1234.56, id="float-simple"), - pytest.param("1.234,56", 1234.56, id="float-european-format"), - pytest.param("1,234.56", 1234.56, id="float-us-format"), - pytest.param("1234,56", 1234.56, id="float-comma-only"), + pytest.param("1234.56", EN, 1234.56, id="float-simple"), + pytest.param("1.234,56", DE, 1234.56, id="float-european-format"), + pytest.param("1,234.56", EN, 1234.56, id="float-us-format"), + pytest.param("1234,56", DE, 1234.56, id="float-comma-only"), + pytest.param("1,234", DE, 1.234, id="float-comma-is-decimal-in-german"), + pytest.param("1,234", EN, 1234.0, id="float-comma-is-thousands-in-english"), + pytest.param("1 234,56", FR, 1234.56, id="float-plain-space-in-french"), + pytest.param( + "1\N{NARROW NO-BREAK SPACE}234,56", + FR, + 1234.56, + id="float-narrow-space-in-french", + ), + pytest.param( + "\N{MINUS SIGN}1.5", + EN, + -1.5, + id="float-unicode-minus", + ), ], ) - def test_float_conversion(self, value: str, expected: float) -> None: + def test_float_conversion( + self, + value: str, + locale: Locale, + expected: float, + ) -> None: """ GIVEN: - - A numeric string in one of several decimal formats + - A numeric string in one of several decimal formats and a locale WHEN: - The value is converted to a FLOAT field value THEN: - The result is the expected float """ - assert _convert_value(value, CustomField.FieldDataType.FLOAT) == expected + assert ( + _convert_value(value, CustomField.FieldDataType.FLOAT, locale) == expected + ) @pytest.mark.parametrize( - ("value", "data_type"), + ("value", "data_type", "locale"), [ pytest.param( "abc", CustomField.FieldDataType.INT, + EN, id="int-empty-returns-none", ), pytest.param( "-", CustomField.FieldDataType.INT, + EN, id="int-only-dash-returns-none", ), + pytest.param( + "12.50", + CustomField.FieldDataType.INT, + EN, + id="int-fraction-returns-none", + ), + pytest.param( + "1.5", + CustomField.FieldDataType.INT, + EN, + id="int-short-fraction-returns-none", + ), + pytest.param( + "0.500", + CustomField.FieldDataType.INT, + EN, + id="int-leading-zero-fraction-returns-none", + ), + pytest.param( + "1..2", + CustomField.FieldDataType.INT, + EN, + id="int-empty-group-returns-none", + ), + pytest.param( + "5-3", + CustomField.FieldDataType.INT, + EN, + id="int-inner-dash-returns-none", + ), + pytest.param( + "1,234.567", + CustomField.FieldDataType.INT, + EN, + id="int-thousands-and-fraction-returns-none", + ), + pytest.param( + "12.3,00", + CustomField.FieldDataType.INT, + EN, + id="int-bad-grouping-before-fraction-returns-none", + ), + pytest.param( + "1,2345.00", + CustomField.FieldDataType.INT, + EN, + id="int-long-group-before-fraction-returns-none", + ), pytest.param( "abc", CustomField.FieldDataType.FLOAT, + EN, id="float-empty-returns-none", ), pytest.param( ",", CustomField.FieldDataType.FLOAT, + EN, id="float-only-separator-returns-none", ), pytest.param( "not a date", CustomField.FieldDataType.DATE, + EN, id="date-invalid-returns-none", ), pytest.param( "2026-13-32", CustomField.FieldDataType.DATE, + EN, id="date-invalid-values-returns-none", ), pytest.param( "CHF", CustomField.FieldDataType.MONETARY, + EN, id="monetary-empty-returns-none", ), pytest.param( "maybe", CustomField.FieldDataType.BOOL, + EN, id="bool-unknown-returns-none", ), pytest.param( "test", CustomField.FieldDataType.DOCUMENTLINK, + EN, id="unsupported-type-documentlink-returns-none", ), pytest.param( "test", CustomField.FieldDataType.SELECT, + EN, id="unsupported-type-select-returns-none", ), pytest.param( "", CustomField.FieldDataType.STRING, + EN, id="empty-string-returns-none", ), pytest.param( "9" * 400, CustomField.FieldDataType.FLOAT, + EN, id="float-overflow-is-not-stored-as-infinity", ), + pytest.param( + "1.234", + CustomField.FieldDataType.INT, + EN, + id="int-german-thousands-is-a-fraction-in-english", + ), + pytest.param( + "12.50", + CustomField.FieldDataType.INT, + DE, + id="int-english-decimal-is-malformed-in-german", + ), + pytest.param( + "1.234.567", + CustomField.FieldDataType.FLOAT, + EN, + id="float-multiple-dots-in-english", + ), ], ) - def test_returns_none(self, value: str, data_type: str) -> None: + def test_returns_none(self, value: str, data_type: str, locale: Locale) -> None: """ GIVEN: - - A value that cannot be converted to the given data type + - A value that cannot be converted to the given data type in a locale WHEN: - The value is converted THEN: - None is returned """ - assert _convert_value(value, data_type) is None + assert _convert_value(value, data_type, locale) is None @pytest.mark.parametrize( "data_type", @@ -1698,7 +2018,7 @@ class TestConvertValue: THEN: - None is returned """ - assert _convert_value(value, data_type) is None + assert _convert_value(value, data_type, EN) is None @pytest.mark.parametrize( "value", @@ -1721,7 +2041,7 @@ class TestConvertValue: THEN: - The result is True """ - assert _convert_value(value, CustomField.FieldDataType.BOOL) is True + assert _convert_value(value, CustomField.FieldDataType.BOOL, EN) is True @pytest.mark.parametrize( "value", @@ -1742,7 +2062,7 @@ class TestConvertValue: THEN: - The result is False """ - assert _convert_value(value, CustomField.FieldDataType.BOOL) is False + assert _convert_value(value, CustomField.FieldDataType.BOOL, EN) is False class TestRenderPages: diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index a594fca3aa..b487bf8604 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -6,6 +6,7 @@ import re import string import subprocess import tempfile +import unicodedata from dataclasses import dataclass from datetime import date from datetime import datetime @@ -18,6 +19,12 @@ from typing import TYPE_CHECKING import dateparser import magic import zxingcpp +from babel import Locale +from babel import UnknownLocaleError +from babel.numbers import NumberFormatError +from babel.numbers import get_decimal_symbol +from babel.numbers import get_group_symbol +from babel.numbers import parse_decimal from django.conf import settings from django.core.exceptions import ValidationError from django.db import transaction @@ -34,12 +41,16 @@ from documents.permissions import permitted_object_ids from documents.regex import safe_regex_fullmatch from documents.signals.handlers import add_to_index from documents.versioning import get_root_document +from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES if TYPE_CHECKING: from django.contrib.auth.models import User logger = logging.getLogger("paperless.zone_ocr") +# Unicode general category for decimal digits in any script. +UNICODE_DECIMAL_DIGIT = "Nd" # codespell:ignore + @dataclass(frozen=True) class ZoneExtraction: @@ -569,7 +580,11 @@ def _prepare_field_update( return None custom_field = zone.custom_field field_name = CustomFieldInstance.get_value_field_name(custom_field.data_type) - typed_value = _convert_value(value, custom_field.data_type) + typed_value = _convert_value( + value, + custom_field.data_type, + _number_locale(zone.ocr_language), + ) model = CustomFieldInstance elif zone.target == OcrTemplateZone.TargetType.TITLE: field_name, typed_value = "title", value[:128] @@ -621,22 +636,75 @@ def _apply_field_updates(document: Document, updates: list[FieldUpdate]) -> None raise -def _normalize_number(value: str) -> str: - cleaned = re.sub(r"[^\d.,\-]", "", value) - # When both separators occur, the rightmost one is the decimal separator. - if "," in cleaned and "." in cleaned: - if cleaned.rindex(",") > cleaned.rindex("."): - cleaned = cleaned.replace(".", "").replace(",", ".") - else: - cleaned = cleaned.replace(",", "") - else: - cleaned = cleaned.replace(",", ".") - Decimal(cleaned) - return cleaned +def _number_locale(ocr_language: str) -> Locale: + """Derive the locale used to read numbers from a tesseract language code. + + Only the first language counts, so "deu+eng" reads numbers the German way. + Languages Babel does not know fall back to English. + """ + tesseract_code = ocr_language.split("+", maxsplit=1)[0].split("_", maxsplit=1)[0] + code = OCR_TO_DATEPARSER_LANGUAGES.get(tesseract_code) + if code is not None: + try: + return Locale.parse(code) + except (UnknownLocaleError, ValueError): + pass + return Locale.parse("en") -def _convert_value(value: str, data_type: str) -> object | None: - """Convert an extracted OCR string to the appropriate type for the custom field.""" +def _parse_number(value: str, locale: Locale) -> Decimal | None: + """Parse OCR text as a decimal number using the locale's number format. + + Digits of any script are read as their ASCII equivalents, and characters + that are not part of a number, such as letters or currency symbols, are + dropped. A dash (also a unicode minus or en dash) makes the number negative + when it comes before the first digit and does not directly follow a letter + or digit, so "-5", "EUR -5" and "$-5" are negative while "INV-123" is + just 123. Any other dash makes the text invalid. Returns None when the + text is not a well-formed number in the locale. + """ + text = "".join( + str(unicodedata.digit(char)) + if unicodedata.category(char) == UNICODE_DECIMAL_DIGIT + else char + for char in value.strip() + ) + text = ( + text.replace("\N{MINUS SIGN}", "-") + .replace("\N{EN DASH}", "-") + .replace("\N{ARABIC DECIMAL SEPARATOR}", ".") + .replace("\N{ARABIC THOUSANDS SEPARATOR}", ",") + ) + sign = re.search(r"(? object | None: + """Convert an extracted OCR string to the appropriate type for the custom field. + + Numeric types are read with the number format of the given locale. + """ if not value: return None @@ -651,16 +719,16 @@ def _convert_value(value: str, data_type: str) -> object | None: return value elif data_type == CustomField.FieldDataType.INT: - digits = re.sub(r"[^\d\-]", "", value) - digits = digits.lstrip("-") - if not digits: + integer = _parse_number(value, locale) + if integer is None or integer != integer.to_integral_value(): return None - if value.strip().startswith("-"): - digits = "-" + digits - return int(digits) + return int(integer) elif data_type == CustomField.FieldDataType.FLOAT: - number = float(_normalize_number(value)) + decimal = _parse_number(value, locale) + if decimal is None: + return None + number = float(decimal) return number if math.isfinite(number) else None elif data_type == CustomField.FieldDataType.DATE: @@ -668,7 +736,8 @@ def _convert_value(value: str, data_type: str) -> object | None: return parsed.isoformat() if parsed else None elif data_type == CustomField.FieldDataType.MONETARY: - return _normalize_number(value) + monetary = _parse_number(value, locale) + return None if monetary is None else format(monetary, "f") elif data_type == CustomField.FieldDataType.BOOL: lower = value.lower().strip()