From 6fc45269476b1c87c132288423c24903506b581d Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:34:16 -0800 Subject: [PATCH] Fix: keep the currency and validate monetary zone values OCR dropped currency codes and stored amounts the API would reject. Keep an ISO code or locale currency symbol, read parentheses as negative, and validate with the same rule as the API. --- src/documents/serialisers.py | 16 +--- src/documents/tests/test_validators.py | 57 +++++++++++++ src/documents/tests/test_zone_ocr.py | 106 +++++++++++++++++++++--- src/documents/validators.py | 28 +++++++ src/documents/zone_ocr.py | 109 ++++++++++++++++++++++++- 5 files changed, 287 insertions(+), 29 deletions(-) create mode 100644 src/documents/tests/test_validators.py diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index acdfc21e72..3f4b49d68d 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -5,7 +5,6 @@ import math import re from datetime import datetime from datetime import timedelta -from decimal import Decimal from typing import TYPE_CHECKING from typing import Any from typing import TypedDict @@ -16,12 +15,10 @@ from django.contrib.auth.models import Group from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError -from django.core.validators import DecimalValidator from django.core.validators import EmailValidator from django.core.validators import MaxLengthValidator from django.core.validators import MaxValueValidator from django.core.validators import MinValueValidator -from django.core.validators import RegexValidator from django.core.validators import integer_validator from django.db import transaction from django.db.models import Count @@ -95,6 +92,7 @@ from documents.regex import validate_regex_pattern from documents.templating.filepath import validate_filepath_template_and_render from documents.templating.utils import convert_format_str_to_template_format from documents.templating.workflows import validate_workflow_template +from documents.validators import monetary_validator from documents.validators import uri_validator from documents.validators import url_validator from documents.versioning import has_prefetched_effective_content @@ -908,17 +906,7 @@ class CustomFieldInstanceSerializer(serializers.ModelSerializer[CustomFieldInsta field.data_type == CustomField.FieldDataType.MONETARY and data["value"] != "" ): - try: - # First try to validate as a number from legacy format - DecimalValidator(max_digits=12, decimal_places=2)( - Decimal(str(data["value"])), - ) - except Exception: - # If that fails, try to validate as a monetary string - RegexValidator( - regex=r"^[A-Z]{3}-?\d+(\.\d{1,2})$", - message="Must be a two-decimal number with optional currency code e.g. GBP123.45", - )(data["value"]) + monetary_validator(data["value"]) elif field.data_type == CustomField.FieldDataType.STRING: MaxLengthValidator(limit_value=128)(data["value"]) elif field.data_type == CustomField.FieldDataType.SELECT: diff --git a/src/documents/tests/test_validators.py b/src/documents/tests/test_validators.py new file mode 100644 index 0000000000..bac3d42b9d --- /dev/null +++ b/src/documents/tests/test_validators.py @@ -0,0 +1,57 @@ +import pytest +from django.core.exceptions import ValidationError + +from documents.validators import monetary_validator + + +class TestMonetaryValidator: + """Tests for the monetary_validator function.""" + + @pytest.mark.parametrize( + "value", + [ + pytest.param("12", id="whole-number"), + pytest.param("12.5", id="one-decimal"), + pytest.param("12.50", id="two-decimals"), + pytest.param("-12.50", id="negative"), + pytest.param("9999999999.99", id="twelve-digits"), + pytest.param("GBP123.45", id="code-with-amount"), + pytest.param("USD-12.50", id="code-with-negative"), + pytest.param("EUR12.5", id="code-with-one-decimal"), + ], + ) + def test_valid_values(self, value: str) -> None: + """ + GIVEN: + - A plain decimal or an amount prefixed with a currency code + WHEN: + - The value is validated + THEN: + - No error is raised + """ + monetary_validator(value) + + @pytest.mark.parametrize( + "value", + [ + pytest.param("12.345", id="too-many-decimals"), + pytest.param("99999999999.99", id="too-many-digits"), + pytest.param("GBP12.345", id="code-with-too-many-decimals"), + pytest.param("GBP12", id="code-without-decimals"), + pytest.param("gbp12.50", id="lowercase-code"), + pytest.param("GB12.50", id="short-code"), + pytest.param("abc", id="not-a-number"), + pytest.param("NaN", id="nan"), + ], + ) + def test_invalid_values(self, value: str) -> None: + """ + GIVEN: + - A value that is neither a valid decimal nor a coded amount + WHEN: + - The value is validated + THEN: + - A ValidationError is raised + """ + with pytest.raises(ValidationError): + monetary_validator(value) diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index a622aedb34..d7ec706822 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -31,6 +31,7 @@ from documents.zone_ocr import _detect_mime from documents.zone_ocr import _extract_zone from documents.zone_ocr import _number_locale from documents.zone_ocr import _parse_date +from documents.zone_ocr import _parse_monetary from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages @@ -2284,6 +2285,92 @@ class TestNumberLocale: assert _number_locale(ocr_language) == Locale.parse(expected) +class TestParseMonetary: + """Tests for the _parse_monetary function.""" + + @pytest.mark.parametrize( + ("value", "locale", "expected"), + [ + pytest.param("12.50", EN, "12.50", id="plain-amount"), + pytest.param("12", EN, "12.00", id="whole-number-gets-two-decimals"), + pytest.param("12.500", EN, "12.50", id="trailing-zero-is-fine"), + pytest.param("USD 12.50", EN, "USD12.50", id="code-before-amount"), + pytest.param("12,50 EUR", DE, "EUR12.50", id="code-after-amount"), + pytest.param("USD12.50", EN, "USD12.50", id="code-attached-to-amount"), + pytest.param("$12.50", EN, "USD12.50", id="dollar-symbol-en"), + pytest.param("$12.50", FR, "12.50", id="dollar-symbol-fr-has-no-code"), + pytest.param( + "\N{EURO SIGN}12.50", + EN, + "EUR12.50", + id="euro-symbol", + ), + pytest.param( + "\N{POUND SIGN}12.50", + EN, + "GBP12.50", + id="pound-symbol", + ), + pytest.param( + "USD $12.50", + EN, + "USD12.50", + id="code-wins-over-symbol", + ), + pytest.param("INV 12.50", EN, "12.50", id="non-currency-word-ignored"), + pytest.param("usd 12.50", EN, "12.50", id="lowercase-code-ignored"), + pytest.param("(12.50)", EN, "-12.50", id="parentheses-negative"), + pytest.param("($12.50)", EN, "USD-12.50", id="parentheses-with-symbol"), + pytest.param("(12,50 EUR)", DE, "EUR-12.50", id="parentheses-with-code"), + pytest.param("(0.00)", EN, "0.00", id="parentheses-zero-not-negative"), + pytest.param("(USD) 12.50", EN, "USD12.50", id="empty-parentheses-ignored"), + pytest.param("USD -12.50", EN, "USD-12.50", id="code-with-negative"), + pytest.param("-12.50", EN, "-12.50", id="plain-negative"), + pytest.param("1,234.56", EN, "1234.56", id="grouped-amount"), + pytest.param("USD 12.50 EUR", EN, None, id="two-codes"), + pytest.param("$12.50 \N{EURO SIGN}", EN, None, id="two-symbols"), + pytest.param("(-12.50)", EN, None, id="parentheses-and-minus"), + pytest.param("12.345", EN, None, id="too-many-decimals"), + pytest.param("12345678901.12", EN, None, id="more-than-12-digits"), + pytest.param( + "USD 12345678901.12", + EN, + None, + id="more-than-12-digits-with-code", + ), + pytest.param("100 (19% VAT)", EN, None, id="digits-outside-parentheses"), + pytest.param("(12) (13)", EN, None, id="two-parenthesised-numbers"), + pytest.param("-0.00", EN, "0.00", id="negative-zero-not-negative"), + pytest.param("CHF 12", DE, "CHF12.00", id="code-in-german"), + pytest.param( + "1\N{NARROW NO-BREAK SPACE}234,56 \N{EURO SIGN}", + FR, + "EUR1234.56", + id="french-space-grouping", + ), + pytest.param("USD", EN, None, id="code-without-amount"), + pytest.param("garbage", EN, None, id="garbage"), + pytest.param("", EN, None, id="empty"), + ], + ) + def test_parse_monetary( + self, + value: str, + locale: Locale, + expected: str | None, + ) -> None: + """ + GIVEN: + - OCR text containing an amount, maybe with a currency or parentheses + WHEN: + - The text is parsed as a monetary value + THEN: + - The amount has two decimals and a leading currency code if one was + found, or None is returned when the text is not a valid value + """ + assert _parse_monetary(value, locale) == expected + + class TestConvertValue: """Tests for the _convert_value function.""" @@ -2406,42 +2493,35 @@ class TestConvertValue: pytest.param( "\u20ac1,234.56", CustomField.FieldDataType.MONETARY, - "1234.56", + "EUR1234.56", EN, id="monetary-with-currency-symbol", ), pytest.param( "12345678901234567890,12345678901234567890", CustomField.FieldDataType.MONETARY, - "12345678901234567890.12345678901234567890", + None, DE, - id="monetary-preserves-decimal-precision", - ), - pytest.param( - "-12345678901234567890,12345678901234567890", - CustomField.FieldDataType.MONETARY, - "-12345678901234567890.12345678901234567890", - DE, - id="monetary-negative-preserves-decimal-precision", + id="monetary-too-precise-returns-none", ), pytest.param( "\u20ac-12.50", CustomField.FieldDataType.MONETARY, - "-12.50", + "EUR-12.50", EN, id="monetary-sign-after-currency-symbol", ), pytest.param( "-\u20ac12.50", CustomField.FieldDataType.MONETARY, - "-12.50", + "EUR-12.50", EN, id="monetary-sign-before-currency-symbol", ), pytest.param( "EUR -12.50", CustomField.FieldDataType.MONETARY, - "-12.50", + "EUR-12.50", EN, id="monetary-sign-after-currency-code", ), diff --git a/src/documents/validators.py b/src/documents/validators.py index bec7252bf1..a28f8fe210 100644 --- a/src/documents/validators.py +++ b/src/documents/validators.py @@ -1,6 +1,9 @@ +from decimal import Decimal from urllib.parse import urlparse from django.core.exceptions import ValidationError +from django.core.validators import DecimalValidator +from django.core.validators import RegexValidator from django.utils.translation import gettext_lazy as _ @@ -58,3 +61,28 @@ def url_validator(value) -> None: ValidationError: If the URL is invalid or not using http/https scheme """ uri_validator(value, allowed_schemes={"http", "https"}) + + +def monetary_validator(value: str) -> None: + """ + Validates the stored value of a monetary custom field. + + Accepts either a plain decimal with at most 12 digits and 2 decimal places + (the legacy format), or a two-decimal number prefixed by a three letter + uppercase currency code, e.g. GBP123.45 or USD-12.50. + + Args: + value: The monetary value to validate + + Raises: + ValidationError: If the value matches neither format + """ + try: + # First try to validate as a number from legacy format + DecimalValidator(max_digits=12, decimal_places=2)(Decimal(str(value))) + except Exception: + # If that fails, try to validate as a monetary string + RegexValidator( + regex=r"^[A-Z]{3}-?\d+(\.\d{1,2})$", + message="Must be a two-decimal number with optional currency code e.g. GBP123.45", + )(value) diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 300a35538e..76bc70d6d4 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -12,6 +12,7 @@ from datetime import date from datetime import datetime from decimal import Decimal from decimal import InvalidOperation +from functools import lru_cache from functools import partial from pathlib import Path from typing import TYPE_CHECKING @@ -24,6 +25,7 @@ from babel import UnknownLocaleError from babel.numbers import NumberFormatError from babel.numbers import get_decimal_symbol from babel.numbers import get_group_symbol +from babel.numbers import is_currency from babel.numbers import parse_decimal from django.conf import settings from django.core.exceptions import ValidationError @@ -42,6 +44,7 @@ from documents.parsers import rasterize_pdf_page_to_png from documents.permissions import permitted_object_ids from documents.regex import safe_regex_fullmatch from documents.signals.handlers import add_to_index +from documents.validators import monetary_validator from documents.versioning import get_root_document from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES @@ -811,6 +814,109 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None: return number.copy_negate() if negative else number +@lru_cache(maxsize=32) +def _currency_symbol_codes(locale: Locale) -> dict[str, str]: + """Map the unambiguous currency symbols of a locale to their currency code. + + Symbols shared by several currencies, and symbols made only of letters or + digits (which includes plain three letter codes), are left out. + """ + codes_by_symbol: dict[str, set[str]] = {} + for code, symbol in locale.currency_symbols.items(): + if not symbol.isalnum(): + codes_by_symbol.setdefault(symbol, set()).add(code) + return { + symbol: next(iter(codes)) + for symbol, codes in codes_by_symbol.items() + if len(codes) == 1 + } + + +def _find_currency(text: str, locale: Locale) -> set[str]: + """Find the currency codes mentioned in OCR text. + + Three letter uppercase tokens that are ISO 4217 codes win. Only when there + are none are the locale's currency symbols looked for, longest first so + "A$" is not also read as "$". + """ + codes = { + code + for code in re.findall(r"(? str | None: + """Parse OCR text as a monetary value in the stored custom field format. + + The result is the amount with exactly two decimals, prefixed by the + currency code when the text names one ("USD12.50", "-12.50"). The rules are: + + - The currency is an ISO 4217 code written as an uppercase word, or else a + currency symbol of the locale that belongs to a single currency. Text + naming two different currencies is rejected, and text naming none gives + an amount without a code. Zone crops are expected to be short, but an + uppercase word that happens to be an ISO code (e.g. "ALL" or "TOP") is + still read as a currency. + - A number enclosed in parentheses is negative, "(12.50)". Digits outside + the parentheses, as in "100 (19% VAT)", or a minus sign combined with + them are rejected. + - The amount is read like any other number of the locale, must not need + more than two decimals ("12.500" is fine, "12.345" is not) and has at + most 12 digits, with or without a currency code. + - The result must pass the same validation as values sent through the API. + + Returns None when the text is not a valid monetary value. + """ + group = re.search(r"\([^()]*\d[^()]*\)", value) + parenthesised = group is not None + text = value + if group is not None: + outside = value[: group.start()] + value[group.end() :] + if re.search(r"\d", outside) or re.search( + "[-\N{MINUS SIGN}\N{EN DASH}]", + value, + ): + return None + text = value.replace("(", " ").replace(")", " ") + + currencies = _find_currency(text, locale) + if len(currencies) > 1: + return None + + amount = _parse_number(text, locale) + if amount is None: + return None + if parenthesised: + amount = amount.copy_negate() + try: + rounded = amount.quantize(Decimal("0.01")) + except InvalidOperation: + return None + if rounded != amount or len(rounded.as_tuple().digits) > 12: + return None + if rounded.is_zero(): + rounded = abs(rounded) + + result = f"{next(iter(currencies), '')}{rounded:f}" + try: + monetary_validator(result) + except ValidationError: + return None + return result + + def _convert_value( value: str, data_type: str, @@ -853,8 +959,7 @@ def _convert_value( return parsed.isoformat() if parsed else None elif data_type == CustomField.FieldDataType.MONETARY: - monetary = _parse_number(value, locale) - return None if monetary is None else format(monetary, "f") + return _parse_monetary(value, locale) elif data_type == CustomField.FieldDataType.BOOL: lower = value.lower().strip()