diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index acdfc21e72..3f4b49d68d 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -5,7 +5,6 @@ import math import re from datetime import datetime from datetime import timedelta -from decimal import Decimal from typing import TYPE_CHECKING from typing import Any from typing import TypedDict @@ -16,12 +15,10 @@ from django.contrib.auth.models import Group from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError -from django.core.validators import DecimalValidator from django.core.validators import EmailValidator from django.core.validators import MaxLengthValidator from django.core.validators import MaxValueValidator from django.core.validators import MinValueValidator -from django.core.validators import RegexValidator from django.core.validators import integer_validator from django.db import transaction from django.db.models import Count @@ -95,6 +92,7 @@ from documents.regex import validate_regex_pattern from documents.templating.filepath import validate_filepath_template_and_render from documents.templating.utils import convert_format_str_to_template_format from documents.templating.workflows import validate_workflow_template +from documents.validators import monetary_validator from documents.validators import uri_validator from documents.validators import url_validator from documents.versioning import has_prefetched_effective_content @@ -908,17 +906,7 @@ class CustomFieldInstanceSerializer(serializers.ModelSerializer[CustomFieldInsta field.data_type == CustomField.FieldDataType.MONETARY and data["value"] != "" ): - try: - # First try to validate as a number from legacy format - DecimalValidator(max_digits=12, decimal_places=2)( - Decimal(str(data["value"])), - ) - except Exception: - # If that fails, try to validate as a monetary string - RegexValidator( - regex=r"^[A-Z]{3}-?\d+(\.\d{1,2})$", - message="Must be a two-decimal number with optional currency code e.g. GBP123.45", - )(data["value"]) + monetary_validator(data["value"]) elif field.data_type == CustomField.FieldDataType.STRING: MaxLengthValidator(limit_value=128)(data["value"]) elif field.data_type == CustomField.FieldDataType.SELECT: diff --git a/src/documents/tests/test_validators.py b/src/documents/tests/test_validators.py new file mode 100644 index 0000000000..bac3d42b9d --- /dev/null +++ b/src/documents/tests/test_validators.py @@ -0,0 +1,57 @@ +import pytest +from django.core.exceptions import ValidationError + +from documents.validators import monetary_validator + + +class TestMonetaryValidator: + """Tests for the monetary_validator function.""" + + @pytest.mark.parametrize( + "value", + [ + pytest.param("12", id="whole-number"), + pytest.param("12.5", id="one-decimal"), + pytest.param("12.50", id="two-decimals"), + pytest.param("-12.50", id="negative"), + pytest.param("9999999999.99", id="twelve-digits"), + pytest.param("GBP123.45", id="code-with-amount"), + pytest.param("USD-12.50", id="code-with-negative"), + pytest.param("EUR12.5", id="code-with-one-decimal"), + ], + ) + def test_valid_values(self, value: str) -> None: + """ + GIVEN: + - A plain decimal or an amount prefixed with a currency code + WHEN: + - The value is validated + THEN: + - No error is raised + """ + monetary_validator(value) + + @pytest.mark.parametrize( + "value", + [ + pytest.param("12.345", id="too-many-decimals"), + pytest.param("99999999999.99", id="too-many-digits"), + pytest.param("GBP12.345", id="code-with-too-many-decimals"), + pytest.param("GBP12", id="code-without-decimals"), + pytest.param("gbp12.50", id="lowercase-code"), + pytest.param("GB12.50", id="short-code"), + pytest.param("abc", id="not-a-number"), + pytest.param("NaN", id="nan"), + ], + ) + def test_invalid_values(self, value: str) -> None: + """ + GIVEN: + - A value that is neither a valid decimal nor a coded amount + WHEN: + - The value is validated + THEN: + - A ValidationError is raised + """ + with pytest.raises(ValidationError): + monetary_validator(value) diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index a622aedb34..d7ec706822 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -31,6 +31,7 @@ from documents.zone_ocr import _detect_mime from documents.zone_ocr import _extract_zone from documents.zone_ocr import _number_locale from documents.zone_ocr import _parse_date +from documents.zone_ocr import _parse_monetary from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages @@ -2284,6 +2285,92 @@ class TestNumberLocale: assert _number_locale(ocr_language) == Locale.parse(expected) +class TestParseMonetary: + """Tests for the _parse_monetary function.""" + + @pytest.mark.parametrize( + ("value", "locale", "expected"), + [ + pytest.param("12.50", EN, "12.50", id="plain-amount"), + pytest.param("12", EN, "12.00", id="whole-number-gets-two-decimals"), + pytest.param("12.500", EN, "12.50", id="trailing-zero-is-fine"), + pytest.param("USD 12.50", EN, "USD12.50", id="code-before-amount"), + pytest.param("12,50 EUR", DE, "EUR12.50", id="code-after-amount"), + pytest.param("USD12.50", EN, "USD12.50", id="code-attached-to-amount"), + pytest.param("$12.50", EN, "USD12.50", id="dollar-symbol-en"), + pytest.param("$12.50", FR, "12.50", id="dollar-symbol-fr-has-no-code"), + pytest.param( + "\N{EURO SIGN}12.50", + EN, + "EUR12.50", + id="euro-symbol", + ), + pytest.param( + "\N{POUND SIGN}12.50", + EN, + "GBP12.50", + id="pound-symbol", + ), + pytest.param( + "USD $12.50", + EN, + "USD12.50", + id="code-wins-over-symbol", + ), + pytest.param("INV 12.50", EN, "12.50", id="non-currency-word-ignored"), + pytest.param("usd 12.50", EN, "12.50", id="lowercase-code-ignored"), + pytest.param("(12.50)", EN, "-12.50", id="parentheses-negative"), + pytest.param("($12.50)", EN, "USD-12.50", id="parentheses-with-symbol"), + pytest.param("(12,50 EUR)", DE, "EUR-12.50", id="parentheses-with-code"), + pytest.param("(0.00)", EN, "0.00", id="parentheses-zero-not-negative"), + pytest.param("(USD) 12.50", EN, "USD12.50", id="empty-parentheses-ignored"), + pytest.param("USD -12.50", EN, "USD-12.50", id="code-with-negative"), + pytest.param("-12.50", EN, "-12.50", id="plain-negative"), + pytest.param("1,234.56", EN, "1234.56", id="grouped-amount"), + pytest.param("USD 12.50 EUR", EN, None, id="two-codes"), + pytest.param("$12.50 \N{EURO SIGN}", EN, None, id="two-symbols"), + pytest.param("(-12.50)", EN, None, id="parentheses-and-minus"), + pytest.param("12.345", EN, None, id="too-many-decimals"), + pytest.param("12345678901.12", EN, None, id="more-than-12-digits"), + pytest.param( + "USD 12345678901.12", + EN, + None, + id="more-than-12-digits-with-code", + ), + pytest.param("100 (19% VAT)", EN, None, id="digits-outside-parentheses"), + pytest.param("(12) (13)", EN, None, id="two-parenthesised-numbers"), + pytest.param("-0.00", EN, "0.00", id="negative-zero-not-negative"), + pytest.param("CHF 12", DE, "CHF12.00", id="code-in-german"), + pytest.param( + "1\N{NARROW NO-BREAK SPACE}234,56 \N{EURO SIGN}", + FR, + "EUR1234.56", + id="french-space-grouping", + ), + pytest.param("USD", EN, None, id="code-without-amount"), + pytest.param("garbage", EN, None, id="garbage"), + pytest.param("", EN, None, id="empty"), + ], + ) + def test_parse_monetary( + self, + value: str, + locale: Locale, + expected: str | None, + ) -> None: + """ + GIVEN: + - OCR text containing an amount, maybe with a currency or parentheses + WHEN: + - The text is parsed as a monetary value + THEN: + - The amount has two decimals and a leading currency code if one was + found, or None is returned when the text is not a valid value + """ + assert _parse_monetary(value, locale) == expected + + class TestConvertValue: """Tests for the _convert_value function.""" @@ -2406,42 +2493,35 @@ class TestConvertValue: pytest.param( "\u20ac1,234.56", CustomField.FieldDataType.MONETARY, - "1234.56", + "EUR1234.56", EN, id="monetary-with-currency-symbol", ), pytest.param( "12345678901234567890,12345678901234567890", CustomField.FieldDataType.MONETARY, - "12345678901234567890.12345678901234567890", + None, DE, - id="monetary-preserves-decimal-precision", - ), - pytest.param( - "-12345678901234567890,12345678901234567890", - CustomField.FieldDataType.MONETARY, - "-12345678901234567890.12345678901234567890", - DE, - id="monetary-negative-preserves-decimal-precision", + id="monetary-too-precise-returns-none", ), pytest.param( "\u20ac-12.50", CustomField.FieldDataType.MONETARY, - "-12.50", + "EUR-12.50", EN, id="monetary-sign-after-currency-symbol", ), pytest.param( "-\u20ac12.50", CustomField.FieldDataType.MONETARY, - "-12.50", + "EUR-12.50", EN, id="monetary-sign-before-currency-symbol", ), pytest.param( "EUR -12.50", CustomField.FieldDataType.MONETARY, - "-12.50", + "EUR-12.50", EN, id="monetary-sign-after-currency-code", ), diff --git a/src/documents/validators.py b/src/documents/validators.py index bec7252bf1..a28f8fe210 100644 --- a/src/documents/validators.py +++ b/src/documents/validators.py @@ -1,6 +1,9 @@ +from decimal import Decimal from urllib.parse import urlparse from django.core.exceptions import ValidationError +from django.core.validators import DecimalValidator +from django.core.validators import RegexValidator from django.utils.translation import gettext_lazy as _ @@ -58,3 +61,28 @@ def url_validator(value) -> None: ValidationError: If the URL is invalid or not using http/https scheme """ uri_validator(value, allowed_schemes={"http", "https"}) + + +def monetary_validator(value: str) -> None: + """ + Validates the stored value of a monetary custom field. + + Accepts either a plain decimal with at most 12 digits and 2 decimal places + (the legacy format), or a two-decimal number prefixed by a three letter + uppercase currency code, e.g. GBP123.45 or USD-12.50. + + Args: + value: The monetary value to validate + + Raises: + ValidationError: If the value matches neither format + """ + try: + # First try to validate as a number from legacy format + DecimalValidator(max_digits=12, decimal_places=2)(Decimal(str(value))) + except Exception: + # If that fails, try to validate as a monetary string + RegexValidator( + regex=r"^[A-Z]{3}-?\d+(\.\d{1,2})$", + message="Must be a two-decimal number with optional currency code e.g. GBP123.45", + )(value) diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 300a35538e..76bc70d6d4 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -12,6 +12,7 @@ from datetime import date from datetime import datetime from decimal import Decimal from decimal import InvalidOperation +from functools import lru_cache from functools import partial from pathlib import Path from typing import TYPE_CHECKING @@ -24,6 +25,7 @@ from babel import UnknownLocaleError from babel.numbers import NumberFormatError from babel.numbers import get_decimal_symbol from babel.numbers import get_group_symbol +from babel.numbers import is_currency from babel.numbers import parse_decimal from django.conf import settings from django.core.exceptions import ValidationError @@ -42,6 +44,7 @@ from documents.parsers import rasterize_pdf_page_to_png from documents.permissions import permitted_object_ids from documents.regex import safe_regex_fullmatch from documents.signals.handlers import add_to_index +from documents.validators import monetary_validator from documents.versioning import get_root_document from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES @@ -811,6 +814,109 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None: return number.copy_negate() if negative else number +@lru_cache(maxsize=32) +def _currency_symbol_codes(locale: Locale) -> dict[str, str]: + """Map the unambiguous currency symbols of a locale to their currency code. + + Symbols shared by several currencies, and symbols made only of letters or + digits (which includes plain three letter codes), are left out. + """ + codes_by_symbol: dict[str, set[str]] = {} + for code, symbol in locale.currency_symbols.items(): + if not symbol.isalnum(): + codes_by_symbol.setdefault(symbol, set()).add(code) + return { + symbol: next(iter(codes)) + for symbol, codes in codes_by_symbol.items() + if len(codes) == 1 + } + + +def _find_currency(text: str, locale: Locale) -> set[str]: + """Find the currency codes mentioned in OCR text. + + Three letter uppercase tokens that are ISO 4217 codes win. Only when there + are none are the locale's currency symbols looked for, longest first so + "A$" is not also read as "$". + """ + codes = { + code + for code in re.findall(r"(? str | None: + """Parse OCR text as a monetary value in the stored custom field format. + + The result is the amount with exactly two decimals, prefixed by the + currency code when the text names one ("USD12.50", "-12.50"). The rules are: + + - The currency is an ISO 4217 code written as an uppercase word, or else a + currency symbol of the locale that belongs to a single currency. Text + naming two different currencies is rejected, and text naming none gives + an amount without a code. Zone crops are expected to be short, but an + uppercase word that happens to be an ISO code (e.g. "ALL" or "TOP") is + still read as a currency. + - A number enclosed in parentheses is negative, "(12.50)". Digits outside + the parentheses, as in "100 (19% VAT)", or a minus sign combined with + them are rejected. + - The amount is read like any other number of the locale, must not need + more than two decimals ("12.500" is fine, "12.345" is not) and has at + most 12 digits, with or without a currency code. + - The result must pass the same validation as values sent through the API. + + Returns None when the text is not a valid monetary value. + """ + group = re.search(r"\([^()]*\d[^()]*\)", value) + parenthesised = group is not None + text = value + if group is not None: + outside = value[: group.start()] + value[group.end() :] + if re.search(r"\d", outside) or re.search( + "[-\N{MINUS SIGN}\N{EN DASH}]", + value, + ): + return None + text = value.replace("(", " ").replace(")", " ") + + currencies = _find_currency(text, locale) + if len(currencies) > 1: + return None + + amount = _parse_number(text, locale) + if amount is None: + return None + if parenthesised: + amount = amount.copy_negate() + try: + rounded = amount.quantize(Decimal("0.01")) + except InvalidOperation: + return None + if rounded != amount or len(rounded.as_tuple().digits) > 12: + return None + if rounded.is_zero(): + rounded = abs(rounded) + + result = f"{next(iter(currencies), '')}{rounded:f}" + try: + monetary_validator(result) + except ValidationError: + return None + return result + + def _convert_value( value: str, data_type: str, @@ -853,8 +959,7 @@ def _convert_value( return parsed.isoformat() if parsed else None elif data_type == CustomField.FieldDataType.MONETARY: - monetary = _parse_number(value, locale) - return None if monetary is None else format(monetary, "f") + return _parse_monetary(value, locale) elif data_type == CustomField.FieldDataType.BOOL: lower = value.lower().strip()