Fix: keep the currency and validate monetary zone values

OCR dropped currency codes and stored amounts the API would reject. Keep an
ISO code or locale currency symbol, read parentheses as negative, and validate
with the same rule as the API.
This commit is contained in:
stumpylog committed 2026-11-19 07:34:16 -08:00
1 parent 218122452c
commit 6fc4526947
5 files changed
+287 -29

No files matched your search

+2 -14
View File
@@ -5,7 +5,6 @@ import math
import re
from datetime import datetime
from datetime import timedelta
from decimal import Decimal
from typing import TYPE_CHECKING
from typing import Any
from typing import TypedDict
@@ -16,12 +15,10 @@ from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.validators import DecimalValidator
from django.core.validators import EmailValidator
from django.core.validators import MaxLengthValidator
from django.core.validators import MaxValueValidator
from django.core.validators import MinValueValidator
from django.core.validators import RegexValidator
from django.core.validators import integer_validator
from django.db import transaction
from django.db.models import Count
@@ -95,6 +92,7 @@ from documents.regex import validate_regex_pattern
from documents.templating.filepath import validate_filepath_template_and_render
from documents.templating.utils import convert_format_str_to_template_format
from documents.templating.workflows import validate_workflow_template
from documents.validators import monetary_validator
from documents.validators import uri_validator
from documents.validators import url_validator
from documents.versioning import has_prefetched_effective_content
@@ -908,17 +906,7 @@ class CustomFieldInstanceSerializer(serializers.ModelSerializer[CustomFieldInsta
field.data_type == CustomField.FieldDataType.MONETARY
and data["value"] != ""
):
try:
# First try to validate as a number from legacy format
DecimalValidator(max_digits=12, decimal_places=2)(
Decimal(str(data["value"])),
)
except Exception:
# If that fails, try to validate as a monetary string
RegexValidator(
regex=r"^[A-Z]{3}-?\d+(\.\d{1,2})$",
message="Must be a two-decimal number with optional currency code e.g. GBP123.45",
)(data["value"])
monetary_validator(data["value"])
elif field.data_type == CustomField.FieldDataType.STRING:
MaxLengthValidator(limit_value=128)(data["value"])
elif field.data_type == CustomField.FieldDataType.SELECT:
+57
View File
@@ -0,0 +1,57 @@
import pytest
from django.core.exceptions import ValidationError
from documents.validators import monetary_validator
class TestMonetaryValidator:
"""Tests for the monetary_validator function."""
@pytest.mark.parametrize(
"value",
[
pytest.param("12", id="whole-number"),
pytest.param("12.5", id="one-decimal"),
pytest.param("12.50", id="two-decimals"),
pytest.param("-12.50", id="negative"),
pytest.param("9999999999.99", id="twelve-digits"),
pytest.param("GBP123.45", id="code-with-amount"),
pytest.param("USD-12.50", id="code-with-negative"),
pytest.param("EUR12.5", id="code-with-one-decimal"),
],
)
def test_valid_values(self, value: str) -> None:
"""
GIVEN:
- A plain decimal or an amount prefixed with a currency code
WHEN:
- The value is validated
THEN:
- No error is raised
"""
monetary_validator(value)
@pytest.mark.parametrize(
"value",
[
pytest.param("12.345", id="too-many-decimals"),
pytest.param("99999999999.99", id="too-many-digits"),
pytest.param("GBP12.345", id="code-with-too-many-decimals"),
pytest.param("GBP12", id="code-without-decimals"),
pytest.param("gbp12.50", id="lowercase-code"),
pytest.param("GB12.50", id="short-code"),
pytest.param("abc", id="not-a-number"),
pytest.param("NaN", id="nan"),
],
)
def test_invalid_values(self, value: str) -> None:
"""
GIVEN:
- A value that is neither a valid decimal nor a coded amount
WHEN:
- The value is validated
THEN:
- A ValidationError is raised
"""
with pytest.raises(ValidationError):
monetary_validator(value)
+93 -13
View File
@@ -31,6 +31,7 @@ from documents.zone_ocr import _detect_mime
from documents.zone_ocr import _extract_zone
from documents.zone_ocr import _number_locale
from documents.zone_ocr import _parse_date
from documents.zone_ocr import _parse_monetary
from documents.zone_ocr import _prepare_field_update
from documents.zone_ocr import _process_template
from documents.zone_ocr import _render_pages
@@ -2284,6 +2285,92 @@ class TestNumberLocale:
assert _number_locale(ocr_language) == Locale.parse(expected)
class TestParseMonetary:
"""Tests for the _parse_monetary function."""
@pytest.mark.parametrize(
("value", "locale", "expected"),
[
pytest.param("12.50", EN, "12.50", id="plain-amount"),
pytest.param("12", EN, "12.00", id="whole-number-gets-two-decimals"),
pytest.param("12.500", EN, "12.50", id="trailing-zero-is-fine"),
pytest.param("USD 12.50", EN, "USD12.50", id="code-before-amount"),
pytest.param("12,50 EUR", DE, "EUR12.50", id="code-after-amount"),
pytest.param("USD12.50", EN, "USD12.50", id="code-attached-to-amount"),
pytest.param("$12.50", EN, "USD12.50", id="dollar-symbol-en"),
pytest.param("$12.50", FR, "12.50", id="dollar-symbol-fr-has-no-code"),
pytest.param(
"\N{EURO SIGN}12.50",
EN,
"EUR12.50",
id="euro-symbol",
),
pytest.param(
"\N{POUND SIGN}12.50",
EN,
"GBP12.50",
id="pound-symbol",
),
pytest.param(
"USD $12.50",
EN,
"USD12.50",
id="code-wins-over-symbol",
),
pytest.param("INV 12.50", EN, "12.50", id="non-currency-word-ignored"),
pytest.param("usd 12.50", EN, "12.50", id="lowercase-code-ignored"),
pytest.param("(12.50)", EN, "-12.50", id="parentheses-negative"),
pytest.param("($12.50)", EN, "USD-12.50", id="parentheses-with-symbol"),
pytest.param("(12,50 EUR)", DE, "EUR-12.50", id="parentheses-with-code"),
pytest.param("(0.00)", EN, "0.00", id="parentheses-zero-not-negative"),
pytest.param("(USD) 12.50", EN, "USD12.50", id="empty-parentheses-ignored"),
pytest.param("USD -12.50", EN, "USD-12.50", id="code-with-negative"),
pytest.param("-12.50", EN, "-12.50", id="plain-negative"),
pytest.param("1,234.56", EN, "1234.56", id="grouped-amount"),
pytest.param("USD 12.50 EUR", EN, None, id="two-codes"),
pytest.param("$12.50 \N{EURO SIGN}", EN, None, id="two-symbols"),
pytest.param("(-12.50)", EN, None, id="parentheses-and-minus"),
pytest.param("12.345", EN, None, id="too-many-decimals"),
pytest.param("12345678901.12", EN, None, id="more-than-12-digits"),
pytest.param(
"USD 12345678901.12",
EN,
None,
id="more-than-12-digits-with-code",
),
pytest.param("100 (19% VAT)", EN, None, id="digits-outside-parentheses"),
pytest.param("(12) (13)", EN, None, id="two-parenthesised-numbers"),
pytest.param("-0.00", EN, "0.00", id="negative-zero-not-negative"),
pytest.param("CHF 12", DE, "CHF12.00", id="code-in-german"),
pytest.param(
"1\N{NARROW NO-BREAK SPACE}234,56 \N{EURO SIGN}",
FR,
"EUR1234.56",
id="french-space-grouping",
),
pytest.param("USD", EN, None, id="code-without-amount"),
pytest.param("garbage", EN, None, id="garbage"),
pytest.param("", EN, None, id="empty"),
],
)
def test_parse_monetary(
self,
value: str,
locale: Locale,
expected: str | None,
) -> None:
"""
GIVEN:
- OCR text containing an amount, maybe with a currency or parentheses
WHEN:
- The text is parsed as a monetary value
THEN:
- The amount has two decimals and a leading currency code if one was
found, or None is returned when the text is not a valid value
"""
assert _parse_monetary(value, locale) == expected
class TestConvertValue:
"""Tests for the _convert_value function."""
@@ -2406,42 +2493,35 @@ class TestConvertValue:
pytest.param(
"\u20ac1,234.56",
CustomField.FieldDataType.MONETARY,
"1234.56",
"EUR1234.56",
EN,
id="monetary-with-currency-symbol",
),
pytest.param(
"12345678901234567890,12345678901234567890",
CustomField.FieldDataType.MONETARY,
"12345678901234567890.12345678901234567890",
None,
DE,
id="monetary-preserves-decimal-precision",
),
pytest.param(
"-12345678901234567890,12345678901234567890",
CustomField.FieldDataType.MONETARY,
"-12345678901234567890.12345678901234567890",
DE,
id="monetary-negative-preserves-decimal-precision",
id="monetary-too-precise-returns-none",
),
pytest.param(
"\u20ac-12.50",
CustomField.FieldDataType.MONETARY,
"-12.50",
"EUR-12.50",
EN,
id="monetary-sign-after-currency-symbol",
),
pytest.param(
"-\u20ac12.50",
CustomField.FieldDataType.MONETARY,
"-12.50",
"EUR-12.50",
EN,
id="monetary-sign-before-currency-symbol",
),
pytest.param(
"EUR -12.50",
CustomField.FieldDataType.MONETARY,
"-12.50",
"EUR-12.50",
EN,
id="monetary-sign-after-currency-code",
),
+28
View File
@@ -1,6 +1,9 @@
from decimal import Decimal
from urllib.parse import urlparse
from django.core.exceptions import ValidationError
from django.core.validators import DecimalValidator
from django.core.validators import RegexValidator
from django.utils.translation import gettext_lazy as _
@@ -58,3 +61,28 @@ def url_validator(value) -> None:
ValidationError: If the URL is invalid or not using http/https scheme
"""
uri_validator(value, allowed_schemes={"http", "https"})
def monetary_validator(value: str) -> None:
"""
Validates the stored value of a monetary custom field.
Accepts either a plain decimal with at most 12 digits and 2 decimal places
(the legacy format), or a two-decimal number prefixed by a three letter
uppercase currency code, e.g. GBP123.45 or USD-12.50.
Args:
value: The monetary value to validate
Raises:
ValidationError: If the value matches neither format
"""
try:
# First try to validate as a number from legacy format
DecimalValidator(max_digits=12, decimal_places=2)(Decimal(str(value)))
except Exception:
# If that fails, try to validate as a monetary string
RegexValidator(
regex=r"^[A-Z]{3}-?\d+(\.\d{1,2})$",
message="Must be a two-decimal number with optional currency code e.g. GBP123.45",
)(value)
+107 -2
View File
@@ -12,6 +12,7 @@ from datetime import date
from datetime import datetime
from decimal import Decimal
from decimal import InvalidOperation
from functools import lru_cache
from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING
@@ -24,6 +25,7 @@ from babel import UnknownLocaleError
from babel.numbers import NumberFormatError
from babel.numbers import get_decimal_symbol
from babel.numbers import get_group_symbol
from babel.numbers import is_currency
from babel.numbers import parse_decimal
from django.conf import settings
from django.core.exceptions import ValidationError
@@ -42,6 +44,7 @@ from documents.parsers import rasterize_pdf_page_to_png
from documents.permissions import permitted_object_ids
from documents.regex import safe_regex_fullmatch
from documents.signals.handlers import add_to_index
from documents.validators import monetary_validator
from documents.versioning import get_root_document
from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES
@@ -811,6 +814,109 @@ def _parse_number(value: str, locale: Locale) -> Decimal | None:
return number.copy_negate() if negative else number
@lru_cache(maxsize=32)
def _currency_symbol_codes(locale: Locale) -> dict[str, str]:
"""Map the unambiguous currency symbols of a locale to their currency code.
Symbols shared by several currencies, and symbols made only of letters or
digits (which includes plain three letter codes), are left out.
"""
codes_by_symbol: dict[str, set[str]] = {}
for code, symbol in locale.currency_symbols.items():
if not symbol.isalnum():
codes_by_symbol.setdefault(symbol, set()).add(code)
return {
symbol: next(iter(codes))
for symbol, codes in codes_by_symbol.items()
if len(codes) == 1
}
def _find_currency(text: str, locale: Locale) -> set[str]:
"""Find the currency codes mentioned in OCR text.
Three letter uppercase tokens that are ISO 4217 codes win. Only when there
are none are the locale's currency symbols looked for, longest first so
"A$" is not also read as "$".
"""
codes = {
code
for code in re.findall(r"(?<![A-Za-z])[A-Z]{3}(?![A-Za-z])", text)
if is_currency(code)
}
if codes:
return codes
for symbol, code in sorted(
_currency_symbol_codes(locale).items(),
key=lambda item: len(item[0]),
reverse=True,
):
if symbol in text:
codes.add(code)
text = text.replace(symbol, " ")
return codes
def _parse_monetary(value: str, locale: Locale) -> str | None:
"""Parse OCR text as a monetary value in the stored custom field format.
The result is the amount with exactly two decimals, prefixed by the
currency code when the text names one ("USD12.50", "-12.50"). The rules are:
- The currency is an ISO 4217 code written as an uppercase word, or else a
currency symbol of the locale that belongs to a single currency. Text
naming two different currencies is rejected, and text naming none gives
an amount without a code. Zone crops are expected to be short, but an
uppercase word that happens to be an ISO code (e.g. "ALL" or "TOP") is
still read as a currency.
- A number enclosed in parentheses is negative, "(12.50)". Digits outside
the parentheses, as in "100 (19% VAT)", or a minus sign combined with
them are rejected.
- The amount is read like any other number of the locale, must not need
more than two decimals ("12.500" is fine, "12.345" is not) and has at
most 12 digits, with or without a currency code.
- The result must pass the same validation as values sent through the API.
Returns None when the text is not a valid monetary value.
"""
group = re.search(r"\([^()]*\d[^()]*\)", value)
parenthesised = group is not None
text = value
if group is not None:
outside = value[: group.start()] + value[group.end() :]
if re.search(r"\d", outside) or re.search(
"[-\N{MINUS SIGN}\N{EN DASH}]",
value,
):
return None
text = value.replace("(", " ").replace(")", " ")
currencies = _find_currency(text, locale)
if len(currencies) > 1:
return None
amount = _parse_number(text, locale)
if amount is None:
return None
if parenthesised:
amount = amount.copy_negate()
try:
rounded = amount.quantize(Decimal("0.01"))
except InvalidOperation:
return None
if rounded != amount or len(rounded.as_tuple().digits) > 12:
return None
if rounded.is_zero():
rounded = abs(rounded)
result = f"{next(iter(currencies), '')}{rounded:f}"
try:
monetary_validator(result)
except ValidationError:
return None
return result
def _convert_value(
value: str,
data_type: str,
@@ -853,8 +959,7 @@ def _convert_value(
return parsed.isoformat() if parsed else None
elif data_type == CustomField.FieldDataType.MONETARY:
monetary = _parse_number(value, locale)
return None if monetary is None else format(monetary, "f")
return _parse_monetary(value, locale)
elif data_type == CustomField.FieldDataType.BOOL:
lower = value.lower().strip()