mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-20 20:46:53 +00:00
OCR dropped currency codes and stored amounts the API would reject. Keep an ISO code or locale currency symbol, read parentheses as negative, and validate with the same rule as the API.
3398 lines
107 KiB
Python
3398 lines
107 KiB
Python
"""Tests for the zone-based OCR extraction engine."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import subprocess
|
|
from collections import Counter
|
|
from datetime import date
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from babel import Locale
|
|
from django.contrib.auth.models import Group
|
|
from PIL import Image
|
|
|
|
from documents.models import CustomField
|
|
from documents.models import CustomFieldInstance
|
|
from documents.models import Document
|
|
from documents.models import MatchingModel
|
|
from documents.models import OcrTemplate
|
|
from documents.models import OcrTemplateZone
|
|
from documents.parsers import ParseError
|
|
from documents.zone_ocr import _apply_transform
|
|
from documents.zone_ocr import _combine_field_value
|
|
from documents.zone_ocr import _convert_value
|
|
from documents.zone_ocr import _crop_zone
|
|
from documents.zone_ocr import _date_rules
|
|
from documents.zone_ocr import _detect_mime
|
|
from documents.zone_ocr import _extract_zone
|
|
from documents.zone_ocr import _number_locale
|
|
from documents.zone_ocr import _parse_date
|
|
from documents.zone_ocr import _parse_monetary
|
|
from documents.zone_ocr import _prepare_field_update
|
|
from documents.zone_ocr import _process_template
|
|
from documents.zone_ocr import _render_pages
|
|
from documents.zone_ocr import _resolve_doc_path
|
|
from documents.zone_ocr import _zone_language
|
|
from documents.zone_ocr import extract_zone_preview
|
|
from documents.zone_ocr import get_templates_for_document
|
|
from documents.zone_ocr import run_zone_extraction
|
|
from paperless_testing.factories import CustomFieldFactory
|
|
from paperless_testing.factories import DocumentFactory
|
|
from paperless_testing.factories import OcrTemplateFactory
|
|
from paperless_testing.factories import OcrTemplateZoneFactory
|
|
from paperless_testing.factories import UserFactory
|
|
from paperless_testing.permissions import grant_object
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable
|
|
from unittest.mock import MagicMock
|
|
|
|
from django.contrib.auth.models import User
|
|
from pytest_django.fixtures import SettingsWrapper
|
|
from pytest_mock import MockerFixture
|
|
|
|
from paperless_testing.dirs import PaperlessDirs
|
|
|
|
|
|
EN = Locale.parse("en")
|
|
DE = Locale.parse("de")
|
|
ENG_DATES = _date_rules("eng")
|
|
FR = Locale.parse("fr")
|
|
|
|
|
|
def make_template(name: str, **kwargs: Any) -> OcrTemplate:
|
|
"""Build a saved template, 100x50 by default."""
|
|
kwargs.setdefault("source_width", 100)
|
|
kwargs.setdefault("source_height", 50)
|
|
return OcrTemplateFactory(name=name, **kwargs)
|
|
|
|
|
|
def make_zone(
|
|
template: OcrTemplate,
|
|
name: str,
|
|
target: str = OcrTemplateZone.TargetType.TITLE,
|
|
**kwargs: Any,
|
|
) -> OcrTemplateZone:
|
|
"""Build a 20x10 zone at the origin of the template."""
|
|
return OcrTemplateZoneFactory(
|
|
template=template,
|
|
name=name,
|
|
target=target,
|
|
width=20,
|
|
height=10,
|
|
**kwargs,
|
|
)
|
|
|
|
|
|
def ocr_outputs(*values: str) -> list[subprocess.CompletedProcess[str]]:
|
|
"""Build one successful fake OCR process result per extracted text."""
|
|
return [
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
|
|
for value in values
|
|
]
|
|
|
|
|
|
def make_visible_template(name: str, **kwargs: Any) -> OcrTemplate:
|
|
"""Build a 100x100 template matching "Northwind"."""
|
|
kwargs.setdefault("match", "Northwind")
|
|
kwargs.setdefault("source_height", 100)
|
|
return make_template(name, **kwargs)
|
|
|
|
|
|
class TestZoneExtraction:
|
|
"""Tests for single-zone cropping, extraction and preview."""
|
|
|
|
@pytest.fixture
|
|
def page(self, tmp_path: Path) -> Path:
|
|
page = tmp_path / "page.png"
|
|
with Image.new("RGB", (200, 100), "white") as image:
|
|
image.paste("black", (20, 10, 60, 30))
|
|
image.save(page)
|
|
return page
|
|
|
|
@pytest.fixture
|
|
def zone(self) -> OcrTemplateZone:
|
|
return OcrTemplateZone(
|
|
name="Account",
|
|
x=10,
|
|
y=5,
|
|
width=20,
|
|
height=10,
|
|
transform="uppercase",
|
|
ocr_language="eng",
|
|
)
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def scratch_dir(self, settings: SettingsWrapper, tmp_path: Path) -> None:
|
|
settings.SCRATCH_DIR = tmp_path
|
|
|
|
def test_crop_scales_coordinates(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone and a page image rendered at a different size than the zone's
|
|
WHEN:
|
|
- The zone is cropped
|
|
THEN:
|
|
- The zone coordinates are scaled to the image and the crop is black
|
|
"""
|
|
with _crop_zone(page, zone, 100, 50) as cropped:
|
|
assert cropped.size == (40, 20)
|
|
assert cropped.getextrema() == ((0, 0), (0, 0), (0, 0))
|
|
|
|
def test_crop_defaults_to_image_dimensions(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone and a page with unknown (zero) reference dimensions
|
|
WHEN:
|
|
- The zone is cropped
|
|
THEN:
|
|
- The image's own dimensions are used for the crop
|
|
"""
|
|
with _crop_zone(page, zone, 0, 0) as cropped:
|
|
assert cropped.size == (20, 10)
|
|
assert cropped.getpixel((0, 0)) == (255, 255, 255)
|
|
assert cropped.getpixel((19, 9)) == (0, 0, 0)
|
|
|
|
def test_crop_clamps_to_image(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone that extends past the edge of the page image
|
|
WHEN:
|
|
- The zone is cropped
|
|
THEN:
|
|
- The crop is clamped to the image bounds
|
|
"""
|
|
zone.x, zone.y = 90, 40
|
|
with _crop_zone(page, zone, 100, 50) as cropped:
|
|
assert cropped.size == (20, 20)
|
|
|
|
def test_tiny_crop_skips_reader(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
tmp_path: Path,
|
|
mock_zone_ocr_run: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone with zero width
|
|
WHEN:
|
|
- The zone is extracted
|
|
THEN:
|
|
- No text or value is returned and the reader is never run
|
|
"""
|
|
zone.width = 0
|
|
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
|
assert result.raw_text is None
|
|
assert result.value is None
|
|
mock_zone_ocr_run.assert_not_called()
|
|
|
|
def test_unreadable_image_skips_reader(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
tmp_path: Path,
|
|
mock_zone_ocr_run: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A page file that is not a valid image
|
|
WHEN:
|
|
- The zone is extracted
|
|
THEN:
|
|
- No text or value is returned and the reader is never run
|
|
"""
|
|
page.write_bytes(b"not an image")
|
|
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
|
assert result.raw_text is None
|
|
assert result.value is None
|
|
mock_zone_ocr_run.assert_not_called()
|
|
|
|
def test_text_extraction_retains_raw_text_and_transforms(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
tmp_path: Path,
|
|
mock_zone_ocr_run: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone with an uppercase transform and a reader returning padded text
|
|
WHEN:
|
|
- The zone is extracted
|
|
THEN:
|
|
- The reader receives the cropped image with the expected arguments
|
|
- The stripped raw text is retained and the transformed value returned
|
|
"""
|
|
|
|
def read_crop(
|
|
args: list[str],
|
|
**kwargs: Any,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
with Image.open(args[1]) as crop:
|
|
assert crop.size == (40, 20)
|
|
assert crop.getpixel((0, 0)) == (0, 0, 0)
|
|
return subprocess.CompletedProcess(
|
|
args=[],
|
|
returncode=0,
|
|
stdout=" account abc\n",
|
|
stderr="",
|
|
)
|
|
|
|
mock_zone_ocr_run.side_effect = read_crop
|
|
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
|
assert result.raw_text == "account abc"
|
|
assert result.value == "ACCOUNT ABC"
|
|
assert mock_zone_ocr_run.call_args.args[0][-4:] == ["-l", "eng", "--psm", "6"]
|
|
assert mock_zone_ocr_run.call_args.kwargs["timeout"] == 30
|
|
|
|
def test_no_text_returns_no_value(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
tmp_path: Path,
|
|
mock_zone_ocr_run: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A reader that returns only whitespace
|
|
WHEN:
|
|
- The zone is extracted
|
|
THEN:
|
|
- No text or value is returned
|
|
"""
|
|
mock_zone_ocr_run.return_value.stdout = " \n"
|
|
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
|
assert result.raw_text is None
|
|
assert result.value is None
|
|
|
|
@pytest.mark.parametrize(
|
|
"error",
|
|
[
|
|
pytest.param(subprocess.TimeoutExpired("tesseract", 30), id="timeout"),
|
|
pytest.param(
|
|
subprocess.CalledProcessError(1, "tesseract", stderr="failure"),
|
|
id="called-process-error-with-stderr",
|
|
),
|
|
pytest.param(
|
|
subprocess.CalledProcessError(1, "tesseract"),
|
|
id="called-process-error-without-stderr",
|
|
),
|
|
],
|
|
)
|
|
def test_reader_failures_return_no_value(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
tmp_path: Path,
|
|
mock_zone_ocr_run: MagicMock,
|
|
error: Exception,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A reader that fails with a timeout or process error
|
|
WHEN:
|
|
- The zone is extracted
|
|
THEN:
|
|
- No text or value is returned
|
|
"""
|
|
mock_zone_ocr_run.side_effect = error
|
|
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
|
assert result.raw_text is None
|
|
assert result.value is None
|
|
|
|
@pytest.mark.parametrize(
|
|
"reader_kwargs",
|
|
[
|
|
pytest.param(
|
|
{"side_effect": RuntimeError("reader failed")},
|
|
id="reader-raises",
|
|
),
|
|
pytest.param({"return_value": []}, id="nothing-found"),
|
|
],
|
|
)
|
|
def test_barcode_without_value_returns_no_value(
|
|
self,
|
|
mocker: MockerFixture,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
tmp_path: Path,
|
|
reader_kwargs: dict[str, Any],
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A QR code zone and a barcode reader that either raises or finds
|
|
no barcode
|
|
WHEN:
|
|
- The zone is extracted
|
|
THEN:
|
|
- No text or value is returned
|
|
"""
|
|
mocker.patch("zxingcpp.read_barcodes", **reader_kwargs)
|
|
zone.transform = "qr_code"
|
|
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
|
assert result.raw_text is None
|
|
assert result.value is None
|
|
|
|
@pytest.mark.parametrize(
|
|
("attribute", "value"),
|
|
[
|
|
pytest.param("width", 0, id="zero-width"),
|
|
pytest.param("page", 2, id="unavailable-page"),
|
|
],
|
|
)
|
|
def test_preview_unreadable_zone_returns_empty_result(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
mock_zone_ocr_run: MagicMock,
|
|
attribute: str,
|
|
value: int,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone that cannot be read, either because its crop has zero
|
|
width or because it targets a page that is not available
|
|
WHEN:
|
|
- A preview is requested
|
|
THEN:
|
|
- An empty result is returned and the reader is never run
|
|
"""
|
|
setattr(zone, attribute, value)
|
|
assert extract_zone_preview(page, zone, 100, 50, 1) == {
|
|
"raw_text": None,
|
|
"value": None,
|
|
}
|
|
mock_zone_ocr_run.assert_not_called()
|
|
|
|
def test_date_transform_uses_zone_format(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
tmp_path: Path,
|
|
mock_zone_ocr_run: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A date zone with a custom date format
|
|
WHEN:
|
|
- The zone is extracted from text in that format
|
|
THEN:
|
|
- The raw text is retained and the value is an ISO date
|
|
"""
|
|
zone.transform = "date"
|
|
zone.date_format = "%d/%m/%Y"
|
|
mock_zone_ocr_run.return_value.stdout = "25/12/2025"
|
|
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
|
assert result.raw_text == "25/12/2025"
|
|
assert result.value == "2025-12-25"
|
|
|
|
def test_barcode_extraction_bypasses_tesseract(
|
|
self,
|
|
mocker: MockerFixture,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
tmp_path: Path,
|
|
mock_zone_ocr_run: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A QR code zone and a barcode reader returning padded text
|
|
WHEN:
|
|
- The zone is extracted
|
|
THEN:
|
|
- The barcode reader receives the cropped image
|
|
- The raw text is kept as read and the value is stripped
|
|
- The text reader is never run and the image is closed afterwards
|
|
"""
|
|
read_barcodes = mocker.patch("zxingcpp.read_barcodes")
|
|
zone.transform = "qr_code"
|
|
|
|
def read_crop(image: Image.Image) -> list[MagicMock]:
|
|
assert image.size == (40, 20)
|
|
assert image.getpixel((0, 0)) == (0, 0, 0)
|
|
return [mocker.MagicMock(text=" https://example.com ")]
|
|
|
|
read_barcodes.side_effect = read_crop
|
|
result = _extract_zone(page, zone, 100, 50, tmp_path)
|
|
assert result.raw_text == " https://example.com "
|
|
assert result.value == "https://example.com"
|
|
mock_zone_ocr_run.assert_not_called()
|
|
with pytest.raises(ValueError):
|
|
read_barcodes.call_args.args[0].getpixel((0, 0))
|
|
|
|
def test_preview_uses_image_dimensions_without_writing(
|
|
self,
|
|
page: Path,
|
|
zone: OcrTemplateZone,
|
|
mock_zone_ocr_run: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone and a reader returning text
|
|
WHEN:
|
|
- A preview is requested with unknown (zero) reference dimensions
|
|
THEN:
|
|
- The raw text and transformed value are returned
|
|
"""
|
|
mock_zone_ocr_run.return_value.stdout = "account abc"
|
|
assert extract_zone_preview(page, zone, 0, 0, 1) == {
|
|
"raw_text": "account abc",
|
|
"value": "ACCOUNT ABC",
|
|
}
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@pytest.mark.usefixtures("paperless_dirs")
|
|
class TestSavedZoneExtraction:
|
|
"""Tests for extracting zones from a saved template and writing the results."""
|
|
|
|
@pytest.fixture
|
|
def page(self, paperless_dirs: PaperlessDirs) -> Path:
|
|
page = paperless_dirs.scratch_dir / "page.png"
|
|
with Image.new("RGB", (200, 100), "white") as image:
|
|
image.save(page)
|
|
return page
|
|
|
|
@pytest.fixture
|
|
def document(self) -> Document:
|
|
return DocumentFactory(title="Original")
|
|
|
|
@pytest.fixture
|
|
def template(self) -> OcrTemplate:
|
|
return make_template("Account")
|
|
|
|
def test_combines_zones_and_writes_each_field_once(
|
|
self,
|
|
mocker: MockerFixture,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Two title zones and a combine format that joins them
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- The combined value is written to the title with a single save
|
|
"""
|
|
zones = [make_zone(template, "First"), make_zone(template, "Second")]
|
|
template.combine_formats = {"title": "{Second} / {First}"}
|
|
mock_zone_ocr_run.side_effect = [
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout="one", stderr=""),
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout="two", stderr=""),
|
|
]
|
|
save = mocker.patch.object(document, "save", wraps=document.save)
|
|
|
|
_process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
zones,
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
document.refresh_from_db()
|
|
assert document.title == "two / one"
|
|
save.assert_called_once_with(update_fields=["title"])
|
|
|
|
def test_converts_custom_and_builtin_fields_before_saving(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Zones for the title, ASN, created date and a boolean custom field
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- Each raw value is converted to its field type and saved
|
|
"""
|
|
field = CustomFieldFactory(name="Paid", data_type="boolean")
|
|
zones = [
|
|
make_zone(template, "Title"),
|
|
make_zone(template, "ASN", "asn"),
|
|
make_zone(template, "Created", "created"),
|
|
make_zone(template, "Paid", "custom_field", custom_field=field),
|
|
]
|
|
mock_zone_ocr_run.side_effect = [
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
|
|
for value in ["New title", "ASN 42", "2026-04-13", "no"]
|
|
]
|
|
|
|
_process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
zones,
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
document.refresh_from_db()
|
|
assert document.title == "New title"
|
|
assert document.archive_serial_number == 42
|
|
assert document.created == date(2026, 4, 13)
|
|
assert not document.custom_fields.get(field=field).value
|
|
|
|
def test_invalid_field_values_preserve_existing_values(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document with an existing custom field value
|
|
- Zones whose ASN, created date and custom field values are invalid
|
|
or unassigned
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- Only the valid title is written and the other values are preserved
|
|
"""
|
|
field = CustomFieldFactory(name="Number", data_type="integer")
|
|
CustomFieldInstance.objects.create(
|
|
document=document,
|
|
field=field,
|
|
value_int=12,
|
|
)
|
|
zones = [
|
|
make_zone(template, "Title"),
|
|
make_zone(template, "ASN", "asn"),
|
|
make_zone(template, "Created", "created"),
|
|
make_zone(template, "Number", "custom_field", custom_field=field),
|
|
make_zone(template, "Unassigned", "custom_field"),
|
|
]
|
|
original_created = document.created
|
|
mock_zone_ocr_run.side_effect = [
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
|
|
for value in [
|
|
"Valid",
|
|
"9999999999999999999999999999999",
|
|
"invalid",
|
|
"abc",
|
|
"text",
|
|
]
|
|
]
|
|
|
|
_process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
zones,
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
document.refresh_from_db()
|
|
assert document.title == "Valid"
|
|
assert document.archive_serial_number is None
|
|
assert document.created == original_created
|
|
assert document.custom_fields.get(field=field).value == 12
|
|
|
|
def test_write_failure_rolls_back_template_and_restores_document(
|
|
self,
|
|
mocker: MockerFixture,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A title zone and a custom field zone
|
|
- A document save that raises after writing
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- The error propagates, the write is rolled back and the in-memory
|
|
document is restored
|
|
"""
|
|
field = CustomFieldFactory(name="Number", data_type="integer")
|
|
zones = [
|
|
make_zone(template, "Title"),
|
|
make_zone(template, "Number", "custom_field", custom_field=field),
|
|
]
|
|
mock_zone_ocr_run.side_effect = [
|
|
subprocess.CompletedProcess(
|
|
args=[],
|
|
returncode=0,
|
|
stdout="New title",
|
|
stderr="",
|
|
),
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout="42", stderr=""),
|
|
]
|
|
save = document.save
|
|
|
|
def fail_after_save(**kwargs: Any) -> None:
|
|
save(**kwargs)
|
|
raise RuntimeError("save failed")
|
|
|
|
mocker.patch.object(document, "save", side_effect=fail_after_save)
|
|
|
|
with pytest.raises(RuntimeError, match="save failed"):
|
|
_process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
zones,
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert document.title == "Original"
|
|
document.refresh_from_db()
|
|
assert document.title == "Original"
|
|
assert not document.custom_fields.exists()
|
|
|
|
def test_last_successful_template_wins(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Three templates writing the title, the last of which rejects its value
|
|
WHEN:
|
|
- Zone extraction runs over all of them
|
|
THEN:
|
|
- The title comes from the last template that produced a value
|
|
"""
|
|
make_zone(template, "First")
|
|
second = make_template("Second")
|
|
make_zone(second, "Second")
|
|
third = make_template("Third")
|
|
make_zone(third, "Third", validation_regex="accepted")
|
|
templates = list(OcrTemplate.objects.order_by("pk"))
|
|
mock_zone_ocr_run.side_effect = [
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
|
|
for value in ["First", "Second", "rejected"]
|
|
]
|
|
|
|
run_zone_extraction(document, page, templates=templates)
|
|
|
|
document.refresh_from_db()
|
|
assert document.title == "Second"
|
|
|
|
def test_document_run_renders_each_page_once_across_templates(
|
|
self,
|
|
mocker: MockerFixture,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Two templates with zones on the first, last and second page of a
|
|
two page document
|
|
WHEN:
|
|
- Zone extraction runs over both templates
|
|
THEN:
|
|
- Each page is rendered only once and the rendered files are removed
|
|
"""
|
|
document.page_count = 2
|
|
make_zone(template, "First", page=1)
|
|
make_zone(template, "Last", page=-1)
|
|
second = make_template("Second")
|
|
make_zone(second, "Last again", page=2)
|
|
pdf = paperless_dirs.scratch_dir / "document.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4")
|
|
rendered = []
|
|
|
|
def render(in_path: Path, out_path: Path, **kwargs: Any) -> None:
|
|
with Image.new("RGB", (100, 50), "white") as image:
|
|
image.save(out_path)
|
|
rendered.append(out_path)
|
|
|
|
mock_rasterize = mocker.patch(
|
|
"documents.zone_ocr.rasterize_pdf_page_to_png",
|
|
side_effect=render,
|
|
)
|
|
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
|
|
[],
|
|
0,
|
|
stdout="extracted",
|
|
)
|
|
|
|
results = run_zone_extraction(document, pdf, templates=[template, second])
|
|
|
|
assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2]
|
|
assert [result.reason for result in results] == [None, None, None]
|
|
assert not any(path.exists() for path in rendered)
|
|
|
|
def test_zone_dimensions_override_template_dimensions(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone with its own source dimensions, differing from the template's
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- The zone is cropped using the zone's dimensions
|
|
"""
|
|
zone = make_zone(
|
|
template,
|
|
"Title",
|
|
zone_source_width=200,
|
|
zone_source_height=100,
|
|
)
|
|
|
|
def read_crop(
|
|
args: list[str],
|
|
**kwargs: Any,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
with Image.open(args[1]) as crop:
|
|
assert crop.size == (20, 10)
|
|
return subprocess.CompletedProcess(args, 0, stdout="extracted")
|
|
|
|
mock_zone_ocr_run.side_effect = read_crop
|
|
|
|
_process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
[zone],
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert document.title == "extracted"
|
|
|
|
def test_missing_page_returns_skipped_result(
|
|
self,
|
|
paperless_dirs: PaperlessDirs,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone on the last page and no rendered pages
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- The zone result is skipped as a missing page with no value
|
|
"""
|
|
zone = make_zone(template, "Last", page=-1)
|
|
|
|
results = _process_template(
|
|
document,
|
|
{},
|
|
template,
|
|
[zone],
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert results[0].reason == "missing_page"
|
|
assert results[0].value is None
|
|
|
|
def test_missing_document_file_returns_skipped_results(
|
|
self,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A template with a zone and no document file
|
|
WHEN:
|
|
- Zone extraction runs
|
|
THEN:
|
|
- A single result is skipped as a missing file with no value
|
|
"""
|
|
make_zone(template, "Title")
|
|
|
|
results = run_zone_extraction(document, None, templates=[template])
|
|
|
|
assert len(results) == 1
|
|
assert results[0].reason == "missing_file"
|
|
assert results[0].value is None
|
|
|
|
def test_empty_combination_cannot_clear_a_field(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A combine format that references an unknown zone and so is empty
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- The result is an invalid value and the title is not cleared
|
|
"""
|
|
zone = make_zone(template, "Title")
|
|
template.combine_formats = {"title": "{Unknown}"}
|
|
mock_zone_ocr_run.return_value.stdout = "extracted"
|
|
|
|
results = _process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
[zone],
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert results[0].reason == "invalid_value"
|
|
assert results[0].value is None
|
|
assert document.title == "Original"
|
|
|
|
def test_skipped_results_never_report_existing_document_values(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Zones that are empty, rejected by regex, invalid or unassigned
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- Each result carries its skip reason and no value, and the title
|
|
is unchanged
|
|
"""
|
|
zones = [
|
|
make_zone(template, "Empty"),
|
|
make_zone(template, "Rejected", validation_regex="accepted"),
|
|
make_zone(template, "Invalid", "asn"),
|
|
make_zone(template, "Unassigned", "custom_field"),
|
|
]
|
|
mock_zone_ocr_run.side_effect = [
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
|
|
for value in ["", "rejected", "abc", "orphan"]
|
|
]
|
|
|
|
results = _process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
zones,
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert [result.reason for result in results] == [
|
|
"no_value",
|
|
"regex_mismatch",
|
|
"invalid_value",
|
|
"unassigned",
|
|
]
|
|
assert [result.value for result in results] == [None] * 4
|
|
assert document.title == "Original"
|
|
|
|
def test_failed_template_reports_rollback_and_next_template_still_runs(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A template whose ASN zone conflicts with another document's ASN
|
|
- A second template that would succeed
|
|
WHEN:
|
|
- Zone extraction runs over both templates
|
|
THEN:
|
|
- The first template is reported as failed and rolled back, and the
|
|
second template is still applied
|
|
"""
|
|
DocumentFactory(archive_serial_number=42)
|
|
make_zone(template, "Title")
|
|
make_zone(template, "Conflict", "asn")
|
|
second = make_template("Second")
|
|
make_zone(second, "Second")
|
|
mock_zone_ocr_run.side_effect = [
|
|
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
|
|
for value in ["Rolled back", "42", "Applied"]
|
|
]
|
|
|
|
results = run_zone_extraction(document, page, templates=[template, second])
|
|
|
|
assert [result.reason for result in results] == [
|
|
"template_failed",
|
|
"template_failed",
|
|
None,
|
|
]
|
|
assert [result.value for result in results] == [None, None, "Applied"]
|
|
document.refresh_from_db()
|
|
assert document.title == "Applied"
|
|
assert document.archive_serial_number is None
|
|
|
|
def test_literal_format_does_not_write_when_every_zone_failed(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A combine format with literal text around a zone that reads nothing
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- The result has no value and the title is not written
|
|
"""
|
|
zone = make_zone(template, "Title")
|
|
template.combine_formats = {"title": "Invoice {Title}"}
|
|
mock_zone_ocr_run.return_value.stdout = ""
|
|
|
|
results = _process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
[zone],
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert results[0].reason == "no_value"
|
|
assert results[0].value is None
|
|
assert document.title == "Original"
|
|
|
|
def test_saved_extraction_matches_preview_and_rejects_invalid_value(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone that uppercases its text and validates it against a regex
|
|
WHEN:
|
|
- A valid value and then an invalid value are previewed and saved in
|
|
turn
|
|
THEN:
|
|
- The preview matches the transformed text for both, the valid value
|
|
is saved, and the invalid value leaves the saved title unchanged
|
|
"""
|
|
zone = make_zone(
|
|
template,
|
|
"Title",
|
|
transform="uppercase",
|
|
validation_regex="ACCOUNT [A-Z]+",
|
|
)
|
|
|
|
def preview_and_save(raw_text: str) -> None:
|
|
mock_zone_ocr_run.return_value.stdout = raw_text
|
|
preview = extract_zone_preview(page, zone, 100, 50, 1)
|
|
assert preview["raw_text"] == raw_text
|
|
assert preview["value"] == raw_text.upper()
|
|
_process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
[zone],
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
document.refresh_from_db()
|
|
|
|
preview_and_save("account abc")
|
|
assert document.title == "ACCOUNT ABC"
|
|
preview_and_save("invalid")
|
|
assert document.title == "ACCOUNT ABC"
|
|
|
|
@pytest.mark.parametrize(
|
|
("failing_zone", "expected_failure"),
|
|
[
|
|
pytest.param(
|
|
{"validation_regex": r"\d{2}"},
|
|
"regex_mismatch",
|
|
id="regex-mismatch",
|
|
),
|
|
pytest.param({"page": 2}, "missing_page", id="missing-page"),
|
|
],
|
|
)
|
|
def test_failed_zone_blocks_combined_field(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
failing_zone: dict[str, Any],
|
|
expected_failure: str,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Three title zones combined as a date, where the first one fails
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- Nothing is written, the failing zone reports its own reason and
|
|
the accepted zones report incomplete
|
|
"""
|
|
zones = [
|
|
make_zone(template, "Day", **failing_zone),
|
|
make_zone(template, "Month"),
|
|
make_zone(template, "Year"),
|
|
]
|
|
template.combine_formats = {"title": "{Day}.{Month}.{Year}"}
|
|
# A zone on a missing page is never read.
|
|
mock_zone_ocr_run.side_effect = ocr_outputs(
|
|
*(
|
|
["x", "05", "2024"]
|
|
if "validation_regex" in failing_zone
|
|
else ["05", "2024"]
|
|
),
|
|
)
|
|
|
|
results = _process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
zones,
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert [result.reason for result in results] == [
|
|
expected_failure,
|
|
"incomplete",
|
|
"incomplete",
|
|
]
|
|
assert [result.value for result in results] == [None] * 3
|
|
document.refresh_from_db()
|
|
assert document.title == "Original"
|
|
|
|
def test_zone_with_no_value_is_an_optional_token(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Two title zones combined with a separator, where the first reads
|
|
nothing
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- The remaining value is still written and reported
|
|
"""
|
|
zones = [make_zone(template, "First"), make_zone(template, "Second")]
|
|
template.combine_formats = {"title": "{First} / {Second}"}
|
|
mock_zone_ocr_run.side_effect = ocr_outputs("", "two")
|
|
|
|
results = _process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
zones,
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert [result.reason for result in results] == ["no_value", None]
|
|
assert results[1].value == "two"
|
|
document.refresh_from_db()
|
|
assert document.title == "two"
|
|
|
|
def test_failed_single_zone_field_does_not_affect_other_fields(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
paperless_dirs: PaperlessDirs,
|
|
page: Path,
|
|
document: Document,
|
|
template: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A title zone and a separate ASN zone, where the ASN fails its
|
|
validation regex
|
|
WHEN:
|
|
- The template is processed
|
|
THEN:
|
|
- The title is written and the failing zone keeps regex_mismatch
|
|
"""
|
|
zones = [
|
|
make_zone(template, "Title"),
|
|
make_zone(template, "ASN", "asn", validation_regex=r"\d+"),
|
|
]
|
|
mock_zone_ocr_run.side_effect = ocr_outputs("New title", "abc")
|
|
|
|
results = _process_template(
|
|
document,
|
|
{0: page},
|
|
template,
|
|
zones,
|
|
paperless_dirs.scratch_dir,
|
|
)
|
|
|
|
assert [result.reason for result in results] == [None, "regex_mismatch"]
|
|
document.refresh_from_db()
|
|
assert document.title == "New title"
|
|
|
|
|
|
class TestCombineFieldValue:
|
|
"""Tests for combining zone values into one field value."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("fmt", "zones", "expected"),
|
|
[
|
|
pytest.param(
|
|
"",
|
|
[("Same", "one"), ("Same", None), ("Same", "two")],
|
|
"one two",
|
|
id="default-format-preserves-zone-order-and-duplicate-names",
|
|
),
|
|
pytest.param(
|
|
"{ First } / {Second} / {Unknown}",
|
|
[("First", "one"), ("Second", None)],
|
|
"one",
|
|
id="format-preserves-existing-missing-token-cleanup",
|
|
),
|
|
pytest.param(
|
|
"{Amount}",
|
|
[("Amount", "-12.50")],
|
|
"-12.50",
|
|
id="negative-amount-sign-kept",
|
|
),
|
|
pytest.param(
|
|
"{Url}",
|
|
[("Url", "http://x.com/a//b")],
|
|
"http://x.com/a//b",
|
|
id="url-double-slash-kept-verbatim",
|
|
),
|
|
pytest.param(
|
|
"{Value}",
|
|
[("Value", "A--B")],
|
|
"A--B",
|
|
id="repeated-punctuation-in-value-kept",
|
|
),
|
|
pytest.param(
|
|
"{Name}",
|
|
[("Name", "Dr. Smith Jr.")],
|
|
"Dr. Smith Jr.",
|
|
id="trailing-period-in-value-kept",
|
|
),
|
|
pytest.param(
|
|
"{Value}",
|
|
[("Value", "wait...")],
|
|
"wait...",
|
|
id="trailing-ellipsis-in-value-kept",
|
|
),
|
|
pytest.param(
|
|
"{A} / {B} / {C}",
|
|
[("A", "a"), ("B", None), ("C", "c")],
|
|
"a / c",
|
|
id="middle-empty-token-drops-one-separator",
|
|
),
|
|
pytest.param(
|
|
"{A} / {B} / {C}",
|
|
[("A", None), ("B", "b"), ("C", "c")],
|
|
"b / c",
|
|
id="leading-empty-token-drops-leading-separator",
|
|
),
|
|
pytest.param(
|
|
"{A} / {B} / {C}",
|
|
[("A", "a"), ("B", "b"), ("C", None)],
|
|
"a / b",
|
|
id="trailing-empty-token-drops-trailing-separator",
|
|
),
|
|
pytest.param(
|
|
"{A} / {B} / {C}",
|
|
[("A", "a"), ("B", None), ("C", None)],
|
|
"a",
|
|
id="trailing-run-of-empty-tokens-drops-nearest-separator",
|
|
),
|
|
pytest.param(
|
|
"Total: {A} / {B}",
|
|
[("A", None), ("B", "b")],
|
|
"Total: b",
|
|
id="leading-literal-kept-before-empty-first-token",
|
|
),
|
|
pytest.param(
|
|
"- {A} {B}",
|
|
[("A", None), ("B", "b")],
|
|
"b",
|
|
id="leading-separator-dropped-before-empty-first-token",
|
|
),
|
|
pytest.param(
|
|
"{Day}.{Month}.{Year}",
|
|
[("Day", "12"), ("Month", "05"), ("Year", "2024")],
|
|
"12.05.2024",
|
|
id="all-tokens-present",
|
|
),
|
|
pytest.param(
|
|
"INV-{Number}",
|
|
[("Number", "123")],
|
|
"INV-123",
|
|
id="literal-prefix-kept-with-value",
|
|
),
|
|
pytest.param(
|
|
"INV-{Number}",
|
|
[("Number", None)],
|
|
"",
|
|
id="literal-prefix-dropped-when-every-token-empty",
|
|
),
|
|
pytest.param(
|
|
"{A} / {B}",
|
|
[("A", None), ("B", " ")],
|
|
"",
|
|
id="every-token-empty",
|
|
),
|
|
],
|
|
)
|
|
def test_combine_field_value(
|
|
self,
|
|
fmt: str,
|
|
zones: list[tuple[str, str | None]],
|
|
expected: str,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A combine format and a list of (zone name, extracted value) pairs
|
|
WHEN:
|
|
- The zone values are combined into a single field value
|
|
THEN:
|
|
- The combined string matches the expected result
|
|
"""
|
|
assert _combine_field_value(fmt, zones) == expected
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@pytest.mark.usefixtures("paperless_dirs")
|
|
class TestTemplateVisibility:
|
|
"""Tests for which templates apply to a document and its requester."""
|
|
|
|
@pytest.fixture
|
|
def owner(self) -> User:
|
|
return UserFactory(username="document_owner")
|
|
|
|
@pytest.fixture
|
|
def other(self) -> User:
|
|
return UserFactory(username="other_owner")
|
|
|
|
@pytest.fixture
|
|
def document(self, owner: User) -> Document:
|
|
return DocumentFactory(
|
|
owner=owner,
|
|
content="Northwind Power & Light statement",
|
|
)
|
|
|
|
@pytest.fixture
|
|
def mine(self, owner: User) -> OcrTemplate:
|
|
return make_visible_template("Mine", owner=owner)
|
|
|
|
@pytest.fixture
|
|
def unowned(self) -> OcrTemplate:
|
|
return make_visible_template("Unowned")
|
|
|
|
@pytest.fixture
|
|
def private(self, other: User) -> OcrTemplate:
|
|
return make_visible_template("Private", owner=other)
|
|
|
|
@pytest.mark.usefixtures("private")
|
|
def test_only_matching_owned_unowned_and_shared_templates_apply(
|
|
self,
|
|
owner: User,
|
|
other: User,
|
|
document: Document,
|
|
mine: OcrTemplate,
|
|
unowned: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Templates owned by the document owner, unowned, private to another
|
|
user, shared with the owner, disabled, for another sender and
|
|
with no matching
|
|
WHEN:
|
|
- Templates are selected for the document
|
|
THEN:
|
|
- Only the enabled, matching, owned, unowned and shared templates apply
|
|
"""
|
|
shared = make_visible_template("Shared", owner=other)
|
|
grant_object(owner, shared, "view_ocrtemplate")
|
|
make_visible_template("Disabled", owner=owner, enabled=False)
|
|
make_visible_template("Other sender", owner=owner, match="Contoso")
|
|
make_visible_template(
|
|
"No matching",
|
|
owner=owner,
|
|
matching_algorithm=MatchingModel.MATCH_NONE,
|
|
)
|
|
|
|
assert Counter(get_templates_for_document(document)) == Counter(
|
|
[mine, unowned, shared],
|
|
)
|
|
|
|
@pytest.mark.usefixtures("mine", "unowned")
|
|
def test_group_sharing_applies(
|
|
self,
|
|
owner: User,
|
|
document: Document,
|
|
private: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A private template shared with a group the document owner is in
|
|
WHEN:
|
|
- Templates are selected for the document
|
|
THEN:
|
|
- The group-shared template applies
|
|
"""
|
|
group = Group.objects.create(name="OCR users")
|
|
owner.groups.add(group)
|
|
grant_object(group, private, "view_ocrtemplate")
|
|
|
|
assert private in get_templates_for_document(document)
|
|
|
|
@pytest.mark.usefixtures("mine", "private")
|
|
def test_unowned_documents_only_use_unowned_templates(
|
|
self,
|
|
document: Document,
|
|
unowned: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document with no owner
|
|
WHEN:
|
|
- Templates are selected for the document
|
|
THEN:
|
|
- Only unowned templates apply
|
|
"""
|
|
document.owner = None
|
|
|
|
assert get_templates_for_document(document) == [unowned]
|
|
|
|
@pytest.mark.usefixtures("private")
|
|
def test_versions_use_the_root_documents_owner(
|
|
self,
|
|
other: User,
|
|
document: Document,
|
|
mine: OcrTemplate,
|
|
unowned: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A version document owned by another user than its root document
|
|
WHEN:
|
|
- Templates are selected for the version
|
|
THEN:
|
|
- The root document's owner decides the visible templates
|
|
"""
|
|
version = DocumentFactory(
|
|
root_document=document,
|
|
owner=other,
|
|
content=document.content,
|
|
)
|
|
|
|
assert Counter(get_templates_for_document(version)) == Counter(
|
|
[mine, unowned],
|
|
)
|
|
|
|
@pytest.mark.usefixtures("private")
|
|
def test_matching_uses_the_selected_files_content(
|
|
self,
|
|
document: Document,
|
|
mine: OcrTemplate,
|
|
unowned: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A root document and a version with different content
|
|
- A template matching only the version's content
|
|
WHEN:
|
|
- Templates are selected for each
|
|
THEN:
|
|
- Each is matched against its own content
|
|
"""
|
|
version = DocumentFactory(root_document=document, content="Contoso")
|
|
contoso = make_visible_template("Contoso", match="Contoso")
|
|
|
|
assert Counter(get_templates_for_document(document)) == Counter(
|
|
[mine, unowned],
|
|
)
|
|
assert get_templates_for_document(version) == [contoso]
|
|
|
|
def test_superuser_owner_can_use_all_visible_templates(
|
|
self,
|
|
owner: User,
|
|
document: Document,
|
|
mine: OcrTemplate,
|
|
unowned: OcrTemplate,
|
|
private: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document owned by a superuser
|
|
WHEN:
|
|
- Templates are selected for the document
|
|
THEN:
|
|
- Every matching template applies, including private ones
|
|
"""
|
|
owner.is_superuser = True
|
|
|
|
assert Counter(get_templates_for_document(document)) == Counter(
|
|
[mine, unowned, private],
|
|
)
|
|
|
|
@pytest.mark.usefixtures("mine", "private")
|
|
def test_requester_limits_the_owners_templates(
|
|
self,
|
|
owner: User,
|
|
other: User,
|
|
document: Document,
|
|
unowned: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Templates owned by the document owner, unowned, private to
|
|
another user, and owned by the owner but shared with the requester
|
|
WHEN:
|
|
- Templates are selected for a manual run by the requester
|
|
THEN:
|
|
- Only templates both the owner and the requester can view are used
|
|
"""
|
|
shared = make_visible_template("Shared", owner=owner)
|
|
grant_object(other, shared, "view_ocrtemplate")
|
|
|
|
assert Counter(
|
|
get_templates_for_document(document, requester=other),
|
|
) == Counter([unowned, shared])
|
|
|
|
@pytest.mark.usefixtures("mine", "unowned", "private")
|
|
def test_inactive_owner_has_no_templates(
|
|
self,
|
|
owner: User,
|
|
document: Document,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document whose owner is inactive
|
|
WHEN:
|
|
- Templates are selected for the document
|
|
THEN:
|
|
- No templates apply
|
|
"""
|
|
owner.is_active = False
|
|
|
|
assert get_templates_for_document(document) == []
|
|
|
|
def test_extraction_never_processes_private_templates(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
document: Document,
|
|
mine: OcrTemplate,
|
|
unowned: OcrTemplate,
|
|
private: OcrTemplate,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Owned, unowned and private templates each with a zone
|
|
- A blank image as the document's file
|
|
WHEN:
|
|
- Zone extraction runs for the document
|
|
THEN:
|
|
- Only the owned and unowned templates report results
|
|
"""
|
|
for template in (mine, unowned, private):
|
|
OcrTemplateZoneFactory(
|
|
template=template,
|
|
name="Title",
|
|
width=20,
|
|
height=20,
|
|
)
|
|
document.mime_type = "image/png"
|
|
with Image.new("RGB", (100, 100), "white") as image:
|
|
image.save(document.source_path, format="PNG")
|
|
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
|
|
args=[],
|
|
returncode=0,
|
|
stdout="extracted",
|
|
stderr="",
|
|
)
|
|
|
|
results = run_zone_extraction(document, None)
|
|
|
|
assert Counter(result.template for result in results) == Counter(
|
|
[mine.name, unowned.name],
|
|
)
|
|
|
|
|
|
class TestApplyTransform:
|
|
"""Tests for the _apply_transform function."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("text", "transform", "date_format", "expected"),
|
|
[
|
|
pytest.param(" hello ", "strip", "", "hello", id="strip"),
|
|
pytest.param(" hello ", "none", "", "hello", id="none-transform"),
|
|
pytest.param(
|
|
"hello world",
|
|
"uppercase",
|
|
"",
|
|
"HELLO WORLD",
|
|
id="uppercase",
|
|
),
|
|
pytest.param(
|
|
"HELLO WORLD",
|
|
"lowercase",
|
|
"",
|
|
"hello world",
|
|
id="lowercase",
|
|
),
|
|
pytest.param(
|
|
"Invoice 2026-001",
|
|
"numeric",
|
|
"",
|
|
"2026-001",
|
|
id="numeric-basic",
|
|
),
|
|
pytest.param(
|
|
"\u20ac1,234.56",
|
|
"numeric",
|
|
"",
|
|
"1,234.56",
|
|
id="numeric-with-currency",
|
|
),
|
|
pytest.param(
|
|
"abc",
|
|
"numeric",
|
|
"",
|
|
"abc",
|
|
id="numeric-empty-result-falls-back",
|
|
),
|
|
pytest.param(
|
|
"13.04.2026",
|
|
"date",
|
|
"%d.%m.%Y",
|
|
"2026-04-13",
|
|
id="date-with-format-dotted",
|
|
),
|
|
pytest.param(
|
|
"01/12/2025",
|
|
"date",
|
|
"%d/%m/%Y",
|
|
"2025-12-01",
|
|
id="date-with-format-slashed",
|
|
),
|
|
pytest.param(
|
|
"13.04.26",
|
|
"date",
|
|
"%d.%m.%y",
|
|
"2026-04-13",
|
|
id="date-two-digit-year",
|
|
),
|
|
pytest.param(
|
|
"2026-04-13",
|
|
"date",
|
|
"",
|
|
"2026-04-13",
|
|
id="date-auto-detect-iso",
|
|
),
|
|
pytest.param(
|
|
"2026/04/13",
|
|
"date",
|
|
"",
|
|
"2026-04-13",
|
|
id="date-auto-detect-slashed",
|
|
),
|
|
pytest.param(
|
|
"13 April 2026",
|
|
"date",
|
|
"",
|
|
"2026-04-13",
|
|
id="date-auto-detect-month-name",
|
|
),
|
|
pytest.param(
|
|
"2026-04-13",
|
|
"date",
|
|
"%d.%m.%Y",
|
|
"2026-04-13",
|
|
id="date-format-mismatch-falls-back-to-auto-detect",
|
|
),
|
|
pytest.param(
|
|
"32.13.2026",
|
|
"date",
|
|
"%d.%m.%Y",
|
|
"32.13.2026",
|
|
id="date-invalid-with-format-returns-text",
|
|
),
|
|
pytest.param(
|
|
"2026-13-32",
|
|
"date",
|
|
"",
|
|
"2026-13-32",
|
|
id="date-invalid-auto-detect-returns-text",
|
|
),
|
|
pytest.param(
|
|
"not a date",
|
|
"date",
|
|
"",
|
|
"not a date",
|
|
id="date-not-a-date-returns-text",
|
|
),
|
|
pytest.param("", "strip", "", "", id="empty-string"),
|
|
pytest.param(" ", "strip", "", "", id="whitespace-only"),
|
|
pytest.param(
|
|
" hello ",
|
|
"unknown",
|
|
"",
|
|
"hello",
|
|
id="unknown-transform-strips",
|
|
),
|
|
pytest.param(
|
|
" -- Account #123. -- ",
|
|
"strip_punctuation",
|
|
"",
|
|
"Account #123",
|
|
id="strip-punctuation",
|
|
),
|
|
],
|
|
)
|
|
def test_apply_transform(
|
|
self,
|
|
text: str,
|
|
transform: str,
|
|
date_format: str,
|
|
expected: str,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Extracted text, a transform name and an optional date format
|
|
- A zone reading English
|
|
WHEN:
|
|
- The transform is applied
|
|
THEN:
|
|
- The transformed text matches the expected result
|
|
"""
|
|
rules = _date_rules("eng", date_format)
|
|
assert _apply_transform(text, transform, rules) == expected
|
|
|
|
def test_unparseable_date_preserves_text(self, mocker: MockerFixture) -> None:
|
|
"""
|
|
GIVEN:
|
|
- dateparser raises OverflowError for the input
|
|
WHEN:
|
|
- A date transform is applied
|
|
THEN:
|
|
- The original text is returned unchanged
|
|
"""
|
|
mocker.patch("documents.zone_ocr.dateparser.parse", side_effect=OverflowError)
|
|
assert (
|
|
_apply_transform("unparsable", "date", _date_rules("eng")) == "unparsable"
|
|
)
|
|
|
|
|
|
class TestZoneLanguage:
|
|
"""Tests for mapping the OCR language to a dateparser language."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("ocr_language", "expected"),
|
|
[
|
|
pytest.param("deu", "de", id="single-language"),
|
|
pytest.param("deu+eng", "de", id="first-of-several-languages"),
|
|
pytest.param("aze_Cyrl", "az", id="script-suffix-is-dropped"),
|
|
pytest.param("xyz", None, id="unmapped-language-is-none"),
|
|
],
|
|
)
|
|
def test_zone_language(self, ocr_language: str, expected: str | None) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A tesseract language code, possibly several or with a script
|
|
WHEN:
|
|
- The zone language is derived
|
|
THEN:
|
|
- The first language is mapped, or None when it is unknown
|
|
"""
|
|
assert _zone_language(ocr_language) == expected
|
|
|
|
|
|
class TestParseDate:
|
|
"""Tests for the zone date parser."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("text", "ocr_language", "expected"),
|
|
[
|
|
pytest.param("12.05.2024", "deu", date(2024, 5, 12), id="german-day-first"),
|
|
pytest.param(
|
|
"05/12/2024",
|
|
"eng",
|
|
date(2024, 5, 12),
|
|
id="english-month-first",
|
|
),
|
|
pytest.param(
|
|
"12. Mai 2024",
|
|
"deu",
|
|
date(2024, 5, 12),
|
|
id="german-month-name",
|
|
),
|
|
pytest.param(
|
|
"13 April 2026",
|
|
"eng",
|
|
date(2026, 4, 13),
|
|
id="english-month-name",
|
|
),
|
|
pytest.param(
|
|
"2026/04/13",
|
|
"eng",
|
|
date(2026, 4, 13),
|
|
id="year-first-slashed",
|
|
),
|
|
pytest.param(
|
|
"01.01.2999",
|
|
"deu",
|
|
date(2999, 1, 1),
|
|
id="future-date-allowed",
|
|
),
|
|
],
|
|
)
|
|
def test_locale_decides_the_order(
|
|
self,
|
|
text: str,
|
|
ocr_language: str,
|
|
expected: date,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A date written in the convention of the zone's language
|
|
WHEN:
|
|
- The date is parsed
|
|
THEN:
|
|
- The locale's day and month order is used
|
|
"""
|
|
assert _parse_date(text, _date_rules(ocr_language)) == expected
|
|
|
|
@pytest.mark.parametrize(
|
|
("date_order", "expected"),
|
|
[
|
|
pytest.param("DMY", date(2024, 5, 12), id="day-first"),
|
|
pytest.param("MDY", date(2024, 12, 5), id="month-first"),
|
|
],
|
|
)
|
|
def test_unmapped_language_uses_the_configured_date_order(
|
|
self,
|
|
settings: SettingsWrapper,
|
|
date_order: str,
|
|
expected: date,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone whose language has no dateparser mapping
|
|
- A configured PAPERLESS_DATE_ORDER
|
|
WHEN:
|
|
- An ambiguous numeric date is parsed
|
|
THEN:
|
|
- The configured date order decides the result
|
|
"""
|
|
settings.DATE_ORDER = date_order
|
|
assert _parse_date("12.05.2024", _date_rules("xyz")) == expected
|
|
|
|
@pytest.mark.parametrize(
|
|
"text",
|
|
[
|
|
pytest.param("now", id="now"),
|
|
pytest.param("yesterday", id="yesterday"),
|
|
pytest.param("today", id="today"),
|
|
pytest.param("in 2 days", id="relative-offset"),
|
|
pytest.param("Mon", id="weekday"),
|
|
pytest.param("March", id="month-only"),
|
|
pytest.param("2024", id="year-only"),
|
|
pytest.param("5", id="day-only"),
|
|
pytest.param("12", id="two-digits"),
|
|
pytest.param("not a date", id="words"),
|
|
],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"ocr_language",
|
|
[
|
|
pytest.param("eng", id="mapped-language"),
|
|
pytest.param("xyz", id="unmapped-language"),
|
|
],
|
|
)
|
|
def test_non_dates_are_rejected(self, text: str, ocr_language: str) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Text that is not a complete calendar date
|
|
WHEN:
|
|
- The date is parsed
|
|
THEN:
|
|
- No date is returned
|
|
"""
|
|
assert _parse_date(text, _date_rules(ocr_language)) is None
|
|
|
|
@pytest.mark.parametrize(
|
|
("text", "ocr_language", "expected"),
|
|
[
|
|
pytest.param(
|
|
"Datum: 13.04.2026",
|
|
"deu",
|
|
date(2026, 4, 13),
|
|
id="german-label-colon",
|
|
),
|
|
pytest.param(
|
|
"Rechnungsdatum 12.05.2024",
|
|
"deu",
|
|
date(2024, 5, 12),
|
|
id="german-compound-label",
|
|
),
|
|
pytest.param(
|
|
"Invoice date 05/12/2024",
|
|
"eng",
|
|
date(2024, 5, 12),
|
|
id="english-label-numeric",
|
|
),
|
|
pytest.param(
|
|
"Date: 13 April 2026",
|
|
"eng",
|
|
date(2026, 4, 13),
|
|
id="english-label-month-name",
|
|
),
|
|
pytest.param(
|
|
"Rechnung 2024-001 vom 12.05.2024",
|
|
"deu",
|
|
date(2024, 5, 12),
|
|
id="german-label-before-invoice-number",
|
|
),
|
|
pytest.param(
|
|
"No. 12 Datum 13.04.2026",
|
|
"deu",
|
|
date(2026, 4, 13),
|
|
id="german-label-after-number",
|
|
),
|
|
pytest.param(
|
|
"Kd-Nr 12345 05.06.2024",
|
|
"deu",
|
|
date(2024, 6, 5),
|
|
id="german-label-after-customer-number",
|
|
),
|
|
pytest.param(
|
|
"Lieferdatum: 12. Mai 2024",
|
|
"deu",
|
|
date(2024, 5, 12),
|
|
id="german-label-month-name",
|
|
),
|
|
pytest.param(
|
|
"Total 5",
|
|
"eng",
|
|
None,
|
|
id="single-number-after-label",
|
|
),
|
|
pytest.param(
|
|
"Total 5 items 2024",
|
|
"eng",
|
|
None,
|
|
id="number-run-without-month",
|
|
),
|
|
pytest.param(
|
|
"Page 3 of 2024",
|
|
"eng",
|
|
None,
|
|
id="page-number-without-month",
|
|
),
|
|
pytest.param(
|
|
"Invoice 12.5",
|
|
"eng",
|
|
None,
|
|
id="partial-numeric-date",
|
|
),
|
|
pytest.param(
|
|
"Page 3 of 12 2024",
|
|
"eng",
|
|
None,
|
|
id="page-of-number-run-with-year",
|
|
),
|
|
pytest.param(
|
|
"Total 3 10 2024",
|
|
"eng",
|
|
None,
|
|
id="space-separated-numbers-with-year",
|
|
),
|
|
pytest.param(
|
|
"Order 12 3 2024",
|
|
"eng",
|
|
None,
|
|
id="order-numbers-with-year",
|
|
),
|
|
pytest.param(
|
|
"Ref 1 2 2024",
|
|
"eng",
|
|
None,
|
|
id="ref-numbers-with-year",
|
|
),
|
|
pytest.param(
|
|
"Invoice #2024 from 5 May",
|
|
"eng",
|
|
None,
|
|
id="year-token-before-day-month",
|
|
),
|
|
pytest.param(
|
|
"Page 1 of 3 03 2024",
|
|
"eng",
|
|
None,
|
|
id="page-numbers-ending-in-year",
|
|
),
|
|
pytest.param(
|
|
"Version 1.2.34",
|
|
"deu",
|
|
None,
|
|
id="version-number-with-two-digit-tail",
|
|
),
|
|
pytest.param(
|
|
"v1.2.24",
|
|
"eng",
|
|
None,
|
|
id="short-version-number",
|
|
),
|
|
pytest.param(
|
|
"Art. 12.3.45",
|
|
"deu",
|
|
None,
|
|
id="article-number",
|
|
),
|
|
pytest.param(
|
|
"Ticket 1-2-34",
|
|
"eng",
|
|
None,
|
|
id="dashed-ticket-number",
|
|
),
|
|
],
|
|
)
|
|
def test_labelled_text_yields_the_date_after_the_label(
|
|
self,
|
|
text: str,
|
|
ocr_language: str,
|
|
expected: date | None,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- OCR text with a label before a date, or a label-like number
|
|
with no complete date
|
|
WHEN:
|
|
- The date is parsed
|
|
THEN:
|
|
- The date after the label is returned in the zone's day/month order
|
|
- Text without a complete date after a label returns no date
|
|
"""
|
|
assert _parse_date(text, _date_rules(ocr_language)) == expected
|
|
|
|
@pytest.mark.parametrize(
|
|
("text", "date_format", "expected"),
|
|
[
|
|
pytest.param(
|
|
"12.05.2024",
|
|
"%m.%d.%Y",
|
|
date(2024, 12, 5),
|
|
id="format-overrides-locale-order",
|
|
),
|
|
pytest.param(
|
|
"12.05.2024",
|
|
"%Y/%m/%d",
|
|
date(2024, 5, 12),
|
|
id="invalid-format-falls-back",
|
|
),
|
|
],
|
|
)
|
|
def test_date_format_is_tried_first(
|
|
self,
|
|
text: str,
|
|
date_format: str,
|
|
expected: date,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A German zone with a date format
|
|
WHEN:
|
|
- A date is parsed
|
|
THEN:
|
|
- A matching format is used, otherwise parsing falls back to the locale
|
|
"""
|
|
assert _parse_date(text, _date_rules("deu", date_format)) == expected
|
|
|
|
@pytest.mark.parametrize(
|
|
("text", "expected"),
|
|
[
|
|
pytest.param("2024-05-06", date(2024, 5, 6), id="plain-iso"),
|
|
pytest.param("Due 2024-05-06.", date(2024, 5, 6), id="iso-inside-text"),
|
|
pytest.param("No. 12345-06-07", None, id="digit-before-is-not-iso"),
|
|
pytest.param("2024-05-061", None, id="digit-after-is-not-iso"),
|
|
pytest.param(
|
|
"2024-13-45 and 2024-05-06",
|
|
date(2024, 5, 6),
|
|
id="invalid-match-is-skipped",
|
|
),
|
|
pytest.param("2024-13-45", None, id="only-invalid-match"),
|
|
],
|
|
)
|
|
def test_iso_dates_are_anchored(self, text: str, expected: date | None) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Text containing ISO-like dates, some invalid or part of longer numbers
|
|
WHEN:
|
|
- The date is parsed
|
|
THEN:
|
|
- Only whole, valid ISO dates are returned, trying every match
|
|
"""
|
|
assert _parse_date(text, _date_rules("eng")) == expected
|
|
|
|
@pytest.mark.parametrize(
|
|
"text",
|
|
[
|
|
pytest.param("2026/04/13", id="slashed"),
|
|
pytest.param("2026.04.13", id="dotted"),
|
|
pytest.param("2026-04-13", id="dashed"),
|
|
],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
("ocr_language", "date_order"),
|
|
[
|
|
pytest.param("xyz", "DMY", id="unmapped-language-day-first"),
|
|
pytest.param("xyz", "MDY", id="unmapped-language-month-first"),
|
|
pytest.param("deu", "DMY", id="mapped-language"),
|
|
],
|
|
)
|
|
def test_year_first_dates_ignore_the_date_order(
|
|
self,
|
|
settings: SettingsWrapper,
|
|
text: str,
|
|
ocr_language: str,
|
|
date_order: str,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A year-first date separated by a dash, slash or dot
|
|
- A mapped or unmapped zone language and a configured date order
|
|
WHEN:
|
|
- The date is parsed
|
|
THEN:
|
|
- The date is read as year, month, day regardless of the order
|
|
"""
|
|
settings.DATE_ORDER = date_order
|
|
assert _parse_date(text, _date_rules(ocr_language)) == date(2026, 4, 13)
|
|
|
|
@pytest.mark.parametrize(
|
|
"text",
|
|
[
|
|
pytest.param("1.2026/04/133", id="digit-after"),
|
|
pytest.param("12026/04/13", id="digit-before"),
|
|
pytest.param("2026/13/45", id="invalid-month"),
|
|
],
|
|
)
|
|
def test_year_first_dates_are_anchored_and_validated(self, text: str) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Year-first looking text that is part of longer digits or invalid
|
|
WHEN:
|
|
- The date is parsed
|
|
THEN:
|
|
- No date is returned
|
|
"""
|
|
assert _parse_date(text, _date_rules("xyz")) is None
|
|
|
|
@pytest.mark.parametrize(
|
|
"text",
|
|
[
|
|
pytest.param("1899-12-31", id="iso"),
|
|
pytest.param("31.12.1899", id="locale-numeric"),
|
|
],
|
|
)
|
|
def test_years_before_1900_are_rejected(self, text: str) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A valid calendar date before the year 1900
|
|
WHEN:
|
|
- The date is parsed
|
|
THEN:
|
|
- No date is returned
|
|
"""
|
|
assert _parse_date(text, _date_rules("deu")) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
class TestPrepareFieldUpdate:
|
|
"""Tests for converting extracted values into field updates."""
|
|
|
|
def test_unknown_target_cannot_produce_a_field_update(self) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone whose target is not a known target
|
|
WHEN:
|
|
- A field update is prepared for it
|
|
THEN:
|
|
- No field update is produced
|
|
"""
|
|
assert _prepare_field_update(OcrTemplateZone(target="unknown"), "value") is None
|
|
|
|
@pytest.mark.parametrize(
|
|
("ocr_language", "expected"),
|
|
[
|
|
pytest.param("deu", 1234, id="german-reads-dot-as-thousands"),
|
|
pytest.param("eng", None, id="english-reads-dot-as-fraction"),
|
|
],
|
|
)
|
|
def test_int_field_uses_the_zone_language_for_numbers(
|
|
self,
|
|
ocr_language: str,
|
|
expected: int | None,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone targeting an integer custom field with a given OCR language
|
|
WHEN:
|
|
- A field update is prepared for the text "1.234"
|
|
THEN:
|
|
- A German zone reads 1234, while an English zone reads a fraction
|
|
and produces no update
|
|
"""
|
|
field = CustomFieldFactory(data_type=CustomField.FieldDataType.INT)
|
|
zone = OcrTemplateZoneFactory(
|
|
target=OcrTemplateZone.TargetType.CUSTOM_FIELD,
|
|
custom_field=field,
|
|
ocr_language=ocr_language,
|
|
)
|
|
|
|
update = _prepare_field_update(zone, "1.234")
|
|
|
|
assert (update.value if update else None) == expected
|
|
|
|
@pytest.mark.parametrize(
|
|
"target",
|
|
[
|
|
pytest.param(OcrTemplateZone.TargetType.CREATED, id="created"),
|
|
pytest.param(OcrTemplateZone.TargetType.CUSTOM_FIELD, id="date-field"),
|
|
],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
("ocr_language", "date_format", "text", "expected"),
|
|
[
|
|
pytest.param("deu", "", "12.05.2024", date(2024, 5, 12), id="german-order"),
|
|
pytest.param(
|
|
"eng",
|
|
"",
|
|
"05/12/2024",
|
|
date(2024, 5, 12),
|
|
id="english-order",
|
|
),
|
|
pytest.param(
|
|
"deu",
|
|
"",
|
|
"12. Mai 2024",
|
|
date(2024, 5, 12),
|
|
id="german-month-name",
|
|
),
|
|
pytest.param(
|
|
"deu",
|
|
"%m.%d.%Y",
|
|
"12.05.2024",
|
|
date(2024, 12, 5),
|
|
id="zone-date-format",
|
|
),
|
|
pytest.param(
|
|
"deu",
|
|
"%Y/%m/%d",
|
|
"12.05.2024",
|
|
date(2024, 5, 12),
|
|
id="invalid-format-falls-back",
|
|
),
|
|
pytest.param("eng", "", "2024", None, id="year-only-is-rejected"),
|
|
],
|
|
)
|
|
def test_date_targets_parse_alike(
|
|
self,
|
|
target: str,
|
|
ocr_language: str,
|
|
date_format: str,
|
|
text: str,
|
|
expected: date | None,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A zone targeting the created date or a date custom field
|
|
- An OCR language and optionally a date format
|
|
WHEN:
|
|
- A field update is prepared for non-ISO date text
|
|
THEN:
|
|
- Both targets read the text the same way, honoring the zone's
|
|
language and date format
|
|
"""
|
|
field = (
|
|
CustomFieldFactory(data_type=CustomField.FieldDataType.DATE)
|
|
if target == OcrTemplateZone.TargetType.CUSTOM_FIELD
|
|
else None
|
|
)
|
|
zone = OcrTemplateZoneFactory(
|
|
target=target,
|
|
custom_field=field,
|
|
ocr_language=ocr_language,
|
|
date_format=date_format,
|
|
)
|
|
|
|
update = _prepare_field_update(zone, text)
|
|
|
|
assert (update.value if update else None) == expected
|
|
|
|
|
|
class TestNumberLocale:
|
|
"""Tests for deriving a number locale from the OCR language."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("ocr_language", "expected"),
|
|
[
|
|
pytest.param("deu", "de", id="single-language"),
|
|
pytest.param("deu+eng", "de", id="first-of-several-languages"),
|
|
pytest.param("aze_Cyrl", "az", id="script-suffix-is-dropped"),
|
|
pytest.param("xyz", "en", id="unknown-language-falls-back-to-english"),
|
|
],
|
|
)
|
|
def test_number_locale(self, ocr_language: str, expected: str) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A tesseract language code, possibly several or with a script
|
|
WHEN:
|
|
- The number locale is derived
|
|
THEN:
|
|
- The first language decides the locale, falling back to English
|
|
"""
|
|
assert _number_locale(ocr_language) == Locale.parse(expected)
|
|
|
|
|
|
class TestParseMonetary:
|
|
"""Tests for the _parse_monetary function."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("value", "locale", "expected"),
|
|
[
|
|
pytest.param("12.50", EN, "12.50", id="plain-amount"),
|
|
pytest.param("12", EN, "12.00", id="whole-number-gets-two-decimals"),
|
|
pytest.param("12.500", EN, "12.50", id="trailing-zero-is-fine"),
|
|
pytest.param("USD 12.50", EN, "USD12.50", id="code-before-amount"),
|
|
pytest.param("12,50 EUR", DE, "EUR12.50", id="code-after-amount"),
|
|
pytest.param("USD12.50", EN, "USD12.50", id="code-attached-to-amount"),
|
|
pytest.param("$12.50", EN, "USD12.50", id="dollar-symbol-en"),
|
|
pytest.param("$12.50", FR, "12.50", id="dollar-symbol-fr-has-no-code"),
|
|
pytest.param(
|
|
"\N{EURO SIGN}12.50",
|
|
EN,
|
|
"EUR12.50",
|
|
id="euro-symbol",
|
|
),
|
|
pytest.param(
|
|
"\N{POUND SIGN}12.50",
|
|
EN,
|
|
"GBP12.50",
|
|
id="pound-symbol",
|
|
),
|
|
pytest.param(
|
|
"USD $12.50",
|
|
EN,
|
|
"USD12.50",
|
|
id="code-wins-over-symbol",
|
|
),
|
|
pytest.param("INV 12.50", EN, "12.50", id="non-currency-word-ignored"),
|
|
pytest.param("usd 12.50", EN, "12.50", id="lowercase-code-ignored"),
|
|
pytest.param("(12.50)", EN, "-12.50", id="parentheses-negative"),
|
|
pytest.param("($12.50)", EN, "USD-12.50", id="parentheses-with-symbol"),
|
|
pytest.param("(12,50 EUR)", DE, "EUR-12.50", id="parentheses-with-code"),
|
|
pytest.param("(0.00)", EN, "0.00", id="parentheses-zero-not-negative"),
|
|
pytest.param("(USD) 12.50", EN, "USD12.50", id="empty-parentheses-ignored"),
|
|
pytest.param("USD -12.50", EN, "USD-12.50", id="code-with-negative"),
|
|
pytest.param("-12.50", EN, "-12.50", id="plain-negative"),
|
|
pytest.param("1,234.56", EN, "1234.56", id="grouped-amount"),
|
|
pytest.param("USD 12.50 EUR", EN, None, id="two-codes"),
|
|
pytest.param("$12.50 \N{EURO SIGN}", EN, None, id="two-symbols"),
|
|
pytest.param("(-12.50)", EN, None, id="parentheses-and-minus"),
|
|
pytest.param("12.345", EN, None, id="too-many-decimals"),
|
|
pytest.param("12345678901.12", EN, None, id="more-than-12-digits"),
|
|
pytest.param(
|
|
"USD 12345678901.12",
|
|
EN,
|
|
None,
|
|
id="more-than-12-digits-with-code",
|
|
),
|
|
pytest.param("100 (19% VAT)", EN, None, id="digits-outside-parentheses"),
|
|
pytest.param("(12) (13)", EN, None, id="two-parenthesised-numbers"),
|
|
pytest.param("-0.00", EN, "0.00", id="negative-zero-not-negative"),
|
|
pytest.param("CHF 12", DE, "CHF12.00", id="code-in-german"),
|
|
pytest.param(
|
|
"1\N{NARROW NO-BREAK SPACE}234,56 \N{EURO SIGN}",
|
|
FR,
|
|
"EUR1234.56",
|
|
id="french-space-grouping",
|
|
),
|
|
pytest.param("USD", EN, None, id="code-without-amount"),
|
|
pytest.param("garbage", EN, None, id="garbage"),
|
|
pytest.param("", EN, None, id="empty"),
|
|
],
|
|
)
|
|
def test_parse_monetary(
|
|
self,
|
|
value: str,
|
|
locale: Locale,
|
|
expected: str | None,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- OCR text containing an amount, maybe with a currency or parentheses
|
|
WHEN:
|
|
- The text is parsed as a monetary value
|
|
THEN:
|
|
- The amount has two decimals and a leading currency code if one was
|
|
found, or None is returned when the text is not a valid value
|
|
"""
|
|
assert _parse_monetary(value, locale) == expected
|
|
|
|
|
|
class TestConvertValue:
|
|
"""Tests for the _convert_value function."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("value", "data_type", "expected", "locale"),
|
|
[
|
|
pytest.param(
|
|
"Hello",
|
|
CustomField.FieldDataType.STRING,
|
|
"Hello",
|
|
EN,
|
|
id="string",
|
|
),
|
|
pytest.param(
|
|
"https://example.com",
|
|
CustomField.FieldDataType.URL,
|
|
"https://example.com",
|
|
EN,
|
|
id="url",
|
|
),
|
|
pytest.param(
|
|
"x" * 500,
|
|
CustomField.FieldDataType.LONG_TEXT,
|
|
"x" * 500,
|
|
EN,
|
|
id="long-text",
|
|
),
|
|
pytest.param("42", CustomField.FieldDataType.INT, 42, EN, id="int-simple"),
|
|
pytest.param(
|
|
"INV-123",
|
|
CustomField.FieldDataType.INT,
|
|
123,
|
|
EN,
|
|
id="int-with-noise",
|
|
),
|
|
pytest.param(
|
|
"-42",
|
|
CustomField.FieldDataType.INT,
|
|
-42,
|
|
EN,
|
|
id="int-negative",
|
|
),
|
|
pytest.param(
|
|
"\N{MINUS SIGN}5",
|
|
CustomField.FieldDataType.INT,
|
|
-5,
|
|
EN,
|
|
id="int-unicode-minus",
|
|
),
|
|
pytest.param(
|
|
"1,234",
|
|
CustomField.FieldDataType.INT,
|
|
1234,
|
|
EN,
|
|
id="int-comma-thousands",
|
|
),
|
|
pytest.param(
|
|
"1.234.567",
|
|
CustomField.FieldDataType.INT,
|
|
1234567,
|
|
DE,
|
|
id="int-dot-thousands",
|
|
),
|
|
pytest.param(
|
|
"12.00",
|
|
CustomField.FieldDataType.INT,
|
|
12,
|
|
EN,
|
|
id="int-zero-fraction",
|
|
),
|
|
pytest.param(
|
|
"1.234,00",
|
|
CustomField.FieldDataType.INT,
|
|
1234,
|
|
DE,
|
|
id="int-thousands-and-zero-fraction",
|
|
),
|
|
pytest.param(
|
|
"1234.00",
|
|
CustomField.FieldDataType.INT,
|
|
1234,
|
|
EN,
|
|
id="int-ungrouped-with-zero-fraction",
|
|
),
|
|
pytest.param(
|
|
"1.234",
|
|
CustomField.FieldDataType.INT,
|
|
1234,
|
|
DE,
|
|
id="int-dot-is-thousands-in-german",
|
|
),
|
|
pytest.param(
|
|
"1\N{NARROW NO-BREAK SPACE}234",
|
|
CustomField.FieldDataType.INT,
|
|
1234,
|
|
FR,
|
|
id="int-narrow-space-thousands-in-french",
|
|
),
|
|
pytest.param(
|
|
"2026-04-13",
|
|
CustomField.FieldDataType.DATE,
|
|
"2026-04-13",
|
|
EN,
|
|
id="date-iso",
|
|
),
|
|
pytest.param(
|
|
"123.45",
|
|
CustomField.FieldDataType.MONETARY,
|
|
"123.45",
|
|
EN,
|
|
id="monetary-simple",
|
|
),
|
|
pytest.param(
|
|
"1.234,56",
|
|
CustomField.FieldDataType.MONETARY,
|
|
"1234.56",
|
|
DE,
|
|
id="monetary-european",
|
|
),
|
|
pytest.param(
|
|
"\u20ac1,234.56",
|
|
CustomField.FieldDataType.MONETARY,
|
|
"EUR1234.56",
|
|
EN,
|
|
id="monetary-with-currency-symbol",
|
|
),
|
|
pytest.param(
|
|
"12345678901234567890,12345678901234567890",
|
|
CustomField.FieldDataType.MONETARY,
|
|
None,
|
|
DE,
|
|
id="monetary-too-precise-returns-none",
|
|
),
|
|
pytest.param(
|
|
"\u20ac-12.50",
|
|
CustomField.FieldDataType.MONETARY,
|
|
"EUR-12.50",
|
|
EN,
|
|
id="monetary-sign-after-currency-symbol",
|
|
),
|
|
pytest.param(
|
|
"-\u20ac12.50",
|
|
CustomField.FieldDataType.MONETARY,
|
|
"EUR-12.50",
|
|
EN,
|
|
id="monetary-sign-before-currency-symbol",
|
|
),
|
|
pytest.param(
|
|
"EUR -12.50",
|
|
CustomField.FieldDataType.MONETARY,
|
|
"EUR-12.50",
|
|
EN,
|
|
id="monetary-sign-after-currency-code",
|
|
),
|
|
pytest.param(
|
|
"Total: -5",
|
|
CustomField.FieldDataType.INT,
|
|
-5,
|
|
EN,
|
|
id="int-sign-after-label",
|
|
),
|
|
pytest.param(
|
|
"\u20ac-12.50",
|
|
CustomField.FieldDataType.FLOAT,
|
|
-12.5,
|
|
EN,
|
|
id="float-sign-after-currency-symbol",
|
|
),
|
|
pytest.param(
|
|
"\u0661\u0662\u0663",
|
|
CustomField.FieldDataType.INT,
|
|
123,
|
|
Locale.parse("ar"),
|
|
id="int-arabic-indic-digits",
|
|
),
|
|
pytest.param(
|
|
"\u0661\u066c\u0662\u0663\u0664\u066b\u0665",
|
|
CustomField.FieldDataType.FLOAT,
|
|
1234.5,
|
|
Locale.parse("ar"),
|
|
id="float-arabic-separators",
|
|
),
|
|
pytest.param(
|
|
"\u06f1\u06f2\u06f3",
|
|
CustomField.FieldDataType.INT,
|
|
123,
|
|
Locale.parse("fa"),
|
|
id="int-persian-digits",
|
|
),
|
|
pytest.param(
|
|
"\u0967\u0968\u0969",
|
|
CustomField.FieldDataType.INT,
|
|
123,
|
|
Locale.parse("hi"),
|
|
id="int-devanagari-digits",
|
|
),
|
|
pytest.param(
|
|
"1 234,56",
|
|
CustomField.FieldDataType.FLOAT,
|
|
1234.56,
|
|
Locale.parse("ru"),
|
|
id="float-plain-space-in-russian",
|
|
),
|
|
pytest.param(
|
|
"1\N{RIGHT SINGLE QUOTATION MARK}234.56",
|
|
CustomField.FieldDataType.FLOAT,
|
|
1234.56,
|
|
Locale.parse("de_CH"),
|
|
id="float-apostrophe-group-in-swiss-german",
|
|
),
|
|
],
|
|
)
|
|
def test_converts_to_exact_value(
|
|
self,
|
|
value: str,
|
|
data_type: str,
|
|
expected: object,
|
|
locale: Locale,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- An extracted string, a custom field data type and a number locale
|
|
WHEN:
|
|
- The value is converted
|
|
THEN:
|
|
- The converted value equals the expected value exactly
|
|
"""
|
|
assert _convert_value(value, data_type, locale, ENG_DATES) == expected
|
|
|
|
def test_string_truncation(self) -> None:
|
|
"""
|
|
GIVEN:
|
|
- An extracted string longer than the string field limit
|
|
WHEN:
|
|
- The value is converted to a STRING field value
|
|
THEN:
|
|
- The result is truncated to 128 characters
|
|
"""
|
|
result = _convert_value(
|
|
"x" * 200,
|
|
CustomField.FieldDataType.STRING,
|
|
EN,
|
|
ENG_DATES,
|
|
)
|
|
assert isinstance(result, str)
|
|
assert len(result) == 128
|
|
|
|
@pytest.mark.parametrize(
|
|
("value", "locale", "expected"),
|
|
[
|
|
pytest.param("1234.56", EN, 1234.56, id="float-simple"),
|
|
pytest.param("1.234,56", DE, 1234.56, id="float-european-format"),
|
|
pytest.param("1,234.56", EN, 1234.56, id="float-us-format"),
|
|
pytest.param("1234,56", DE, 1234.56, id="float-comma-only"),
|
|
pytest.param("1,234", DE, 1.234, id="float-comma-is-decimal-in-german"),
|
|
pytest.param("1,234", EN, 1234.0, id="float-comma-is-thousands-in-english"),
|
|
pytest.param("1 234,56", FR, 1234.56, id="float-plain-space-in-french"),
|
|
pytest.param(
|
|
"1\N{NARROW NO-BREAK SPACE}234,56",
|
|
FR,
|
|
1234.56,
|
|
id="float-narrow-space-in-french",
|
|
),
|
|
pytest.param(
|
|
"\N{MINUS SIGN}1.5",
|
|
EN,
|
|
-1.5,
|
|
id="float-unicode-minus",
|
|
),
|
|
],
|
|
)
|
|
def test_float_conversion(
|
|
self,
|
|
value: str,
|
|
locale: Locale,
|
|
expected: float,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A numeric string in one of several decimal formats and a locale
|
|
WHEN:
|
|
- The value is converted to a FLOAT field value
|
|
THEN:
|
|
- The result is the expected float
|
|
"""
|
|
assert (
|
|
_convert_value(value, CustomField.FieldDataType.FLOAT, locale, ENG_DATES)
|
|
== expected
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
("value", "data_type", "locale"),
|
|
[
|
|
pytest.param(
|
|
"abc",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-empty-returns-none",
|
|
),
|
|
pytest.param(
|
|
"-",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-only-dash-returns-none",
|
|
),
|
|
pytest.param(
|
|
"12.50",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-fraction-returns-none",
|
|
),
|
|
pytest.param(
|
|
"1.5",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-short-fraction-returns-none",
|
|
),
|
|
pytest.param(
|
|
"0.500",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-leading-zero-fraction-returns-none",
|
|
),
|
|
pytest.param(
|
|
"1..2",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-empty-group-returns-none",
|
|
),
|
|
pytest.param(
|
|
"5-3",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-inner-dash-returns-none",
|
|
),
|
|
pytest.param(
|
|
"1,234.567",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-thousands-and-fraction-returns-none",
|
|
),
|
|
pytest.param(
|
|
"12.3,00",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-bad-grouping-before-fraction-returns-none",
|
|
),
|
|
pytest.param(
|
|
"1,2345.00",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-long-group-before-fraction-returns-none",
|
|
),
|
|
pytest.param(
|
|
"abc",
|
|
CustomField.FieldDataType.FLOAT,
|
|
EN,
|
|
id="float-empty-returns-none",
|
|
),
|
|
pytest.param(
|
|
",",
|
|
CustomField.FieldDataType.FLOAT,
|
|
EN,
|
|
id="float-only-separator-returns-none",
|
|
),
|
|
pytest.param(
|
|
"not a date",
|
|
CustomField.FieldDataType.DATE,
|
|
EN,
|
|
id="date-invalid-returns-none",
|
|
),
|
|
pytest.param(
|
|
"2026-13-32",
|
|
CustomField.FieldDataType.DATE,
|
|
EN,
|
|
id="date-invalid-values-returns-none",
|
|
),
|
|
pytest.param(
|
|
"CHF",
|
|
CustomField.FieldDataType.MONETARY,
|
|
EN,
|
|
id="monetary-empty-returns-none",
|
|
),
|
|
pytest.param(
|
|
"maybe",
|
|
CustomField.FieldDataType.BOOL,
|
|
EN,
|
|
id="bool-unknown-returns-none",
|
|
),
|
|
pytest.param(
|
|
"test",
|
|
CustomField.FieldDataType.DOCUMENTLINK,
|
|
EN,
|
|
id="unsupported-type-documentlink-returns-none",
|
|
),
|
|
pytest.param(
|
|
"test",
|
|
CustomField.FieldDataType.SELECT,
|
|
EN,
|
|
id="unsupported-type-select-returns-none",
|
|
),
|
|
pytest.param(
|
|
"",
|
|
CustomField.FieldDataType.STRING,
|
|
EN,
|
|
id="empty-string-returns-none",
|
|
),
|
|
pytest.param(
|
|
"9" * 400,
|
|
CustomField.FieldDataType.FLOAT,
|
|
EN,
|
|
id="float-overflow-is-not-stored-as-infinity",
|
|
),
|
|
pytest.param(
|
|
"1.234",
|
|
CustomField.FieldDataType.INT,
|
|
EN,
|
|
id="int-german-thousands-is-a-fraction-in-english",
|
|
),
|
|
pytest.param(
|
|
"12.50",
|
|
CustomField.FieldDataType.INT,
|
|
DE,
|
|
id="int-english-decimal-is-malformed-in-german",
|
|
),
|
|
pytest.param(
|
|
"1.234.567",
|
|
CustomField.FieldDataType.FLOAT,
|
|
EN,
|
|
id="float-multiple-dots-in-english",
|
|
),
|
|
],
|
|
)
|
|
def test_returns_none(self, value: str, data_type: str, locale: Locale) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A value that cannot be converted to the given data type in a locale
|
|
WHEN:
|
|
- The value is converted
|
|
THEN:
|
|
- None is returned
|
|
"""
|
|
assert _convert_value(value, data_type, locale, ENG_DATES) is None
|
|
|
|
@pytest.mark.parametrize(
|
|
"data_type",
|
|
[
|
|
pytest.param(CustomField.FieldDataType.FLOAT, id="float"),
|
|
pytest.param(CustomField.FieldDataType.MONETARY, id="monetary"),
|
|
],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"value",
|
|
[
|
|
pytest.param("1.2.3", id="multiple-dots"),
|
|
pytest.param("--12", id="double-leading-dash"),
|
|
pytest.param("12-34", id="embedded-dash"),
|
|
pytest.param("", id="empty"),
|
|
pytest.param(".", id="only-dot"),
|
|
pytest.param("-", id="only-dash"),
|
|
pytest.param("not numeric", id="not-numeric"),
|
|
],
|
|
)
|
|
def test_numeric_types_reject_malformed_numbers(
|
|
self,
|
|
data_type: str,
|
|
value: str,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A malformed numeric string and a FLOAT or MONETARY data type
|
|
WHEN:
|
|
- The value is converted
|
|
THEN:
|
|
- None is returned
|
|
"""
|
|
assert _convert_value(value, data_type, EN, ENG_DATES) is None
|
|
|
|
@pytest.mark.parametrize(
|
|
"value",
|
|
[
|
|
pytest.param("true", id="true"),
|
|
pytest.param("True", id="true-titlecase"),
|
|
pytest.param("yes", id="yes"),
|
|
pytest.param("1", id="one"),
|
|
pytest.param("ja", id="ja"),
|
|
pytest.param("x", id="x-lowercase"),
|
|
pytest.param("X", id="x-uppercase"),
|
|
],
|
|
)
|
|
def test_bool_true(self, value: str) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A string that represents a true boolean
|
|
WHEN:
|
|
- The value is converted to a BOOL field value
|
|
THEN:
|
|
- The result is True
|
|
"""
|
|
assert (
|
|
_convert_value(value, CustomField.FieldDataType.BOOL, EN, ENG_DATES) is True
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
"value",
|
|
[
|
|
pytest.param("false", id="false"),
|
|
pytest.param("False", id="false-titlecase"),
|
|
pytest.param("no", id="no"),
|
|
pytest.param("0", id="zero"),
|
|
pytest.param("nein", id="nein"),
|
|
],
|
|
)
|
|
def test_bool_false(self, value: str) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A string that represents a false boolean
|
|
WHEN:
|
|
- The value is converted to a BOOL field value
|
|
THEN:
|
|
- The result is False
|
|
"""
|
|
assert (
|
|
_convert_value(value, CustomField.FieldDataType.BOOL, EN, ENG_DATES)
|
|
is False
|
|
)
|
|
|
|
|
|
class TestRenderPages:
|
|
"""Tests for _render_pages."""
|
|
|
|
@pytest.fixture
|
|
def pdf(self, tmp_path: Path) -> Path:
|
|
pdf = tmp_path / "document.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4")
|
|
return pdf
|
|
|
|
@pytest.fixture
|
|
def mock_rasterize(self, mocker: MockerFixture) -> MagicMock:
|
|
def render(in_path: Path, out_path: Path, **kwargs: Any) -> None:
|
|
out_path.write_bytes(b"rendered")
|
|
|
|
return mocker.patch(
|
|
"documents.zone_ocr.rasterize_pdf_page_to_png",
|
|
side_effect=render,
|
|
)
|
|
|
|
def test_renders_requested_pages_in_order_and_skips_out_of_range(
|
|
self,
|
|
pdf: Path,
|
|
tmp_path: Path,
|
|
mock_rasterize: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A 3 page PDF and requested pages including out of range indexes
|
|
WHEN:
|
|
- The pages are rendered
|
|
THEN:
|
|
- Only the in range pages are rendered, in order, as 1-based pages
|
|
at 300 DPI with a 60 second timeout
|
|
"""
|
|
pages = _render_pages(pdf, {-1, 0, 2, 3}, tmp_path, 3)
|
|
|
|
assert list(pages) == [0, 2]
|
|
assert mock_rasterize.call_count == 2
|
|
for page, call in zip([1, 3], mock_rasterize.call_args_list, strict=True):
|
|
assert call.args[0] == pdf
|
|
assert call.kwargs == {"dpi": 300, "page": page, "timeout": 60}
|
|
|
|
def test_missing_render_output_is_not_a_page(
|
|
self,
|
|
pdf: Path,
|
|
tmp_path: Path,
|
|
mock_rasterize: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A renderer that produces no output file
|
|
WHEN:
|
|
- A page is rendered
|
|
THEN:
|
|
- No pages are returned
|
|
"""
|
|
mock_rasterize.side_effect = None
|
|
|
|
assert _render_pages(pdf, {0}, tmp_path, None) == {}
|
|
|
|
@pytest.mark.parametrize(
|
|
"error",
|
|
[
|
|
pytest.param(ParseError("pdftoppm timed out after 60s"), id="timeout"),
|
|
pytest.param(
|
|
ParseError("pdftoppm failed at ['pdftoppm']"),
|
|
id="render-failure",
|
|
),
|
|
],
|
|
)
|
|
def test_render_failure_does_not_prevent_other_pages(
|
|
self,
|
|
pdf: Path,
|
|
tmp_path: Path,
|
|
mock_rasterize: MagicMock,
|
|
error: Exception,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A renderer that fails for the first page only
|
|
WHEN:
|
|
- Two pages are rendered
|
|
THEN:
|
|
- The failing page is skipped and the other page is still rendered
|
|
"""
|
|
|
|
def render(in_path: Path, out_path: Path, *, page: int, **kwargs: Any) -> None:
|
|
if page == 1:
|
|
raise error
|
|
out_path.write_bytes(b"rendered")
|
|
|
|
mock_rasterize.side_effect = render
|
|
|
|
assert list(_render_pages(pdf, {0, 1}, tmp_path, 2)) == [1]
|
|
|
|
def test_image_is_used_directly_only_for_first_page(
|
|
self,
|
|
tmp_path: Path,
|
|
mock_rasterize: MagicMock,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A single image file
|
|
WHEN:
|
|
- The first page, then the second page, is requested
|
|
THEN:
|
|
- The image itself is returned for the first page only
|
|
- The renderer is never run
|
|
"""
|
|
image_path = tmp_path / "image.png"
|
|
with Image.new("RGB", (10, 10)) as image:
|
|
image.save(image_path)
|
|
assert _render_pages(image_path, {0}, tmp_path, 1) == {0: image_path}
|
|
assert _render_pages(image_path, {1}, tmp_path, 1) == {}
|
|
mock_rasterize.assert_not_called()
|
|
|
|
|
|
class TestDetectMime:
|
|
"""Tests for _detect_mime."""
|
|
|
|
@pytest.mark.parametrize(
|
|
("filename", "expected"),
|
|
[
|
|
pytest.param("test.pdf", "application/pdf", id="pdf-extension"),
|
|
pytest.param("test.png", "image/png", id="png-extension"),
|
|
pytest.param("test.jpg", "image/jpeg", id="jpg-extension"),
|
|
pytest.param("test.xyz", None, id="unknown-extension"),
|
|
pytest.param("test.webp", "image/webp", id="webp-extension"),
|
|
],
|
|
)
|
|
def test_detect_mime(self, filename: str, expected: str | None) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A path with a given file extension
|
|
WHEN:
|
|
- The MIME type is detected
|
|
THEN:
|
|
- The expected MIME type, or None for an unknown extension, is returned
|
|
"""
|
|
assert _detect_mime(Path(filename)) == expected
|
|
|
|
|
|
class TestResolveDocPath:
|
|
"""Tests for _resolve_doc_path."""
|
|
|
|
@pytest.fixture
|
|
def document(self, mocker: MockerFixture) -> MagicMock:
|
|
doc = mocker.MagicMock()
|
|
doc.has_archive_version = False
|
|
doc.source_path = Path("/nonexistent/source.pdf")
|
|
return doc
|
|
|
|
def test_returns_none_when_no_files_exist(self, document: MagicMock) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document with no archive version and a missing source file
|
|
WHEN:
|
|
- The document path is resolved without an original file
|
|
THEN:
|
|
- None is returned
|
|
"""
|
|
assert _resolve_doc_path(document, None) is None
|
|
|
|
def test_prefers_archive_and_falls_back_to_source(
|
|
self,
|
|
mocker: MockerFixture,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document with existing archive and source files
|
|
WHEN:
|
|
- The path is resolved, then the archive file is removed and it is
|
|
resolved again
|
|
THEN:
|
|
- The archive path is returned first, then the source path
|
|
"""
|
|
source = tmp_path / "source.pdf"
|
|
archive = tmp_path / "archive.pdf"
|
|
source.touch()
|
|
archive.touch()
|
|
document = mocker.MagicMock(
|
|
has_archive_version=True,
|
|
archive_path=archive,
|
|
source_path=source,
|
|
)
|
|
assert _resolve_doc_path(document, None) == archive
|
|
archive.unlink()
|
|
assert _resolve_doc_path(document, None) == source
|
|
|
|
def test_returns_original_file_as_fallback(
|
|
self,
|
|
document: MagicMock,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document whose own files are missing and an existing original file
|
|
WHEN:
|
|
- The document path is resolved with the original file
|
|
THEN:
|
|
- The original file path is returned
|
|
"""
|
|
original = tmp_path / "original.pdf"
|
|
original.touch()
|
|
assert _resolve_doc_path(document, original) == original
|
|
|
|
def test_returns_none_for_missing_original_file(
|
|
self,
|
|
document: MagicMock,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A document whose own files are missing and an original file path
|
|
that does not exist
|
|
WHEN:
|
|
- The document path is resolved with the original file
|
|
THEN:
|
|
- None is returned
|
|
"""
|
|
assert _resolve_doc_path(document, tmp_path / "missing.pdf") is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@pytest.mark.usefixtures("paperless_dirs")
|
|
class TestRunZoneExtraction:
|
|
"""Tests for the full extraction pipeline."""
|
|
|
|
@pytest.fixture
|
|
def document(self) -> Document:
|
|
return DocumentFactory(
|
|
title="Test",
|
|
content="test",
|
|
mime_type="image/png",
|
|
)
|
|
|
|
@pytest.fixture
|
|
def source(self, paperless_dirs: PaperlessDirs) -> Path:
|
|
source = paperless_dirs.scratch_dir / "document.png"
|
|
with Image.new("RGB", (200, 200), "white") as image:
|
|
image.save(source)
|
|
return source
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def ocr_reads_text(self, mock_zone_ocr_run: MagicMock) -> None:
|
|
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
|
|
args=[],
|
|
returncode=0,
|
|
stdout="extracted",
|
|
stderr="",
|
|
)
|
|
|
|
@pytest.fixture
|
|
def template_with_zone(self) -> Callable[..., OcrTemplate]:
|
|
"""Build a template with one default zone, matching "test" by default."""
|
|
|
|
def _make(name: str, **kwargs: Any) -> OcrTemplate:
|
|
kwargs.setdefault("match", "test")
|
|
template = OcrTemplateFactory(name=name, **kwargs)
|
|
OcrTemplateZoneFactory(template=template)
|
|
return template
|
|
|
|
return _make
|
|
|
|
@pytest.mark.parametrize(
|
|
("template_name", "template_kwargs"),
|
|
[
|
|
pytest.param("Other sender", {"match": "Contoso"}, id="non-matching"),
|
|
pytest.param("Disabled", {"enabled": False}, id="disabled"),
|
|
],
|
|
)
|
|
def test_skips_ineligible_template(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
template_with_zone: Callable[..., OcrTemplate],
|
|
document: Document,
|
|
source: Path,
|
|
template_name: str,
|
|
template_kwargs: dict[str, Any],
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A template whose match text is not in the document content, or a
|
|
disabled template that would otherwise match
|
|
WHEN:
|
|
- Zone extraction runs against a real document file
|
|
THEN:
|
|
- No results are returned, no text is read and the title is unchanged
|
|
"""
|
|
template_with_zone(template_name, **template_kwargs)
|
|
|
|
results = run_zone_extraction(document, source)
|
|
|
|
assert results == []
|
|
mock_zone_ocr_run.assert_not_called()
|
|
document.refresh_from_db()
|
|
assert document.title == "Test"
|
|
|
|
def test_skips_template_with_no_zones(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
document: Document,
|
|
source: Path,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- An enabled matching template with no zones
|
|
WHEN:
|
|
- Zone extraction runs
|
|
THEN:
|
|
- No results are returned, no text is read and the title is unchanged
|
|
"""
|
|
OcrTemplateFactory(
|
|
name="Empty",
|
|
match="test",
|
|
)
|
|
|
|
results = run_zone_extraction(document, source)
|
|
|
|
assert results == []
|
|
mock_zone_ocr_run.assert_not_called()
|
|
document.refresh_from_db()
|
|
assert document.title == "Test"
|
|
|
|
def test_handles_process_exception_gracefully(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
template_with_zone: Callable[..., OcrTemplate],
|
|
document: Document,
|
|
source: Path,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- Two matching templates, where reading the first one's text raises
|
|
an unexpected error
|
|
WHEN:
|
|
- Zone extraction runs
|
|
THEN:
|
|
- The error does not propagate, the failing template's zone is
|
|
reported as template_failed and the other template still runs
|
|
"""
|
|
mock_zone_ocr_run.side_effect = [
|
|
RuntimeError("test error"),
|
|
subprocess.CompletedProcess(
|
|
args=[],
|
|
returncode=0,
|
|
stdout="extracted",
|
|
stderr="",
|
|
),
|
|
]
|
|
template_with_zone("First", enabled=True)
|
|
template_with_zone("Second", enabled=True)
|
|
|
|
results = run_zone_extraction(document, source)
|
|
|
|
assert Counter((r.value, r.reason) for r in results) == Counter(
|
|
[(None, "template_failed"), ("extracted", None)],
|
|
)
|
|
assert {r.template for r in results} == {"First", "Second"}
|
|
|
|
def test_handles_none_original_file(
|
|
self,
|
|
mock_zone_ocr_run: MagicMock,
|
|
template_with_zone: Callable[..., OcrTemplate],
|
|
document: Document,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A matching template, a document with no accessible file and no
|
|
original file
|
|
WHEN:
|
|
- Zone extraction runs
|
|
THEN:
|
|
- Each zone is reported as missing_file and no text is read
|
|
"""
|
|
template = template_with_zone("Matching", enabled=True)
|
|
zone = template.zones.get()
|
|
|
|
results = run_zone_extraction(document, None)
|
|
|
|
assert [(r.template, r.zone, r.reason) for r in results] == [
|
|
("Matching", zone.name, "missing_file"),
|
|
]
|
|
mock_zone_ocr_run.assert_not_called()
|
|
|
|
@pytest.mark.parametrize(
|
|
"template_names",
|
|
[
|
|
pytest.param(["Active"], id="one-template"),
|
|
pytest.param(
|
|
["Template 0", "Template 1", "Template 2"],
|
|
id="three-templates",
|
|
),
|
|
],
|
|
)
|
|
def test_processes_every_enabled_template(
|
|
self,
|
|
template_with_zone: Callable[..., OcrTemplate],
|
|
document: Document,
|
|
source: Path,
|
|
template_names: list[str],
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- One or more enabled matching templates, each with a zone
|
|
WHEN:
|
|
- Zone extraction runs
|
|
THEN:
|
|
- Each template reports exactly one successful result
|
|
"""
|
|
for name in template_names:
|
|
template_with_zone(name, enabled=True)
|
|
|
|
results = run_zone_extraction(document, source)
|
|
|
|
assert sorted((r.template, r.reason) for r in results) == [
|
|
(name, None) for name in sorted(template_names)
|
|
]
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@pytest.mark.usefixtures("paperless_dirs")
|
|
class TestRealZoneOcr:
|
|
"""End to end zone extraction using the real pdftoppm and tesseract."""
|
|
|
|
@pytest.fixture
|
|
def template(self) -> OcrTemplate:
|
|
template = make_template("Whole page", match="test", source_height=100)
|
|
OcrTemplateZoneFactory(
|
|
template=template,
|
|
name="Title",
|
|
target=OcrTemplateZone.TargetType.TITLE,
|
|
x=0,
|
|
y=0,
|
|
width=100,
|
|
height=100,
|
|
page=1,
|
|
)
|
|
return template
|
|
|
|
@pytest.mark.usefixtures("template")
|
|
def test_pdf_page_is_rendered_and_read(
|
|
self,
|
|
simple_digital_pdf_file: Path,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A one page PDF containing text
|
|
- A template with a title zone covering the whole page
|
|
WHEN:
|
|
- Zone extraction runs without any mocks
|
|
THEN:
|
|
- The page text is read and written to the document title
|
|
"""
|
|
document = DocumentFactory(
|
|
title="Original",
|
|
content="This is a test document.",
|
|
mime_type="application/pdf",
|
|
page_count=1,
|
|
)
|
|
shutil.copy(simple_digital_pdf_file, document.source_path)
|
|
|
|
results = run_zone_extraction(document, None)
|
|
|
|
assert [result.reason for result in results] == [None]
|
|
document.refresh_from_db()
|
|
assert "This is a test document" in document.title
|
|
|
|
@pytest.mark.usefixtures("template")
|
|
def test_image_is_read_without_rendering(self) -> None:
|
|
"""
|
|
GIVEN:
|
|
- An image containing text as the document's file
|
|
- A template with a title zone covering the whole image
|
|
WHEN:
|
|
- Zone extraction runs without any mocks
|
|
THEN:
|
|
- The image text is read and written to the document title
|
|
"""
|
|
document = DocumentFactory(
|
|
title="Original",
|
|
content="This is a test document.",
|
|
mime_type="image/png",
|
|
page_count=1,
|
|
)
|
|
shutil.copy(
|
|
Path(__file__).parent / "samples" / "simple.png",
|
|
document.source_path,
|
|
)
|
|
|
|
results = run_zone_extraction(document, None)
|
|
|
|
assert [result.reason for result in results] == [None]
|
|
document.refresh_from_db()
|
|
assert "This is a test document" in document.title
|