"""Tests for the zone-based OCR extraction engine.""" from __future__ import annotations import shutil import subprocess from collections import Counter from datetime import date from pathlib import Path from typing import TYPE_CHECKING from typing import Any import pytest from babel import Locale from django.contrib.auth.models import Group from PIL import Image from documents.models import CustomField from documents.models import CustomFieldInstance from documents.models import Document from documents.models import MatchingModel from documents.models import OcrTemplate from documents.models import OcrTemplateZone from documents.parsers import ParseError from documents.zone_ocr import _apply_transform from documents.zone_ocr import _combine_field_value from documents.zone_ocr import _convert_value from documents.zone_ocr import _crop_zone from documents.zone_ocr import _date_rules from documents.zone_ocr import _detect_mime from documents.zone_ocr import _extract_zone from documents.zone_ocr import _number_locale from documents.zone_ocr import _parse_date from documents.zone_ocr import _parse_monetary from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages from documents.zone_ocr import _resolve_doc_path from documents.zone_ocr import _zone_language from documents.zone_ocr import extract_zone_preview from documents.zone_ocr import get_templates_for_document from documents.zone_ocr import run_zone_extraction from paperless_testing.factories import CustomFieldFactory from paperless_testing.factories import DocumentFactory from paperless_testing.factories import OcrTemplateFactory from paperless_testing.factories import OcrTemplateZoneFactory from paperless_testing.factories import UserFactory from paperless_testing.permissions import grant_object if TYPE_CHECKING: from collections.abc import Callable from unittest.mock import MagicMock from django.contrib.auth.models import User from pytest_django.fixtures import SettingsWrapper from pytest_mock import MockerFixture from paperless_testing.dirs import PaperlessDirs EN = Locale.parse("en") DE = Locale.parse("de") ENG_DATES = _date_rules("eng") FR = Locale.parse("fr") def make_template(name: str, **kwargs: Any) -> OcrTemplate: """Build a saved template, 100x50 by default.""" kwargs.setdefault("source_width", 100) kwargs.setdefault("source_height", 50) return OcrTemplateFactory(name=name, **kwargs) def make_zone( template: OcrTemplate, name: str, target: str = OcrTemplateZone.TargetType.TITLE, **kwargs: Any, ) -> OcrTemplateZone: """Build a 20x10 zone at the origin of the template.""" return OcrTemplateZoneFactory( template=template, name=name, target=target, width=20, height=10, **kwargs, ) def ocr_outputs(*values: str) -> list[subprocess.CompletedProcess[str]]: """Build one successful fake OCR process result per extracted text.""" return [ subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") for value in values ] def make_visible_template(name: str, **kwargs: Any) -> OcrTemplate: """Build a 100x100 template matching "Northwind".""" kwargs.setdefault("match", "Northwind") kwargs.setdefault("source_height", 100) return make_template(name, **kwargs) class TestZoneExtraction: """Tests for single-zone cropping, extraction and preview.""" @pytest.fixture def page(self, tmp_path: Path) -> Path: page = tmp_path / "page.png" with Image.new("RGB", (200, 100), "white") as image: image.paste("black", (20, 10, 60, 30)) image.save(page) return page @pytest.fixture def zone(self) -> OcrTemplateZone: return OcrTemplateZone( name="Account", x=10, y=5, width=20, height=10, transform="uppercase", ocr_language="eng", ) @pytest.fixture(autouse=True) def scratch_dir(self, settings: SettingsWrapper, tmp_path: Path) -> None: settings.SCRATCH_DIR = tmp_path def test_crop_scales_coordinates( self, page: Path, zone: OcrTemplateZone, ) -> None: """ GIVEN: - A zone and a page image rendered at a different size than the zone's WHEN: - The zone is cropped THEN: - The zone coordinates are scaled to the image and the crop is black """ with _crop_zone(page, zone, 100, 50) as cropped: assert cropped.size == (40, 20) assert cropped.getextrema() == ((0, 0), (0, 0), (0, 0)) def test_crop_defaults_to_image_dimensions( self, page: Path, zone: OcrTemplateZone, ) -> None: """ GIVEN: - A zone and a page with unknown (zero) reference dimensions WHEN: - The zone is cropped THEN: - The image's own dimensions are used for the crop """ with _crop_zone(page, zone, 0, 0) as cropped: assert cropped.size == (20, 10) assert cropped.getpixel((0, 0)) == (255, 255, 255) assert cropped.getpixel((19, 9)) == (0, 0, 0) def test_crop_clamps_to_image( self, page: Path, zone: OcrTemplateZone, ) -> None: """ GIVEN: - A zone that extends past the edge of the page image WHEN: - The zone is cropped THEN: - The crop is clamped to the image bounds """ zone.x, zone.y = 90, 40 with _crop_zone(page, zone, 100, 50) as cropped: assert cropped.size == (20, 20) def test_tiny_crop_skips_reader( self, page: Path, zone: OcrTemplateZone, tmp_path: Path, mock_zone_ocr_run: MagicMock, ) -> None: """ GIVEN: - A zone with zero width WHEN: - The zone is extracted THEN: - No text or value is returned and the reader is never run """ zone.width = 0 result = _extract_zone(page, zone, 100, 50, tmp_path) assert result.raw_text is None assert result.value is None mock_zone_ocr_run.assert_not_called() def test_unreadable_image_skips_reader( self, page: Path, zone: OcrTemplateZone, tmp_path: Path, mock_zone_ocr_run: MagicMock, ) -> None: """ GIVEN: - A page file that is not a valid image WHEN: - The zone is extracted THEN: - No text or value is returned and the reader is never run """ page.write_bytes(b"not an image") result = _extract_zone(page, zone, 100, 50, tmp_path) assert result.raw_text is None assert result.value is None mock_zone_ocr_run.assert_not_called() def test_text_extraction_retains_raw_text_and_transforms( self, page: Path, zone: OcrTemplateZone, tmp_path: Path, mock_zone_ocr_run: MagicMock, ) -> None: """ GIVEN: - A zone with an uppercase transform and a reader returning padded text WHEN: - The zone is extracted THEN: - The reader receives the cropped image with the expected arguments - The stripped raw text is retained and the transformed value returned """ def read_crop( args: list[str], **kwargs: Any, ) -> subprocess.CompletedProcess[str]: with Image.open(args[1]) as crop: assert crop.size == (40, 20) assert crop.getpixel((0, 0)) == (0, 0, 0) return subprocess.CompletedProcess( args=[], returncode=0, stdout=" account abc\n", stderr="", ) mock_zone_ocr_run.side_effect = read_crop result = _extract_zone(page, zone, 100, 50, tmp_path) assert result.raw_text == "account abc" assert result.value == "ACCOUNT ABC" assert mock_zone_ocr_run.call_args.args[0][-4:] == ["-l", "eng", "--psm", "6"] assert mock_zone_ocr_run.call_args.kwargs["timeout"] == 30 def test_no_text_returns_no_value( self, page: Path, zone: OcrTemplateZone, tmp_path: Path, mock_zone_ocr_run: MagicMock, ) -> None: """ GIVEN: - A reader that returns only whitespace WHEN: - The zone is extracted THEN: - No text or value is returned """ mock_zone_ocr_run.return_value.stdout = " \n" result = _extract_zone(page, zone, 100, 50, tmp_path) assert result.raw_text is None assert result.value is None @pytest.mark.parametrize( "error", [ pytest.param(subprocess.TimeoutExpired("tesseract", 30), id="timeout"), pytest.param( subprocess.CalledProcessError(1, "tesseract", stderr="failure"), id="called-process-error-with-stderr", ), pytest.param( subprocess.CalledProcessError(1, "tesseract"), id="called-process-error-without-stderr", ), ], ) def test_reader_failures_return_no_value( self, page: Path, zone: OcrTemplateZone, tmp_path: Path, mock_zone_ocr_run: MagicMock, error: Exception, ) -> None: """ GIVEN: - A reader that fails with a timeout or process error WHEN: - The zone is extracted THEN: - No text or value is returned """ mock_zone_ocr_run.side_effect = error result = _extract_zone(page, zone, 100, 50, tmp_path) assert result.raw_text is None assert result.value is None @pytest.mark.parametrize( "reader_kwargs", [ pytest.param( {"side_effect": RuntimeError("reader failed")}, id="reader-raises", ), pytest.param({"return_value": []}, id="nothing-found"), ], ) def test_barcode_without_value_returns_no_value( self, mocker: MockerFixture, page: Path, zone: OcrTemplateZone, tmp_path: Path, reader_kwargs: dict[str, Any], ) -> None: """ GIVEN: - A QR code zone and a barcode reader that either raises or finds no barcode WHEN: - The zone is extracted THEN: - No text or value is returned """ mocker.patch("zxingcpp.read_barcodes", **reader_kwargs) zone.transform = "qr_code" result = _extract_zone(page, zone, 100, 50, tmp_path) assert result.raw_text is None assert result.value is None @pytest.mark.parametrize( ("attribute", "value"), [ pytest.param("width", 0, id="zero-width"), pytest.param("page", 2, id="unavailable-page"), ], ) def test_preview_unreadable_zone_returns_empty_result( self, page: Path, zone: OcrTemplateZone, mock_zone_ocr_run: MagicMock, attribute: str, value: int, ) -> None: """ GIVEN: - A zone that cannot be read, either because its crop has zero width or because it targets a page that is not available WHEN: - A preview is requested THEN: - An empty result is returned and the reader is never run """ setattr(zone, attribute, value) assert extract_zone_preview(page, zone, 100, 50, 1) == { "raw_text": None, "value": None, } mock_zone_ocr_run.assert_not_called() def test_date_transform_uses_zone_format( self, page: Path, zone: OcrTemplateZone, tmp_path: Path, mock_zone_ocr_run: MagicMock, ) -> None: """ GIVEN: - A date zone with a custom date format WHEN: - The zone is extracted from text in that format THEN: - The raw text is retained and the value is an ISO date """ zone.transform = "date" zone.date_format = "%d/%m/%Y" mock_zone_ocr_run.return_value.stdout = "25/12/2025" result = _extract_zone(page, zone, 100, 50, tmp_path) assert result.raw_text == "25/12/2025" assert result.value == "2025-12-25" def test_barcode_extraction_bypasses_tesseract( self, mocker: MockerFixture, page: Path, zone: OcrTemplateZone, tmp_path: Path, mock_zone_ocr_run: MagicMock, ) -> None: """ GIVEN: - A QR code zone and a barcode reader returning padded text WHEN: - The zone is extracted THEN: - The barcode reader receives the cropped image - The raw text is kept as read and the value is stripped - The text reader is never run and the image is closed afterwards """ read_barcodes = mocker.patch("zxingcpp.read_barcodes") zone.transform = "qr_code" def read_crop(image: Image.Image) -> list[MagicMock]: assert image.size == (40, 20) assert image.getpixel((0, 0)) == (0, 0, 0) return [mocker.MagicMock(text=" https://example.com ")] read_barcodes.side_effect = read_crop result = _extract_zone(page, zone, 100, 50, tmp_path) assert result.raw_text == " https://example.com " assert result.value == "https://example.com" mock_zone_ocr_run.assert_not_called() with pytest.raises(ValueError): read_barcodes.call_args.args[0].getpixel((0, 0)) def test_preview_uses_image_dimensions_without_writing( self, page: Path, zone: OcrTemplateZone, mock_zone_ocr_run: MagicMock, ) -> None: """ GIVEN: - A zone and a reader returning text WHEN: - A preview is requested with unknown (zero) reference dimensions THEN: - The raw text and transformed value are returned """ mock_zone_ocr_run.return_value.stdout = "account abc" assert extract_zone_preview(page, zone, 0, 0, 1) == { "raw_text": "account abc", "value": "ACCOUNT ABC", } @pytest.mark.django_db @pytest.mark.usefixtures("paperless_dirs") class TestSavedZoneExtraction: """Tests for extracting zones from a saved template and writing the results.""" @pytest.fixture def page(self, paperless_dirs: PaperlessDirs) -> Path: page = paperless_dirs.scratch_dir / "page.png" with Image.new("RGB", (200, 100), "white") as image: image.save(page) return page @pytest.fixture def document(self) -> Document: return DocumentFactory(title="Original") @pytest.fixture def template(self) -> OcrTemplate: return make_template("Account") def test_combines_zones_and_writes_each_field_once( self, mocker: MockerFixture, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - Two title zones and a combine format that joins them WHEN: - The template is processed THEN: - The combined value is written to the title with a single save """ zones = [make_zone(template, "First"), make_zone(template, "Second")] template.combine_formats = {"title": "{Second} / {First}"} mock_zone_ocr_run.side_effect = [ subprocess.CompletedProcess(args=[], returncode=0, stdout="one", stderr=""), subprocess.CompletedProcess(args=[], returncode=0, stdout="two", stderr=""), ] save = mocker.patch.object(document, "save", wraps=document.save) _process_template( document, {0: page}, template, zones, paperless_dirs.scratch_dir, ) document.refresh_from_db() assert document.title == "two / one" save.assert_called_once_with(update_fields=["title"]) def test_converts_custom_and_builtin_fields_before_saving( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - Zones for the title, ASN, created date and a boolean custom field WHEN: - The template is processed THEN: - Each raw value is converted to its field type and saved """ field = CustomFieldFactory(name="Paid", data_type="boolean") zones = [ make_zone(template, "Title"), make_zone(template, "ASN", "asn"), make_zone(template, "Created", "created"), make_zone(template, "Paid", "custom_field", custom_field=field), ] mock_zone_ocr_run.side_effect = [ subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") for value in ["New title", "ASN 42", "2026-04-13", "no"] ] _process_template( document, {0: page}, template, zones, paperless_dirs.scratch_dir, ) document.refresh_from_db() assert document.title == "New title" assert document.archive_serial_number == 42 assert document.created == date(2026, 4, 13) assert not document.custom_fields.get(field=field).value def test_invalid_field_values_preserve_existing_values( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A document with an existing custom field value - Zones whose ASN, created date and custom field values are invalid or unassigned WHEN: - The template is processed THEN: - Only the valid title is written and the other values are preserved """ field = CustomFieldFactory(name="Number", data_type="integer") CustomFieldInstance.objects.create( document=document, field=field, value_int=12, ) zones = [ make_zone(template, "Title"), make_zone(template, "ASN", "asn"), make_zone(template, "Created", "created"), make_zone(template, "Number", "custom_field", custom_field=field), make_zone(template, "Unassigned", "custom_field"), ] original_created = document.created mock_zone_ocr_run.side_effect = [ subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") for value in [ "Valid", "9999999999999999999999999999999", "invalid", "abc", "text", ] ] _process_template( document, {0: page}, template, zones, paperless_dirs.scratch_dir, ) document.refresh_from_db() assert document.title == "Valid" assert document.archive_serial_number is None assert document.created == original_created assert document.custom_fields.get(field=field).value == 12 def test_write_failure_rolls_back_template_and_restores_document( self, mocker: MockerFixture, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A title zone and a custom field zone - A document save that raises after writing WHEN: - The template is processed THEN: - The error propagates, the write is rolled back and the in-memory document is restored """ field = CustomFieldFactory(name="Number", data_type="integer") zones = [ make_zone(template, "Title"), make_zone(template, "Number", "custom_field", custom_field=field), ] mock_zone_ocr_run.side_effect = [ subprocess.CompletedProcess( args=[], returncode=0, stdout="New title", stderr="", ), subprocess.CompletedProcess(args=[], returncode=0, stdout="42", stderr=""), ] save = document.save def fail_after_save(**kwargs: Any) -> None: save(**kwargs) raise RuntimeError("save failed") mocker.patch.object(document, "save", side_effect=fail_after_save) with pytest.raises(RuntimeError, match="save failed"): _process_template( document, {0: page}, template, zones, paperless_dirs.scratch_dir, ) assert document.title == "Original" document.refresh_from_db() assert document.title == "Original" assert not document.custom_fields.exists() def test_last_successful_template_wins( self, mock_zone_ocr_run: MagicMock, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - Three templates writing the title, the last of which rejects its value WHEN: - Zone extraction runs over all of them THEN: - The title comes from the last template that produced a value """ make_zone(template, "First") second = make_template("Second") make_zone(second, "Second") third = make_template("Third") make_zone(third, "Third", validation_regex="accepted") templates = list(OcrTemplate.objects.order_by("pk")) mock_zone_ocr_run.side_effect = [ subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") for value in ["First", "Second", "rejected"] ] run_zone_extraction(document, page, templates=templates) document.refresh_from_db() assert document.title == "Second" def test_document_run_renders_each_page_once_across_templates( self, mocker: MockerFixture, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - Two templates with zones on the first, last and second page of a two page document WHEN: - Zone extraction runs over both templates THEN: - Each page is rendered only once and the rendered files are removed """ document.page_count = 2 make_zone(template, "First", page=1) make_zone(template, "Last", page=-1) second = make_template("Second") make_zone(second, "Last again", page=2) pdf = paperless_dirs.scratch_dir / "document.pdf" pdf.write_bytes(b"%PDF-1.4") rendered = [] def render(in_path: Path, out_path: Path, **kwargs: Any) -> None: with Image.new("RGB", (100, 50), "white") as image: image.save(out_path) rendered.append(out_path) mock_rasterize = mocker.patch( "documents.zone_ocr.rasterize_pdf_page_to_png", side_effect=render, ) mock_zone_ocr_run.return_value = subprocess.CompletedProcess( [], 0, stdout="extracted", ) results = run_zone_extraction(document, pdf, templates=[template, second]) assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2] assert [result.reason for result in results] == [None, None, None] assert not any(path.exists() for path in rendered) def test_zone_dimensions_override_template_dimensions( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A zone with its own source dimensions, differing from the template's WHEN: - The template is processed THEN: - The zone is cropped using the zone's dimensions """ zone = make_zone( template, "Title", zone_source_width=200, zone_source_height=100, ) def read_crop( args: list[str], **kwargs: Any, ) -> subprocess.CompletedProcess[str]: with Image.open(args[1]) as crop: assert crop.size == (20, 10) return subprocess.CompletedProcess(args, 0, stdout="extracted") mock_zone_ocr_run.side_effect = read_crop _process_template( document, {0: page}, template, [zone], paperless_dirs.scratch_dir, ) assert document.title == "extracted" def test_missing_page_returns_skipped_result( self, paperless_dirs: PaperlessDirs, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A zone on the last page and no rendered pages WHEN: - The template is processed THEN: - The zone result is skipped as a missing page with no value """ zone = make_zone(template, "Last", page=-1) results = _process_template( document, {}, template, [zone], paperless_dirs.scratch_dir, ) assert results[0].reason == "missing_page" assert results[0].value is None def test_missing_document_file_returns_skipped_results( self, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A template with a zone and no document file WHEN: - Zone extraction runs THEN: - A single result is skipped as a missing file with no value """ make_zone(template, "Title") results = run_zone_extraction(document, None, templates=[template]) assert len(results) == 1 assert results[0].reason == "missing_file" assert results[0].value is None def test_empty_combination_cannot_clear_a_field( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A combine format that references an unknown zone and so is empty WHEN: - The template is processed THEN: - The result is an invalid value and the title is not cleared """ zone = make_zone(template, "Title") template.combine_formats = {"title": "{Unknown}"} mock_zone_ocr_run.return_value.stdout = "extracted" results = _process_template( document, {0: page}, template, [zone], paperless_dirs.scratch_dir, ) assert results[0].reason == "invalid_value" assert results[0].value is None assert document.title == "Original" def test_skipped_results_never_report_existing_document_values( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - Zones that are empty, rejected by regex, invalid or unassigned WHEN: - The template is processed THEN: - Each result carries its skip reason and no value, and the title is unchanged """ zones = [ make_zone(template, "Empty"), make_zone(template, "Rejected", validation_regex="accepted"), make_zone(template, "Invalid", "asn"), make_zone(template, "Unassigned", "custom_field"), ] mock_zone_ocr_run.side_effect = [ subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") for value in ["", "rejected", "abc", "orphan"] ] results = _process_template( document, {0: page}, template, zones, paperless_dirs.scratch_dir, ) assert [result.reason for result in results] == [ "no_value", "regex_mismatch", "invalid_value", "unassigned", ] assert [result.value for result in results] == [None] * 4 assert document.title == "Original" def test_failed_template_reports_rollback_and_next_template_still_runs( self, mock_zone_ocr_run: MagicMock, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A template whose ASN zone conflicts with another document's ASN - A second template that would succeed WHEN: - Zone extraction runs over both templates THEN: - The first template is reported as failed and rolled back, and the second template is still applied """ DocumentFactory(archive_serial_number=42) make_zone(template, "Title") make_zone(template, "Conflict", "asn") second = make_template("Second") make_zone(second, "Second") mock_zone_ocr_run.side_effect = [ subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") for value in ["Rolled back", "42", "Applied"] ] results = run_zone_extraction(document, page, templates=[template, second]) assert [result.reason for result in results] == [ "template_failed", "template_failed", None, ] assert [result.value for result in results] == [None, None, "Applied"] document.refresh_from_db() assert document.title == "Applied" assert document.archive_serial_number is None def test_literal_format_does_not_write_when_every_zone_failed( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A combine format with literal text around a zone that reads nothing WHEN: - The template is processed THEN: - The result has no value and the title is not written """ zone = make_zone(template, "Title") template.combine_formats = {"title": "Invoice {Title}"} mock_zone_ocr_run.return_value.stdout = "" results = _process_template( document, {0: page}, template, [zone], paperless_dirs.scratch_dir, ) assert results[0].reason == "no_value" assert results[0].value is None assert document.title == "Original" def test_saved_extraction_matches_preview_and_rejects_invalid_value( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A zone that uppercases its text and validates it against a regex WHEN: - A valid value and then an invalid value are previewed and saved in turn THEN: - The preview matches the transformed text for both, the valid value is saved, and the invalid value leaves the saved title unchanged """ zone = make_zone( template, "Title", transform="uppercase", validation_regex="ACCOUNT [A-Z]+", ) def preview_and_save(raw_text: str) -> None: mock_zone_ocr_run.return_value.stdout = raw_text preview = extract_zone_preview(page, zone, 100, 50, 1) assert preview["raw_text"] == raw_text assert preview["value"] == raw_text.upper() _process_template( document, {0: page}, template, [zone], paperless_dirs.scratch_dir, ) document.refresh_from_db() preview_and_save("account abc") assert document.title == "ACCOUNT ABC" preview_and_save("invalid") assert document.title == "ACCOUNT ABC" @pytest.mark.parametrize( ("failing_zone", "expected_failure"), [ pytest.param( {"validation_regex": r"\d{2}"}, "regex_mismatch", id="regex-mismatch", ), pytest.param({"page": 2}, "missing_page", id="missing-page"), ], ) def test_failed_zone_blocks_combined_field( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, failing_zone: dict[str, Any], expected_failure: str, ) -> None: """ GIVEN: - Three title zones combined as a date, where the first one fails WHEN: - The template is processed THEN: - Nothing is written, the failing zone reports its own reason and the accepted zones report incomplete """ zones = [ make_zone(template, "Day", **failing_zone), make_zone(template, "Month"), make_zone(template, "Year"), ] template.combine_formats = {"title": "{Day}.{Month}.{Year}"} # A zone on a missing page is never read. mock_zone_ocr_run.side_effect = ocr_outputs( *( ["x", "05", "2024"] if "validation_regex" in failing_zone else ["05", "2024"] ), ) results = _process_template( document, {0: page}, template, zones, paperless_dirs.scratch_dir, ) assert [result.reason for result in results] == [ expected_failure, "incomplete", "incomplete", ] assert [result.value for result in results] == [None] * 3 document.refresh_from_db() assert document.title == "Original" def test_zone_with_no_value_is_an_optional_token( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - Two title zones combined with a separator, where the first reads nothing WHEN: - The template is processed THEN: - The remaining value is still written and reported """ zones = [make_zone(template, "First"), make_zone(template, "Second")] template.combine_formats = {"title": "{First} / {Second}"} mock_zone_ocr_run.side_effect = ocr_outputs("", "two") results = _process_template( document, {0: page}, template, zones, paperless_dirs.scratch_dir, ) assert [result.reason for result in results] == ["no_value", None] assert results[1].value == "two" document.refresh_from_db() assert document.title == "two" def test_failed_single_zone_field_does_not_affect_other_fields( self, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, page: Path, document: Document, template: OcrTemplate, ) -> None: """ GIVEN: - A title zone and a separate ASN zone, where the ASN fails its validation regex WHEN: - The template is processed THEN: - The title is written and the failing zone keeps regex_mismatch """ zones = [ make_zone(template, "Title"), make_zone(template, "ASN", "asn", validation_regex=r"\d+"), ] mock_zone_ocr_run.side_effect = ocr_outputs("New title", "abc") results = _process_template( document, {0: page}, template, zones, paperless_dirs.scratch_dir, ) assert [result.reason for result in results] == [None, "regex_mismatch"] document.refresh_from_db() assert document.title == "New title" class TestCombineFieldValue: """Tests for combining zone values into one field value.""" @pytest.mark.parametrize( ("fmt", "zones", "expected"), [ pytest.param( "", [("Same", "one"), ("Same", None), ("Same", "two")], "one two", id="default-format-preserves-zone-order-and-duplicate-names", ), pytest.param( "{ First } / {Second} / {Unknown}", [("First", "one"), ("Second", None)], "one", id="format-preserves-existing-missing-token-cleanup", ), pytest.param( "{Amount}", [("Amount", "-12.50")], "-12.50", id="negative-amount-sign-kept", ), pytest.param( "{Url}", [("Url", "http://x.com/a//b")], "http://x.com/a//b", id="url-double-slash-kept-verbatim", ), pytest.param( "{Value}", [("Value", "A--B")], "A--B", id="repeated-punctuation-in-value-kept", ), pytest.param( "{Name}", [("Name", "Dr. Smith Jr.")], "Dr. Smith Jr.", id="trailing-period-in-value-kept", ), pytest.param( "{Value}", [("Value", "wait...")], "wait...", id="trailing-ellipsis-in-value-kept", ), pytest.param( "{A} / {B} / {C}", [("A", "a"), ("B", None), ("C", "c")], "a / c", id="middle-empty-token-drops-one-separator", ), pytest.param( "{A} / {B} / {C}", [("A", None), ("B", "b"), ("C", "c")], "b / c", id="leading-empty-token-drops-leading-separator", ), pytest.param( "{A} / {B} / {C}", [("A", "a"), ("B", "b"), ("C", None)], "a / b", id="trailing-empty-token-drops-trailing-separator", ), pytest.param( "{A} / {B} / {C}", [("A", "a"), ("B", None), ("C", None)], "a", id="trailing-run-of-empty-tokens-drops-nearest-separator", ), pytest.param( "Total: {A} / {B}", [("A", None), ("B", "b")], "Total: b", id="leading-literal-kept-before-empty-first-token", ), pytest.param( "- {A} {B}", [("A", None), ("B", "b")], "b", id="leading-separator-dropped-before-empty-first-token", ), pytest.param( "{Day}.{Month}.{Year}", [("Day", "12"), ("Month", "05"), ("Year", "2024")], "12.05.2024", id="all-tokens-present", ), pytest.param( "INV-{Number}", [("Number", "123")], "INV-123", id="literal-prefix-kept-with-value", ), pytest.param( "INV-{Number}", [("Number", None)], "", id="literal-prefix-dropped-when-every-token-empty", ), pytest.param( "{A} / {B}", [("A", None), ("B", " ")], "", id="every-token-empty", ), ], ) def test_combine_field_value( self, fmt: str, zones: list[tuple[str, str | None]], expected: str, ) -> None: """ GIVEN: - A combine format and a list of (zone name, extracted value) pairs WHEN: - The zone values are combined into a single field value THEN: - The combined string matches the expected result """ assert _combine_field_value(fmt, zones) == expected @pytest.mark.django_db @pytest.mark.usefixtures("paperless_dirs") class TestTemplateVisibility: """Tests for which templates apply to a document and its requester.""" @pytest.fixture def owner(self) -> User: return UserFactory(username="document_owner") @pytest.fixture def other(self) -> User: return UserFactory(username="other_owner") @pytest.fixture def document(self, owner: User) -> Document: return DocumentFactory( owner=owner, content="Northwind Power & Light statement", ) @pytest.fixture def mine(self, owner: User) -> OcrTemplate: return make_visible_template("Mine", owner=owner) @pytest.fixture def unowned(self) -> OcrTemplate: return make_visible_template("Unowned") @pytest.fixture def private(self, other: User) -> OcrTemplate: return make_visible_template("Private", owner=other) @pytest.mark.usefixtures("private") def test_only_matching_owned_unowned_and_shared_templates_apply( self, owner: User, other: User, document: Document, mine: OcrTemplate, unowned: OcrTemplate, ) -> None: """ GIVEN: - Templates owned by the document owner, unowned, private to another user, shared with the owner, disabled, for another sender and with no matching WHEN: - Templates are selected for the document THEN: - Only the enabled, matching, owned, unowned and shared templates apply """ shared = make_visible_template("Shared", owner=other) grant_object(owner, shared, "view_ocrtemplate") make_visible_template("Disabled", owner=owner, enabled=False) make_visible_template("Other sender", owner=owner, match="Contoso") make_visible_template( "No matching", owner=owner, matching_algorithm=MatchingModel.MATCH_NONE, ) assert Counter(get_templates_for_document(document)) == Counter( [mine, unowned, shared], ) @pytest.mark.usefixtures("mine", "unowned") def test_group_sharing_applies( self, owner: User, document: Document, private: OcrTemplate, ) -> None: """ GIVEN: - A private template shared with a group the document owner is in WHEN: - Templates are selected for the document THEN: - The group-shared template applies """ group = Group.objects.create(name="OCR users") owner.groups.add(group) grant_object(group, private, "view_ocrtemplate") assert private in get_templates_for_document(document) @pytest.mark.usefixtures("mine", "private") def test_unowned_documents_only_use_unowned_templates( self, document: Document, unowned: OcrTemplate, ) -> None: """ GIVEN: - A document with no owner WHEN: - Templates are selected for the document THEN: - Only unowned templates apply """ document.owner = None assert get_templates_for_document(document) == [unowned] @pytest.mark.usefixtures("private") def test_versions_use_the_root_documents_owner( self, other: User, document: Document, mine: OcrTemplate, unowned: OcrTemplate, ) -> None: """ GIVEN: - A version document owned by another user than its root document WHEN: - Templates are selected for the version THEN: - The root document's owner decides the visible templates """ version = DocumentFactory( root_document=document, owner=other, content=document.content, ) assert Counter(get_templates_for_document(version)) == Counter( [mine, unowned], ) @pytest.mark.usefixtures("private") def test_matching_uses_the_selected_files_content( self, document: Document, mine: OcrTemplate, unowned: OcrTemplate, ) -> None: """ GIVEN: - A root document and a version with different content - A template matching only the version's content WHEN: - Templates are selected for each THEN: - Each is matched against its own content """ version = DocumentFactory(root_document=document, content="Contoso") contoso = make_visible_template("Contoso", match="Contoso") assert Counter(get_templates_for_document(document)) == Counter( [mine, unowned], ) assert get_templates_for_document(version) == [contoso] def test_superuser_owner_can_use_all_visible_templates( self, owner: User, document: Document, mine: OcrTemplate, unowned: OcrTemplate, private: OcrTemplate, ) -> None: """ GIVEN: - A document owned by a superuser WHEN: - Templates are selected for the document THEN: - Every matching template applies, including private ones """ owner.is_superuser = True assert Counter(get_templates_for_document(document)) == Counter( [mine, unowned, private], ) @pytest.mark.usefixtures("mine", "private") def test_requester_limits_the_owners_templates( self, owner: User, other: User, document: Document, unowned: OcrTemplate, ) -> None: """ GIVEN: - Templates owned by the document owner, unowned, private to another user, and owned by the owner but shared with the requester WHEN: - Templates are selected for a manual run by the requester THEN: - Only templates both the owner and the requester can view are used """ shared = make_visible_template("Shared", owner=owner) grant_object(other, shared, "view_ocrtemplate") assert Counter( get_templates_for_document(document, requester=other), ) == Counter([unowned, shared]) @pytest.mark.usefixtures("mine", "unowned", "private") def test_inactive_owner_has_no_templates( self, owner: User, document: Document, ) -> None: """ GIVEN: - A document whose owner is inactive WHEN: - Templates are selected for the document THEN: - No templates apply """ owner.is_active = False assert get_templates_for_document(document) == [] def test_extraction_never_processes_private_templates( self, mock_zone_ocr_run: MagicMock, document: Document, mine: OcrTemplate, unowned: OcrTemplate, private: OcrTemplate, ) -> None: """ GIVEN: - Owned, unowned and private templates each with a zone - A blank image as the document's file WHEN: - Zone extraction runs for the document THEN: - Only the owned and unowned templates report results """ for template in (mine, unowned, private): OcrTemplateZoneFactory( template=template, name="Title", width=20, height=20, ) document.mime_type = "image/png" with Image.new("RGB", (100, 100), "white") as image: image.save(document.source_path, format="PNG") mock_zone_ocr_run.return_value = subprocess.CompletedProcess( args=[], returncode=0, stdout="extracted", stderr="", ) results = run_zone_extraction(document, None) assert Counter(result.template for result in results) == Counter( [mine.name, unowned.name], ) class TestApplyTransform: """Tests for the _apply_transform function.""" @pytest.mark.parametrize( ("text", "transform", "date_format", "expected"), [ pytest.param(" hello ", "strip", "", "hello", id="strip"), pytest.param(" hello ", "none", "", "hello", id="none-transform"), pytest.param( "hello world", "uppercase", "", "HELLO WORLD", id="uppercase", ), pytest.param( "HELLO WORLD", "lowercase", "", "hello world", id="lowercase", ), pytest.param( "Invoice 2026-001", "numeric", "", "2026-001", id="numeric-basic", ), pytest.param( "\u20ac1,234.56", "numeric", "", "1,234.56", id="numeric-with-currency", ), pytest.param( "abc", "numeric", "", "abc", id="numeric-empty-result-falls-back", ), pytest.param( "13.04.2026", "date", "%d.%m.%Y", "2026-04-13", id="date-with-format-dotted", ), pytest.param( "01/12/2025", "date", "%d/%m/%Y", "2025-12-01", id="date-with-format-slashed", ), pytest.param( "13.04.26", "date", "%d.%m.%y", "2026-04-13", id="date-two-digit-year", ), pytest.param( "2026-04-13", "date", "", "2026-04-13", id="date-auto-detect-iso", ), pytest.param( "2026/04/13", "date", "", "2026-04-13", id="date-auto-detect-slashed", ), pytest.param( "13 April 2026", "date", "", "2026-04-13", id="date-auto-detect-month-name", ), pytest.param( "2026-04-13", "date", "%d.%m.%Y", "2026-04-13", id="date-format-mismatch-falls-back-to-auto-detect", ), pytest.param( "32.13.2026", "date", "%d.%m.%Y", "32.13.2026", id="date-invalid-with-format-returns-text", ), pytest.param( "2026-13-32", "date", "", "2026-13-32", id="date-invalid-auto-detect-returns-text", ), pytest.param( "not a date", "date", "", "not a date", id="date-not-a-date-returns-text", ), pytest.param("", "strip", "", "", id="empty-string"), pytest.param(" ", "strip", "", "", id="whitespace-only"), pytest.param( " hello ", "unknown", "", "hello", id="unknown-transform-strips", ), pytest.param( " -- Account #123. -- ", "strip_punctuation", "", "Account #123", id="strip-punctuation", ), ], ) def test_apply_transform( self, text: str, transform: str, date_format: str, expected: str, ) -> None: """ GIVEN: - Extracted text, a transform name and an optional date format - A zone reading English WHEN: - The transform is applied THEN: - The transformed text matches the expected result """ rules = _date_rules("eng", date_format) assert _apply_transform(text, transform, rules) == expected def test_unparseable_date_preserves_text(self, mocker: MockerFixture) -> None: """ GIVEN: - dateparser raises OverflowError for the input WHEN: - A date transform is applied THEN: - The original text is returned unchanged """ mocker.patch("documents.zone_ocr.dateparser.parse", side_effect=OverflowError) assert ( _apply_transform("unparsable", "date", _date_rules("eng")) == "unparsable" ) class TestZoneLanguage: """Tests for mapping the OCR language to a dateparser language.""" @pytest.mark.parametrize( ("ocr_language", "expected"), [ pytest.param("deu", "de", id="single-language"), pytest.param("deu+eng", "de", id="first-of-several-languages"), pytest.param("aze_Cyrl", "az", id="script-suffix-is-dropped"), pytest.param("xyz", None, id="unmapped-language-is-none"), ], ) def test_zone_language(self, ocr_language: str, expected: str | None) -> None: """ GIVEN: - A tesseract language code, possibly several or with a script WHEN: - The zone language is derived THEN: - The first language is mapped, or None when it is unknown """ assert _zone_language(ocr_language) == expected class TestParseDate: """Tests for the zone date parser.""" @pytest.mark.parametrize( ("text", "ocr_language", "expected"), [ pytest.param("12.05.2024", "deu", date(2024, 5, 12), id="german-day-first"), pytest.param( "05/12/2024", "eng", date(2024, 5, 12), id="english-month-first", ), pytest.param( "12. Mai 2024", "deu", date(2024, 5, 12), id="german-month-name", ), pytest.param( "13 April 2026", "eng", date(2026, 4, 13), id="english-month-name", ), pytest.param( "2026/04/13", "eng", date(2026, 4, 13), id="year-first-slashed", ), pytest.param( "01.01.2999", "deu", date(2999, 1, 1), id="future-date-allowed", ), ], ) def test_locale_decides_the_order( self, text: str, ocr_language: str, expected: date, ) -> None: """ GIVEN: - A date written in the convention of the zone's language WHEN: - The date is parsed THEN: - The locale's day and month order is used """ assert _parse_date(text, _date_rules(ocr_language)) == expected @pytest.mark.parametrize( ("date_order", "expected"), [ pytest.param("DMY", date(2024, 5, 12), id="day-first"), pytest.param("MDY", date(2024, 12, 5), id="month-first"), ], ) def test_unmapped_language_uses_the_configured_date_order( self, settings: SettingsWrapper, date_order: str, expected: date, ) -> None: """ GIVEN: - A zone whose language has no dateparser mapping - A configured PAPERLESS_DATE_ORDER WHEN: - An ambiguous numeric date is parsed THEN: - The configured date order decides the result """ settings.DATE_ORDER = date_order assert _parse_date("12.05.2024", _date_rules("xyz")) == expected @pytest.mark.parametrize( "text", [ pytest.param("now", id="now"), pytest.param("yesterday", id="yesterday"), pytest.param("today", id="today"), pytest.param("in 2 days", id="relative-offset"), pytest.param("Mon", id="weekday"), pytest.param("March", id="month-only"), pytest.param("2024", id="year-only"), pytest.param("5", id="day-only"), pytest.param("12", id="two-digits"), pytest.param("not a date", id="words"), ], ) @pytest.mark.parametrize( "ocr_language", [ pytest.param("eng", id="mapped-language"), pytest.param("xyz", id="unmapped-language"), ], ) def test_non_dates_are_rejected(self, text: str, ocr_language: str) -> None: """ GIVEN: - Text that is not a complete calendar date WHEN: - The date is parsed THEN: - No date is returned """ assert _parse_date(text, _date_rules(ocr_language)) is None @pytest.mark.parametrize( ("text", "ocr_language", "expected"), [ pytest.param( "Datum: 13.04.2026", "deu", date(2026, 4, 13), id="german-label-colon", ), pytest.param( "Rechnungsdatum 12.05.2024", "deu", date(2024, 5, 12), id="german-compound-label", ), pytest.param( "Invoice date 05/12/2024", "eng", date(2024, 5, 12), id="english-label-numeric", ), pytest.param( "Date: 13 April 2026", "eng", date(2026, 4, 13), id="english-label-month-name", ), pytest.param( "Rechnung 2024-001 vom 12.05.2024", "deu", date(2024, 5, 12), id="german-label-before-invoice-number", ), pytest.param( "No. 12 Datum 13.04.2026", "deu", date(2026, 4, 13), id="german-label-after-number", ), pytest.param( "Kd-Nr 12345 05.06.2024", "deu", date(2024, 6, 5), id="german-label-after-customer-number", ), pytest.param( "Lieferdatum: 12. Mai 2024", "deu", date(2024, 5, 12), id="german-label-month-name", ), pytest.param( "Total 5", "eng", None, id="single-number-after-label", ), pytest.param( "Total 5 items 2024", "eng", None, id="number-run-without-month", ), pytest.param( "Page 3 of 2024", "eng", None, id="page-number-without-month", ), pytest.param( "Invoice 12.5", "eng", None, id="partial-numeric-date", ), pytest.param( "Page 3 of 12 2024", "eng", None, id="page-of-number-run-with-year", ), pytest.param( "Total 3 10 2024", "eng", None, id="space-separated-numbers-with-year", ), pytest.param( "Order 12 3 2024", "eng", None, id="order-numbers-with-year", ), pytest.param( "Ref 1 2 2024", "eng", None, id="ref-numbers-with-year", ), pytest.param( "Invoice #2024 from 5 May", "eng", None, id="year-token-before-day-month", ), pytest.param( "Page 1 of 3 03 2024", "eng", None, id="page-numbers-ending-in-year", ), pytest.param( "Version 1.2.34", "deu", None, id="version-number-with-two-digit-tail", ), pytest.param( "v1.2.24", "eng", None, id="short-version-number", ), pytest.param( "Art. 12.3.45", "deu", None, id="article-number", ), pytest.param( "Ticket 1-2-34", "eng", None, id="dashed-ticket-number", ), ], ) def test_labelled_text_yields_the_date_after_the_label( self, text: str, ocr_language: str, expected: date | None, ) -> None: """ GIVEN: - OCR text with a label before a date, or a label-like number with no complete date WHEN: - The date is parsed THEN: - The date after the label is returned in the zone's day/month order - Text without a complete date after a label returns no date """ assert _parse_date(text, _date_rules(ocr_language)) == expected @pytest.mark.parametrize( ("text", "date_format", "expected"), [ pytest.param( "12.05.2024", "%m.%d.%Y", date(2024, 12, 5), id="format-overrides-locale-order", ), pytest.param( "12.05.2024", "%Y/%m/%d", date(2024, 5, 12), id="invalid-format-falls-back", ), ], ) def test_date_format_is_tried_first( self, text: str, date_format: str, expected: date, ) -> None: """ GIVEN: - A German zone with a date format WHEN: - A date is parsed THEN: - A matching format is used, otherwise parsing falls back to the locale """ assert _parse_date(text, _date_rules("deu", date_format)) == expected @pytest.mark.parametrize( ("text", "expected"), [ pytest.param("2024-05-06", date(2024, 5, 6), id="plain-iso"), pytest.param("Due 2024-05-06.", date(2024, 5, 6), id="iso-inside-text"), pytest.param("No. 12345-06-07", None, id="digit-before-is-not-iso"), pytest.param("2024-05-061", None, id="digit-after-is-not-iso"), pytest.param( "2024-13-45 and 2024-05-06", date(2024, 5, 6), id="invalid-match-is-skipped", ), pytest.param("2024-13-45", None, id="only-invalid-match"), ], ) def test_iso_dates_are_anchored(self, text: str, expected: date | None) -> None: """ GIVEN: - Text containing ISO-like dates, some invalid or part of longer numbers WHEN: - The date is parsed THEN: - Only whole, valid ISO dates are returned, trying every match """ assert _parse_date(text, _date_rules("eng")) == expected @pytest.mark.parametrize( "text", [ pytest.param("2026/04/13", id="slashed"), pytest.param("2026.04.13", id="dotted"), pytest.param("2026-04-13", id="dashed"), ], ) @pytest.mark.parametrize( ("ocr_language", "date_order"), [ pytest.param("xyz", "DMY", id="unmapped-language-day-first"), pytest.param("xyz", "MDY", id="unmapped-language-month-first"), pytest.param("deu", "DMY", id="mapped-language"), ], ) def test_year_first_dates_ignore_the_date_order( self, settings: SettingsWrapper, text: str, ocr_language: str, date_order: str, ) -> None: """ GIVEN: - A year-first date separated by a dash, slash or dot - A mapped or unmapped zone language and a configured date order WHEN: - The date is parsed THEN: - The date is read as year, month, day regardless of the order """ settings.DATE_ORDER = date_order assert _parse_date(text, _date_rules(ocr_language)) == date(2026, 4, 13) @pytest.mark.parametrize( "text", [ pytest.param("1.2026/04/133", id="digit-after"), pytest.param("12026/04/13", id="digit-before"), pytest.param("2026/13/45", id="invalid-month"), ], ) def test_year_first_dates_are_anchored_and_validated(self, text: str) -> None: """ GIVEN: - Year-first looking text that is part of longer digits or invalid WHEN: - The date is parsed THEN: - No date is returned """ assert _parse_date(text, _date_rules("xyz")) is None @pytest.mark.parametrize( "text", [ pytest.param("1899-12-31", id="iso"), pytest.param("31.12.1899", id="locale-numeric"), ], ) def test_years_before_1900_are_rejected(self, text: str) -> None: """ GIVEN: - A valid calendar date before the year 1900 WHEN: - The date is parsed THEN: - No date is returned """ assert _parse_date(text, _date_rules("deu")) is None @pytest.mark.django_db class TestPrepareFieldUpdate: """Tests for converting extracted values into field updates.""" def test_unknown_target_cannot_produce_a_field_update(self) -> None: """ GIVEN: - A zone whose target is not a known target WHEN: - A field update is prepared for it THEN: - No field update is produced """ assert _prepare_field_update(OcrTemplateZone(target="unknown"), "value") is None @pytest.mark.parametrize( ("ocr_language", "expected"), [ pytest.param("deu", 1234, id="german-reads-dot-as-thousands"), pytest.param("eng", None, id="english-reads-dot-as-fraction"), ], ) def test_int_field_uses_the_zone_language_for_numbers( self, ocr_language: str, expected: int | None, ) -> None: """ GIVEN: - A zone targeting an integer custom field with a given OCR language WHEN: - A field update is prepared for the text "1.234" THEN: - A German zone reads 1234, while an English zone reads a fraction and produces no update """ field = CustomFieldFactory(data_type=CustomField.FieldDataType.INT) zone = OcrTemplateZoneFactory( target=OcrTemplateZone.TargetType.CUSTOM_FIELD, custom_field=field, ocr_language=ocr_language, ) update = _prepare_field_update(zone, "1.234") assert (update.value if update else None) == expected @pytest.mark.parametrize( "target", [ pytest.param(OcrTemplateZone.TargetType.CREATED, id="created"), pytest.param(OcrTemplateZone.TargetType.CUSTOM_FIELD, id="date-field"), ], ) @pytest.mark.parametrize( ("ocr_language", "date_format", "text", "expected"), [ pytest.param("deu", "", "12.05.2024", date(2024, 5, 12), id="german-order"), pytest.param( "eng", "", "05/12/2024", date(2024, 5, 12), id="english-order", ), pytest.param( "deu", "", "12. Mai 2024", date(2024, 5, 12), id="german-month-name", ), pytest.param( "deu", "%m.%d.%Y", "12.05.2024", date(2024, 12, 5), id="zone-date-format", ), pytest.param( "deu", "%Y/%m/%d", "12.05.2024", date(2024, 5, 12), id="invalid-format-falls-back", ), pytest.param("eng", "", "2024", None, id="year-only-is-rejected"), ], ) def test_date_targets_parse_alike( self, target: str, ocr_language: str, date_format: str, text: str, expected: date | None, ) -> None: """ GIVEN: - A zone targeting the created date or a date custom field - An OCR language and optionally a date format WHEN: - A field update is prepared for non-ISO date text THEN: - Both targets read the text the same way, honoring the zone's language and date format """ field = ( CustomFieldFactory(data_type=CustomField.FieldDataType.DATE) if target == OcrTemplateZone.TargetType.CUSTOM_FIELD else None ) zone = OcrTemplateZoneFactory( target=target, custom_field=field, ocr_language=ocr_language, date_format=date_format, ) update = _prepare_field_update(zone, text) assert (update.value if update else None) == expected class TestNumberLocale: """Tests for deriving a number locale from the OCR language.""" @pytest.mark.parametrize( ("ocr_language", "expected"), [ pytest.param("deu", "de", id="single-language"), pytest.param("deu+eng", "de", id="first-of-several-languages"), pytest.param("aze_Cyrl", "az", id="script-suffix-is-dropped"), pytest.param("xyz", "en", id="unknown-language-falls-back-to-english"), ], ) def test_number_locale(self, ocr_language: str, expected: str) -> None: """ GIVEN: - A tesseract language code, possibly several or with a script WHEN: - The number locale is derived THEN: - The first language decides the locale, falling back to English """ assert _number_locale(ocr_language) == Locale.parse(expected) class TestParseMonetary: """Tests for the _parse_monetary function.""" @pytest.mark.parametrize( ("value", "locale", "expected"), [ pytest.param("12.50", EN, "12.50", id="plain-amount"), pytest.param("12", EN, "12.00", id="whole-number-gets-two-decimals"), pytest.param("12.500", EN, "12.50", id="trailing-zero-is-fine"), pytest.param("USD 12.50", EN, "USD12.50", id="code-before-amount"), pytest.param("12,50 EUR", DE, "EUR12.50", id="code-after-amount"), pytest.param("USD12.50", EN, "USD12.50", id="code-attached-to-amount"), pytest.param("$12.50", EN, "USD12.50", id="dollar-symbol-en"), pytest.param("$12.50", FR, "12.50", id="dollar-symbol-fr-has-no-code"), pytest.param( "\N{EURO SIGN}12.50", EN, "EUR12.50", id="euro-symbol", ), pytest.param( "\N{POUND SIGN}12.50", EN, "GBP12.50", id="pound-symbol", ), pytest.param( "USD $12.50", EN, "USD12.50", id="code-wins-over-symbol", ), pytest.param("INV 12.50", EN, "12.50", id="non-currency-word-ignored"), pytest.param("usd 12.50", EN, "12.50", id="lowercase-code-ignored"), pytest.param("(12.50)", EN, "-12.50", id="parentheses-negative"), pytest.param("($12.50)", EN, "USD-12.50", id="parentheses-with-symbol"), pytest.param("(12,50 EUR)", DE, "EUR-12.50", id="parentheses-with-code"), pytest.param("(0.00)", EN, "0.00", id="parentheses-zero-not-negative"), pytest.param("(USD) 12.50", EN, "USD12.50", id="empty-parentheses-ignored"), pytest.param("USD -12.50", EN, "USD-12.50", id="code-with-negative"), pytest.param("-12.50", EN, "-12.50", id="plain-negative"), pytest.param("1,234.56", EN, "1234.56", id="grouped-amount"), pytest.param("USD 12.50 EUR", EN, None, id="two-codes"), pytest.param("$12.50 \N{EURO SIGN}", EN, None, id="two-symbols"), pytest.param("(-12.50)", EN, None, id="parentheses-and-minus"), pytest.param("12.345", EN, None, id="too-many-decimals"), pytest.param("12345678901.12", EN, None, id="more-than-12-digits"), pytest.param( "USD 12345678901.12", EN, None, id="more-than-12-digits-with-code", ), pytest.param("100 (19% VAT)", EN, None, id="digits-outside-parentheses"), pytest.param("(12) (13)", EN, None, id="two-parenthesised-numbers"), pytest.param("-0.00", EN, "0.00", id="negative-zero-not-negative"), pytest.param("CHF 12", DE, "CHF12.00", id="code-in-german"), pytest.param( "1\N{NARROW NO-BREAK SPACE}234,56 \N{EURO SIGN}", FR, "EUR1234.56", id="french-space-grouping", ), pytest.param("USD", EN, None, id="code-without-amount"), pytest.param("garbage", EN, None, id="garbage"), pytest.param("", EN, None, id="empty"), ], ) def test_parse_monetary( self, value: str, locale: Locale, expected: str | None, ) -> None: """ GIVEN: - OCR text containing an amount, maybe with a currency or parentheses WHEN: - The text is parsed as a monetary value THEN: - The amount has two decimals and a leading currency code if one was found, or None is returned when the text is not a valid value """ assert _parse_monetary(value, locale) == expected class TestConvertValue: """Tests for the _convert_value function.""" @pytest.mark.parametrize( ("value", "data_type", "expected", "locale"), [ pytest.param( "Hello", CustomField.FieldDataType.STRING, "Hello", EN, id="string", ), pytest.param( "https://example.com", CustomField.FieldDataType.URL, "https://example.com", EN, id="url", ), pytest.param( "x" * 500, CustomField.FieldDataType.LONG_TEXT, "x" * 500, EN, id="long-text", ), pytest.param("42", CustomField.FieldDataType.INT, 42, EN, id="int-simple"), pytest.param( "INV-123", CustomField.FieldDataType.INT, 123, EN, id="int-with-noise", ), pytest.param( "-42", CustomField.FieldDataType.INT, -42, EN, id="int-negative", ), pytest.param( "\N{MINUS SIGN}5", CustomField.FieldDataType.INT, -5, EN, id="int-unicode-minus", ), pytest.param( "1,234", CustomField.FieldDataType.INT, 1234, EN, id="int-comma-thousands", ), pytest.param( "1.234.567", CustomField.FieldDataType.INT, 1234567, DE, id="int-dot-thousands", ), pytest.param( "12.00", CustomField.FieldDataType.INT, 12, EN, id="int-zero-fraction", ), pytest.param( "1.234,00", CustomField.FieldDataType.INT, 1234, DE, id="int-thousands-and-zero-fraction", ), pytest.param( "1234.00", CustomField.FieldDataType.INT, 1234, EN, id="int-ungrouped-with-zero-fraction", ), pytest.param( "1.234", CustomField.FieldDataType.INT, 1234, DE, id="int-dot-is-thousands-in-german", ), pytest.param( "1\N{NARROW NO-BREAK SPACE}234", CustomField.FieldDataType.INT, 1234, FR, id="int-narrow-space-thousands-in-french", ), pytest.param( "2026-04-13", CustomField.FieldDataType.DATE, "2026-04-13", EN, id="date-iso", ), pytest.param( "123.45", CustomField.FieldDataType.MONETARY, "123.45", EN, id="monetary-simple", ), pytest.param( "1.234,56", CustomField.FieldDataType.MONETARY, "1234.56", DE, id="monetary-european", ), pytest.param( "\u20ac1,234.56", CustomField.FieldDataType.MONETARY, "EUR1234.56", EN, id="monetary-with-currency-symbol", ), pytest.param( "12345678901234567890,12345678901234567890", CustomField.FieldDataType.MONETARY, None, DE, id="monetary-too-precise-returns-none", ), pytest.param( "\u20ac-12.50", CustomField.FieldDataType.MONETARY, "EUR-12.50", EN, id="monetary-sign-after-currency-symbol", ), pytest.param( "-\u20ac12.50", CustomField.FieldDataType.MONETARY, "EUR-12.50", EN, id="monetary-sign-before-currency-symbol", ), pytest.param( "EUR -12.50", CustomField.FieldDataType.MONETARY, "EUR-12.50", EN, id="monetary-sign-after-currency-code", ), pytest.param( "Total: -5", CustomField.FieldDataType.INT, -5, EN, id="int-sign-after-label", ), pytest.param( "\u20ac-12.50", CustomField.FieldDataType.FLOAT, -12.5, EN, id="float-sign-after-currency-symbol", ), pytest.param( "\u0661\u0662\u0663", CustomField.FieldDataType.INT, 123, Locale.parse("ar"), id="int-arabic-indic-digits", ), pytest.param( "\u0661\u066c\u0662\u0663\u0664\u066b\u0665", CustomField.FieldDataType.FLOAT, 1234.5, Locale.parse("ar"), id="float-arabic-separators", ), pytest.param( "\u06f1\u06f2\u06f3", CustomField.FieldDataType.INT, 123, Locale.parse("fa"), id="int-persian-digits", ), pytest.param( "\u0967\u0968\u0969", CustomField.FieldDataType.INT, 123, Locale.parse("hi"), id="int-devanagari-digits", ), pytest.param( "1 234,56", CustomField.FieldDataType.FLOAT, 1234.56, Locale.parse("ru"), id="float-plain-space-in-russian", ), pytest.param( "1\N{RIGHT SINGLE QUOTATION MARK}234.56", CustomField.FieldDataType.FLOAT, 1234.56, Locale.parse("de_CH"), id="float-apostrophe-group-in-swiss-german", ), ], ) def test_converts_to_exact_value( self, value: str, data_type: str, expected: object, locale: Locale, ) -> None: """ GIVEN: - An extracted string, a custom field data type and a number locale WHEN: - The value is converted THEN: - The converted value equals the expected value exactly """ assert _convert_value(value, data_type, locale, ENG_DATES) == expected def test_string_truncation(self) -> None: """ GIVEN: - An extracted string longer than the string field limit WHEN: - The value is converted to a STRING field value THEN: - The result is truncated to 128 characters """ result = _convert_value( "x" * 200, CustomField.FieldDataType.STRING, EN, ENG_DATES, ) assert isinstance(result, str) assert len(result) == 128 @pytest.mark.parametrize( ("value", "locale", "expected"), [ pytest.param("1234.56", EN, 1234.56, id="float-simple"), pytest.param("1.234,56", DE, 1234.56, id="float-european-format"), pytest.param("1,234.56", EN, 1234.56, id="float-us-format"), pytest.param("1234,56", DE, 1234.56, id="float-comma-only"), pytest.param("1,234", DE, 1.234, id="float-comma-is-decimal-in-german"), pytest.param("1,234", EN, 1234.0, id="float-comma-is-thousands-in-english"), pytest.param("1 234,56", FR, 1234.56, id="float-plain-space-in-french"), pytest.param( "1\N{NARROW NO-BREAK SPACE}234,56", FR, 1234.56, id="float-narrow-space-in-french", ), pytest.param( "\N{MINUS SIGN}1.5", EN, -1.5, id="float-unicode-minus", ), ], ) def test_float_conversion( self, value: str, locale: Locale, expected: float, ) -> None: """ GIVEN: - A numeric string in one of several decimal formats and a locale WHEN: - The value is converted to a FLOAT field value THEN: - The result is the expected float """ assert ( _convert_value(value, CustomField.FieldDataType.FLOAT, locale, ENG_DATES) == expected ) @pytest.mark.parametrize( ("value", "data_type", "locale"), [ pytest.param( "abc", CustomField.FieldDataType.INT, EN, id="int-empty-returns-none", ), pytest.param( "-", CustomField.FieldDataType.INT, EN, id="int-only-dash-returns-none", ), pytest.param( "12.50", CustomField.FieldDataType.INT, EN, id="int-fraction-returns-none", ), pytest.param( "1.5", CustomField.FieldDataType.INT, EN, id="int-short-fraction-returns-none", ), pytest.param( "0.500", CustomField.FieldDataType.INT, EN, id="int-leading-zero-fraction-returns-none", ), pytest.param( "1..2", CustomField.FieldDataType.INT, EN, id="int-empty-group-returns-none", ), pytest.param( "5-3", CustomField.FieldDataType.INT, EN, id="int-inner-dash-returns-none", ), pytest.param( "1,234.567", CustomField.FieldDataType.INT, EN, id="int-thousands-and-fraction-returns-none", ), pytest.param( "12.3,00", CustomField.FieldDataType.INT, EN, id="int-bad-grouping-before-fraction-returns-none", ), pytest.param( "1,2345.00", CustomField.FieldDataType.INT, EN, id="int-long-group-before-fraction-returns-none", ), pytest.param( "abc", CustomField.FieldDataType.FLOAT, EN, id="float-empty-returns-none", ), pytest.param( ",", CustomField.FieldDataType.FLOAT, EN, id="float-only-separator-returns-none", ), pytest.param( "not a date", CustomField.FieldDataType.DATE, EN, id="date-invalid-returns-none", ), pytest.param( "2026-13-32", CustomField.FieldDataType.DATE, EN, id="date-invalid-values-returns-none", ), pytest.param( "CHF", CustomField.FieldDataType.MONETARY, EN, id="monetary-empty-returns-none", ), pytest.param( "maybe", CustomField.FieldDataType.BOOL, EN, id="bool-unknown-returns-none", ), pytest.param( "test", CustomField.FieldDataType.DOCUMENTLINK, EN, id="unsupported-type-documentlink-returns-none", ), pytest.param( "test", CustomField.FieldDataType.SELECT, EN, id="unsupported-type-select-returns-none", ), pytest.param( "", CustomField.FieldDataType.STRING, EN, id="empty-string-returns-none", ), pytest.param( "9" * 400, CustomField.FieldDataType.FLOAT, EN, id="float-overflow-is-not-stored-as-infinity", ), pytest.param( "1.234", CustomField.FieldDataType.INT, EN, id="int-german-thousands-is-a-fraction-in-english", ), pytest.param( "12.50", CustomField.FieldDataType.INT, DE, id="int-english-decimal-is-malformed-in-german", ), pytest.param( "1.234.567", CustomField.FieldDataType.FLOAT, EN, id="float-multiple-dots-in-english", ), ], ) def test_returns_none(self, value: str, data_type: str, locale: Locale) -> None: """ GIVEN: - A value that cannot be converted to the given data type in a locale WHEN: - The value is converted THEN: - None is returned """ assert _convert_value(value, data_type, locale, ENG_DATES) is None @pytest.mark.parametrize( "data_type", [ pytest.param(CustomField.FieldDataType.FLOAT, id="float"), pytest.param(CustomField.FieldDataType.MONETARY, id="monetary"), ], ) @pytest.mark.parametrize( "value", [ pytest.param("1.2.3", id="multiple-dots"), pytest.param("--12", id="double-leading-dash"), pytest.param("12-34", id="embedded-dash"), pytest.param("", id="empty"), pytest.param(".", id="only-dot"), pytest.param("-", id="only-dash"), pytest.param("not numeric", id="not-numeric"), ], ) def test_numeric_types_reject_malformed_numbers( self, data_type: str, value: str, ) -> None: """ GIVEN: - A malformed numeric string and a FLOAT or MONETARY data type WHEN: - The value is converted THEN: - None is returned """ assert _convert_value(value, data_type, EN, ENG_DATES) is None @pytest.mark.parametrize( "value", [ pytest.param("true", id="true"), pytest.param("True", id="true-titlecase"), pytest.param("yes", id="yes"), pytest.param("1", id="one"), pytest.param("ja", id="ja"), pytest.param("x", id="x-lowercase"), pytest.param("X", id="x-uppercase"), ], ) def test_bool_true(self, value: str) -> None: """ GIVEN: - A string that represents a true boolean WHEN: - The value is converted to a BOOL field value THEN: - The result is True """ assert ( _convert_value(value, CustomField.FieldDataType.BOOL, EN, ENG_DATES) is True ) @pytest.mark.parametrize( "value", [ pytest.param("false", id="false"), pytest.param("False", id="false-titlecase"), pytest.param("no", id="no"), pytest.param("0", id="zero"), pytest.param("nein", id="nein"), ], ) def test_bool_false(self, value: str) -> None: """ GIVEN: - A string that represents a false boolean WHEN: - The value is converted to a BOOL field value THEN: - The result is False """ assert ( _convert_value(value, CustomField.FieldDataType.BOOL, EN, ENG_DATES) is False ) class TestRenderPages: """Tests for _render_pages.""" @pytest.fixture def pdf(self, tmp_path: Path) -> Path: pdf = tmp_path / "document.pdf" pdf.write_bytes(b"%PDF-1.4") return pdf @pytest.fixture def mock_rasterize(self, mocker: MockerFixture) -> MagicMock: def render(in_path: Path, out_path: Path, **kwargs: Any) -> None: out_path.write_bytes(b"rendered") return mocker.patch( "documents.zone_ocr.rasterize_pdf_page_to_png", side_effect=render, ) def test_renders_requested_pages_in_order_and_skips_out_of_range( self, pdf: Path, tmp_path: Path, mock_rasterize: MagicMock, ) -> None: """ GIVEN: - A 3 page PDF and requested pages including out of range indexes WHEN: - The pages are rendered THEN: - Only the in range pages are rendered, in order, as 1-based pages at 300 DPI with a 60 second timeout """ pages = _render_pages(pdf, {-1, 0, 2, 3}, tmp_path, 3) assert list(pages) == [0, 2] assert mock_rasterize.call_count == 2 for page, call in zip([1, 3], mock_rasterize.call_args_list, strict=True): assert call.args[0] == pdf assert call.kwargs == {"dpi": 300, "page": page, "timeout": 60} def test_missing_render_output_is_not_a_page( self, pdf: Path, tmp_path: Path, mock_rasterize: MagicMock, ) -> None: """ GIVEN: - A renderer that produces no output file WHEN: - A page is rendered THEN: - No pages are returned """ mock_rasterize.side_effect = None assert _render_pages(pdf, {0}, tmp_path, None) == {} @pytest.mark.parametrize( "error", [ pytest.param(ParseError("pdftoppm timed out after 60s"), id="timeout"), pytest.param( ParseError("pdftoppm failed at ['pdftoppm']"), id="render-failure", ), ], ) def test_render_failure_does_not_prevent_other_pages( self, pdf: Path, tmp_path: Path, mock_rasterize: MagicMock, error: Exception, ) -> None: """ GIVEN: - A renderer that fails for the first page only WHEN: - Two pages are rendered THEN: - The failing page is skipped and the other page is still rendered """ def render(in_path: Path, out_path: Path, *, page: int, **kwargs: Any) -> None: if page == 1: raise error out_path.write_bytes(b"rendered") mock_rasterize.side_effect = render assert list(_render_pages(pdf, {0, 1}, tmp_path, 2)) == [1] def test_image_is_used_directly_only_for_first_page( self, tmp_path: Path, mock_rasterize: MagicMock, ) -> None: """ GIVEN: - A single image file WHEN: - The first page, then the second page, is requested THEN: - The image itself is returned for the first page only - The renderer is never run """ image_path = tmp_path / "image.png" with Image.new("RGB", (10, 10)) as image: image.save(image_path) assert _render_pages(image_path, {0}, tmp_path, 1) == {0: image_path} assert _render_pages(image_path, {1}, tmp_path, 1) == {} mock_rasterize.assert_not_called() class TestDetectMime: """Tests for _detect_mime.""" @pytest.mark.parametrize( ("filename", "expected"), [ pytest.param("test.pdf", "application/pdf", id="pdf-extension"), pytest.param("test.png", "image/png", id="png-extension"), pytest.param("test.jpg", "image/jpeg", id="jpg-extension"), pytest.param("test.xyz", None, id="unknown-extension"), pytest.param("test.webp", "image/webp", id="webp-extension"), ], ) def test_detect_mime(self, filename: str, expected: str | None) -> None: """ GIVEN: - A path with a given file extension WHEN: - The MIME type is detected THEN: - The expected MIME type, or None for an unknown extension, is returned """ assert _detect_mime(Path(filename)) == expected class TestResolveDocPath: """Tests for _resolve_doc_path.""" @pytest.fixture def document(self, mocker: MockerFixture) -> MagicMock: doc = mocker.MagicMock() doc.has_archive_version = False doc.source_path = Path("/nonexistent/source.pdf") return doc def test_returns_none_when_no_files_exist(self, document: MagicMock) -> None: """ GIVEN: - A document with no archive version and a missing source file WHEN: - The document path is resolved without an original file THEN: - None is returned """ assert _resolve_doc_path(document, None) is None def test_prefers_archive_and_falls_back_to_source( self, mocker: MockerFixture, tmp_path: Path, ) -> None: """ GIVEN: - A document with existing archive and source files WHEN: - The path is resolved, then the archive file is removed and it is resolved again THEN: - The archive path is returned first, then the source path """ source = tmp_path / "source.pdf" archive = tmp_path / "archive.pdf" source.touch() archive.touch() document = mocker.MagicMock( has_archive_version=True, archive_path=archive, source_path=source, ) assert _resolve_doc_path(document, None) == archive archive.unlink() assert _resolve_doc_path(document, None) == source def test_returns_original_file_as_fallback( self, document: MagicMock, tmp_path: Path, ) -> None: """ GIVEN: - A document whose own files are missing and an existing original file WHEN: - The document path is resolved with the original file THEN: - The original file path is returned """ original = tmp_path / "original.pdf" original.touch() assert _resolve_doc_path(document, original) == original def test_returns_none_for_missing_original_file( self, document: MagicMock, tmp_path: Path, ) -> None: """ GIVEN: - A document whose own files are missing and an original file path that does not exist WHEN: - The document path is resolved with the original file THEN: - None is returned """ assert _resolve_doc_path(document, tmp_path / "missing.pdf") is None @pytest.mark.django_db @pytest.mark.usefixtures("paperless_dirs") class TestRunZoneExtraction: """Tests for the full extraction pipeline.""" @pytest.fixture def document(self) -> Document: return DocumentFactory( title="Test", content="test", mime_type="image/png", ) @pytest.fixture def source(self, paperless_dirs: PaperlessDirs) -> Path: source = paperless_dirs.scratch_dir / "document.png" with Image.new("RGB", (200, 200), "white") as image: image.save(source) return source @pytest.fixture(autouse=True) def ocr_reads_text(self, mock_zone_ocr_run: MagicMock) -> None: mock_zone_ocr_run.return_value = subprocess.CompletedProcess( args=[], returncode=0, stdout="extracted", stderr="", ) @pytest.fixture def template_with_zone(self) -> Callable[..., OcrTemplate]: """Build a template with one default zone, matching "test" by default.""" def _make(name: str, **kwargs: Any) -> OcrTemplate: kwargs.setdefault("match", "test") template = OcrTemplateFactory(name=name, **kwargs) OcrTemplateZoneFactory(template=template) return template return _make @pytest.mark.parametrize( ("template_name", "template_kwargs"), [ pytest.param("Other sender", {"match": "Contoso"}, id="non-matching"), pytest.param("Disabled", {"enabled": False}, id="disabled"), ], ) def test_skips_ineligible_template( self, mock_zone_ocr_run: MagicMock, template_with_zone: Callable[..., OcrTemplate], document: Document, source: Path, template_name: str, template_kwargs: dict[str, Any], ) -> None: """ GIVEN: - A template whose match text is not in the document content, or a disabled template that would otherwise match WHEN: - Zone extraction runs against a real document file THEN: - No results are returned, no text is read and the title is unchanged """ template_with_zone(template_name, **template_kwargs) results = run_zone_extraction(document, source) assert results == [] mock_zone_ocr_run.assert_not_called() document.refresh_from_db() assert document.title == "Test" def test_skips_template_with_no_zones( self, mock_zone_ocr_run: MagicMock, document: Document, source: Path, ) -> None: """ GIVEN: - An enabled matching template with no zones WHEN: - Zone extraction runs THEN: - No results are returned, no text is read and the title is unchanged """ OcrTemplateFactory( name="Empty", match="test", ) results = run_zone_extraction(document, source) assert results == [] mock_zone_ocr_run.assert_not_called() document.refresh_from_db() assert document.title == "Test" def test_handles_process_exception_gracefully( self, mock_zone_ocr_run: MagicMock, template_with_zone: Callable[..., OcrTemplate], document: Document, source: Path, ) -> None: """ GIVEN: - Two matching templates, where reading the first one's text raises an unexpected error WHEN: - Zone extraction runs THEN: - The error does not propagate, the failing template's zone is reported as template_failed and the other template still runs """ mock_zone_ocr_run.side_effect = [ RuntimeError("test error"), subprocess.CompletedProcess( args=[], returncode=0, stdout="extracted", stderr="", ), ] template_with_zone("First", enabled=True) template_with_zone("Second", enabled=True) results = run_zone_extraction(document, source) assert Counter((r.value, r.reason) for r in results) == Counter( [(None, "template_failed"), ("extracted", None)], ) assert {r.template for r in results} == {"First", "Second"} def test_handles_none_original_file( self, mock_zone_ocr_run: MagicMock, template_with_zone: Callable[..., OcrTemplate], document: Document, ) -> None: """ GIVEN: - A matching template, a document with no accessible file and no original file WHEN: - Zone extraction runs THEN: - Each zone is reported as missing_file and no text is read """ template = template_with_zone("Matching", enabled=True) zone = template.zones.get() results = run_zone_extraction(document, None) assert [(r.template, r.zone, r.reason) for r in results] == [ ("Matching", zone.name, "missing_file"), ] mock_zone_ocr_run.assert_not_called() @pytest.mark.parametrize( "template_names", [ pytest.param(["Active"], id="one-template"), pytest.param( ["Template 0", "Template 1", "Template 2"], id="three-templates", ), ], ) def test_processes_every_enabled_template( self, template_with_zone: Callable[..., OcrTemplate], document: Document, source: Path, template_names: list[str], ) -> None: """ GIVEN: - One or more enabled matching templates, each with a zone WHEN: - Zone extraction runs THEN: - Each template reports exactly one successful result """ for name in template_names: template_with_zone(name, enabled=True) results = run_zone_extraction(document, source) assert sorted((r.template, r.reason) for r in results) == [ (name, None) for name in sorted(template_names) ] @pytest.mark.django_db @pytest.mark.usefixtures("paperless_dirs") class TestRealZoneOcr: """End to end zone extraction using the real pdftoppm and tesseract.""" @pytest.fixture def template(self) -> OcrTemplate: template = make_template("Whole page", match="test", source_height=100) OcrTemplateZoneFactory( template=template, name="Title", target=OcrTemplateZone.TargetType.TITLE, x=0, y=0, width=100, height=100, page=1, ) return template @pytest.mark.usefixtures("template") def test_pdf_page_is_rendered_and_read( self, simple_digital_pdf_file: Path, ) -> None: """ GIVEN: - A one page PDF containing text - A template with a title zone covering the whole page WHEN: - Zone extraction runs without any mocks THEN: - The page text is read and written to the document title """ document = DocumentFactory( title="Original", content="This is a test document.", mime_type="application/pdf", page_count=1, ) shutil.copy(simple_digital_pdf_file, document.source_path) results = run_zone_extraction(document, None) assert [result.reason for result in results] == [None] document.refresh_from_db() assert "This is a test document" in document.title @pytest.mark.usefixtures("template") def test_image_is_read_without_rendering(self) -> None: """ GIVEN: - An image containing text as the document's file - A template with a title zone covering the whole image WHEN: - Zone extraction runs without any mocks THEN: - The image text is read and written to the document title """ document = DocumentFactory( title="Original", content="This is a test document.", mime_type="image/png", page_count=1, ) shutil.copy( Path(__file__).parent / "samples" / "simple.png", document.source_path, ) results = run_zone_extraction(document, None) assert [result.reason for result in results] == [None] document.refresh_from_db() assert "This is a test document" in document.title