diff --git a/docs/usage.md b/docs/usage.md index bb6eea22ae..39764e5b2b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -872,10 +872,18 @@ is left empty, the values are joined with spaces in zone order. ### When templates run -When a document is added, every enabled template that matches its content runs automatically. A template is -only used for a document if the document's owner can view the template, so users' templates apply to their own -documents, and unowned templates apply to everyone's. Templates run in name order, so if two templates fill the -same field, the later one wins. +When a document is added, every enabled template that matches its content runs automatically, reading the +zones from the archived file, or from the original PDF or image when there is no archived file. A template is +only used if the document's owner can view it, so users' templates apply to their own documents, and unowned +templates apply to everyone's. The owner here is the one set by the upload, the mail rule or a consumption +workflow. A document without an owner at that point, for example from the consume folder or a mail rule +without an owner, considers every enabled template. An owner assigned later by a "Document Added" workflow +does not change which templates ran. Templates run in name order, so if two templates fill the same field, the +later one wins. + +Zone OCR keeps values supplied when the document is consumed: an uploaded title, ASN, created date or custom +field value, a mail rule's title, a barcode ASN, and values assigned by a consumption workflow. Zones fill the +remaining fields before "Document Added" workflows run, so those workflows can use or overwrite the values. Turn off **Enabled** to pause a template without deleting it. Disabled templates never run automatically, but can still be run manually. @@ -897,7 +905,8 @@ the latest version of each document and are processed in the background. !!! warning - Zone OCR overwrites the fields it fills. A zone that finds no value leaves its field unchanged. + Running zone OCR manually or in bulk overwrites the fields it fills. A zone that finds no value leaves its + field unchanged. Managing OCR templates requires the OCR template permissions. Running zone OCR on documents requires permission to change them, and choosing a template requires permission to view it. diff --git a/src/documents/apps.py b/src/documents/apps.py index 4a3aa9be9e..50b6ac2ce4 100644 --- a/src/documents/apps.py +++ b/src/documents/apps.py @@ -15,7 +15,6 @@ class DocumentsConfig(AppConfig): from documents.signals.handlers import add_to_index from documents.signals.handlers import run_workflows_added from documents.signals.handlers import run_workflows_updated - from documents.signals.handlers import run_zone_ocr_extraction from documents.signals.handlers import send_websocket_document_updated from documents.signals.handlers import set_correspondent from documents.signals.handlers import set_document_type @@ -28,7 +27,6 @@ class DocumentsConfig(AppConfig): document_consumption_finished.connect(set_tags) document_consumption_finished.connect(set_storage_path) document_consumption_finished.connect(run_workflows_added) - document_consumption_finished.connect(run_zone_ocr_extraction) document_consumption_finished.connect(add_to_index) document_consumption_finished.connect(add_or_update_document_in_llm_index) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index c060368b4d..3a5cd8de14 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -34,6 +34,7 @@ from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentBarcode from documents.models import DocumentType +from documents.models import OcrTemplateZone from documents.models import StoragePath from documents.models import Tag from documents.models import WorkflowTrigger @@ -56,6 +57,13 @@ from documents.utils import compute_checksum from documents.utils import copy_basic_file_stats from documents.utils import copy_file_with_basic_stats from documents.utils import run_subprocess +from documents.versioning import get_root_document +from documents.zone_ocr import FieldKey +from documents.zone_ocr import TemplatePlan +from documents.zone_ocr import apply_zone_plans +from documents.zone_ocr import field_key +from documents.zone_ocr import get_templates_for_content +from documents.zone_ocr import plan_zone_extraction from paperless.config import BarcodeConfig from paperless.config import OcrConfig from paperless.config import RemoteOCRConfig @@ -588,6 +596,16 @@ class ConsumerPlugin( exception=e, ) + # Zone OCR renders and reads pages, so it runs here, before the + # transaction opens. Its values are written once the document + # is stored. + zone_plans = self._plan_zone_ocr( + text=text, + archive_path=archive_path, + mime_type=mime_type, + page_count=page_count, + ) + # Prepare the document classifier. # TODO: I don't really like to do this here, but this way we avoid @@ -672,6 +690,11 @@ class ConsumerPlugin( mime_type=mime_type, ) + # Before the consumption finished signal, so DOCUMENT_ADDED + # workflows and the other handlers of that signal see the + # zone values. + self._apply_zone_ocr(document, zone_plans) + # If we get here, it was successful. Proceed with post-consume # hooks. If they fail, nothing will get changed. @@ -986,6 +1009,107 @@ class ConsumerPlugin( self._store_barcodes(document) + def _plan_zone_ocr( + self, + *, + text: str | None, + archive_path: Path | None, + mime_type: str, + page_count: int | None, + ) -> list[TemplatePlan]: + """ + Select the OCR templates matching the parsed text and compute their zone + values, without writing anything. Zones are read from the archive, the + file the template editor draws on, or from the working copy of a PDF or + image without one. Any error is logged and yields no zone values. + """ + try: + if self.input_doc.root_document_id is not None: + root_doc = Document.objects.select_related("owner").get( + pk=self.input_doc.root_document_id, + ) + owner = get_root_document(root_doc).owner + elif self.metadata.owner_id is not None: + owner = User.objects.get(pk=self.metadata.owner_id) + else: + owner = None + templates = get_templates_for_content( + text or "", + owner, + title=self.filename, + ) + if not templates: + return [] + if archive_path is not None and Path(archive_path).is_file(): + doc_path: Path | None = Path(archive_path) + elif mime_type == "application/pdf" or mime_type.startswith("image/"): + doc_path = self.working_copy + else: + doc_path = None + return plan_zone_extraction( + doc_path, + page_count, + templates, + label=self.filename, + ) + except Exception: + self.log.exception( + f"Zone OCR failed for {self.filename}, continuing without zone values", + ) + return [] + + def _apply_zone_ocr(self, document: Document, plans: list[TemplatePlan]) -> None: + """ + Write the planned zone values to the stored document, or to the root of a + new version. For a new document, a value supplied explicitly in the + metadata overrides is kept rather than replaced by a zone. + """ + if not plans: + return + try: + protected = ( + frozenset() + if self.input_doc.root_document_id is not None + else self._explicit_zone_targets() + ) + root_doc = get_root_document(document) + results = apply_zone_plans( + root_doc, + plans, + label=f"document {root_doc.pk}", + protected=protected, + ) + except Exception: + self.log.exception( + f"Zone OCR could not write values for {self.filename}", + ) + return + for result in results: + self.log.debug( + f"Zone OCR: template '{result.template}' zone '{result.zone}' " + f"for {result.custom_field}: " + f"{result.reason if result.reason is not None else 'written'}", + ) + + def _explicit_zone_targets(self) -> frozenset[FieldKey]: + """The zone targets given a value by the metadata overrides. + + A custom field listed without a value is not a supplied value. + """ + targets = set() + if self.metadata.title is not None: + targets.add(field_key(OcrTemplateZone.TargetType.TITLE, None)) + if self.metadata.asn is not None: + targets.add(field_key(OcrTemplateZone.TargetType.ASN, None)) + if self.metadata.created is not None: + targets.add(field_key(OcrTemplateZone.TargetType.CREATED, None)) + for field_id, value in (self.metadata.custom_fields or {}).items(): + if value is not None: + targets.add( + field_key(OcrTemplateZone.TargetType.CUSTOM_FIELD, int(field_id)), + ) + return frozenset(targets) + def _read_version_barcodes(self, mime_type: str, work_dir: Path) -> None: barcode_settings = BarcodeConfig() if not barcode_settings.barcode_store_values: diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 70619be8a9..a1d8f659ce 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -1478,24 +1478,6 @@ def close_connection_pool_on_worker_init(**kwargs) -> None: conn.close_pool() -def run_zone_ocr_extraction( - sender: Any, - document: Document, - original_file: Path | None = None, - **kwargs: Any, -) -> None: - """Apply matching templates without failing consumption if extraction fails.""" - try: - from documents.zone_ocr import run_zone_extraction - - run_zone_extraction(document, Path(original_file) if original_file else None) - except Exception: - logger.exception( - "Zone OCR extraction failed for document %s", - document.pk, - ) - - @worker_process_shutdown.connect def close_connection_pool_on_worker_shutdown(**kwargs) -> None: # pragma: no cover """ diff --git a/src/documents/tests/test_api_ocr_templates.py b/src/documents/tests/test_api_ocr_templates.py index c41b6dead6..c5a6e0e2cd 100644 --- a/src/documents/tests/test_api_ocr_templates.py +++ b/src/documents/tests/test_api_ocr_templates.py @@ -29,6 +29,8 @@ from documents.tasks import bulk_update_documents from documents.zone_ocr import ZoneExtraction from documents.zone_ocr import ZoneOcrReason from documents.zone_ocr import ZoneOcrResult +from documents.zone_ocr import apply_zone_plans +from documents.zone_ocr import plan_zone_extraction from paperless_testing.factories import CustomFieldFactory from paperless_testing.factories import DocumentFactory from paperless_testing.factories import OcrTemplateFactory @@ -323,6 +325,14 @@ class TestZoneOcrIndexing: ) with django_capture_on_commit_callbacks(execute=True): if entry_point == "ingestion": + # The consumer writes the planned values, then sends the signal. + plans = plan_zone_extraction( + file_document.source_path, + file_document.page_count, + [template], + label="ingestion", + ) + apply_zone_plans(document, plans, label="ingestion") document_consumption_finished.send(sender=None, document=file_document) elif entry_point == "single": response = admin_client.post( diff --git a/src/documents/tests/test_consumer_zone_ocr.py b/src/documents/tests/test_consumer_zone_ocr.py new file mode 100644 index 0000000000..3b435181b9 --- /dev/null +++ b/src/documents/tests/test_consumer_zone_ocr.py @@ -0,0 +1,676 @@ +"""Tests for zone OCR during consumption.""" + +from __future__ import annotations + +import json +import logging +import shutil +import tempfile +from datetime import date +from pathlib import Path +from typing import TYPE_CHECKING +from typing import Any +from typing import Self + +import pytest +from django.conf import settings +from django.db import transaction +from PIL import Image + +from documents import consumer as consumer_module +from documents.consumer import ConsumerPlugin +from documents.data_models import ConsumableDocument +from documents.data_models import DocumentMetadataOverrides +from documents.data_models import DocumentSource +from documents.models import CustomField +from documents.models import CustomFieldInstance +from documents.models import Document +from documents.models import OcrTemplate +from documents.models import OcrTemplateZone +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger +from paperless_testing.factories import CustomFieldFactory +from paperless_testing.factories import DocumentFactory +from paperless_testing.factories import OcrTemplateFactory +from paperless_testing.factories import OcrTemplateZoneFactory +from paperless_testing.factories import TagFactory +from paperless_testing.factories import UserFactory +from paperless_testing.fakes.progress import FakeProgressManager +from paperless_testing.samples import SIMPLE_DIGITAL_PDF + +if TYPE_CHECKING: + from collections.abc import Callable + from unittest.mock import MagicMock + + from pytest_django.fixtures import SettingsWrapper + from pytest_mock import MockerFixture + + from documents.zone_ocr import TemplatePlan + from paperless_testing.dirs import PaperlessDirs + +CONTENT = "Northwind Power & Light statement" + +# The text each zone's fake OCR returns, by zone name. +ZONE_TEXT = { + "Title": "Zone title", + "ASN": "ASN 777", + "Created": "2024-03-05", + "Code": "INV42", +} + +MIME_TYPES = { + ".pdf": "application/pdf", + ".png": "image/png", + ".eml": "message/rfc822", +} + + +class ZoneTestParser: + """A parser returning CONTENT, with a PDF archive when one is requested.""" + + name = "zone-test-parser" + version = "0.1" + author = "test" + url = "test" + + @classmethod + def supported_mime_types(cls) -> dict[str, str]: + return {mime: suffix for suffix, mime in MIME_TYPES.items()} + + @classmethod + def score(cls, mime_type: str, filename: str, path: Path | None = None) -> int: + return 0 + + @property + def can_produce_archive(self) -> bool: + return True + + @property + def requires_pdf_rendition(self) -> bool: + return False + + def __enter__(self) -> Self: + self._tmpdir = Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) + self._archive: Path | None = None + self._thumb = self._tmpdir / "thumb.webp" + self._thumb.write_bytes(b"thumbnail") + return self + + def __exit__(self, *args: object) -> None: + shutil.rmtree(self._tmpdir, ignore_errors=True) + + def configure(self, context: object) -> None: + """The test parser ignores its context.""" + + def parse( + self, + document_path: Path, + mime_type: str, + *, + produce_archive: bool = True, + ) -> None: + if produce_archive: + self._archive = self._tmpdir / "archive.pdf" + shutil.copy(SIMPLE_DIGITAL_PDF, self._archive) + + def get_text(self) -> str: + return CONTENT + + def get_date(self) -> None: + return None + + def get_archive_path(self) -> Path | None: + return self._archive + + def get_thumbnail(self, document_path: Path, mime_type: str) -> Path: + return self._thumb + + def get_page_count(self, document_path: Path, mime_type: str) -> int: + return 1 + + def extract_metadata(self, document_path: Path, mime_type: str) -> list[Any]: + return [] + + +@pytest.fixture +def page_image(tmp_path: Path) -> Path: + page = tmp_path / "page.png" + with Image.new("RGB", (200, 200), "white") as image: + image.save(page) + return page + + +@pytest.fixture +def mock_render(mocker: MockerFixture, page_image: Path) -> MagicMock: + """Fake page rendering, recording whether a transaction was open.""" + render = mocker.patch("documents.zone_ocr._render_pages") + render.in_transaction = [] + + def record(*args: Any, **kwargs: Any) -> dict[int, Path]: + render.in_transaction.append(transaction.get_connection().in_atomic_block) + return {0: page_image} + + render.side_effect = record + return render + + +@pytest.fixture +def mock_ocr(mocker: MockerFixture) -> MagicMock: + """Fake zone OCR returning ZONE_TEXT, recording whether a transaction was open.""" + ocr = mocker.patch("documents.zone_ocr._ocr_text") + ocr.in_transaction = [] + + def read(cropped: Image.Image, zone: OcrTemplateZone, tmp_dir: Path) -> str: + ocr.in_transaction.append(transaction.get_connection().in_atomic_block) + return ZONE_TEXT[zone.name] + + ocr.side_effect = read + return ocr + + +@pytest.fixture +def consume( + paperless_dirs: PaperlessDirs, + mocker: MockerFixture, +) -> Callable[..., Document]: + """Consume a file of the given suffix through the ConsumerPlugin.""" + registry = mocker.patch("documents.consumer.get_parser_registry") + registry.return_value.get_parser_for_file.return_value = ZoneTestParser + mocker.patch( + "documents.consumer.magic.from_file", + side_effect=lambda path, *, mime=False: MIME_TYPES[Path(path).suffix], + ) + + def run( + suffix: str = ".pdf", + overrides: DocumentMetadataOverrides | None = None, + root_document_id: int | None = None, + ) -> Document: + source = paperless_dirs.scratch_dir / f"upload{suffix}" + if suffix == ".pdf": + shutil.copy(SIMPLE_DIGITAL_PDF, source) + else: + source.write_bytes(f"{suffix} {source.name}".encode()) + plugin = ConsumerPlugin( + ConsumableDocument( + DocumentSource.ApiUpload, + original_file=source, + root_document_id=root_document_id, + ), + overrides or DocumentMetadataOverrides(), + FakeProgressManager(source.name, None), + paperless_dirs.scratch_dir, + "task-id", + ) + plugin.setup() + try: + result = plugin.run() + finally: + plugin.cleanup() + return Document.objects.get(pk=result["document_id"]) + + return run + + +def make_template( + name: str = "Invoice", + targets: tuple[str, ...] = ("Title",), + custom_field: CustomField | None = None, + **kwargs: Any, +) -> OcrTemplate: + """A template matching CONTENT with one zone per named target.""" + template = OcrTemplateFactory( + name=name, + match="Northwind", + source_width=100, + source_height=100, + **kwargs, + ) + zone_targets = { + "Title": OcrTemplateZone.TargetType.TITLE, + "ASN": OcrTemplateZone.TargetType.ASN, + "Created": OcrTemplateZone.TargetType.CREATED, + "Code": OcrTemplateZone.TargetType.CUSTOM_FIELD, + } + for order, zone_name in enumerate(targets): + OcrTemplateZoneFactory( + template=template, + name=zone_name, + target=zone_targets[zone_name], + custom_field=custom_field if zone_name == "Code" else None, + y=order * 20, + width=50, + height=20, + order=order, + ) + return template + + +def custom_field_value(document: Document, field: CustomField) -> Any: + instance = CustomFieldInstance.objects.filter( + document=document, + field=field, + ).first() + return instance.value if instance is not None else None + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.usefixtures("mock_ocr") +class TestConsumerZoneOcrTransaction: + """Zone OCR runs outside the consumer's database transaction.""" + + def test_ocr_runs_before_the_transaction_opens( + self, + consume: Callable[..., Document], + mock_render: MagicMock, + mock_ocr: MagicMock, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + WHEN: + - The document is consumed + THEN: + - Pages are rendered and zones are read with no transaction open + - The zone value is written to the document + """ + make_template() + + document = consume() + + assert mock_render.in_transaction == [False] + assert mock_ocr.in_transaction == [False] + assert document.title == "Zone title" + + +@pytest.mark.django_db +@pytest.mark.usefixtures("mock_ocr") +class TestConsumerZoneOcr: + """Zone values are computed before and written during consumption.""" + + @pytest.fixture + def code_field(self) -> CustomField: + return CustomFieldFactory( + name="Invoice code", + data_type=CustomField.FieldDataType.STRING, + ) + + @pytest.mark.parametrize( + ("suffix", "generation", "expected_file"), + [ + pytest.param(".pdf", "always", "archive.pdf", id="pdf-archive"), + pytest.param(".eml", "always", "archive.pdf", id="mail-archive"), + pytest.param(".pdf", "never", "upload.pdf", id="pdf-working-copy"), + pytest.param(".png", "never", "upload.png", id="image-working-copy"), + ], + ) + def test_zones_are_read_from_the_archive_or_working_copy( + self, + consume: Callable[..., Document], + mock_render: MagicMock, + settings: SettingsWrapper, + suffix: str, + generation: str, + expected_file: str, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + WHEN: + - A document is consumed with or without an archive + THEN: + - Zones are read from the archive when there is one, otherwise + from the working copy of the PDF or image + """ + settings.ARCHIVE_FILE_GENERATION = generation + make_template() + + document = consume(suffix) + + mock_render.assert_called_once() + assert mock_render.call_args.args[0].name == expected_file + assert document.title == "Zone title" + + def test_mail_without_archive_is_not_rendered( + self, + consume: Callable[..., Document], + mock_render: MagicMock, + settings: SettingsWrapper, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + WHEN: + - An email is consumed without an archive + THEN: + - No page is rendered from the email file + - The document is consumed without zone values + """ + settings.ARCHIVE_FILE_GENERATION = "never" + make_template() + + document = consume(".eml") + + mock_render.assert_not_called() + assert document.title == "upload" + + @pytest.mark.usefixtures("mock_render") + def test_values_are_written_before_document_added_workflows( + self, + consume: Callable[..., Document], + code_field: CustomField, + ) -> None: + """ + GIVEN: + - A template writing a custom field + - A DOCUMENT_ADDED workflow filtering on that custom field's value + and assigning a tag + WHEN: + - A matching document is consumed + THEN: + - The workflow sees the zone value and assigns its tag + """ + make_template(targets=("Code",), custom_field=code_field) + tag = TagFactory(name="Zone matched") + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_custom_field_query=json.dumps( + ["AND", [[code_field.pk, "exact", "INV42"]]], + ), + ) + action = WorkflowAction.objects.create() + action.assign_tags.add(tag) + workflow = Workflow.objects.create(name="On zone value", order=0) + workflow.triggers.add(trigger) + workflow.actions.add(action) + + document = consume() + + assert custom_field_value(document, code_field) == "INV42" + assert list(document.tags.all()) == [tag] + + @pytest.mark.parametrize( + "overridden", + [ + pytest.param("title", id="title"), + pytest.param("asn", id="asn"), + pytest.param("created", id="created"), + pytest.param("custom_field", id="custom-field"), + ], + ) + @pytest.mark.usefixtures("mock_render") + def test_explicit_values_are_not_overwritten( + self, + consume: Callable[..., Document], + code_field: CustomField, + overridden: str, + ) -> None: + """ + GIVEN: + - A template with title, ASN, created and custom field zones + - Metadata overrides giving one of those fields a value + WHEN: + - A matching document is consumed + THEN: + - The supplied value is kept + - Every other zone value is written + """ + make_template( + targets=("Title", "ASN", "Created", "Code"), + custom_field=code_field, + ) + explicit = { + "title": "Upload title", + "asn": 123, + "created": date(2020, 1, 2), + "custom_field": "Given", + } + overrides = { + "title": DocumentMetadataOverrides(title="Upload title"), + "asn": DocumentMetadataOverrides(asn=123), + "created": DocumentMetadataOverrides(created=date(2020, 1, 2)), + "custom_field": DocumentMetadataOverrides( + custom_fields={code_field.pk: "Given"}, + ), + }[overridden] + expected = { + "title": "Zone title", + "asn": 777, + "created": date(2024, 3, 5), + "custom_field": "INV42", + } | {overridden: explicit[overridden]} + + document = consume(overrides=overrides) + + assert { + "title": document.title, + "asn": document.archive_serial_number, + "created": document.created, + "custom_field": custom_field_value(document, code_field), + } == expected + + @pytest.mark.usefixtures("mock_render") + def test_custom_field_listed_without_a_value_is_filled( + self, + consume: Callable[..., Document], + code_field: CustomField, + ) -> None: + """ + GIVEN: + - A template with a custom field zone + - Metadata overrides listing that custom field without a value + WHEN: + - A matching document is consumed + THEN: + - The zone value is written to the custom field + """ + make_template(targets=("Code",), custom_field=code_field) + + document = consume( + overrides=DocumentMetadataOverrides(custom_fields={code_field.pk: None}), + ) + + assert custom_field_value(document, code_field) == "INV42" + + @pytest.mark.usefixtures("mock_render") + def test_custom_field_deleted_before_writing_is_skipped( + self, + consume: Callable[..., Document], + code_field: CustomField, + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + ) -> None: + """ + GIVEN: + - A template with title and custom field zones + - The custom field deleted after the zone values are planned + WHEN: + - A matching document is consumed + THEN: + - The document is stored with the title + - The custom field zone is reported as unassigned + """ + make_template(targets=("Title", "Code"), custom_field=code_field) + plan = consumer_module.plan_zone_extraction + + def plan_then_delete(*args: Any, **kwargs: Any) -> list[TemplatePlan]: + plans = plan(*args, **kwargs) + CustomField.objects.filter(pk=code_field.pk).delete() + return plans + + mocker.patch( + "documents.consumer.plan_zone_extraction", + side_effect=plan_then_delete, + ) + caplog.set_level(logging.DEBUG, logger="paperless.consumer") + + document = consume() + + assert document.title == "Zone title" + assert not CustomFieldInstance.objects.filter(document=document).exists() + assert "zone 'Code' for Invoice code: unassigned" in caplog.text + + @pytest.mark.parametrize( + ("with_owner", "expected_title", "expected_code"), + [ + pytest.param(True, "Zone title", None, id="owner-sees-own-templates"), + pytest.param(False, "Zone title", "INV42", id="no-owner-sees-all"), + ], + ) + @pytest.mark.usefixtures("mock_render") + def test_templates_are_selected_by_the_metadata_owner( + self, + consume: Callable[..., Document], + code_field: CustomField, + with_owner: bool, # noqa: FBT001 + expected_title: str, + expected_code: str | None, + ) -> None: + """ + GIVEN: + - A template owned by the upload's owner writing the title + - A private template of another user writing a custom field + WHEN: + - A matching document is consumed with or without an owner + THEN: + - With an owner, only the templates that owner can view apply + - Without an owner, every template applies + """ + owner = UserFactory() + make_template("Mine", owner=owner) + make_template( + "Private", + targets=("Code",), + custom_field=code_field, + owner=UserFactory(), + ) + + document = consume( + overrides=DocumentMetadataOverrides( + owner_id=owner.pk if with_owner else None, + ), + ) + + assert document.title == expected_title + assert custom_field_value(document, code_field) == expected_code + + @pytest.mark.usefixtures("mock_render") + def test_duplicate_asn_is_checked_when_writing( + self, + consume: Callable[..., Document], + ) -> None: + """ + GIVEN: + - A template with title and ASN zones + - Another document already using the zone's ASN + WHEN: + - A matching document is consumed + THEN: + - The ASN is not written + - The title from the same template is written + """ + DocumentFactory(archive_serial_number=777) + make_template(targets=("Title", "ASN")) + + document = consume() + + assert document.archive_serial_number is None + assert document.title == "Zone title" + + @pytest.mark.parametrize( + "failing", + [ + pytest.param( + "documents.consumer.get_templates_for_content", + id="template-selection", + ), + pytest.param("documents.consumer.plan_zone_extraction", id="extraction"), + ], + ) + @pytest.mark.usefixtures("mock_render") + def test_read_phase_failure_does_not_fail_consumption( + self, + consume: Callable[..., Document], + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + failing: str, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + - Template selection or zone extraction raising an error + WHEN: + - The document is consumed + THEN: + - The document is stored without zone values + - The failure is logged + """ + make_template() + mocker.patch(failing, side_effect=RuntimeError("zone OCR broke")) + + document = consume() + + assert document.title == "upload" + assert "Zone OCR failed for upload.pdf" in caplog.text + + @pytest.mark.usefixtures("mock_render") + def test_write_failure_does_not_fail_consumption( + self, + consume: Callable[..., Document], + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + - Writing the template's values raising an error + WHEN: + - The document is consumed + THEN: + - The document is stored without the template's values + - The failed template is logged + """ + make_template() + mocker.patch( + "documents.zone_ocr._apply_field_updates", + side_effect=RuntimeError("write failed"), + ) + + document = consume() + + assert document.title == "upload" + assert "error processing template 'Invoice'" in caplog.text + + @pytest.mark.usefixtures("mock_render") + def test_version_values_are_written_to_the_root( + self, + consume: Callable[..., Document], + code_field: CustomField, + ) -> None: + """ + GIVEN: + - A root document whose owner can view a template writing the title + and a custom field + - A private template of another user + WHEN: + - A new version of the root document is consumed + THEN: + - The zone values are written to the root document + - The version keeps the root's previous title and gets no custom field + - The other user's template does not apply + """ + owner = UserFactory() + root = DocumentFactory(title="Root title", owner=owner, content=CONTENT) + make_template(targets=("Title", "Code"), custom_field=code_field, owner=owner) + make_template("Private", targets=("ASN",), owner=UserFactory()) + + version = consume(root_document_id=root.pk) + + root.refresh_from_db() + assert version.root_document_id == root.pk + assert root.title == "Zone title" + assert custom_field_value(root, code_field) == "INV42" + assert root.archive_serial_number is None + assert version.title == "Root title" + assert custom_field_value(version, code_field) is None diff --git a/src/documents/tests/test_task_signals.py b/src/documents/tests/test_task_signals.py index 2a44c9788c..2b8b84904c 100644 --- a/src/documents/tests/test_task_signals.py +++ b/src/documents/tests/test_task_signals.py @@ -13,81 +13,14 @@ from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentSource from documents.models import PaperlessTask from documents.signals.handlers import before_task_publish_handler -from documents.signals.handlers import run_zone_ocr_extraction from documents.signals.handlers import task_failure_handler from documents.signals.handlers import task_postrun_handler from documents.signals.handlers import task_prerun_handler from documents.signals.handlers import task_revoked_handler -from paperless_testing.factories import DocumentFactory from paperless_testing.factories import PaperlessTaskFactory from paperless_testing.factories import UserFactory -@pytest.mark.django_db -class TestRunZoneOcrExtraction: - """Zone OCR extraction triggered by the consumption finished signal.""" - - @pytest.mark.parametrize( - "original_file", - [ - pytest.param(None, id="no-original-file"), - pytest.param("/tmp/original.pdf", id="str-path"), - pytest.param(Path("/tmp/original.pdf"), id="path-object"), - ], - ) - def test_consumption_runs_zone_extraction( - self, - original_file: str | Path | None, - mocker: pytest_mock.MockerFixture, - ) -> None: - """ - GIVEN: - - A consumed document and an original file given as None, a str, or a Path - WHEN: - - The consumption finished handler runs zone OCR extraction - THEN: - - Zone extraction is called once with the document and the - original file normalised to a Path (or None) - """ - extract = mocker.patch("documents.zone_ocr.run_zone_extraction") - document = DocumentFactory() - - run_zone_ocr_extraction( - sender=None, - document=document, - original_file=original_file, - ) - - extract.assert_called_once_with( - document, - Path(original_file) if original_file else None, - ) - - def test_zone_extraction_failure_does_not_fail_consumption( - self, - mocker: pytest_mock.MockerFixture, - caplog: pytest.LogCaptureFixture, - ) -> None: - """ - GIVEN: - - A consumed document whose zone extraction raises an error - WHEN: - - The consumption finished handler runs zone OCR extraction - THEN: - - The error is not propagated - - The failure is logged with the document id - """ - mocker.patch( - "documents.zone_ocr.run_zone_extraction", - side_effect=RuntimeError("OCR failed"), - ) - document = DocumentFactory() - - run_zone_ocr_extraction(sender=None, document=document) - - assert f"Zone OCR extraction failed for document {document.pk}" in caplog.text - - @pytest.fixture def consume_input_doc(): doc = mock.MagicMock(spec=ConsumableDocument) diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index c558be8897..8e87ac2dbc 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -52,6 +52,7 @@ from documents.zone_ocr import _render_pages from documents.zone_ocr import _resolve_doc_path from documents.zone_ocr import _zone_language from documents.zone_ocr import extract_zone_preview +from documents.zone_ocr import get_templates_for_content from documents.zone_ocr import get_templates_for_document from documents.zone_ocr import run_zone_extraction from paperless_testing.factories import CustomFieldFactory @@ -1707,6 +1708,39 @@ class TestTemplateVisibility: get_templates_for_document(document, requester=other), ) == Counter([unowned, private]) + @pytest.mark.parametrize( + ("has_owner", "expected_names"), + [ + pytest.param(True, {"Mine", "Unowned"}, id="owner"), + pytest.param(False, {"Mine", "Unowned", "Private"}, id="no-owner"), + ], + ) + @pytest.mark.usefixtures("mine", "unowned", "private") + def test_content_selection_follows_the_owner_rule( + self, + owner: User, + has_owner: bool, # noqa: FBT001 + expected_names: set[str], + ) -> None: + """ + GIVEN: + - Templates owned by two users and an unowned template, all + matching "Northwind", and one matching only other text + WHEN: + - Templates are selected for unsaved text, with or without an owner + THEN: + - Only matching templates apply + - An owner sees owned and unowned templates, no owner sees all + """ + make_visible_template("Other sender", match="Contoso") + + templates = get_templates_for_content( + "Northwind Power & Light statement", + owner if has_owner else None, + ) + + assert {template.name for template in templates} == expected_names + @pytest.mark.usefixtures("private") def test_versions_use_the_root_documents_owner( self, diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 2717df68b6..c7a830e340 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -57,6 +57,7 @@ if TYPE_CHECKING: from typing import Any from django.contrib.auth.models import User + from django.db.models import QuerySet logger = logging.getLogger("paperless.zone_ocr") @@ -146,6 +147,25 @@ class ZoneOcrResult: reason: ZoneOcrReason | None = None +@dataclass(kw_only=True, slots=True) +class PlannedUpdate: + """A field update and the accepted zone results it was combined from.""" + + key: FieldKey + update: FieldUpdate + results: list[ZoneOcrResult] + + +@dataclass(kw_only=True, slots=True) +class TemplatePlan: + """A template's zone results and the field updates they produce, unwritten.""" + + template: OcrTemplate + zones: list[OcrTemplateZone] + results: list[ZoneOcrResult] + updates: list[PlannedUpdate] + + # Result labels for zones that target a built-in field. BUILTIN_TARGET_LABELS: dict[OcrTemplateZone.TargetType, str] = { OcrTemplateZone.TargetType.TITLE: "Title", @@ -187,20 +207,7 @@ def get_templates_for_document( view, so a user editing someone else's document never runs or sees that owner's private templates. """ - owner = get_root_document(document).owner - templates = OcrTemplate.objects.filter(enabled=True) - if owner is not None: - templates = templates.filter( - id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"), - ) - if requester is not None: - templates = templates.filter( - id__in=permitted_object_ids(requester, OcrTemplate, "view_ocrtemplate"), - ) - templates = templates.order_by("name", "pk").prefetch_related( - "zones", - "zones__custom_field", - ) + templates = _visible_templates(get_root_document(document).owner, requester) match_document = document if document.root_document_id is None and templates: # The original file can be selected even when newer versions exist. @@ -215,6 +222,43 @@ def get_templates_for_document( return [template for template in templates if matches(template, match_document)] +def get_templates_for_content( + content: str, + owner: User | None, + *, + title: str = "", +) -> list[OcrTemplate]: + """ + The enabled templates that match the given text and that the owner can view, + for a document that is not saved yet. A missing owner considers every + template, as get_templates_for_document does. The title only labels the + matching log messages. + """ + templates = _visible_templates(owner, None) + match_document = Document(title=title, content=content) + return [template for template in templates if matches(template, match_document)] + + +def _visible_templates( + owner: User | None, + requester: User | None, +) -> QuerySet[OcrTemplate]: + """The enabled templates both the owner and the requester can view, if given.""" + templates = OcrTemplate.objects.filter(enabled=True) + if owner is not None: + templates = templates.filter( + id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"), + ) + if requester is not None: + templates = templates.filter( + id__in=permitted_object_ids(requester, OcrTemplate, "view_ocrtemplate"), + ) + return templates.order_by("name", "pk").prefetch_related( + "zones", + "zones__custom_field", + ) + + def run_zone_extraction( document: Document, original_file: Path | None = None, @@ -232,46 +276,14 @@ def run_zone_extraction( if not templates: return [] - doc_path = _resolve_doc_path(document, original_file) - if doc_path is None: - logger.warning( - "Zone OCR: no accessible file for document %d", - document.pk, - ) - return [ - _zone_result(template, zone, reason=ZoneOcrReason.MISSING_FILE) - for template in templates - for zone in template.zones.all() - ] - - template_zones = [(template, list(template.zones.all())) for template in templates] - pages = { - _resolve_page_idx(zone.page, document.page_count) - for _, zones in template_zones - for zone in zones - } - if not pages: - return [] - results = [] - with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: - tmp_path = Path(tmp_dir) - page_images = _render_pages(doc_path, pages, tmp_path, document.page_count) - for template, zones in template_zones: - try: - results.extend( - _process_template(document, page_images, template, zones, tmp_path), - ) - except Exception: - # One failed template must not prevent the remaining templates from running. - logger.exception( - "Zone OCR: error processing template '%s' for document %d", - template.name, - document.pk, - ) - results.extend( - _zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED) - for zone in zones - ) + label = f"document {document.pk}" + plans = plan_zone_extraction( + _resolve_doc_path(document, original_file), + document.page_count, + templates, + label=label, + ) + results = apply_zone_plans(get_root_document(document), plans, label=label) if zone_results_written(results): Document.objects.filter(pk=get_root_document(document).pk).update( modified=timezone.now(), @@ -279,6 +291,124 @@ def run_zone_extraction( return results +def plan_zone_extraction( + doc_path: Path | None, + page_count: int | None, + templates: list[OcrTemplate], + *, + label: str, +) -> list[TemplatePlan]: + """ + Render the zones' pages from doc_path and compute each template's field + updates without writing anything. A template that fails yields only + TEMPLATE_FAILED results, and a missing file yields MISSING_FILE results for + every zone. The label names the document in log messages. + """ + template_zones = [(template, list(template.zones.all())) for template in templates] + if doc_path is None: + logger.warning("Zone OCR: no accessible file for %s", label) + return [ + _failed_plan(template, zones, ZoneOcrReason.MISSING_FILE) + for template, zones in template_zones + ] + + pages = { + _resolve_page_idx(zone.page, page_count) + for _, zones in template_zones + for zone in zones + } + if not pages: + return [] + plans = [] + with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: + tmp_path = Path(tmp_dir) + page_images = _render_pages(doc_path, pages, tmp_path, page_count) + for template, zones in template_zones: + try: + plans.append( + _plan_template(page_count, page_images, template, zones, tmp_path), + ) + except Exception: + # One failed template must not prevent the remaining templates + # from running. + logger.exception( + "Zone OCR: error processing template '%s' for %s", + template.name, + label, + ) + plans.append( + _failed_plan(template, zones, ZoneOcrReason.TEMPLATE_FAILED), + ) + return plans + + +def apply_zone_plans( + root_document: Document, + plans: list[TemplatePlan], + *, + label: str, + protected: frozenset[FieldKey] = frozenset(), +) -> list[ZoneOcrResult]: + """ + Write each template's planned updates to the root document, each template + in its own atomic block, and return the zone results. Fields whose key is in + protected are not written, and the results of every zone targeting them are + left out. A template whose writes fail gets TEMPLATE_FAILED results and the + others still apply. + """ + results: list[ZoneOcrResult] = [] + for plan in plans: + for planned in plan.updates: + if planned.key in protected: + logger.info( + "Zone OCR: template '%s' kept the supplied %s of %s", + plan.template.name, + planned.update.field_name, + label, + ) + updates = [planned for planned in plan.updates if planned.key not in protected] + try: + if updates: + _apply_template_updates(root_document, updates) + plan_results = plan.results + except Exception: + logger.exception( + "Zone OCR: error processing template '%s' for %s", + plan.template.name, + label, + ) + plan_results = _failed_plan( + plan.template, + plan.zones, + ZoneOcrReason.TEMPLATE_FAILED, + ).results + # Each zone has exactly one result, in zone order. + results.extend( + result + for zone, result in zip(plan.zones, plan_results, strict=True) + if _zone_key(zone) not in protected + ) + return results + + +def _zone_key(zone: OcrTemplateZone) -> FieldKey: + """The combine_formats key of the field a zone fills.""" + return field_key(OcrTemplateZone.TargetType(zone.target), zone.custom_field_id) + + +def _failed_plan( + template: OcrTemplate, + zones: list[OcrTemplateZone], + reason: ZoneOcrReason, +) -> TemplatePlan: + return TemplatePlan( + template=template, + zones=zones, + results=[_zone_result(template, zone, reason=reason) for zone in zones], + updates=[], + ) + + def zone_results_written(results: list[ZoneOcrResult]) -> bool: """Whether at least one zone's value was written to the root document.""" return any(result.reason is None for result in results) @@ -288,7 +418,7 @@ def _resolve_doc_path( document: Document, original_file: Path | None, ) -> Path | None: - """Prefer the archive, then the source, then the ingestion signal's file.""" + """Prefer the archive, then the source, then the given file.""" candidates: list[Path | None] = [] if document.has_archive_version: candidates.append(document.archive_path) @@ -316,7 +446,21 @@ def _process_template( zones: list[OcrTemplateZone], tmp_path: Path, ) -> list[ZoneOcrResult]: - """Extract and combine zone values before applying a template's updates.""" + """Extract and combine zone values, then apply a template's updates.""" + plan = _plan_template(document.page_count, page_images, template, zones, tmp_path) + if plan.updates: + _apply_template_updates(get_root_document(document), plan.updates) + return plan.results + + +def _plan_template( + page_count: int | None, + page_images: dict[int, Path], + template: OcrTemplate, + zones: list[OcrTemplateZone], + tmp_path: Path, +) -> TemplatePlan: + """Extract and combine zone values into a template's updates, unwritten.""" # Each zone's result and its accepted text, by target field. grouped: dict[ FieldKey, @@ -324,7 +468,7 @@ def _process_template( ] = {} results = [] for zone in zones: - page_idx = _resolve_page_idx(zone.page, document.page_count) + page_idx = _resolve_page_idx(zone.page, page_count) extracted = None reason: ZoneOcrReason | None = ZoneOcrReason.MISSING_PAGE if page_idx in page_images: @@ -351,16 +495,12 @@ def _process_template( reason = ZoneOcrReason.REGEX_MISMATCH result = _zone_result(template, zone, reason=reason) results.append(result) - key = field_key( - OcrTemplateZone.TargetType(zone.target), - zone.custom_field_id, - ) + key = _zone_key(zone) grouped.setdefault(key, []).append( (zone, result, extracted if reason is None else None), ) updates = [] - root_document = get_root_document(document) combine_formats: dict[FieldKey, str] = template.combine_formats or {} for key, field_zones in grouped.items(): accepted = [result for _, result, _ in field_zones if result.reason is None] @@ -381,15 +521,8 @@ def _process_template( ) target_zone = field_zones[0][0] update = _prepare_field_update(target_zone, value) if value else None - if update is not None and _asn_in_use(update, root_document): - # Saving would violate the unique ASN and roll back every other - # write of this template, so only this field is dropped. - for result in accepted: - result.value = None - result.reason = ZoneOcrReason.DUPLICATE_ASN - continue if update is not None: - updates.append(update) + updates.append(PlannedUpdate(key=key, update=update, results=accepted)) for result in accepted: result.value = update.value if update else None if update is None: @@ -398,8 +531,57 @@ def _process_template( if key == UNASSIGNED_FIELD_KEY else ZoneOcrReason.INVALID_VALUE ) - _apply_field_updates(root_document, updates) - return results + return TemplatePlan( + template=template, + zones=zones, + results=results, + updates=updates, + ) + + +def _apply_template_updates( + root_document: Document, + updates: list[PlannedUpdate], +) -> None: + """Write one template's updates to the root document in one atomic block. + + The ASN duplicate check and the custom field existence check run here, at + write time, so they see the database as it is when the values are saved. + """ + field_ids = { + planned.update.custom_field.pk + for planned in updates + if planned.update.custom_field is not None + } + with transaction.atomic(): + existing_field_ids = ( + set( + CustomField.objects.filter(pk__in=field_ids).values_list( + "pk", + flat=True, + ), + ) + if field_ids + else set() + ) + writable = [] + for planned in updates: + custom_field = planned.update.custom_field + if custom_field is not None and custom_field.pk not in existing_field_ids: + # The field was deleted after the values were planned. + for result in planned.results: + result.value = None + result.reason = ZoneOcrReason.UNASSIGNED + continue + if _asn_in_use(planned.update, root_document): + # Saving would violate the unique ASN and roll back every other + # write of this template, so only this field is dropped. + for result in planned.results: + result.value = None + result.reason = ZoneOcrReason.DUPLICATE_ASN + continue + writable.append(planned.update) + _apply_field_updates(root_document, writable) def _asn_in_use(update: FieldUpdate, root_document: Document) -> bool: