From a6b81b2d648a0f26b0f60b7f0956db2aa55704a9 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:09:41 -0800 Subject: [PATCH] Fix: read zones before the consume transaction and write them before workflows Zone OCR ran inside the consume transaction, read the working copy instead of the archive, and ran after the document added workflows. Zones are now read from the archive before the transaction and written just before those workflows, keeping values supplied at consume time. --- docs/usage.md | 19 +- src/documents/apps.py | 2 - src/documents/consumer.py | 124 ++++ src/documents/signals/handlers.py | 18 - src/documents/tests/test_api_ocr_templates.py | 10 + src/documents/tests/test_consumer_zone_ocr.py | 676 ++++++++++++++++++ src/documents/tests/test_task_signals.py | 67 -- src/documents/tests/test_zone_ocr.py | 34 + src/documents/zone_ocr.py | 326 +++++++-- 9 files changed, 1112 insertions(+), 164 deletions(-) create mode 100644 src/documents/tests/test_consumer_zone_ocr.py diff --git a/docs/usage.md b/docs/usage.md index bb6eea22ae..39764e5b2b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -872,10 +872,18 @@ is left empty, the values are joined with spaces in zone order. ### When templates run -When a document is added, every enabled template that matches its content runs automatically. A template is -only used for a document if the document's owner can view the template, so users' templates apply to their own -documents, and unowned templates apply to everyone's. Templates run in name order, so if two templates fill the -same field, the later one wins. +When a document is added, every enabled template that matches its content runs automatically, reading the +zones from the archived file, or from the original PDF or image when there is no archived file. A template is +only used if the document's owner can view it, so users' templates apply to their own documents, and unowned +templates apply to everyone's. The owner here is the one set by the upload, the mail rule or a consumption +workflow. A document without an owner at that point, for example from the consume folder or a mail rule +without an owner, considers every enabled template. An owner assigned later by a "Document Added" workflow +does not change which templates ran. Templates run in name order, so if two templates fill the same field, the +later one wins. + +Zone OCR keeps values supplied when the document is consumed: an uploaded title, ASN, created date or custom +field value, a mail rule's title, a barcode ASN, and values assigned by a consumption workflow. Zones fill the +remaining fields before "Document Added" workflows run, so those workflows can use or overwrite the values. Turn off **Enabled** to pause a template without deleting it. Disabled templates never run automatically, but can still be run manually. @@ -897,7 +905,8 @@ the latest version of each document and are processed in the background. !!! warning - Zone OCR overwrites the fields it fills. A zone that finds no value leaves its field unchanged. + Running zone OCR manually or in bulk overwrites the fields it fills. A zone that finds no value leaves its + field unchanged. Managing OCR templates requires the OCR template permissions. Running zone OCR on documents requires permission to change them, and choosing a template requires permission to view it. diff --git a/src/documents/apps.py b/src/documents/apps.py index 4a3aa9be9e..50b6ac2ce4 100644 --- a/src/documents/apps.py +++ b/src/documents/apps.py @@ -15,7 +15,6 @@ class DocumentsConfig(AppConfig): from documents.signals.handlers import add_to_index from documents.signals.handlers import run_workflows_added from documents.signals.handlers import run_workflows_updated - from documents.signals.handlers import run_zone_ocr_extraction from documents.signals.handlers import send_websocket_document_updated from documents.signals.handlers import set_correspondent from documents.signals.handlers import set_document_type @@ -28,7 +27,6 @@ class DocumentsConfig(AppConfig): document_consumption_finished.connect(set_tags) document_consumption_finished.connect(set_storage_path) document_consumption_finished.connect(run_workflows_added) - document_consumption_finished.connect(run_zone_ocr_extraction) document_consumption_finished.connect(add_to_index) document_consumption_finished.connect(add_or_update_document_in_llm_index) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index c060368b4d..3a5cd8de14 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -34,6 +34,7 @@ from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentBarcode from documents.models import DocumentType +from documents.models import OcrTemplateZone from documents.models import StoragePath from documents.models import Tag from documents.models import WorkflowTrigger @@ -56,6 +57,13 @@ from documents.utils import compute_checksum from documents.utils import copy_basic_file_stats from documents.utils import copy_file_with_basic_stats from documents.utils import run_subprocess +from documents.versioning import get_root_document +from documents.zone_ocr import FieldKey +from documents.zone_ocr import TemplatePlan +from documents.zone_ocr import apply_zone_plans +from documents.zone_ocr import field_key +from documents.zone_ocr import get_templates_for_content +from documents.zone_ocr import plan_zone_extraction from paperless.config import BarcodeConfig from paperless.config import OcrConfig from paperless.config import RemoteOCRConfig @@ -588,6 +596,16 @@ class ConsumerPlugin( exception=e, ) + # Zone OCR renders and reads pages, so it runs here, before the + # transaction opens. Its values are written once the document + # is stored. + zone_plans = self._plan_zone_ocr( + text=text, + archive_path=archive_path, + mime_type=mime_type, + page_count=page_count, + ) + # Prepare the document classifier. # TODO: I don't really like to do this here, but this way we avoid @@ -672,6 +690,11 @@ class ConsumerPlugin( mime_type=mime_type, ) + # Before the consumption finished signal, so DOCUMENT_ADDED + # workflows and the other handlers of that signal see the + # zone values. + self._apply_zone_ocr(document, zone_plans) + # If we get here, it was successful. Proceed with post-consume # hooks. If they fail, nothing will get changed. @@ -986,6 +1009,107 @@ class ConsumerPlugin( self._store_barcodes(document) + def _plan_zone_ocr( + self, + *, + text: str | None, + archive_path: Path | None, + mime_type: str, + page_count: int | None, + ) -> list[TemplatePlan]: + """ + Select the OCR templates matching the parsed text and compute their zone + values, without writing anything. Zones are read from the archive, the + file the template editor draws on, or from the working copy of a PDF or + image without one. Any error is logged and yields no zone values. + """ + try: + if self.input_doc.root_document_id is not None: + root_doc = Document.objects.select_related("owner").get( + pk=self.input_doc.root_document_id, + ) + owner = get_root_document(root_doc).owner + elif self.metadata.owner_id is not None: + owner = User.objects.get(pk=self.metadata.owner_id) + else: + owner = None + templates = get_templates_for_content( + text or "", + owner, + title=self.filename, + ) + if not templates: + return [] + if archive_path is not None and Path(archive_path).is_file(): + doc_path: Path | None = Path(archive_path) + elif mime_type == "application/pdf" or mime_type.startswith("image/"): + doc_path = self.working_copy + else: + doc_path = None + return plan_zone_extraction( + doc_path, + page_count, + templates, + label=self.filename, + ) + except Exception: + self.log.exception( + f"Zone OCR failed for {self.filename}, continuing without zone values", + ) + return [] + + def _apply_zone_ocr(self, document: Document, plans: list[TemplatePlan]) -> None: + """ + Write the planned zone values to the stored document, or to the root of a + new version. For a new document, a value supplied explicitly in the + metadata overrides is kept rather than replaced by a zone. + """ + if not plans: + return + try: + protected = ( + frozenset() + if self.input_doc.root_document_id is not None + else self._explicit_zone_targets() + ) + root_doc = get_root_document(document) + results = apply_zone_plans( + root_doc, + plans, + label=f"document {root_doc.pk}", + protected=protected, + ) + except Exception: + self.log.exception( + f"Zone OCR could not write values for {self.filename}", + ) + return + for result in results: + self.log.debug( + f"Zone OCR: template '{result.template}' zone '{result.zone}' " + f"for {result.custom_field}: " + f"{result.reason if result.reason is not None else 'written'}", + ) + + def _explicit_zone_targets(self) -> frozenset[FieldKey]: + """The zone targets given a value by the metadata overrides. + + A custom field listed without a value is not a supplied value. + """ + targets = set() + if self.metadata.title is not None: + targets.add(field_key(OcrTemplateZone.TargetType.TITLE, None)) + if self.metadata.asn is not None: + targets.add(field_key(OcrTemplateZone.TargetType.ASN, None)) + if self.metadata.created is not None: + targets.add(field_key(OcrTemplateZone.TargetType.CREATED, None)) + for field_id, value in (self.metadata.custom_fields or {}).items(): + if value is not None: + targets.add( + field_key(OcrTemplateZone.TargetType.CUSTOM_FIELD, int(field_id)), + ) + return frozenset(targets) + def _read_version_barcodes(self, mime_type: str, work_dir: Path) -> None: barcode_settings = BarcodeConfig() if not barcode_settings.barcode_store_values: diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 70619be8a9..a1d8f659ce 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -1478,24 +1478,6 @@ def close_connection_pool_on_worker_init(**kwargs) -> None: conn.close_pool() -def run_zone_ocr_extraction( - sender: Any, - document: Document, - original_file: Path | None = None, - **kwargs: Any, -) -> None: - """Apply matching templates without failing consumption if extraction fails.""" - try: - from documents.zone_ocr import run_zone_extraction - - run_zone_extraction(document, Path(original_file) if original_file else None) - except Exception: - logger.exception( - "Zone OCR extraction failed for document %s", - document.pk, - ) - - @worker_process_shutdown.connect def close_connection_pool_on_worker_shutdown(**kwargs) -> None: # pragma: no cover """ diff --git a/src/documents/tests/test_api_ocr_templates.py b/src/documents/tests/test_api_ocr_templates.py index c41b6dead6..c5a6e0e2cd 100644 --- a/src/documents/tests/test_api_ocr_templates.py +++ b/src/documents/tests/test_api_ocr_templates.py @@ -29,6 +29,8 @@ from documents.tasks import bulk_update_documents from documents.zone_ocr import ZoneExtraction from documents.zone_ocr import ZoneOcrReason from documents.zone_ocr import ZoneOcrResult +from documents.zone_ocr import apply_zone_plans +from documents.zone_ocr import plan_zone_extraction from paperless_testing.factories import CustomFieldFactory from paperless_testing.factories import DocumentFactory from paperless_testing.factories import OcrTemplateFactory @@ -323,6 +325,14 @@ class TestZoneOcrIndexing: ) with django_capture_on_commit_callbacks(execute=True): if entry_point == "ingestion": + # The consumer writes the planned values, then sends the signal. + plans = plan_zone_extraction( + file_document.source_path, + file_document.page_count, + [template], + label="ingestion", + ) + apply_zone_plans(document, plans, label="ingestion") document_consumption_finished.send(sender=None, document=file_document) elif entry_point == "single": response = admin_client.post( diff --git a/src/documents/tests/test_consumer_zone_ocr.py b/src/documents/tests/test_consumer_zone_ocr.py new file mode 100644 index 0000000000..3b435181b9 --- /dev/null +++ b/src/documents/tests/test_consumer_zone_ocr.py @@ -0,0 +1,676 @@ +"""Tests for zone OCR during consumption.""" + +from __future__ import annotations + +import json +import logging +import shutil +import tempfile +from datetime import date +from pathlib import Path +from typing import TYPE_CHECKING +from typing import Any +from typing import Self + +import pytest +from django.conf import settings +from django.db import transaction +from PIL import Image + +from documents import consumer as consumer_module +from documents.consumer import ConsumerPlugin +from documents.data_models import ConsumableDocument +from documents.data_models import DocumentMetadataOverrides +from documents.data_models import DocumentSource +from documents.models import CustomField +from documents.models import CustomFieldInstance +from documents.models import Document +from documents.models import OcrTemplate +from documents.models import OcrTemplateZone +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger +from paperless_testing.factories import CustomFieldFactory +from paperless_testing.factories import DocumentFactory +from paperless_testing.factories import OcrTemplateFactory +from paperless_testing.factories import OcrTemplateZoneFactory +from paperless_testing.factories import TagFactory +from paperless_testing.factories import UserFactory +from paperless_testing.fakes.progress import FakeProgressManager +from paperless_testing.samples import SIMPLE_DIGITAL_PDF + +if TYPE_CHECKING: + from collections.abc import Callable + from unittest.mock import MagicMock + + from pytest_django.fixtures import SettingsWrapper + from pytest_mock import MockerFixture + + from documents.zone_ocr import TemplatePlan + from paperless_testing.dirs import PaperlessDirs + +CONTENT = "Northwind Power & Light statement" + +# The text each zone's fake OCR returns, by zone name. +ZONE_TEXT = { + "Title": "Zone title", + "ASN": "ASN 777", + "Created": "2024-03-05", + "Code": "INV42", +} + +MIME_TYPES = { + ".pdf": "application/pdf", + ".png": "image/png", + ".eml": "message/rfc822", +} + + +class ZoneTestParser: + """A parser returning CONTENT, with a PDF archive when one is requested.""" + + name = "zone-test-parser" + version = "0.1" + author = "test" + url = "test" + + @classmethod + def supported_mime_types(cls) -> dict[str, str]: + return {mime: suffix for suffix, mime in MIME_TYPES.items()} + + @classmethod + def score(cls, mime_type: str, filename: str, path: Path | None = None) -> int: + return 0 + + @property + def can_produce_archive(self) -> bool: + return True + + @property + def requires_pdf_rendition(self) -> bool: + return False + + def __enter__(self) -> Self: + self._tmpdir = Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) + self._archive: Path | None = None + self._thumb = self._tmpdir / "thumb.webp" + self._thumb.write_bytes(b"thumbnail") + return self + + def __exit__(self, *args: object) -> None: + shutil.rmtree(self._tmpdir, ignore_errors=True) + + def configure(self, context: object) -> None: + """The test parser ignores its context.""" + + def parse( + self, + document_path: Path, + mime_type: str, + *, + produce_archive: bool = True, + ) -> None: + if produce_archive: + self._archive = self._tmpdir / "archive.pdf" + shutil.copy(SIMPLE_DIGITAL_PDF, self._archive) + + def get_text(self) -> str: + return CONTENT + + def get_date(self) -> None: + return None + + def get_archive_path(self) -> Path | None: + return self._archive + + def get_thumbnail(self, document_path: Path, mime_type: str) -> Path: + return self._thumb + + def get_page_count(self, document_path: Path, mime_type: str) -> int: + return 1 + + def extract_metadata(self, document_path: Path, mime_type: str) -> list[Any]: + return [] + + +@pytest.fixture +def page_image(tmp_path: Path) -> Path: + page = tmp_path / "page.png" + with Image.new("RGB", (200, 200), "white") as image: + image.save(page) + return page + + +@pytest.fixture +def mock_render(mocker: MockerFixture, page_image: Path) -> MagicMock: + """Fake page rendering, recording whether a transaction was open.""" + render = mocker.patch("documents.zone_ocr._render_pages") + render.in_transaction = [] + + def record(*args: Any, **kwargs: Any) -> dict[int, Path]: + render.in_transaction.append(transaction.get_connection().in_atomic_block) + return {0: page_image} + + render.side_effect = record + return render + + +@pytest.fixture +def mock_ocr(mocker: MockerFixture) -> MagicMock: + """Fake zone OCR returning ZONE_TEXT, recording whether a transaction was open.""" + ocr = mocker.patch("documents.zone_ocr._ocr_text") + ocr.in_transaction = [] + + def read(cropped: Image.Image, zone: OcrTemplateZone, tmp_dir: Path) -> str: + ocr.in_transaction.append(transaction.get_connection().in_atomic_block) + return ZONE_TEXT[zone.name] + + ocr.side_effect = read + return ocr + + +@pytest.fixture +def consume( + paperless_dirs: PaperlessDirs, + mocker: MockerFixture, +) -> Callable[..., Document]: + """Consume a file of the given suffix through the ConsumerPlugin.""" + registry = mocker.patch("documents.consumer.get_parser_registry") + registry.return_value.get_parser_for_file.return_value = ZoneTestParser + mocker.patch( + "documents.consumer.magic.from_file", + side_effect=lambda path, *, mime=False: MIME_TYPES[Path(path).suffix], + ) + + def run( + suffix: str = ".pdf", + overrides: DocumentMetadataOverrides | None = None, + root_document_id: int | None = None, + ) -> Document: + source = paperless_dirs.scratch_dir / f"upload{suffix}" + if suffix == ".pdf": + shutil.copy(SIMPLE_DIGITAL_PDF, source) + else: + source.write_bytes(f"{suffix} {source.name}".encode()) + plugin = ConsumerPlugin( + ConsumableDocument( + DocumentSource.ApiUpload, + original_file=source, + root_document_id=root_document_id, + ), + overrides or DocumentMetadataOverrides(), + FakeProgressManager(source.name, None), + paperless_dirs.scratch_dir, + "task-id", + ) + plugin.setup() + try: + result = plugin.run() + finally: + plugin.cleanup() + return Document.objects.get(pk=result["document_id"]) + + return run + + +def make_template( + name: str = "Invoice", + targets: tuple[str, ...] = ("Title",), + custom_field: CustomField | None = None, + **kwargs: Any, +) -> OcrTemplate: + """A template matching CONTENT with one zone per named target.""" + template = OcrTemplateFactory( + name=name, + match="Northwind", + source_width=100, + source_height=100, + **kwargs, + ) + zone_targets = { + "Title": OcrTemplateZone.TargetType.TITLE, + "ASN": OcrTemplateZone.TargetType.ASN, + "Created": OcrTemplateZone.TargetType.CREATED, + "Code": OcrTemplateZone.TargetType.CUSTOM_FIELD, + } + for order, zone_name in enumerate(targets): + OcrTemplateZoneFactory( + template=template, + name=zone_name, + target=zone_targets[zone_name], + custom_field=custom_field if zone_name == "Code" else None, + y=order * 20, + width=50, + height=20, + order=order, + ) + return template + + +def custom_field_value(document: Document, field: CustomField) -> Any: + instance = CustomFieldInstance.objects.filter( + document=document, + field=field, + ).first() + return instance.value if instance is not None else None + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.usefixtures("mock_ocr") +class TestConsumerZoneOcrTransaction: + """Zone OCR runs outside the consumer's database transaction.""" + + def test_ocr_runs_before_the_transaction_opens( + self, + consume: Callable[..., Document], + mock_render: MagicMock, + mock_ocr: MagicMock, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + WHEN: + - The document is consumed + THEN: + - Pages are rendered and zones are read with no transaction open + - The zone value is written to the document + """ + make_template() + + document = consume() + + assert mock_render.in_transaction == [False] + assert mock_ocr.in_transaction == [False] + assert document.title == "Zone title" + + +@pytest.mark.django_db +@pytest.mark.usefixtures("mock_ocr") +class TestConsumerZoneOcr: + """Zone values are computed before and written during consumption.""" + + @pytest.fixture + def code_field(self) -> CustomField: + return CustomFieldFactory( + name="Invoice code", + data_type=CustomField.FieldDataType.STRING, + ) + + @pytest.mark.parametrize( + ("suffix", "generation", "expected_file"), + [ + pytest.param(".pdf", "always", "archive.pdf", id="pdf-archive"), + pytest.param(".eml", "always", "archive.pdf", id="mail-archive"), + pytest.param(".pdf", "never", "upload.pdf", id="pdf-working-copy"), + pytest.param(".png", "never", "upload.png", id="image-working-copy"), + ], + ) + def test_zones_are_read_from_the_archive_or_working_copy( + self, + consume: Callable[..., Document], + mock_render: MagicMock, + settings: SettingsWrapper, + suffix: str, + generation: str, + expected_file: str, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + WHEN: + - A document is consumed with or without an archive + THEN: + - Zones are read from the archive when there is one, otherwise + from the working copy of the PDF or image + """ + settings.ARCHIVE_FILE_GENERATION = generation + make_template() + + document = consume(suffix) + + mock_render.assert_called_once() + assert mock_render.call_args.args[0].name == expected_file + assert document.title == "Zone title" + + def test_mail_without_archive_is_not_rendered( + self, + consume: Callable[..., Document], + mock_render: MagicMock, + settings: SettingsWrapper, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + WHEN: + - An email is consumed without an archive + THEN: + - No page is rendered from the email file + - The document is consumed without zone values + """ + settings.ARCHIVE_FILE_GENERATION = "never" + make_template() + + document = consume(".eml") + + mock_render.assert_not_called() + assert document.title == "upload" + + @pytest.mark.usefixtures("mock_render") + def test_values_are_written_before_document_added_workflows( + self, + consume: Callable[..., Document], + code_field: CustomField, + ) -> None: + """ + GIVEN: + - A template writing a custom field + - A DOCUMENT_ADDED workflow filtering on that custom field's value + and assigning a tag + WHEN: + - A matching document is consumed + THEN: + - The workflow sees the zone value and assigns its tag + """ + make_template(targets=("Code",), custom_field=code_field) + tag = TagFactory(name="Zone matched") + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_custom_field_query=json.dumps( + ["AND", [[code_field.pk, "exact", "INV42"]]], + ), + ) + action = WorkflowAction.objects.create() + action.assign_tags.add(tag) + workflow = Workflow.objects.create(name="On zone value", order=0) + workflow.triggers.add(trigger) + workflow.actions.add(action) + + document = consume() + + assert custom_field_value(document, code_field) == "INV42" + assert list(document.tags.all()) == [tag] + + @pytest.mark.parametrize( + "overridden", + [ + pytest.param("title", id="title"), + pytest.param("asn", id="asn"), + pytest.param("created", id="created"), + pytest.param("custom_field", id="custom-field"), + ], + ) + @pytest.mark.usefixtures("mock_render") + def test_explicit_values_are_not_overwritten( + self, + consume: Callable[..., Document], + code_field: CustomField, + overridden: str, + ) -> None: + """ + GIVEN: + - A template with title, ASN, created and custom field zones + - Metadata overrides giving one of those fields a value + WHEN: + - A matching document is consumed + THEN: + - The supplied value is kept + - Every other zone value is written + """ + make_template( + targets=("Title", "ASN", "Created", "Code"), + custom_field=code_field, + ) + explicit = { + "title": "Upload title", + "asn": 123, + "created": date(2020, 1, 2), + "custom_field": "Given", + } + overrides = { + "title": DocumentMetadataOverrides(title="Upload title"), + "asn": DocumentMetadataOverrides(asn=123), + "created": DocumentMetadataOverrides(created=date(2020, 1, 2)), + "custom_field": DocumentMetadataOverrides( + custom_fields={code_field.pk: "Given"}, + ), + }[overridden] + expected = { + "title": "Zone title", + "asn": 777, + "created": date(2024, 3, 5), + "custom_field": "INV42", + } | {overridden: explicit[overridden]} + + document = consume(overrides=overrides) + + assert { + "title": document.title, + "asn": document.archive_serial_number, + "created": document.created, + "custom_field": custom_field_value(document, code_field), + } == expected + + @pytest.mark.usefixtures("mock_render") + def test_custom_field_listed_without_a_value_is_filled( + self, + consume: Callable[..., Document], + code_field: CustomField, + ) -> None: + """ + GIVEN: + - A template with a custom field zone + - Metadata overrides listing that custom field without a value + WHEN: + - A matching document is consumed + THEN: + - The zone value is written to the custom field + """ + make_template(targets=("Code",), custom_field=code_field) + + document = consume( + overrides=DocumentMetadataOverrides(custom_fields={code_field.pk: None}), + ) + + assert custom_field_value(document, code_field) == "INV42" + + @pytest.mark.usefixtures("mock_render") + def test_custom_field_deleted_before_writing_is_skipped( + self, + consume: Callable[..., Document], + code_field: CustomField, + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + ) -> None: + """ + GIVEN: + - A template with title and custom field zones + - The custom field deleted after the zone values are planned + WHEN: + - A matching document is consumed + THEN: + - The document is stored with the title + - The custom field zone is reported as unassigned + """ + make_template(targets=("Title", "Code"), custom_field=code_field) + plan = consumer_module.plan_zone_extraction + + def plan_then_delete(*args: Any, **kwargs: Any) -> list[TemplatePlan]: + plans = plan(*args, **kwargs) + CustomField.objects.filter(pk=code_field.pk).delete() + return plans + + mocker.patch( + "documents.consumer.plan_zone_extraction", + side_effect=plan_then_delete, + ) + caplog.set_level(logging.DEBUG, logger="paperless.consumer") + + document = consume() + + assert document.title == "Zone title" + assert not CustomFieldInstance.objects.filter(document=document).exists() + assert "zone 'Code' for Invoice code: unassigned" in caplog.text + + @pytest.mark.parametrize( + ("with_owner", "expected_title", "expected_code"), + [ + pytest.param(True, "Zone title", None, id="owner-sees-own-templates"), + pytest.param(False, "Zone title", "INV42", id="no-owner-sees-all"), + ], + ) + @pytest.mark.usefixtures("mock_render") + def test_templates_are_selected_by_the_metadata_owner( + self, + consume: Callable[..., Document], + code_field: CustomField, + with_owner: bool, # noqa: FBT001 + expected_title: str, + expected_code: str | None, + ) -> None: + """ + GIVEN: + - A template owned by the upload's owner writing the title + - A private template of another user writing a custom field + WHEN: + - A matching document is consumed with or without an owner + THEN: + - With an owner, only the templates that owner can view apply + - Without an owner, every template applies + """ + owner = UserFactory() + make_template("Mine", owner=owner) + make_template( + "Private", + targets=("Code",), + custom_field=code_field, + owner=UserFactory(), + ) + + document = consume( + overrides=DocumentMetadataOverrides( + owner_id=owner.pk if with_owner else None, + ), + ) + + assert document.title == expected_title + assert custom_field_value(document, code_field) == expected_code + + @pytest.mark.usefixtures("mock_render") + def test_duplicate_asn_is_checked_when_writing( + self, + consume: Callable[..., Document], + ) -> None: + """ + GIVEN: + - A template with title and ASN zones + - Another document already using the zone's ASN + WHEN: + - A matching document is consumed + THEN: + - The ASN is not written + - The title from the same template is written + """ + DocumentFactory(archive_serial_number=777) + make_template(targets=("Title", "ASN")) + + document = consume() + + assert document.archive_serial_number is None + assert document.title == "Zone title" + + @pytest.mark.parametrize( + "failing", + [ + pytest.param( + "documents.consumer.get_templates_for_content", + id="template-selection", + ), + pytest.param("documents.consumer.plan_zone_extraction", id="extraction"), + ], + ) + @pytest.mark.usefixtures("mock_render") + def test_read_phase_failure_does_not_fail_consumption( + self, + consume: Callable[..., Document], + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + failing: str, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + - Template selection or zone extraction raising an error + WHEN: + - The document is consumed + THEN: + - The document is stored without zone values + - The failure is logged + """ + make_template() + mocker.patch(failing, side_effect=RuntimeError("zone OCR broke")) + + document = consume() + + assert document.title == "upload" + assert "Zone OCR failed for upload.pdf" in caplog.text + + @pytest.mark.usefixtures("mock_render") + def test_write_failure_does_not_fail_consumption( + self, + consume: Callable[..., Document], + mocker: MockerFixture, + caplog: pytest.LogCaptureFixture, + ) -> None: + """ + GIVEN: + - A template matching the consumed document's text + - Writing the template's values raising an error + WHEN: + - The document is consumed + THEN: + - The document is stored without the template's values + - The failed template is logged + """ + make_template() + mocker.patch( + "documents.zone_ocr._apply_field_updates", + side_effect=RuntimeError("write failed"), + ) + + document = consume() + + assert document.title == "upload" + assert "error processing template 'Invoice'" in caplog.text + + @pytest.mark.usefixtures("mock_render") + def test_version_values_are_written_to_the_root( + self, + consume: Callable[..., Document], + code_field: CustomField, + ) -> None: + """ + GIVEN: + - A root document whose owner can view a template writing the title + and a custom field + - A private template of another user + WHEN: + - A new version of the root document is consumed + THEN: + - The zone values are written to the root document + - The version keeps the root's previous title and gets no custom field + - The other user's template does not apply + """ + owner = UserFactory() + root = DocumentFactory(title="Root title", owner=owner, content=CONTENT) + make_template(targets=("Title", "Code"), custom_field=code_field, owner=owner) + make_template("Private", targets=("ASN",), owner=UserFactory()) + + version = consume(root_document_id=root.pk) + + root.refresh_from_db() + assert version.root_document_id == root.pk + assert root.title == "Zone title" + assert custom_field_value(root, code_field) == "INV42" + assert root.archive_serial_number is None + assert version.title == "Root title" + assert custom_field_value(version, code_field) is None diff --git a/src/documents/tests/test_task_signals.py b/src/documents/tests/test_task_signals.py index 2a44c9788c..2b8b84904c 100644 --- a/src/documents/tests/test_task_signals.py +++ b/src/documents/tests/test_task_signals.py @@ -13,81 +13,14 @@ from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentSource from documents.models import PaperlessTask from documents.signals.handlers import before_task_publish_handler -from documents.signals.handlers import run_zone_ocr_extraction from documents.signals.handlers import task_failure_handler from documents.signals.handlers import task_postrun_handler from documents.signals.handlers import task_prerun_handler from documents.signals.handlers import task_revoked_handler -from paperless_testing.factories import DocumentFactory from paperless_testing.factories import PaperlessTaskFactory from paperless_testing.factories import UserFactory -@pytest.mark.django_db -class TestRunZoneOcrExtraction: - """Zone OCR extraction triggered by the consumption finished signal.""" - - @pytest.mark.parametrize( - "original_file", - [ - pytest.param(None, id="no-original-file"), - pytest.param("/tmp/original.pdf", id="str-path"), - pytest.param(Path("/tmp/original.pdf"), id="path-object"), - ], - ) - def test_consumption_runs_zone_extraction( - self, - original_file: str | Path | None, - mocker: pytest_mock.MockerFixture, - ) -> None: - """ - GIVEN: - - A consumed document and an original file given as None, a str, or a Path - WHEN: - - The consumption finished handler runs zone OCR extraction - THEN: - - Zone extraction is called once with the document and the - original file normalised to a Path (or None) - """ - extract = mocker.patch("documents.zone_ocr.run_zone_extraction") - document = DocumentFactory() - - run_zone_ocr_extraction( - sender=None, - document=document, - original_file=original_file, - ) - - extract.assert_called_once_with( - document, - Path(original_file) if original_file else None, - ) - - def test_zone_extraction_failure_does_not_fail_consumption( - self, - mocker: pytest_mock.MockerFixture, - caplog: pytest.LogCaptureFixture, - ) -> None: - """ - GIVEN: - - A consumed document whose zone extraction raises an error - WHEN: - - The consumption finished handler runs zone OCR extraction - THEN: - - The error is not propagated - - The failure is logged with the document id - """ - mocker.patch( - "documents.zone_ocr.run_zone_extraction", - side_effect=RuntimeError("OCR failed"), - ) - document = DocumentFactory() - - run_zone_ocr_extraction(sender=None, document=document) - - assert f"Zone OCR extraction failed for document {document.pk}" in caplog.text - - @pytest.fixture def consume_input_doc(): doc = mock.MagicMock(spec=ConsumableDocument) diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index c558be8897..8e87ac2dbc 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -52,6 +52,7 @@ from documents.zone_ocr import _render_pages from documents.zone_ocr import _resolve_doc_path from documents.zone_ocr import _zone_language from documents.zone_ocr import extract_zone_preview +from documents.zone_ocr import get_templates_for_content from documents.zone_ocr import get_templates_for_document from documents.zone_ocr import run_zone_extraction from paperless_testing.factories import CustomFieldFactory @@ -1707,6 +1708,39 @@ class TestTemplateVisibility: get_templates_for_document(document, requester=other), ) == Counter([unowned, private]) + @pytest.mark.parametrize( + ("has_owner", "expected_names"), + [ + pytest.param(True, {"Mine", "Unowned"}, id="owner"), + pytest.param(False, {"Mine", "Unowned", "Private"}, id="no-owner"), + ], + ) + @pytest.mark.usefixtures("mine", "unowned", "private") + def test_content_selection_follows_the_owner_rule( + self, + owner: User, + has_owner: bool, # noqa: FBT001 + expected_names: set[str], + ) -> None: + """ + GIVEN: + - Templates owned by two users and an unowned template, all + matching "Northwind", and one matching only other text + WHEN: + - Templates are selected for unsaved text, with or without an owner + THEN: + - Only matching templates apply + - An owner sees owned and unowned templates, no owner sees all + """ + make_visible_template("Other sender", match="Contoso") + + templates = get_templates_for_content( + "Northwind Power & Light statement", + owner if has_owner else None, + ) + + assert {template.name for template in templates} == expected_names + @pytest.mark.usefixtures("private") def test_versions_use_the_root_documents_owner( self, diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 2717df68b6..c7a830e340 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -57,6 +57,7 @@ if TYPE_CHECKING: from typing import Any from django.contrib.auth.models import User + from django.db.models import QuerySet logger = logging.getLogger("paperless.zone_ocr") @@ -146,6 +147,25 @@ class ZoneOcrResult: reason: ZoneOcrReason | None = None +@dataclass(kw_only=True, slots=True) +class PlannedUpdate: + """A field update and the accepted zone results it was combined from.""" + + key: FieldKey + update: FieldUpdate + results: list[ZoneOcrResult] + + +@dataclass(kw_only=True, slots=True) +class TemplatePlan: + """A template's zone results and the field updates they produce, unwritten.""" + + template: OcrTemplate + zones: list[OcrTemplateZone] + results: list[ZoneOcrResult] + updates: list[PlannedUpdate] + + # Result labels for zones that target a built-in field. BUILTIN_TARGET_LABELS: dict[OcrTemplateZone.TargetType, str] = { OcrTemplateZone.TargetType.TITLE: "Title", @@ -187,20 +207,7 @@ def get_templates_for_document( view, so a user editing someone else's document never runs or sees that owner's private templates. """ - owner = get_root_document(document).owner - templates = OcrTemplate.objects.filter(enabled=True) - if owner is not None: - templates = templates.filter( - id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"), - ) - if requester is not None: - templates = templates.filter( - id__in=permitted_object_ids(requester, OcrTemplate, "view_ocrtemplate"), - ) - templates = templates.order_by("name", "pk").prefetch_related( - "zones", - "zones__custom_field", - ) + templates = _visible_templates(get_root_document(document).owner, requester) match_document = document if document.root_document_id is None and templates: # The original file can be selected even when newer versions exist. @@ -215,6 +222,43 @@ def get_templates_for_document( return [template for template in templates if matches(template, match_document)] +def get_templates_for_content( + content: str, + owner: User | None, + *, + title: str = "", +) -> list[OcrTemplate]: + """ + The enabled templates that match the given text and that the owner can view, + for a document that is not saved yet. A missing owner considers every + template, as get_templates_for_document does. The title only labels the + matching log messages. + """ + templates = _visible_templates(owner, None) + match_document = Document(title=title, content=content) + return [template for template in templates if matches(template, match_document)] + + +def _visible_templates( + owner: User | None, + requester: User | None, +) -> QuerySet[OcrTemplate]: + """The enabled templates both the owner and the requester can view, if given.""" + templates = OcrTemplate.objects.filter(enabled=True) + if owner is not None: + templates = templates.filter( + id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"), + ) + if requester is not None: + templates = templates.filter( + id__in=permitted_object_ids(requester, OcrTemplate, "view_ocrtemplate"), + ) + return templates.order_by("name", "pk").prefetch_related( + "zones", + "zones__custom_field", + ) + + def run_zone_extraction( document: Document, original_file: Path | None = None, @@ -232,46 +276,14 @@ def run_zone_extraction( if not templates: return [] - doc_path = _resolve_doc_path(document, original_file) - if doc_path is None: - logger.warning( - "Zone OCR: no accessible file for document %d", - document.pk, - ) - return [ - _zone_result(template, zone, reason=ZoneOcrReason.MISSING_FILE) - for template in templates - for zone in template.zones.all() - ] - - template_zones = [(template, list(template.zones.all())) for template in templates] - pages = { - _resolve_page_idx(zone.page, document.page_count) - for _, zones in template_zones - for zone in zones - } - if not pages: - return [] - results = [] - with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: - tmp_path = Path(tmp_dir) - page_images = _render_pages(doc_path, pages, tmp_path, document.page_count) - for template, zones in template_zones: - try: - results.extend( - _process_template(document, page_images, template, zones, tmp_path), - ) - except Exception: - # One failed template must not prevent the remaining templates from running. - logger.exception( - "Zone OCR: error processing template '%s' for document %d", - template.name, - document.pk, - ) - results.extend( - _zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED) - for zone in zones - ) + label = f"document {document.pk}" + plans = plan_zone_extraction( + _resolve_doc_path(document, original_file), + document.page_count, + templates, + label=label, + ) + results = apply_zone_plans(get_root_document(document), plans, label=label) if zone_results_written(results): Document.objects.filter(pk=get_root_document(document).pk).update( modified=timezone.now(), @@ -279,6 +291,124 @@ def run_zone_extraction( return results +def plan_zone_extraction( + doc_path: Path | None, + page_count: int | None, + templates: list[OcrTemplate], + *, + label: str, +) -> list[TemplatePlan]: + """ + Render the zones' pages from doc_path and compute each template's field + updates without writing anything. A template that fails yields only + TEMPLATE_FAILED results, and a missing file yields MISSING_FILE results for + every zone. The label names the document in log messages. + """ + template_zones = [(template, list(template.zones.all())) for template in templates] + if doc_path is None: + logger.warning("Zone OCR: no accessible file for %s", label) + return [ + _failed_plan(template, zones, ZoneOcrReason.MISSING_FILE) + for template, zones in template_zones + ] + + pages = { + _resolve_page_idx(zone.page, page_count) + for _, zones in template_zones + for zone in zones + } + if not pages: + return [] + plans = [] + with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: + tmp_path = Path(tmp_dir) + page_images = _render_pages(doc_path, pages, tmp_path, page_count) + for template, zones in template_zones: + try: + plans.append( + _plan_template(page_count, page_images, template, zones, tmp_path), + ) + except Exception: + # One failed template must not prevent the remaining templates + # from running. + logger.exception( + "Zone OCR: error processing template '%s' for %s", + template.name, + label, + ) + plans.append( + _failed_plan(template, zones, ZoneOcrReason.TEMPLATE_FAILED), + ) + return plans + + +def apply_zone_plans( + root_document: Document, + plans: list[TemplatePlan], + *, + label: str, + protected: frozenset[FieldKey] = frozenset(), +) -> list[ZoneOcrResult]: + """ + Write each template's planned updates to the root document, each template + in its own atomic block, and return the zone results. Fields whose key is in + protected are not written, and the results of every zone targeting them are + left out. A template whose writes fail gets TEMPLATE_FAILED results and the + others still apply. + """ + results: list[ZoneOcrResult] = [] + for plan in plans: + for planned in plan.updates: + if planned.key in protected: + logger.info( + "Zone OCR: template '%s' kept the supplied %s of %s", + plan.template.name, + planned.update.field_name, + label, + ) + updates = [planned for planned in plan.updates if planned.key not in protected] + try: + if updates: + _apply_template_updates(root_document, updates) + plan_results = plan.results + except Exception: + logger.exception( + "Zone OCR: error processing template '%s' for %s", + plan.template.name, + label, + ) + plan_results = _failed_plan( + plan.template, + plan.zones, + ZoneOcrReason.TEMPLATE_FAILED, + ).results + # Each zone has exactly one result, in zone order. + results.extend( + result + for zone, result in zip(plan.zones, plan_results, strict=True) + if _zone_key(zone) not in protected + ) + return results + + +def _zone_key(zone: OcrTemplateZone) -> FieldKey: + """The combine_formats key of the field a zone fills.""" + return field_key(OcrTemplateZone.TargetType(zone.target), zone.custom_field_id) + + +def _failed_plan( + template: OcrTemplate, + zones: list[OcrTemplateZone], + reason: ZoneOcrReason, +) -> TemplatePlan: + return TemplatePlan( + template=template, + zones=zones, + results=[_zone_result(template, zone, reason=reason) for zone in zones], + updates=[], + ) + + def zone_results_written(results: list[ZoneOcrResult]) -> bool: """Whether at least one zone's value was written to the root document.""" return any(result.reason is None for result in results) @@ -288,7 +418,7 @@ def _resolve_doc_path( document: Document, original_file: Path | None, ) -> Path | None: - """Prefer the archive, then the source, then the ingestion signal's file.""" + """Prefer the archive, then the source, then the given file.""" candidates: list[Path | None] = [] if document.has_archive_version: candidates.append(document.archive_path) @@ -316,7 +446,21 @@ def _process_template( zones: list[OcrTemplateZone], tmp_path: Path, ) -> list[ZoneOcrResult]: - """Extract and combine zone values before applying a template's updates.""" + """Extract and combine zone values, then apply a template's updates.""" + plan = _plan_template(document.page_count, page_images, template, zones, tmp_path) + if plan.updates: + _apply_template_updates(get_root_document(document), plan.updates) + return plan.results + + +def _plan_template( + page_count: int | None, + page_images: dict[int, Path], + template: OcrTemplate, + zones: list[OcrTemplateZone], + tmp_path: Path, +) -> TemplatePlan: + """Extract and combine zone values into a template's updates, unwritten.""" # Each zone's result and its accepted text, by target field. grouped: dict[ FieldKey, @@ -324,7 +468,7 @@ def _process_template( ] = {} results = [] for zone in zones: - page_idx = _resolve_page_idx(zone.page, document.page_count) + page_idx = _resolve_page_idx(zone.page, page_count) extracted = None reason: ZoneOcrReason | None = ZoneOcrReason.MISSING_PAGE if page_idx in page_images: @@ -351,16 +495,12 @@ def _process_template( reason = ZoneOcrReason.REGEX_MISMATCH result = _zone_result(template, zone, reason=reason) results.append(result) - key = field_key( - OcrTemplateZone.TargetType(zone.target), - zone.custom_field_id, - ) + key = _zone_key(zone) grouped.setdefault(key, []).append( (zone, result, extracted if reason is None else None), ) updates = [] - root_document = get_root_document(document) combine_formats: dict[FieldKey, str] = template.combine_formats or {} for key, field_zones in grouped.items(): accepted = [result for _, result, _ in field_zones if result.reason is None] @@ -381,15 +521,8 @@ def _process_template( ) target_zone = field_zones[0][0] update = _prepare_field_update(target_zone, value) if value else None - if update is not None and _asn_in_use(update, root_document): - # Saving would violate the unique ASN and roll back every other - # write of this template, so only this field is dropped. - for result in accepted: - result.value = None - result.reason = ZoneOcrReason.DUPLICATE_ASN - continue if update is not None: - updates.append(update) + updates.append(PlannedUpdate(key=key, update=update, results=accepted)) for result in accepted: result.value = update.value if update else None if update is None: @@ -398,8 +531,57 @@ def _process_template( if key == UNASSIGNED_FIELD_KEY else ZoneOcrReason.INVALID_VALUE ) - _apply_field_updates(root_document, updates) - return results + return TemplatePlan( + template=template, + zones=zones, + results=results, + updates=updates, + ) + + +def _apply_template_updates( + root_document: Document, + updates: list[PlannedUpdate], +) -> None: + """Write one template's updates to the root document in one atomic block. + + The ASN duplicate check and the custom field existence check run here, at + write time, so they see the database as it is when the values are saved. + """ + field_ids = { + planned.update.custom_field.pk + for planned in updates + if planned.update.custom_field is not None + } + with transaction.atomic(): + existing_field_ids = ( + set( + CustomField.objects.filter(pk__in=field_ids).values_list( + "pk", + flat=True, + ), + ) + if field_ids + else set() + ) + writable = [] + for planned in updates: + custom_field = planned.update.custom_field + if custom_field is not None and custom_field.pk not in existing_field_ids: + # The field was deleted after the values were planned. + for result in planned.results: + result.value = None + result.reason = ZoneOcrReason.UNASSIGNED + continue + if _asn_in_use(planned.update, root_document): + # Saving would violate the unique ASN and roll back every other + # write of this template, so only this field is dropped. + for result in planned.results: + result.value = None + result.reason = ZoneOcrReason.DUPLICATE_ASN + continue + writable.append(planned.update) + _apply_field_updates(root_document, writable) def _asn_in_use(update: FieldUpdate, root_document: Document) -> bool: