From c5fdba73a15e265b04cf6e47b417f763b13bc146 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:25:48 -0800 Subject: [PATCH] Fix: queue single document zone OCR runs instead of running them in the request Rendering and OCR ran inside the HTTP request and could block other requests for minutes. The action now validates, then queues the bulk zone OCR operation for the selected file and returns its task id. --- docs/usage.md | 4 +- src/documents/bulk_edit.py | 5 +- src/documents/serialisers.py | 41 +- src/documents/tasks.py | 9 +- src/documents/tests/test_api_ocr_templates.py | 533 +++++++----------- src/documents/tests/test_bulk_edit.py | 8 +- src/documents/tests/test_bulk_operations.py | 2 + src/documents/tests/test_tasks.py | 63 +++ src/documents/tests/test_zone_ocr.py | 71 +-- src/documents/views.py | 117 ++-- src/documents/zone_ocr.py | 9 +- 11 files changed, 379 insertions(+), 483 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 39764e5b2b..226a189025 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -898,7 +898,9 @@ details page: instead. A template you choose runs even if it does not match the document or is disabled. Both actions use the file version selected in document details. The latest version is selected by default. -Extracted values update the document's metadata, which is shared across its versions. +Extracted values update the document's metadata, which is shared across its versions. If no template +applies to the document, nothing is run. Otherwise the run is processed in the background and appears in the +task list. The document shows the extracted values once the task finishes. The bulk editor's **Run Zone OCR** action opens the same dialog for the selected documents. Bulk runs use the latest version of each document and are processed in the background. diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py index 77644b1475..7b19aeaade 100644 --- a/src/documents/bulk_edit.py +++ b/src/documents/bulk_edit.py @@ -1263,10 +1263,12 @@ def run_zone_ocr( *, template: int | None = None, user: User | None = None, + resolve_latest: bool = True, ) -> Literal["OK"]: """ Run zone OCR within the tracked bulk operation task, then finish once for - the root documents that had a value written. + the root documents that had a value written. Root documents are run on + their newest version unless resolve_latest is False. """ written_ids: list[int] = [] try: @@ -1275,6 +1277,7 @@ def run_zone_ocr( document_id=document_id, template_id=template, requester=user, + resolve_latest=resolve_latest, ) if written_id is not None: written_ids.append(written_id) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index b5781a2e9d..aeb4c2a322 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -99,9 +99,6 @@ from documents.validators import uri_validator from documents.validators import url_validator from documents.versioning import has_prefetched_effective_content from documents.versioning import sort_versions_newest_first -from documents.zone_ocr import ZoneOcrReason -from documents.zone_ocr import ZoneOcrResult -from documents.zone_ocr import ZoneOcrRunStatus from documents.zone_ocr import ZoneOcrSkipReason from documents.zone_ocr import field_key @@ -1819,6 +1816,7 @@ class RunZoneOcrSerializer(serializers.Serializer[Any]): required=False, allow_null=True, ) + from_webui = serializers.BooleanField(required=False, default=False) def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @@ -1833,34 +1831,18 @@ class RunZoneOcrSerializer(serializers.Serializer[Any]): ).prefetch_related("zones", "zones__custom_field") -class ZoneOcrResultSerializer(serializers.Serializer[ZoneOcrResult]): - """The outcome of one zone, mirroring the ZoneOcrResult dataclass.""" - - template = serializers.CharField(read_only=True) - zone = serializers.CharField(read_only=True) - custom_field = serializers.CharField(read_only=True) - value = serializers.JSONField( - read_only=True, - allow_null=True, - help_text=( - "The value written to the target. A string, number, boolean or ISO " - "date string, depending on the target, or null when nothing was written." - ), - ) - reason = serializers.ChoiceField( - choices=[reason.value for reason in ZoneOcrReason], - read_only=True, - allow_null=True, - help_text="Why no value was written, or null when one was.", - ) - - class RunZoneOcrResponseSerializer(serializers.Serializer[dict[str, Any]]): - """The response of running zone OCR on one document.""" + """The response of queueing zone OCR on one document.""" - status = serializers.ChoiceField( - choices=[status.value for status in ZoneOcrRunStatus], + result = serializers.ChoiceField( + choices=["OK", "skipped"], read_only=True, + help_text="OK when the run was queued, skipped when it was not.", + ) + task_id = serializers.CharField( + read_only=True, + required=False, + help_text="The queued task. Absent when the run was skipped.", ) reason = serializers.ChoiceField( choices=[reason.value for reason in ZoneOcrSkipReason], @@ -1868,7 +1850,6 @@ class RunZoneOcrResponseSerializer(serializers.Serializer[dict[str, Any]]): required=False, help_text="Present only when the run was skipped.", ) - results = ZoneOcrResultSerializer(many=True, read_only=True) class OcrErrorSerializer(serializers.Serializer[dict[str, Any]]): @@ -1878,7 +1859,7 @@ class OcrErrorSerializer(serializers.Serializer[dict[str, Any]]): class RunZoneOcrDocumentsSerializer(DocumentSelectionSerializer, RunZoneOcrSerializer): - from_webui = serializers.BooleanField(required=False, default=False) + pass class BulkEditSerializer( diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 85a425aea8..5878747009 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -381,16 +381,23 @@ def extract_zones( document_id: int, template_id: int | None = None, requester: User | None = None, + *, + resolve_latest: bool = True, ) -> int | None: """ Run an OCR template against a stored document, or every template that matches it when no template is given. Matched templates are limited to those the requester can view. + A root document id is mapped to its newest version unless resolve_latest + is False, in which case exactly the given document is processed. + Returns the root document's id when any value was written, otherwise None. The caller finishes the edit, for example with bulk_update_documents. """ - document = latest_version(Document.objects.get(id=document_id)) + document = Document.objects.get(id=document_id) + if resolve_latest: + document = latest_version(document) if template_id is None: templates = get_templates_for_document(document, requester=requester) else: diff --git a/src/documents/tests/test_api_ocr_templates.py b/src/documents/tests/test_api_ocr_templates.py index c5a6e0e2cd..414930e5bc 100644 --- a/src/documents/tests/test_api_ocr_templates.py +++ b/src/documents/tests/test_api_ocr_templates.py @@ -5,6 +5,7 @@ from __future__ import annotations import functools import subprocess from io import BytesIO +from types import SimpleNamespace from typing import TYPE_CHECKING from typing import Any @@ -15,8 +16,8 @@ from PIL import Image from rest_framework import status from documents import bulk_edit +from documents.bulk_operations import execute_bulk_operation from documents.models import CustomField -from documents.models import CustomFieldInstance from documents.models import Document from documents.models import MatchingModel from documents.models import OcrTemplate @@ -25,10 +26,7 @@ from documents.models import PaperlessTask from documents.parsers import ParseError from documents.search import get_backend from documents.signals import document_consumption_finished -from documents.tasks import bulk_update_documents from documents.zone_ocr import ZoneExtraction -from documents.zone_ocr import ZoneOcrReason -from documents.zone_ocr import ZoneOcrResult from documents.zone_ocr import apply_zone_plans from documents.zone_ocr import plan_zone_extraction from paperless_testing.factories import CustomFieldFactory @@ -57,7 +55,8 @@ CONTENT = "Northwind Power & Light statement" ENDPOINT = "/api/ocr_templates/" RUN_ZONE_OCR_URL = "/api/documents/{pk}/run-zone-ocr/" PREVIEW_ZONE_URL = f"{ENDPOINT}test-zone/" -SKIPPED_NO_TEMPLATE = {"status": "skipped", "reason": "no_template", "results": []} +SKIPPED_NO_TEMPLATE = {"result": "skipped", "reason": "no_template"} +TASK_ID = "zone-ocr-task" def template_payload(**overrides: Any) -> dict[str, Any]: @@ -149,14 +148,11 @@ def title_template( @pytest.fixture -def mock_extract(mocker: MockerFixture) -> MagicMock: - return mocker.patch("documents.zone_ocr.run_zone_extraction") - - -@pytest.fixture -def mock_finish(mocker: MockerFixture) -> MagicMock: +def mock_enqueue(mocker: MockerFixture) -> MagicMock: + """Replace queueing of the bulk operation task, returning a fixed task id.""" return mocker.patch( - "documents.views.bulk_edit.bulk_update_documents.apply_async", + "documents.views.execute_bulk_operation.apply_async", + return_value=SimpleNamespace(id=TASK_ID), ) @@ -251,8 +247,8 @@ class TestZoneOcrIndexing: - A document (optionally with an uploaded version) indexed for search - An OCR template with a title zone and a custom field zone WHEN: - - Zone OCR runs through ingestion, the single-document API or the - bulk task + - Zone OCR runs through ingestion, the single-document API (queued + as a bulk operation and run inline) or the bulk task THEN: - The root document title and custom field are updated - The search index reflects the new title and custom field value @@ -318,10 +314,14 @@ class TestZoneOcrIndexing: backend.add_or_update(document) assert backend.search_ids("title:before", user=None) == [document.pk] - # The single-document finish is a queued task, run it inline here. + # The single-document run is a queued bulk operation, run it inline here. + def run_inline(kwargs: dict[str, Any], **_: Any) -> SimpleNamespace: + execute_bulk_operation(**kwargs) + return SimpleNamespace(id="inline") + mocker.patch( - "documents.views.bulk_edit.bulk_update_documents.apply_async", - side_effect=lambda kwargs, **_: bulk_update_documents(**kwargs), + "documents.views.execute_bulk_operation.apply_async", + side_effect=run_inline, ) with django_capture_on_commit_callbacks(execute=True): if entry_point == "ingestion": @@ -1827,9 +1827,9 @@ class TestOcrTemplateDelete: @pytest.mark.django_db -@pytest.mark.usefixtures("paperless_dirs", "mock_finish") +@pytest.mark.usefixtures("paperless_dirs") class TestRunZoneOcr: - """Running zone OCR on documents through the run-zone-ocr endpoint.""" + """Queueing zone OCR on one document through the run-zone-ocr endpoint.""" @pytest.mark.parametrize( "template_kwargs_list", @@ -1848,7 +1848,7 @@ class TestRunZoneOcr: def test_without_enabled_matching_template_is_skipped( self, admin_client: APIClient, - mock_extract: MagicMock, + mock_enqueue: MagicMock, template_kwargs_list: list[dict[str, Any]], ) -> None: """ @@ -1856,9 +1856,9 @@ class TestRunZoneOcr: - A document and either no templates, only a matching disabled template, or only a template that does not match the document WHEN: - - Zone OCR is run on the document + - Zone OCR is run on the document without choosing a template THEN: - - The run is skipped with reason no_template and nothing is extracted + - The run is skipped with reason no_template and nothing is queued """ document = DocumentFactory(content=CONTENT) for template_kwargs in template_kwargs_list: @@ -1868,12 +1868,13 @@ class TestRunZoneOcr: assert response.status_code == status.HTTP_200_OK assert response.json() == SKIPPED_NO_TEMPLATE - mock_extract.assert_not_called() + mock_enqueue.assert_not_called() - def test_applies_a_chosen_template( + def test_queues_a_chosen_template( self, admin_client: APIClient, - mock_extract: MagicMock, + admin_user: User, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: @@ -1881,7 +1882,10 @@ class TestRunZoneOcr: WHEN: - Zone OCR is run on the document with that template chosen THEN: - - The template is applied anyway, since it was chosen explicitly + - The bulk zone OCR operation is queued for the document and the + template, as the requesting user, even though the template does + not match + - The task id is returned """ document = DocumentFactory(content=CONTENT, mime_type="application/pdf") document.source_path.write_bytes(b"test PDF") @@ -1898,107 +1902,137 @@ class TestRunZoneOcr: ) assert response.status_code == status.HTTP_200_OK - assert response.json()["status"] == "completed" - mock_extract.assert_called_once_with(document, templates=[template]) + assert response.json() == {"result": "OK", "task_id": TASK_ID} + mock_enqueue.assert_called_once_with( + kwargs={ + "operation": "run_zone_ocr", + "document_ids": [document.pk], + "parameters": {"template": template.pk, "resolve_latest": False}, + "user_id": admin_user.pk, + "trigger_source": PaperlessTask.TriggerSource.API_UPLOAD, + }, + headers={"trigger_source": PaperlessTask.TriggerSource.API_UPLOAD}, + ) @pytest.mark.parametrize( - ("reason", "finishes"), + ("from_webui", "trigger_source"), [ - pytest.param(None, True, id="written"), - pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"), + pytest.param(True, PaperlessTask.TriggerSource.WEB_UI, id="web-ui"), + pytest.param(False, PaperlessTask.TriggerSource.API_UPLOAD, id="api"), ], ) - def test_finishes_the_edit_only_when_a_value_was_written( + def test_queues_every_matching_template_when_none_is_chosen( self, admin_client: APIClient, - mock_extract: MagicMock, - mock_finish: MagicMock, - reason: ZoneOcrReason | None, - finishes: bool, # noqa: FBT001 + mock_enqueue: MagicMock, + from_webui: bool, # noqa: FBT001 + trigger_source: PaperlessTask.TriggerSource, ) -> None: """ GIVEN: - - A document with a version and a template whose run does or does - not write a value + - A document with a matching template and a stored file WHEN: - - Zone OCR is run on the version through the detail endpoint + - Zone OCR is run on the document without choosing a template, + with or without the web UI flag THEN: - - The shared document update task is queued once for the root - document only when a value was written - - No per-document search reindex is requested directly + - The bulk operation is queued with no template, so the task + picks the matching templates, and with exact document processing + - The trigger source follows the web UI flag """ - document = DocumentFactory(content=CONTENT) - version = DocumentFactory( - root_document=document, - version_index=1, - content=CONTENT, - ) - version.source_path.write_bytes(b"test PDF") - template = OcrTemplateFactory(name="Northwind", match="Northwind") - mock_extract.return_value = [ - ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason), - ] + document = DocumentFactory(content=CONTENT, mime_type="application/pdf") + document.source_path.write_bytes(b"test PDF") + OcrTemplateFactory(name="Northwind", match="Northwind") response = admin_client.post( - f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={version.pk}", - {"template": template.pk}, + RUN_ZONE_OCR_URL.format(pk=document.pk), + {"from_webui": from_webui}, format="json", ) assert response.status_code == status.HTTP_200_OK - if finishes: - mock_finish.assert_called_once_with( - kwargs={"document_ids": [document.pk]}, - headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM}, - ) - else: - mock_finish.assert_not_called() + kwargs = mock_enqueue.call_args.kwargs + assert kwargs["kwargs"]["document_ids"] == [document.pk] + assert kwargs["kwargs"]["parameters"] == {"resolve_latest": False} + assert kwargs["kwargs"]["trigger_source"] == trigger_source + assert kwargs["headers"] == {"trigger_source": trigger_source} @pytest.mark.parametrize( - "version", + ("version", "queued"), [ - pytest.param("root", id="root-version"), - pytest.param("older", id="older-version"), + pytest.param("older", True, id="requested-older-version"), + pytest.param(None, False, id="latest-version-by-default"), ], ) - def test_matches_the_requested_file_version( + def test_matches_and_queues_the_resolved_file_version( self, admin_client: APIClient, - mock_extract: MagicMock, - version: str, + mock_enqueue: MagicMock, + version: str | None, + queued: bool, # noqa: FBT001 ) -> None: """ GIVEN: - - A document with two versions, the newest not matching the template + - A document with an older version matching the template and a + newest version that does not WHEN: - - Zone OCR is run for the root document or an older version + - Zone OCR is run for the older version, or without a version THEN: - - The template is applied to the requested file document + - Templates are matched against the resolved file + - The older version is queued by its own id when requested + - Without a version the latest does not match, so the run is skipped """ - mock_extract.return_value = [] - document = DocumentFactory(content=CONTENT) + document = DocumentFactory(content="Contoso") older = DocumentFactory( root_document=document, version_index=1, content=CONTENT, ) DocumentFactory(root_document=document, version_index=2, content="Contoso") - template = OcrTemplateFactory( - name="Northwind", - match="Northwind", - source_width=100, - source_height=100, - ) - file_document = {"root": document, "older": older}[version] - file_document.source_path.write_bytes(b"test PDF") + older.source_path.write_bytes(b"test PDF") + OcrTemplateFactory(name="Northwind", match="Northwind") + url = RUN_ZONE_OCR_URL.format(pk=document.pk) + if version is not None: + url = f"{url}?version={older.pk}" + + response = admin_client.post(url) + + assert response.status_code == status.HTTP_200_OK + if queued: + assert response.json() == {"result": "OK", "task_id": TASK_ID} + assert mock_enqueue.call_args.kwargs["kwargs"]["document_ids"] == [ + older.pk, + ] + else: + assert response.json() == SKIPPED_NO_TEMPLATE + mock_enqueue.assert_not_called() + + def test_queues_the_root_file_exactly_when_requested( + self, + admin_client: APIClient, + mock_enqueue: MagicMock, + ) -> None: + """ + GIVEN: + - A document with a newer version and a stored original file + WHEN: + - Zone OCR is run with the root document id as the version + THEN: + - The root id is queued with resolve_latest off, so the original + file is processed instead of the newest version + """ + document = DocumentFactory(content=CONTENT) + DocumentFactory(root_document=document, version_index=1, content="Contoso") + document.source_path.write_bytes(b"test PDF") + OcrTemplateFactory(name="Northwind", match="Northwind") response = admin_client.post( - f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={file_document.pk}", + f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={document.pk}", ) assert response.status_code == status.HTTP_200_OK - assert response.json()["status"] == "completed" - mock_extract.assert_called_once_with(file_document, templates=[template]) + queued = mock_enqueue.call_args.kwargs["kwargs"] + assert queued["document_ids"] == [document.pk] + assert queued["parameters"] == {"resolve_latest": False} @pytest.mark.parametrize( "version", @@ -2011,7 +2045,7 @@ class TestRunZoneOcr: def test_rejects_invalid_or_unavailable_versions( self, admin_client: APIClient, - mock_extract: MagicMock, + mock_enqueue: MagicMock, version: str, ) -> None: """ @@ -2020,7 +2054,7 @@ class TestRunZoneOcr: WHEN: - Zone OCR is run with an invalid or unavailable version THEN: - - 404 is returned and nothing is extracted + - 404 is returned and nothing is queued """ document = DocumentFactory() unrelated = DocumentFactory() @@ -2040,16 +2074,20 @@ class TestRunZoneOcr: ) assert response.status_code == status.HTTP_404_NOT_FOUND - mock_extract.assert_not_called() + mock_enqueue.assert_not_called() - def test_rejects_an_unknown_template(self, admin_client: APIClient) -> None: + def test_rejects_an_unknown_template( + self, + admin_client: APIClient, + mock_enqueue: MagicMock, + ) -> None: """ GIVEN: - A document WHEN: - Zone OCR is run with a template id that does not exist THEN: - - 400 is returned + - 400 is returned and nothing is queued """ document = DocumentFactory(content=CONTENT) @@ -2060,24 +2098,30 @@ class TestRunZoneOcr: ) assert response.status_code == status.HTTP_400_BAD_REQUEST + mock_enqueue.assert_not_called() - def test_missing_document_is_an_error(self, admin_client: APIClient) -> None: + def test_missing_document_is_an_error( + self, + admin_client: APIClient, + mock_enqueue: MagicMock, + ) -> None: """ GIVEN: - No document with the requested id WHEN: - Zone OCR is run on it THEN: - - 404 is returned + - 404 is returned and nothing is queued """ response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=99999)) assert response.status_code == status.HTTP_404_NOT_FOUND + mock_enqueue.assert_not_called() def test_missing_file_is_an_error( self, admin_client: APIClient, - mock_extract: MagicMock, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: @@ -2085,7 +2129,7 @@ class TestRunZoneOcr: WHEN: - Zone OCR is run on the document THEN: - - 404 is returned with a file not found error and nothing is extracted + - 404 is returned with a file not found error and nothing is queued """ document = DocumentFactory(content=CONTENT) OcrTemplateFactory( @@ -2097,171 +2141,34 @@ class TestRunZoneOcr: assert response.status_code == status.HTTP_404_NOT_FOUND assert response.json() == {"error": "Document file not found"} - mock_extract.assert_not_called() + mock_enqueue.assert_not_called() - def test_reports_completed_results( + def test_queueing_failure_is_a_bad_request( self, admin_client: APIClient, - mock_zone_ocr_run: MagicMock, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: - - A document and a matching template with a title zone + - A document with a matching template and a stored file + - A task queue that raises when the task is queued WHEN: - Zone OCR is run on the document THEN: - - The completed result lists the extracted title value + - 400 is returned """ - document = DocumentFactory( - content=CONTENT, - mime_type="application/pdf", - ) - write_blank_png(document.source_path) - mock_zone_ocr_run.return_value.stdout = "Extracted title" - template = OcrTemplateFactory( - name="Template", - match="Northwind", - ) - OcrTemplateZoneFactory( - template=template, - name="Title zone", - height=100, - ) + document = DocumentFactory(content=CONTENT, mime_type="application/pdf") + document.source_path.write_bytes(b"test PDF") + OcrTemplateFactory(name="Northwind", match="Northwind") + mock_enqueue.side_effect = RuntimeError("broker down") response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk)) - assert response.status_code == status.HTTP_200_OK - assert response.json() == { - "status": "completed", - "results": [ - { - "template": template.name, - "zone": "Title zone", - "custom_field": "Title", - "value": "Extracted title", - "reason": None, - }, - ], - } - mock_zone_ocr_run.assert_called_once() - - def test_does_not_report_an_old_title_when_extraction_is_empty( - self, - admin_client: APIClient, - mock_zone_ocr_run: MagicMock, - ) -> None: - """ - GIVEN: - - A document with a title and a template whose title zone extracts - nothing - WHEN: - - Zone OCR is run on the document - THEN: - - The result has reason no_value and no value - - The document title is unchanged - """ - document = DocumentFactory(title="Existing title", content=CONTENT) - write_blank_png(document.source_path) - template = OcrTemplateFactory( - name="Template", - match="Northwind", - source_width=100, - source_height=100, - ) - OcrTemplateZoneFactory( - template=template, - name="Title", - width=20, - height=20, - ) - mock_zone_ocr_run.return_value.stdout = "" - - response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk)) - - assert response.status_code == status.HTTP_200_OK - assert response.json()["results"][0]["reason"] == ZoneOcrReason.NO_VALUE - assert response.json()["results"][0]["value"] is None - document.refresh_from_db() - assert document.title == "Existing title" - - def test_reports_all_target_types_and_missing_values( - self, - admin_client: APIClient, - mock_zone_ocr_run: MagicMock, - text_field: CustomField, - date_field: CustomField, - ) -> None: - """ - GIVEN: - - A document with an existing custom field value and ASN - - A template with zones for every target type, some extracting nothing - WHEN: - - Zone OCR is run with that template chosen - THEN: - - Each zone reports the value for its target, or None when missing - """ - document = DocumentFactory(content=CONTENT, archive_serial_number=123) - write_blank_png(document.source_path) - mock_zone_ocr_run.side_effect = [ - subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="") - for value in [ - "New invoice", - "", - "orphan", - "New title", - "456", - "2026-04-13", - ] - ] - template = OcrTemplateFactory( - name="Test", - source_width=100, - source_height=100, - ) - CustomFieldInstance.objects.create( - document=document, - field=text_field, - value_text="INV-123", - ) - for name, target, field in [ - ("Present", "custom_field", text_field), - ("Missing", "custom_field", date_field), - ("Unassigned", "custom_field", None), - ("Title", "title", None), - ("ASN", "asn", None), - ("Created", "created", None), - ]: - OcrTemplateZoneFactory( - template=template, - name=name, - target=target, - custom_field=field, - width=10, - height=10, - ) - - response = admin_client.post( - RUN_ZONE_OCR_URL.format(pk=document.pk), - {"template": template.pk}, - format="json", - ) - - assert response.status_code == status.HTTP_200_OK - values = { - result["zone"]: result["value"] for result in response.json()["results"] - } - assert values == { - "Present": "New invoice", - "Missing": None, - "Unassigned": None, - "Title": "New title", - "ASN": 456, - "Created": "2026-04-13", - } + assert response.status_code == status.HTTP_400_BAD_REQUEST @pytest.mark.django_db -@pytest.mark.usefixtures("paperless_dirs", "mock_finish") +@pytest.mark.usefixtures("paperless_dirs") class TestRunZoneOcrPermissions: """Permission checks for the run-zone-ocr endpoint.""" @@ -2269,7 +2176,7 @@ class TestRunZoneOcrPermissions: self, regular_user: User, user_client: APIClient, - mock_extract: MagicMock, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: @@ -2279,7 +2186,7 @@ class TestRunZoneOcrPermissions: - The editor runs zone OCR choosing that template THEN: - 400 is returned, naming the template field, and nothing is - extracted + queued """ grant_global(regular_user, "view_document", "change_document") document = DocumentFactory(content=CONTENT, owner=regular_user) @@ -2296,13 +2203,13 @@ class TestRunZoneOcrPermissions: assert response.status_code == status.HTTP_400_BAD_REQUEST assert "template" in response.json() - mock_extract.assert_not_called() + mock_enqueue.assert_not_called() def test_requires_change_permission_on_the_document( self, regular_user: User, user_client: APIClient, - mock_extract: MagicMock, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: @@ -2311,7 +2218,7 @@ class TestRunZoneOcrPermissions: WHEN: - The user runs zone OCR on the document THEN: - - 403 is returned and nothing is extracted + - 403 is returned and nothing is queued """ owner = UserFactory() grant_global(regular_user, "view_document", "change_document") @@ -2320,13 +2227,48 @@ class TestRunZoneOcrPermissions: response = user_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk)) assert response.status_code == status.HTTP_403_FORBIDDEN - mock_extract.assert_not_called() + mock_enqueue.assert_not_called() + + def test_a_version_path_id_is_authorized_through_its_root( + self, + regular_user: User, + user_client: APIClient, + mock_enqueue: MagicMock, + ) -> None: + """ + GIVEN: + - A root document owned by another user and shared for editing + with the user, and one of its versions + WHEN: + - The user runs zone OCR using the version id in the path + THEN: + - The request is authorized through the root and the version is + queued + """ + owner = UserFactory() + grant_global(regular_user, "view_document", "change_document") + root = DocumentFactory(content=CONTENT, owner=owner) + version = DocumentFactory( + root_document=root, + version_index=1, + content=CONTENT, + ) + version.source_path.write_bytes(b"test PDF") + grant_object(regular_user, root, "view_document", "change_document") + OcrTemplateFactory(name="Northwind", match="Northwind", owner=None) + + response = user_client.post(RUN_ZONE_OCR_URL.format(pk=version.pk)) + + assert response.status_code == status.HTTP_200_OK + assert mock_enqueue.call_args.kwargs["kwargs"]["document_ids"] == [ + version.pk, + ] def test_requires_global_change_permission( self, regular_user: User, user_client: APIClient, - mock_extract: MagicMock, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: @@ -2335,7 +2277,7 @@ class TestRunZoneOcrPermissions: WHEN: - The user runs zone OCR on the document THEN: - - 403 is returned and nothing is extracted + - 403 is returned and nothing is queued """ grant_global(regular_user, "view_document", "add_document") document = DocumentFactory(content=CONTENT, owner=regular_user) @@ -2343,13 +2285,13 @@ class TestRunZoneOcrPermissions: response = user_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk)) assert response.status_code == status.HTTP_403_FORBIDDEN - mock_extract.assert_not_called() + mock_enqueue.assert_not_called() def test_does_not_require_document_upload_permission( self, regular_user: User, user_client: APIClient, - mock_extract: MagicMock, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: @@ -2358,7 +2300,7 @@ class TestRunZoneOcrPermissions: WHEN: - The user runs zone OCR on the document THEN: - - The request succeeds and extraction runs + - The request succeeds and the run is queued as that user """ grant_global(regular_user, "change_document") document = DocumentFactory(owner=regular_user, content=CONTENT) @@ -2374,13 +2316,14 @@ class TestRunZoneOcrPermissions: response = user_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk)) assert response.status_code == status.HTTP_200_OK - mock_extract.assert_called_once() + mock_enqueue.assert_called_once() + assert mock_enqueue.call_args.kwargs["kwargs"]["user_id"] == regular_user.pk def test_skips_templates_private_to_the_requester( self, admin_client: APIClient, admin_user: User, - mock_extract: MagicMock, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: @@ -2389,7 +2332,7 @@ class TestRunZoneOcrPermissions: WHEN: - The requester runs zone OCR on the document THEN: - - The run is skipped with reason no_template and nothing is extracted + - The run is skipped with reason no_template and nothing is queued """ owner = UserFactory() document = DocumentFactory(content=CONTENT, owner=owner) @@ -2405,96 +2348,34 @@ class TestRunZoneOcrPermissions: assert response.status_code == status.HTTP_200_OK assert response.json() == SKIPPED_NO_TEMPLATE - mock_extract.assert_not_called() + mock_enqueue.assert_not_called() - def test_reports_only_templates_visible_to_the_document_owner( - self, - admin_client: APIClient, - admin_user: User, - mock_zone_ocr_run: MagicMock, - ) -> None: - """ - GIVEN: - - A document owned by another user - - Templates owned by the document owner, unowned, shared with the - document owner, and private to the requester - WHEN: - - The requester runs zone OCR on the document - THEN: - - Only templates visible to the document owner run and are reported - """ - owner = UserFactory() - document = DocumentFactory(content=CONTENT, owner=owner) - write_blank_png(document.source_path) - mock_zone_ocr_run.return_value.stdout = "extracted" - for name, template_owner in [ - ("Owned", owner), - ("Unowned", None), - ("Shared", admin_user), - ("Private", admin_user), - ]: - template = title_template( - name, - template_owner, - source_width=100, - source_height=100, - ) - if name == "Shared": - grant_object(owner, template, "view_ocrtemplate") - - response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk)) - - assert response.status_code == status.HTTP_200_OK - assert sorted(result["template"] for result in response.json()["results"]) == [ - "Owned", - "Shared", - "Unowned", - ] - assert mock_zone_ocr_run.call_count == 3 - - def test_hides_the_owners_private_templates_from_an_editor( + def test_skips_when_only_the_owners_private_templates_match( self, regular_user: User, user_client: APIClient, - mock_zone_ocr_run: MagicMock, + mock_enqueue: MagicMock, ) -> None: """ GIVEN: - A document shared for editing with another user - - Templates private to the owner, unowned, and shared with both + - A matching template private to the document owner WHEN: - The editor runs zone OCR without choosing a template THEN: - - Only templates the editor can also view run and are reported + - The run is skipped with reason no_template and nothing is queued """ owner = UserFactory() grant_global(regular_user, "view_document", "change_document") document = DocumentFactory(content=CONTENT, owner=owner) grant_object(regular_user, document, "view_document", "change_document") - write_blank_png(document.source_path) - mock_zone_ocr_run.return_value.stdout = "extracted" - for name, template_owner in [ - ("Owner private", owner), - ("Unowned", None), - ("Shared", owner), - ]: - template = title_template( - name, - template_owner, - source_width=100, - source_height=100, - ) - if name == "Shared": - grant_object(regular_user, template, "view_ocrtemplate") + title_template("Owner private", owner, source_width=100, source_height=100) response = user_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk)) assert response.status_code == status.HTTP_200_OK - assert sorted(result["template"] for result in response.json()["results"]) == [ - "Shared", - "Unowned", - ] - assert mock_zone_ocr_run.call_count == 2 + assert response.json() == SKIPPED_NO_TEMPLATE + mock_enqueue.assert_not_called() @pytest.mark.django_db diff --git a/src/documents/tests/test_bulk_edit.py b/src/documents/tests/test_bulk_edit.py index fe53789b93..8bff7c359f 100644 --- a/src/documents/tests/test_bulk_edit.py +++ b/src/documents/tests/test_bulk_edit.py @@ -1966,7 +1966,12 @@ class TestBulkEditRunZoneOcr: assert result == "OK" assert mock_extract_zones.call_args_list == [ - mocker.call(document_id=doc.id, template_id=None, requester=user) + mocker.call( + document_id=doc.id, + template_id=None, + requester=user, + resolve_latest=True, + ) for doc in docs ] mock_extract_zones.apply_async.assert_not_called() @@ -1993,6 +1998,7 @@ class TestBulkEditRunZoneOcr: document_id=doc.id, template_id=template.pk, requester=None, + resolve_latest=True, ) mock_extract_zones.apply_async.assert_not_called() diff --git a/src/documents/tests/test_bulk_operations.py b/src/documents/tests/test_bulk_operations.py index 1cd9af4884..343d79ac75 100644 --- a/src/documents/tests/test_bulk_operations.py +++ b/src/documents/tests/test_bulk_operations.py @@ -154,6 +154,7 @@ class TestRunZoneOcrBulkOperation: document_id=document.pk, template_id=template.pk, requester=admin_user, + resolve_latest=True, ) extract.apply_async.assert_not_called() assert result == {"operation": "run_zone_ocr", "document_count": 1} @@ -206,6 +207,7 @@ class TestRunZoneOcrBulkOperation: document_id=document.pk, template_id=None, requester=admin_user, + resolve_latest=True, ) @pytest.mark.parametrize( diff --git a/src/documents/tests/test_tasks.py b/src/documents/tests/test_tasks.py index ff51fdfaaa..6a7c5ae59b 100644 --- a/src/documents/tests/test_tasks.py +++ b/src/documents/tests/test_tasks.py @@ -732,6 +732,69 @@ class TestExtractZonesTask: assert result == (root.pk if returns_root else None) + def test_task_runs_a_version_and_reports_the_root( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - A root document with two versions and a template + WHEN: + - The task runs for the older version id + THEN: + - Extraction runs on that version, not the newest + - The root document id is returned when a value was written + """ + root = DocumentFactory() + older = DocumentFactory(root_document=root, version_index=1) + DocumentFactory(root_document=root, version_index=2) + template = OcrTemplateFactory() + extract = mocker.patch( + "documents.tasks.run_zone_extraction", + return_value=[ + ZoneOcrResult(template="t", zone="z", custom_field="", reason=None), + ], + ) + + result = extract_zones(older.pk, template.pk) + + extract.assert_called_once_with(older, templates=[template]) + assert result == root.pk + + @pytest.mark.parametrize( + ("resolve_latest", "processes_root"), + [ + pytest.param(True, False, id="resolves-latest"), + pytest.param(False, True, id="exact-document"), + ], + ) + def test_task_resolves_latest_only_when_asked( + self, + mocker: MockerFixture, + resolve_latest: bool, # noqa: FBT001 + processes_root: bool, # noqa: FBT001 + ) -> None: + """ + GIVEN: + - A root document with a newer version and a template + WHEN: + - The task runs for the root id with resolve_latest on or off + THEN: + - The newest version is processed when resolve_latest is on + - The root document itself is processed when it is off + """ + root = DocumentFactory() + newest = DocumentFactory(root_document=root, version_index=1) + template = OcrTemplateFactory() + extract = mocker.patch("documents.tasks.run_zone_extraction", return_value=[]) + + extract_zones(root.pk, template.pk, resolve_latest=resolve_latest) + + extract.assert_called_once_with( + root if processes_root else newest, + templates=[template], + ) + def test_task_skips_a_deleted_template(self, mocker: MockerFixture) -> None: """ GIVEN: diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index 8e87ac2dbc..e5944fa88b 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -5,7 +5,6 @@ from __future__ import annotations import shutil import subprocess from collections import Counter -from dataclasses import asdict from datetime import date from decimal import Decimal from pathlib import Path @@ -27,14 +26,10 @@ from documents.models import OcrTemplateZone from documents.parsers import ParseError from documents.serialisers import OcrTemplateTestResultSerializer from documents.serialisers import RunZoneOcrResponseSerializer -from documents.serialisers import ZoneOcrResultSerializer from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH from documents.zone_ocr import ZoneExtraction from documents.zone_ocr import ZoneOcrReason -from documents.zone_ocr import ZoneOcrResult -from documents.zone_ocr import ZoneOcrRunStatus from documents.zone_ocr import ZoneOcrSkipReason -from documents.zone_ocr import ZoneValue from documents.zone_ocr import _apply_transform from documents.zone_ocr import _combine_field_value from documents.zone_ocr import _convert_value @@ -3959,82 +3954,24 @@ class TestRealZoneOcr: assert "This is a test document" in document.title -class TestZoneOcrResultSerializer: - @pytest.mark.parametrize( - "value", - [ - pytest.param("text", id="string"), - pytest.param(42, id="int"), - pytest.param(1.5, id="float"), - pytest.param(True, id="bool"), - pytest.param(date(2025, 12, 25), id="date"), - pytest.param(None, id="none"), - ], - ) - @pytest.mark.parametrize( - "reason", - [pytest.param(None, id="no-reason")] - + [pytest.param(reason, id=reason.value) for reason in ZoneOcrReason], - ) - def test_wire_format_matches_dataclass( - self, - value: ZoneValue | None, - reason: ZoneOcrReason | None, - ) -> None: - """ - GIVEN: - - A zone result with any value type and any reason - WHEN: - - It is serialized with the result serializer and rendered as JSON - THEN: - - The JSON equals the rendering of the dataclass as a dict - """ - result = ZoneOcrResult( - template="Template", - zone="Zone", - custom_field="Title", - value=value, - reason=reason, - ) - - assert JSONRenderer().render( - ZoneOcrResultSerializer(result).data, - ) == JSONRenderer().render(asdict(result)) - - class TestRunZoneOcrResponseSerializer: @pytest.mark.parametrize( "body", [ pytest.param( - { - "status": ZoneOcrRunStatus.SKIPPED, - "reason": ZoneOcrSkipReason.NO_TEMPLATE, - "results": [], - }, + {"result": "skipped", "reason": ZoneOcrSkipReason.NO_TEMPLATE}, id="skipped", ), pytest.param( - { - "status": ZoneOcrRunStatus.COMPLETED, - "results": [ - { - "template": "Template", - "zone": "Zone", - "custom_field": "Title", - "value": "text", - "reason": None, - }, - ], - }, - id="completed", + {"result": "OK", "task_id": "abc-123"}, + id="queued", ), ], ) def test_renders_body_unchanged(self, body: dict[str, Any]) -> None: """ GIVEN: - - A response body with the run status and results fields + - A response body for a skipped or a queued run WHEN: - The body is rendered through the response serializer as JSON THEN: diff --git a/src/documents/views.py b/src/documents/views.py index 9626806501..0d71f41840 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -248,7 +248,6 @@ from documents.serialisers import UiSettingsViewSerializer from documents.serialisers import WorkflowActionSerializer from documents.serialisers import WorkflowSerializer from documents.serialisers import WorkflowTriggerSerializer -from documents.serialisers import ZoneOcrResultSerializer from documents.signals import document_updated from documents.tasks import build_share_link_bundle from documents.tasks import consume_file @@ -2300,8 +2299,11 @@ class DocumentViewSet( ) def run_zone_ocr(self, request: Request, pk: str | None = None) -> HttpResponse: """ - Run zone-based OCR extraction on this document, with the given template - or with every template that matches it. + Queue zone-based OCR extraction on this document, with the given + template or with every template that matches it. The file version + given by the version parameter is run when it is not the root + document. Responds with the task id, or skipped when no template + applies. """ serializer = RunZoneOcrSerializer( data=request.data, @@ -2320,11 +2322,8 @@ class DocumentViewSet( ) or not has_perms_owner_aware(user, "change_document", document): return HttpResponseForbidden("Insufficient permissions") - from documents.zone_ocr import ZoneOcrRunStatus from documents.zone_ocr import ZoneOcrSkipReason from documents.zone_ocr import get_templates_for_document - from documents.zone_ocr import run_zone_extraction - from documents.zone_ocr import zone_results_written resolution = resolve_effective_document(document, request) if resolution.error == VersionResolutionError.INVALID: @@ -2333,17 +2332,14 @@ class DocumentViewSet( if file_document is None: raise Http404 - templates = ( - [template] - if template is not None - else get_templates_for_document(file_document, requester=user) - ) - if not templates: + if template is None and not get_templates_for_document( + file_document, + requester=user, + ): return Response( { - "status": ZoneOcrRunStatus.SKIPPED, + "result": "skipped", "reason": ZoneOcrSkipReason.NO_TEMPLATE, - "results": [], }, ) @@ -2354,18 +2350,22 @@ class DocumentViewSet( status=status.HTTP_404_NOT_FOUND, ) - results = run_zone_extraction(file_document, templates=templates) - if zone_results_written(results): - bulk_edit.bulk_update_documents.apply_async( - kwargs={"document_ids": [get_root_document(file_document).pk]}, - headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM}, + parameters: dict[str, Any] = {"resolve_latest": False} + if template is not None: + parameters["template"] = template.pk + try: + return queue_bulk_operation( + user, + bulk_edit.run_zone_ocr, + [file_document.pk], + parameters, + from_webui=serializer.validated_data["from_webui"], + ) + except Exception as e: + logger.warning(f"An error occurred performing document zone OCR: {e!s}") + return HttpResponseBadRequest( + "Error performing document zone OCR, check logs for more detail.", ) - return Response( - { - "status": ZoneOcrRunStatus.COMPLETED, - "results": ZoneOcrResultSerializer(results, many=True).data, - }, - ) @extend_schema( operation_id="documents_delete_version", @@ -3101,35 +3101,56 @@ class DocumentSelectionMixin: return list(filtered_documents.values_list("pk", flat=True)) +def queue_bulk_operation( + user: User, + method: Callable[..., Any], + documents: list[int], + parameters: dict[str, Any], + *, + from_webui: bool, +) -> Response: + """ + Validate the parameters against the bulk edit method, queue it as a tracked + bulk operation task and return the task id. + """ + if "update_callback" in parameters: + raise ValueError("Unsupported bulk operation parameter: update_callback") + trigger_source = ( + PaperlessTask.TriggerSource.WEB_UI + if from_webui + else PaperlessTask.TriggerSource.API_UPLOAD + ) + final_parameters = parameters.copy() + if method.__name__ in USER_OPERATIONS: + final_parameters["user"] = user + if method.__name__ in TRIGGER_SOURCE_OPERATIONS: + final_parameters["trigger_source"] = trigger_source + signature(method).bind(documents, **final_parameters) + task = execute_bulk_operation.apply_async( + kwargs={ + "operation": method.__name__, + "document_ids": documents, + "parameters": parameters, + "user_id": user.pk, + "trigger_source": trigger_source, + }, + headers={"trigger_source": trigger_source}, + ) + return Response({"result": "OK", "task_id": task.id}) + + class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin): permission_classes = (IsAuthenticated,) parser_classes = (parsers.JSONParser,) def _queue_bulk_operation(self, method, documents, parameters, from_webui): - if "update_callback" in parameters: - raise ValueError("Unsupported bulk operation parameter: update_callback") - trigger_source = ( - PaperlessTask.TriggerSource.WEB_UI - if from_webui - else PaperlessTask.TriggerSource.API_UPLOAD + return queue_bulk_operation( + cast("User", self.request.user), + method, + documents, + parameters, + from_webui=from_webui, ) - final_parameters = parameters.copy() - if method.__name__ in USER_OPERATIONS: - final_parameters["user"] = self.request.user - if method.__name__ in TRIGGER_SOURCE_OPERATIONS: - final_parameters["trigger_source"] = trigger_source - signature(method).bind(documents, **final_parameters) - task = execute_bulk_operation.apply_async( - kwargs={ - "operation": method.__name__, - "document_ids": documents, - "parameters": parameters, - "user_id": self.request.user.pk, - "trigger_source": trigger_source, - }, - headers={"trigger_source": trigger_source}, - ) - return Response({"result": "OK", "task_id": task.id}) def _has_document_permissions( self, diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index c7a830e340..280684f687 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -125,15 +125,8 @@ class ZoneOcrReason(StrEnum): TEMPLATE_FAILED = "template_failed" -class ZoneOcrRunStatus(StrEnum): - """The overall outcome of a zone OCR run on one document.""" - - COMPLETED = "completed" - SKIPPED = "skipped" - - class ZoneOcrSkipReason(StrEnum): - """Why a zone OCR run on one document was skipped.""" + """Why a zone OCR run on one document was not queued.""" NO_TEMPLATE = "no_template"