From 54be9dc9c5395ea7872a568ca96553a0df443b58 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:18:30 -0800 Subject: [PATCH] Fix: finish manual and bulk zone OCR runs like other bulk edits Runs reindexed each document separately, never sent document_updated and did not bump modified for custom field writes. They now finish through bulk_update_documents for the documents that got a value. --- src/documents/bulk_edit.py | 21 +++- src/documents/tasks.py | 14 ++- src/documents/tests/test_api_ocr_templates.py | 87 +++++++++++++--- src/documents/tests/test_bulk_edit.py | 67 ++++++++++++- src/documents/tests/test_bulk_operations.py | 49 ++++++++- src/documents/tests/test_tasks.py | 45 +++++++-- src/documents/tests/test_zone_ocr.py | 99 +++++++++++++++++++ src/documents/views.py | 8 +- src/documents/zone_ocr.py | 23 ++--- 9 files changed, 368 insertions(+), 45 deletions(-) diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py index 7b605a9f0e..77644b1475 100644 --- a/src/documents/bulk_edit.py +++ b/src/documents/bulk_edit.py @@ -1264,8 +1264,23 @@ def run_zone_ocr( template: int | None = None, user: User | None = None, ) -> Literal["OK"]: - """Run zone OCR within the tracked bulk operation task.""" - for document_id in doc_ids: - extract_zones(document_id=document_id, template_id=template, requester=user) + """ + Run zone OCR within the tracked bulk operation task, then finish once for + the root documents that had a value written. + """ + written_ids: list[int] = [] + try: + for document_id in doc_ids: + written_id = extract_zones( + document_id=document_id, + template_id=template, + requester=user, + ) + if written_id is not None: + written_ids.append(written_id) + finally: + # Documents already written are finished even if a later one raises. + if written_ids: + bulk_update_documents(list(dict.fromkeys(written_ids))) return "OK" diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 5bed784787..85a425aea8 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -72,10 +72,12 @@ from documents.utils import IterWrapper from documents.utils import compute_checksum from documents.utils import identity from documents.versioning import annotate_effective_content +from documents.versioning import get_root_document from documents.versioning import latest_version from documents.workflows.utils import get_workflows_for_trigger from documents.zone_ocr import get_templates_for_document from documents.zone_ocr import run_zone_extraction +from documents.zone_ocr import zone_results_written from paperless.config import AIConfig from paperless.config import BarcodeConfig from paperless.config import RemoteOCRConfig @@ -379,11 +381,14 @@ def extract_zones( document_id: int, template_id: int | None = None, requester: User | None = None, -) -> None: +) -> int | None: """ Run an OCR template against a stored document, or every template that matches it when no template is given. Matched templates are limited to those the requester can view. + + Returns the root document's id when any value was written, otherwise None. + The caller finishes the edit, for example with bulk_update_documents. """ document = latest_version(Document.objects.get(id=document_id)) if template_id is None: @@ -399,10 +404,13 @@ def extract_zones( "OCR template %s no longer exists, not running zone OCR", template_id, ) - return + return None templates = [template] - run_zone_extraction(document, templates=templates, reindex=True) + results = run_zone_extraction(document, templates=templates) + if zone_results_written(results): + return get_root_document(document).pk + return None @shared_task diff --git a/src/documents/tests/test_api_ocr_templates.py b/src/documents/tests/test_api_ocr_templates.py index 0a681adba6..c41b6dead6 100644 --- a/src/documents/tests/test_api_ocr_templates.py +++ b/src/documents/tests/test_api_ocr_templates.py @@ -14,18 +14,21 @@ from django.utils import timezone from PIL import Image from rest_framework import status +from documents import bulk_edit from documents.models import CustomField from documents.models import CustomFieldInstance from documents.models import Document from documents.models import MatchingModel from documents.models import OcrTemplate from documents.models import OcrTemplateZone +from documents.models import PaperlessTask from documents.parsers import ParseError from documents.search import get_backend from documents.signals import document_consumption_finished -from documents.tasks import extract_zones +from documents.tasks import bulk_update_documents from documents.zone_ocr import ZoneExtraction from documents.zone_ocr import ZoneOcrReason +from documents.zone_ocr import ZoneOcrResult from paperless_testing.factories import CustomFieldFactory from paperless_testing.factories import DocumentFactory from paperless_testing.factories import OcrTemplateFactory @@ -148,6 +151,13 @@ def mock_extract(mocker: MockerFixture) -> MagicMock: return mocker.patch("documents.zone_ocr.run_zone_extraction") +@pytest.fixture +def mock_finish(mocker: MockerFixture) -> MagicMock: + return mocker.patch( + "documents.views.bulk_edit.bulk_update_documents.apply_async", + ) + + @pytest.fixture def mock_preview(mocker: MockerFixture) -> MagicMock: return mocker.patch("documents.zone_ocr.extract_zone_preview") @@ -232,6 +242,7 @@ class TestZoneOcrIndexing: admin_client: APIClient, django_capture_on_commit_callbacks: DjangoCaptureOnCommitCallbacks, mock_zone_ocr_run: MagicMock, + mocker: MockerFixture, ) -> None: """ GIVEN: @@ -305,6 +316,11 @@ class TestZoneOcrIndexing: backend.add_or_update(document) assert backend.search_ids("title:before", user=None) == [document.pk] + # The single-document finish is a queued task, run it inline here. + mocker.patch( + "documents.views.bulk_edit.bulk_update_documents.apply_async", + side_effect=lambda kwargs, **_: bulk_update_documents(**kwargs), + ) with django_capture_on_commit_callbacks(execute=True): if entry_point == "ingestion": document_consumption_finished.send(sender=None, document=file_document) @@ -314,7 +330,7 @@ class TestZoneOcrIndexing: ) assert response.status_code == status.HTTP_200_OK else: - extract_zones(document.pk, template.pk) + bulk_edit.run_zone_ocr([document.pk], template=template.pk) document.refresh_from_db() assert document.title == "After OCR" @@ -1801,7 +1817,7 @@ class TestOcrTemplateDelete: @pytest.mark.django_db -@pytest.mark.usefixtures("paperless_dirs") +@pytest.mark.usefixtures("paperless_dirs", "mock_finish") class TestRunZoneOcr: """Running zone OCR on documents through the run-zone-ocr endpoint.""" @@ -1873,11 +1889,60 @@ class TestRunZoneOcr: assert response.status_code == status.HTTP_200_OK assert response.json()["status"] == "completed" - mock_extract.assert_called_once_with( - document, - templates=[template], - reindex=True, + mock_extract.assert_called_once_with(document, templates=[template]) + + @pytest.mark.parametrize( + ("reason", "finishes"), + [ + pytest.param(None, True, id="written"), + pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"), + ], + ) + def test_finishes_the_edit_only_when_a_value_was_written( + self, + admin_client: APIClient, + mock_extract: MagicMock, + mock_finish: MagicMock, + reason: ZoneOcrReason | None, + finishes: bool, # noqa: FBT001 + ) -> None: + """ + GIVEN: + - A document with a version and a template whose run does or does + not write a value + WHEN: + - Zone OCR is run on the version through the detail endpoint + THEN: + - The shared document update task is queued once for the root + document only when a value was written + - No per-document search reindex is requested directly + """ + document = DocumentFactory(content=CONTENT) + version = DocumentFactory( + root_document=document, + version_index=1, + content=CONTENT, ) + version.source_path.write_bytes(b"test PDF") + template = OcrTemplateFactory(name="Northwind", match="Northwind") + mock_extract.return_value = [ + ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason), + ] + + response = admin_client.post( + f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={version.pk}", + {"template": template.pk}, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + if finishes: + mock_finish.assert_called_once_with( + kwargs={"document_ids": [document.pk]}, + headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM}, + ) + else: + mock_finish.assert_not_called() @pytest.mark.parametrize( "version", @@ -1923,11 +1988,7 @@ class TestRunZoneOcr: assert response.status_code == status.HTTP_200_OK assert response.json()["status"] == "completed" - mock_extract.assert_called_once_with( - file_document, - templates=[template], - reindex=True, - ) + mock_extract.assert_called_once_with(file_document, templates=[template]) @pytest.mark.parametrize( "version", @@ -2190,7 +2251,7 @@ class TestRunZoneOcr: @pytest.mark.django_db -@pytest.mark.usefixtures("paperless_dirs") +@pytest.mark.usefixtures("paperless_dirs", "mock_finish") class TestRunZoneOcrPermissions: """Permission checks for the run-zone-ocr endpoint.""" diff --git a/src/documents/tests/test_bulk_edit.py b/src/documents/tests/test_bulk_edit.py index 08c3b4a92f..fe53789b93 100644 --- a/src/documents/tests/test_bulk_edit.py +++ b/src/documents/tests/test_bulk_edit.py @@ -1942,7 +1942,7 @@ class TestBulkEditRunZoneOcr: @pytest.fixture def mock_extract_zones(self, mocker: MockerFixture) -> mock.MagicMock: - return mocker.patch("documents.bulk_edit.extract_zones") + return mocker.patch("documents.bulk_edit.extract_zones", return_value=None) def test_runs_every_selected_document_in_the_current_task( self, @@ -1996,6 +1996,71 @@ class TestBulkEditRunZoneOcr: ) mock_extract_zones.apply_async.assert_not_called() + def test_finishes_once_for_documents_with_a_write( + self, + mock_extract_zones: mock.MagicMock, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - Three selected documents, the second of which had nothing written + and the third reporting the same root as the first + WHEN: + - run_zone_ocr is called + THEN: + - bulk_update_documents runs once with the distinct written ids + """ + docs = DocumentFactory.create_batch(3) + finish = mocker.patch("documents.bulk_edit.bulk_update_documents") + mock_extract_zones.side_effect = [docs[0].pk, None, docs[0].pk] + + bulk_edit.run_zone_ocr([doc.pk for doc in docs]) + + finish.assert_called_once_with([docs[0].pk]) + + def test_finishes_written_documents_when_a_later_one_raises( + self, + mock_extract_zones: mock.MagicMock, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - Two selected documents, the second of which raises during extraction + WHEN: + - run_zone_ocr is called + THEN: + - The exception propagates + - bulk_update_documents still runs with the first document's id + """ + docs = DocumentFactory.create_batch(2) + finish = mocker.patch("documents.bulk_edit.bulk_update_documents") + mock_extract_zones.side_effect = [docs[0].pk, RuntimeError("boom")] + + with pytest.raises(RuntimeError, match="boom"): + bulk_edit.run_zone_ocr([doc.pk for doc in docs]) + + finish.assert_called_once_with([docs[0].pk]) + + def test_does_not_finish_when_nothing_was_written( + self, + mock_extract_zones: mock.MagicMock, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - Selected documents with no successful write + WHEN: + - run_zone_ocr is called + THEN: + - bulk_update_documents is not run + """ + docs = DocumentFactory.create_batch(2) + finish = mocker.patch("documents.bulk_edit.bulk_update_documents") + + bulk_edit.run_zone_ocr([doc.pk for doc in docs]) + + finish.assert_not_called() + @pytest.mark.django_db class TestBulkEditReprocess: diff --git a/src/documents/tests/test_bulk_operations.py b/src/documents/tests/test_bulk_operations.py index 20e6926cbc..1cd9af4884 100644 --- a/src/documents/tests/test_bulk_operations.py +++ b/src/documents/tests/test_bulk_operations.py @@ -97,7 +97,10 @@ class TestRunZoneOcrBulkOperation: """ template = OcrTemplateFactory() enqueue = mocker.patch("documents.views.execute_bulk_operation.apply_async") - extract = mocker.patch("documents.bulk_edit.extract_zones") + extract = mocker.patch( + "documents.bulk_edit.extract_zones", + return_value=None, + ) enqueue.return_value.id = "zone-ocr-task" response = admin_client.post( RUN_ZONE_OCR_BULK_URL, @@ -137,7 +140,10 @@ class TestRunZoneOcrBulkOperation: - The result reports the operation and document count """ template = OcrTemplateFactory() - extract = mocker.patch("documents.bulk_edit.extract_zones") + extract = mocker.patch( + "documents.bulk_edit.extract_zones", + return_value=None, + ) result = execute_bulk_operation( "run_zone_ocr", [document.pk], @@ -152,6 +158,30 @@ class TestRunZoneOcrBulkOperation: extract.apply_async.assert_not_called() assert result == {"operation": "run_zone_ocr", "document_count": 1} + def test_finishes_with_the_ids_that_were_written( + self, + mocker: MockerFixture, + admin_user: User, + document: Document, + ) -> None: + """ + GIVEN: + - A queued zone OCR run whose extraction reports a written document + WHEN: + - The tracked task executes it + THEN: + - bulk_update_documents runs once with that document id + """ + mocker.patch( + "documents.bulk_edit.extract_zones", + return_value=document.pk, + ) + finish = mocker.patch("documents.bulk_edit.bulk_update_documents") + + execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk) + + finish.assert_called_once_with([document.pk]) + def test_without_template_selects_for_the_requester( self, mocker: MockerFixture, @@ -167,7 +197,10 @@ class TestRunZoneOcrBulkOperation: - Extraction receives the requesting user so templates are limited to those they can view """ - extract = mocker.patch("documents.bulk_edit.extract_zones") + extract = mocker.patch( + "documents.bulk_edit.extract_zones", + return_value=None, + ) execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk) extract.assert_called_once_with( document_id=document.pk, @@ -223,7 +256,10 @@ class TestRunZoneOcrBulkOperation: template.save(update_fields=["owner"]) else: User.objects.filter(pk=regular_user.pk).update(is_active=False) - extract = mocker.patch("documents.bulk_edit.extract_zones") + extract = mocker.patch( + "documents.bulk_edit.extract_zones", + return_value=None, + ) with pytest.raises(PermissionDenied): execute_bulk_operation(**enqueue.call_args.kwargs["kwargs"]) extract.assert_not_called() @@ -244,7 +280,10 @@ class TestRunZoneOcrBulkOperation: template matching - Extraction never runs """ - extract = mocker.patch("documents.bulk_edit.extract_zones") + extract = mocker.patch( + "documents.bulk_edit.extract_zones", + return_value=None, + ) with pytest.raises(OcrTemplate.DoesNotExist): execute_bulk_operation( "run_zone_ocr", diff --git a/src/documents/tests/test_tasks.py b/src/documents/tests/test_tasks.py index e1430bcaab..ff51fdfaaa 100644 --- a/src/documents/tests/test_tasks.py +++ b/src/documents/tests/test_tasks.py @@ -23,6 +23,8 @@ from documents.sanity_checker import SanityCheckFailedException from documents.sanity_checker import SanityCheckMessages from documents.tasks import extract_zones from documents.tests.helpers import dummy_preprocess +from documents.zone_ocr import ZoneOcrReason +from documents.zone_ocr import ZoneOcrResult from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import DocumentFactory @@ -676,11 +678,7 @@ class TestExtractZonesTask: extract_zones(doc.id, requester=user) select.assert_called_once_with(doc, requester=user) - extract.assert_called_once_with( - doc, - templates=select.return_value, - reindex=True, - ) + extract.assert_called_once_with(doc, templates=select.return_value) def test_task_runs_the_chosen_template(self, mocker: MockerFixture) -> None: """ @@ -697,7 +695,42 @@ class TestExtractZonesTask: extract_zones(doc.id, template.pk) - extract.assert_called_once_with(doc, templates=[template], reindex=True) + extract.assert_called_once_with(doc, templates=[template]) + + @pytest.mark.parametrize( + ("reason", "returns_root"), + [ + pytest.param(None, True, id="written"), + pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"), + ], + ) + def test_task_reports_the_root_when_a_value_was_written( + self, + mocker: MockerFixture, + reason: ZoneOcrReason | None, + returns_root: bool, # noqa: FBT001 + ) -> None: + """ + GIVEN: + - A document version whose extraction does or does not write a value + WHEN: + - The task runs + THEN: + - The root document id is returned only when a value was written + """ + root = DocumentFactory() + DocumentFactory(root_document=root, version_index=1) + template = OcrTemplateFactory() + mocker.patch( + "documents.tasks.run_zone_extraction", + return_value=[ + ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason), + ], + ) + + result = extract_zones(root.pk, template.pk) + + assert result == (root.pk if returns_root else None) def test_task_skips_a_deleted_template(self, mocker: MockerFixture) -> None: """ diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index 40d83fc410..c558be8897 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -66,6 +66,7 @@ if TYPE_CHECKING: from unittest.mock import MagicMock from django.contrib.auth.models import User + from pytest_django.fixtures import DjangoCaptureOnCommitCallbacks from pytest_django.fixtures import SettingsWrapper from pytest_mock import MockerFixture @@ -781,6 +782,104 @@ class TestSavedZoneExtraction: document.refresh_from_db() assert document.title == "Second" + @pytest.mark.parametrize( + "target", + [ + pytest.param("title", id="builtin-field"), + pytest.param("custom_field", id="custom-field-only"), + ], + ) + def test_successful_write_bumps_modified_of_the_root( + self, + mock_zone_ocr_run: MagicMock, + page: Path, + document: Document, + template: OcrTemplate, + target: str, + ) -> None: + """ + GIVEN: + - A document and a template with a zone for a built-in field or + only for a custom field + WHEN: + - Zone extraction writes the value + THEN: + - The root document's modified time is later than before the run + """ + field = CustomFieldFactory(name="Code", data_type="string") + if target == "title": + make_zone(template, "Title") + else: + make_zone(template, "Code", "custom_field", custom_field=field) + mock_zone_ocr_run.return_value = subprocess.CompletedProcess( + [], + 0, + stdout="value", + ) + before = Document.objects.get(pk=document.pk).modified + + results = run_zone_extraction(document, page, templates=[template]) + + assert [result.reason for result in results] == [None] + assert Document.objects.get(pk=document.pk).modified > before + + def test_without_a_write_leaves_modified_untouched( + self, + mock_zone_ocr_run: MagicMock, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - A template whose only zone reads no text + WHEN: + - Zone extraction runs + THEN: + - Nothing is written and modified is unchanged + """ + make_zone(template, "Title") + mock_zone_ocr_run.return_value = subprocess.CompletedProcess( + [], + 0, + stdout="", + ) + before = Document.objects.get(pk=document.pk).modified + + results = run_zone_extraction(document, page, templates=[template]) + + assert [result.reason for result in results] == [ZoneOcrReason.NO_VALUE] + assert Document.objects.get(pk=document.pk).modified == before + + def test_successful_write_does_not_reindex_per_document( + self, + django_capture_on_commit_callbacks: DjangoCaptureOnCommitCallbacks, + mock_zone_ocr_run: MagicMock, + page: Path, + document: Document, + template: OcrTemplate, + ) -> None: + """ + GIVEN: + - A template that writes the title + WHEN: + - Zone extraction runs + THEN: + - No on-commit callbacks are registered; finishing is left to the + caller + """ + make_zone(template, "Title") + mock_zone_ocr_run.return_value = subprocess.CompletedProcess( + [], + 0, + stdout="value", + ) + + with django_capture_on_commit_callbacks() as callbacks: + run_zone_extraction(document, page, templates=[template]) + + assert callbacks == [] + def test_document_run_renders_each_page_once_across_templates( self, mocker: MockerFixture, diff --git a/src/documents/views.py b/src/documents/views.py index bc135fd563..9626806501 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -2324,6 +2324,7 @@ class DocumentViewSet( from documents.zone_ocr import ZoneOcrSkipReason from documents.zone_ocr import get_templates_for_document from documents.zone_ocr import run_zone_extraction + from documents.zone_ocr import zone_results_written resolution = resolve_effective_document(document, request) if resolution.error == VersionResolutionError.INVALID: @@ -2353,7 +2354,12 @@ class DocumentViewSet( status=status.HTTP_404_NOT_FOUND, ) - results = run_zone_extraction(file_document, templates=templates, reindex=True) + results = run_zone_extraction(file_document, templates=templates) + if zone_results_written(results): + bulk_edit.bulk_update_documents.apply_async( + kwargs={"document_ids": [get_root_document(file_document).pk]}, + headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM}, + ) return Response( { "status": ZoneOcrRunStatus.COMPLETED, diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 31c87be12e..2717df68b6 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -14,7 +14,6 @@ from decimal import Decimal from decimal import InvalidOperation from enum import StrEnum from functools import lru_cache -from functools import partial from pathlib import Path from typing import TYPE_CHECKING from typing import NewType @@ -36,6 +35,7 @@ from django.core.exceptions import ValidationError from django.db import transaction from django.db.models import F from django.db.models import Field +from django.utils import timezone from PIL import Image from documents.matching import matches @@ -48,7 +48,6 @@ from documents.parsers import ParseError from documents.parsers import rasterize_pdf_page_to_png from documents.permissions import permitted_object_ids from documents.regex import safe_regex_fullmatch -from documents.signals.handlers import add_to_index from documents.validators import monetary_validator from documents.versioning import get_root_document from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES @@ -220,14 +219,13 @@ def run_zone_extraction( document: Document, original_file: Path | None = None, templates: list[OcrTemplate] | None = None, - *, - reindex: bool = False, ) -> list[ZoneOcrResult]: """ Extract zones from the selected file version and update its root document. Run all matching templates when no templates are given. - Manual runs can request a search refresh after successful writes commit. + The root document's modified time is bumped when any value was written. + Callers finish the edit (index, signals) themselves; see zone_results_written. """ if templates is None: templates = get_templates_for_document(document) @@ -274,17 +272,16 @@ def run_zone_extraction( _zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED) for zone in zones ) - if reindex and any(result.reason is None for result in results): - # queue a reindex of the document after the transaction commits - root_document = get_root_document(document) - transaction.on_commit(partial(_reindex_document, root_document.pk)) + if zone_results_written(results): + Document.objects.filter(pk=get_root_document(document).pk).update( + modified=timezone.now(), + ) return results -def _reindex_document(document_id: int) -> None: - document = Document.objects.filter(pk=document_id).first() - if document is not None: - add_to_index(sender=None, document=document) +def zone_results_written(results: list[ZoneOcrResult]) -> bool: + """Whether at least one zone's value was written to the root document.""" + return any(result.reason is None for result in results) def _resolve_doc_path(