diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py index 7b19aeaade..02ebc3ba58 100644 --- a/src/documents/bulk_edit.py +++ b/src/documents/bulk_edit.py @@ -33,6 +33,7 @@ from documents.models import CustomField from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentType +from documents.models import OcrTemplate from documents.models import PaperlessTask from documents.models import StoragePath from documents.models import Tag @@ -41,13 +42,16 @@ from documents.permissions import set_permissions_for_objects from documents.plugins.helpers import DocumentsStatusManager from documents.tasks import bulk_update_documents from documents.tasks import consume_file -from documents.tasks import extract_zones from documents.tasks import remove_document_from_index from documents.tasks import run_workflows_for_documents from documents.tasks import update_document_content_maybe_archive_file from documents.versioning import get_latest_version_for_root from documents.versioning import get_root_document +from documents.versioning import latest_version from documents.versioning import versions_newest_first +from documents.zone_ocr import get_templates_for_document +from documents.zone_ocr import run_zone_extraction +from documents.zone_ocr import zone_results_written if TYPE_CHECKING: from collections.abc import Callable @@ -443,7 +447,7 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]: A root document with versions reprocesses its latest version, which is the file whose content, archive and thumbnail are shown for it. """ - latest_version = versions_newest_first( + newest_version_id = versions_newest_first( Document.objects.filter(root_document=OuterRef("pk")), ).values("id")[:1] source_ids = ( @@ -451,7 +455,7 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]: .annotate( source_id=Case( When(root_document__isnull=False, then=F("id")), - default=Coalesce(Subquery(latest_version), F("id")), + default=Coalesce(Subquery(newest_version_id), F("id")), ), ) .order_by() @@ -1268,22 +1272,61 @@ def run_zone_ocr( """ Run zone OCR within the tracked bulk operation task, then finish once for the root documents that had a value written. Root documents are run on - their newest version unless resolve_latest is False. + their newest version unless resolve_latest is False. Without a template, + every template matching the document that the user can view is run. + + A failing document is logged and the rest still run; the task then fails. """ - written_ids: list[int] = [] - try: - for document_id in doc_ids: - written_id = extract_zones( - document_id=document_id, - template_id=template, - requester=user, - resolve_latest=resolve_latest, + selected_template: OcrTemplate | None = None + if template is not None: + selected_template = ( + OcrTemplate.objects.filter(pk=template) + .prefetch_related("zones", "zones__custom_field") + .first() + ) + if selected_template is None: + logger.warning( + "OCR template %s no longer exists, not running zone OCR", + template, ) - if written_id is not None: - written_ids.append(written_id) + return "OK" + + documents = {d.pk: d for d in Document.objects.filter(pk__in=doc_ids)} + missing_ids = [pk for pk in dict.fromkeys(doc_ids) if pk not in documents] + if missing_ids: + logger.warning("Documents %s not found, skipping zone OCR", missing_ids) + + targets: dict[int, Document] = {} + for document_id in doc_ids: + document = documents.get(document_id) + if document is None: + continue + if resolve_latest: + document = latest_version(document) + targets.setdefault(document.pk, document) + + written_ids: list[int] = [] + failed_ids: list[int] = [] + try: + for document in targets.values(): + try: + templates = ( + [selected_template] + if selected_template is not None + else get_templates_for_document(document, requester=user) + ) + results = run_zone_extraction(document, templates=templates) + if zone_results_written(results): + written_ids.append(get_root_document(document).pk) + except Exception: + logger.exception("Zone OCR failed for document %s", document.pk) + failed_ids.append(document.pk) finally: - # Documents already written are finished even if a later one raises. if written_ids: bulk_update_documents(list(dict.fromkeys(written_ids))) + if failed_ids: + raise RuntimeError( + f"Zone OCR failed for {len(failed_ids)} of {len(targets)} documents", + ) return "OK" diff --git a/src/documents/bulk_operations.py b/src/documents/bulk_operations.py index 1bc4ba109e..d732c1d1f8 100644 --- a/src/documents/bulk_operations.py +++ b/src/documents/bulk_operations.py @@ -118,8 +118,12 @@ def execute_bulk_operation( raise PermissionDenied("Insufficient permissions to run this workflow") if operation == "run_zone_ocr" and parameters.get("template") is not None: - template = OcrTemplate.objects.get(pk=parameters["template"]) - if not has_perms_owner_aware(user, "view_ocrtemplate", template): + template = OcrTemplate.objects.filter(pk=parameters["template"]).first() + if template is not None and not has_perms_owner_aware( + user, + "view_ocrtemplate", + template, + ): raise PermissionDenied("Insufficient permissions to run this OCR template") actor_context = set_actor(user) if settings.AUDIT_LOG_ENABLED else nullcontext() diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 5878747009..193596f635 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -11,7 +11,6 @@ from tempfile import mkstemp from celery import Task from celery import shared_task from django.conf import settings -from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.contrib.sessions.models import Session from django.db import models @@ -49,7 +48,6 @@ from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentBarcode from documents.models import DocumentType -from documents.models import OcrTemplate from documents.models import PaperlessTask from documents.models import ShareLink from documents.models import ShareLinkBundle @@ -72,12 +70,7 @@ from documents.utils import IterWrapper from documents.utils import compute_checksum from documents.utils import identity from documents.versioning import annotate_effective_content -from documents.versioning import get_root_document -from documents.versioning import latest_version from documents.workflows.utils import get_workflows_for_trigger -from documents.zone_ocr import get_templates_for_document -from documents.zone_ocr import run_zone_extraction -from documents.zone_ocr import zone_results_written from paperless.config import AIConfig from paperless.config import BarcodeConfig from paperless.config import RemoteOCRConfig @@ -376,50 +369,6 @@ def _read_barcodes_for_reprocess(document: Document) -> list[StoredBarcode] | No return None -@shared_task -def extract_zones( - document_id: int, - template_id: int | None = None, - requester: User | None = None, - *, - resolve_latest: bool = True, -) -> int | None: - """ - Run an OCR template against a stored document, or every template that - matches it when no template is given. Matched templates are limited to - those the requester can view. - - A root document id is mapped to its newest version unless resolve_latest - is False, in which case exactly the given document is processed. - - Returns the root document's id when any value was written, otherwise None. - The caller finishes the edit, for example with bulk_update_documents. - """ - document = Document.objects.get(id=document_id) - if resolve_latest: - document = latest_version(document) - if template_id is None: - templates = get_templates_for_document(document, requester=requester) - else: - try: - template = OcrTemplate.objects.prefetch_related( - "zones", - "zones__custom_field", - ).get(id=template_id) - except OcrTemplate.DoesNotExist: - logger.warning( - "OCR template %s no longer exists, not running zone OCR", - template_id, - ) - return None - templates = [template] - - results = run_zone_extraction(document, templates=templates) - if zone_results_written(results): - return get_root_document(document).pk - return None - - @shared_task def update_document_content_maybe_archive_file( document_id, diff --git a/src/documents/tests/test_bulk_edit.py b/src/documents/tests/test_bulk_edit.py index 8bff7c359f..88c5961979 100644 --- a/src/documents/tests/test_bulk_edit.py +++ b/src/documents/tests/test_bulk_edit.py @@ -26,6 +26,8 @@ from documents.models import Tag from documents.pdf_ops import PageSpec from documents.permissions import PermissionsSpec from documents.permissions import set_permissions_for_objects +from documents.zone_ocr import ZoneOcrReason +from documents.zone_ocr import ZoneOcrResult from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import DocumentFactory from paperless_testing.factories import OcrTemplateFactory @@ -1936,17 +1938,34 @@ class TestPDFActions(DirectoriesMixin, TestCase): self.assertIn("Error removing password from document", cm.output[0]) +def _written_result() -> ZoneOcrResult: + return ZoneOcrResult(template="t", zone="z", custom_field="", reason=None) + + +def _unwritten_result() -> ZoneOcrResult: + return ZoneOcrResult( + template="t", + zone="z", + custom_field="", + reason=ZoneOcrReason.NO_VALUE, + ) + + @pytest.mark.django_db class TestBulkEditRunZoneOcr: """Bulk run_zone_ocr runs zone extraction for each selected document.""" @pytest.fixture - def mock_extract_zones(self, mocker: MockerFixture) -> mock.MagicMock: - return mocker.patch("documents.bulk_edit.extract_zones", return_value=None) + def mock_extract(self, mocker: MockerFixture) -> mock.MagicMock: + return mocker.patch("documents.bulk_edit.run_zone_extraction", return_value=[]) - def test_runs_every_selected_document_in_the_current_task( + @pytest.fixture + def mock_finish(self, mocker: MockerFixture) -> mock.MagicMock: + return mocker.patch("documents.bulk_edit.bulk_update_documents") + + def test_selects_templates_for_the_requester( self, - mock_extract_zones: mock.MagicMock, + mock_extract: mock.MagicMock, mocker: MockerFixture, ) -> None: """ @@ -1955,102 +1974,157 @@ class TestBulkEditRunZoneOcr: WHEN: - run_zone_ocr is called for the documents without a template THEN: - - Extraction runs inline for each document with the requester - - Nothing is queued + - Templates are selected for each document with the requester + - Extraction runs inline for each document """ + select = mocker.patch("documents.bulk_edit.get_templates_for_document") docs = DocumentFactory.create_batch(2) - user = UserFactory() result = bulk_edit.run_zone_ocr([doc.id for doc in docs], user=user) assert result == "OK" - assert mock_extract_zones.call_args_list == [ - mocker.call( - document_id=doc.id, - template_id=None, - requester=user, - resolve_latest=True, - ) - for doc in docs + assert select.call_args_list == [ + mocker.call(doc, requester=user) for doc in docs + ] + assert mock_extract.call_args_list == [ + mocker.call(doc, templates=select.return_value) for doc in docs ] - mock_extract_zones.apply_async.assert_not_called() def test_runs_the_chosen_template( self, - mock_extract_zones: mock.MagicMock, + mock_extract: mock.MagicMock, + mocker: MockerFixture, ) -> None: """ GIVEN: - - A document and an OCR template + - Two documents and an OCR template WHEN: - - run_zone_ocr is called for the document with the template + - run_zone_ocr is called for the documents with the template THEN: - - Extraction runs inline with the template and no requester - - Nothing is queued + - Extraction runs for each document with only that template + - Template matching is not used """ - doc = DocumentFactory() + select = mocker.patch("documents.bulk_edit.get_templates_for_document") + docs = DocumentFactory.create_batch(2) template = OcrTemplateFactory() - bulk_edit.run_zone_ocr([doc.id], template=template.pk) + bulk_edit.run_zone_ocr([doc.id for doc in docs], template=template.pk) - mock_extract_zones.assert_called_once_with( - document_id=doc.id, - template_id=template.pk, - requester=None, - resolve_latest=True, - ) - mock_extract_zones.apply_async.assert_not_called() + select.assert_not_called() + assert mock_extract.call_args_list == [ + mocker.call(doc, templates=[template]) for doc in docs + ] - def test_finishes_once_for_documents_with_a_write( + def test_deleted_template_runs_nothing( self, - mock_extract_zones: mock.MagicMock, + mock_extract: mock.MagicMock, + mock_finish: mock.MagicMock, + ) -> None: + """ + GIVEN: + - A chosen template that no longer exists + WHEN: + - run_zone_ocr is called + THEN: + - OK is returned without running extraction or finishing + """ + doc = DocumentFactory() + + result = bulk_edit.run_zone_ocr([doc.pk], template=999) + + assert result == "OK" + mock_extract.assert_not_called() + mock_finish.assert_not_called() + + def test_missing_document_does_not_abort_the_others( + self, + mock_extract: mock.MagicMock, + mock_finish: mock.MagicMock, mocker: MockerFixture, ) -> None: """ GIVEN: - - Three selected documents, the second of which had nothing written - and the third reporting the same root as the first + - A selection holding a nonexistent id and a trashed document + alongside a live document WHEN: - run_zone_ocr is called THEN: - - bulk_update_documents runs once with the distinct written ids + - Only the live document is extracted + - The live document is finished + """ + live = DocumentFactory() + trashed = DocumentFactory() + trashed.delete() + template = OcrTemplateFactory() + mock_extract.return_value = [_written_result()] + + bulk_edit.run_zone_ocr([999999, trashed.pk, live.pk], template=template.pk) + + mock_extract.assert_called_once_with(live, templates=[template]) + mock_finish.assert_called_once_with([live.pk]) + + def test_failing_document_does_not_abort_the_others( + self, + mock_extract: mock.MagicMock, + mock_finish: mock.MagicMock, + ) -> None: + """ + GIVEN: + - Three selected documents, the second of which raises during extraction + WHEN: + - run_zone_ocr is called + THEN: + - The failure is logged and the remaining document still runs + - bulk_update_documents runs once with the written documents + - The task then fails, naming the number of failed documents """ docs = DocumentFactory.create_batch(3) - finish = mocker.patch("documents.bulk_edit.bulk_update_documents") - mock_extract_zones.side_effect = [docs[0].pk, None, docs[0].pk] + template = OcrTemplateFactory() + mock_extract.side_effect = [ + [_written_result()], + RuntimeError("boom"), + [_written_result()], + ] - bulk_edit.run_zone_ocr([doc.pk for doc in docs]) + with pytest.raises(RuntimeError, match="1 of 3 documents"): + bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk) - finish.assert_called_once_with([docs[0].pk]) + assert mock_extract.call_count == 3 + mock_finish.assert_called_once_with([docs[0].pk, docs[2].pk]) - def test_finishes_written_documents_when_a_later_one_raises( + def test_finishes_each_root_once( self, - mock_extract_zones: mock.MagicMock, - mocker: MockerFixture, + mock_extract: mock.MagicMock, + mock_finish: mock.MagicMock, ) -> None: """ GIVEN: - - Two selected documents, the second of which raises during extraction + - Two documents whose extraction writes, one being a version of the other + when not resolving the latest version WHEN: - - run_zone_ocr is called + - run_zone_ocr is called without resolving the latest version THEN: - - The exception propagates - - bulk_update_documents still runs with the first document's id + - bulk_update_documents runs once with the root id listed once """ - docs = DocumentFactory.create_batch(2) - finish = mocker.patch("documents.bulk_edit.bulk_update_documents") - mock_extract_zones.side_effect = [docs[0].pk, RuntimeError("boom")] + root = DocumentFactory() + version = DocumentFactory(root_document=root, version_index=1) + template = OcrTemplateFactory() + mock_extract.return_value = [_written_result()] - with pytest.raises(RuntimeError, match="boom"): - bulk_edit.run_zone_ocr([doc.pk for doc in docs]) + bulk_edit.run_zone_ocr( + [root.pk, version.pk], + template=template.pk, + resolve_latest=False, + ) - finish.assert_called_once_with([docs[0].pk]) + assert mock_extract.call_count == 2 + mock_finish.assert_called_once_with([root.pk]) def test_does_not_finish_when_nothing_was_written( self, - mock_extract_zones: mock.MagicMock, - mocker: MockerFixture, + mock_extract: mock.MagicMock, + mock_finish: mock.MagicMock, ) -> None: """ GIVEN: @@ -2061,11 +2135,122 @@ class TestBulkEditRunZoneOcr: - bulk_update_documents is not run """ docs = DocumentFactory.create_batch(2) - finish = mocker.patch("documents.bulk_edit.bulk_update_documents") + template = OcrTemplateFactory() + mock_extract.return_value = [_unwritten_result()] - bulk_edit.run_zone_ocr([doc.pk for doc in docs]) + bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk) - finish.assert_not_called() + mock_finish.assert_not_called() + + def test_finishes_written_documents_when_the_run_is_interrupted( + self, + mock_extract: mock.MagicMock, + mock_finish: mock.MagicMock, + ) -> None: + """ + GIVEN: + - Two selected documents, the second of which raises an exception + that is not an Exception subclass + WHEN: + - run_zone_ocr is called + THEN: + - The interruption propagates + - bulk_update_documents still runs with the first document's id + """ + docs = DocumentFactory.create_batch(2) + template = OcrTemplateFactory() + mock_extract.side_effect = [[_written_result()], KeyboardInterrupt()] + + with pytest.raises(KeyboardInterrupt): + bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk) + + mock_finish.assert_called_once_with([docs[0].pk]) + + @pytest.mark.parametrize( + ("resolve_latest", "processes_root"), + [ + pytest.param(True, False, id="resolves-latest"), + pytest.param(False, True, id="exact-document"), + ], + ) + def test_resolves_latest_only_when_asked( + self, + mock_extract: mock.MagicMock, + resolve_latest: bool, # noqa: FBT001 + processes_root: bool, # noqa: FBT001 + ) -> None: + """ + GIVEN: + - A root document with a newer version and a template + WHEN: + - run_zone_ocr runs for the root id with resolve_latest on or off + THEN: + - The newest version is processed when resolve_latest is on + - The root document itself is processed when it is off + """ + root = DocumentFactory() + newest = DocumentFactory(root_document=root, version_index=1) + template = OcrTemplateFactory() + + bulk_edit.run_zone_ocr( + [root.pk], + template=template.pk, + resolve_latest=resolve_latest, + ) + + mock_extract.assert_called_once_with( + root if processes_root else newest, + templates=[template], + ) + + def test_root_and_its_version_are_processed_once( + self, + mock_extract: mock.MagicMock, + mock_finish: mock.MagicMock, + ) -> None: + """ + GIVEN: + - A root document with a version, both listed in the selection + WHEN: + - run_zone_ocr runs with resolve_latest on + THEN: + - The newest version is extracted once + - The root is finished once + """ + root = DocumentFactory() + version = DocumentFactory(root_document=root, version_index=1) + template = OcrTemplateFactory() + mock_extract.return_value = [_written_result()] + + bulk_edit.run_zone_ocr([root.pk, version.pk], template=template.pk) + + mock_extract.assert_called_once_with(version, templates=[template]) + mock_finish.assert_called_once_with([root.pk]) + + def test_a_version_reports_its_root( + self, + mock_extract: mock.MagicMock, + mock_finish: mock.MagicMock, + ) -> None: + """ + GIVEN: + - A root with two versions and the older version selected + WHEN: + - run_zone_ocr runs without resolving the latest version + THEN: + - Extraction runs on that version, not the newest + - The root is finished + """ + root = DocumentFactory() + older = DocumentFactory(root_document=root, version_index=1) + DocumentFactory(root_document=root, version_index=2) + template = OcrTemplateFactory() + mock_extract.return_value = [_written_result()] + + bulk_edit.run_zone_ocr([older.pk], template=template.pk, resolve_latest=False) + + mock_extract.assert_called_once_with(older, templates=[template]) + mock_finish.assert_called_once_with([root.pk]) @pytest.mark.django_db diff --git a/src/documents/tests/test_bulk_operations.py b/src/documents/tests/test_bulk_operations.py index 343d79ac75..34f9dc51be 100644 --- a/src/documents/tests/test_bulk_operations.py +++ b/src/documents/tests/test_bulk_operations.py @@ -14,12 +14,12 @@ from rest_framework.test import APIClient from documents.bulk_operations import execute_bulk_operation from documents.models import Correspondent from documents.models import Document -from documents.models import OcrTemplate from documents.models import PaperlessTask from documents.signals.handlers import before_task_publish_handler from documents.signals.handlers import task_failure_handler from documents.signals.handlers import task_postrun_handler from documents.signals.handlers import task_revoked_handler +from documents.zone_ocr import ZoneOcrResult from paperless_testing.factories import DocumentFactory from paperless_testing.factories import OcrTemplateFactory from paperless_testing.factories import UserFactory @@ -98,8 +98,8 @@ class TestRunZoneOcrBulkOperation: template = OcrTemplateFactory() enqueue = mocker.patch("documents.views.execute_bulk_operation.apply_async") extract = mocker.patch( - "documents.bulk_edit.extract_zones", - return_value=None, + "documents.bulk_edit.run_zone_extraction", + return_value=[], ) enqueue.return_value.id = "zone-ocr-task" response = admin_client.post( @@ -135,14 +135,13 @@ class TestRunZoneOcrBulkOperation: WHEN: - The tracked task executes it THEN: - - Extraction is called synchronously with the requester - - No further extraction task is enqueued + - Extraction runs synchronously with only that template - The result reports the operation and document count """ template = OcrTemplateFactory() extract = mocker.patch( - "documents.bulk_edit.extract_zones", - return_value=None, + "documents.bulk_edit.run_zone_extraction", + return_value=[], ) result = execute_bulk_operation( "run_zone_ocr", @@ -150,13 +149,7 @@ class TestRunZoneOcrBulkOperation: {"template": template.pk}, admin_user.pk, ) - extract.assert_called_once_with( - document_id=document.pk, - template_id=template.pk, - requester=admin_user, - resolve_latest=True, - ) - extract.apply_async.assert_not_called() + extract.assert_called_once_with(document, templates=[template]) assert result == {"operation": "run_zone_ocr", "document_count": 1} def test_finishes_with_the_ids_that_were_written( @@ -173,13 +166,21 @@ class TestRunZoneOcrBulkOperation: THEN: - bulk_update_documents runs once with that document id """ + template = OcrTemplateFactory() mocker.patch( - "documents.bulk_edit.extract_zones", - return_value=document.pk, + "documents.bulk_edit.run_zone_extraction", + return_value=[ + ZoneOcrResult(template="t", zone="z", custom_field="", reason=None), + ], ) finish = mocker.patch("documents.bulk_edit.bulk_update_documents") - execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk) + execute_bulk_operation( + "run_zone_ocr", + [document.pk], + {"template": template.pk}, + admin_user.pk, + ) finish.assert_called_once_with([document.pk]) @@ -195,20 +196,17 @@ class TestRunZoneOcrBulkOperation: WHEN: - The tracked task executes it THEN: - - Extraction receives the requesting user so templates are limited to - those they can view + - Templates are selected for the requesting user so they are limited + to those they can view """ + select = mocker.patch("documents.bulk_edit.get_templates_for_document") extract = mocker.patch( - "documents.bulk_edit.extract_zones", - return_value=None, + "documents.bulk_edit.run_zone_extraction", + return_value=[], ) execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk) - extract.assert_called_once_with( - document_id=document.pk, - template_id=None, - requester=admin_user, - resolve_latest=True, - ) + select.assert_called_once_with(document, requester=admin_user) + extract.assert_called_once_with(document, templates=select.return_value) @pytest.mark.parametrize( "revoked", @@ -259,14 +257,14 @@ class TestRunZoneOcrBulkOperation: else: User.objects.filter(pk=regular_user.pk).update(is_active=False) extract = mocker.patch( - "documents.bulk_edit.extract_zones", - return_value=None, + "documents.bulk_edit.run_zone_extraction", + return_value=[], ) with pytest.raises(PermissionDenied): execute_bulk_operation(**enqueue.call_args.kwargs["kwargs"]) extract.assert_not_called() - def test_cannot_fall_back_to_matching_if_chosen_template_is_deleted( + def test_does_not_fall_back_to_matching_if_chosen_template_is_deleted( self, mocker: MockerFixture, admin_user: User, @@ -278,21 +276,22 @@ class TestRunZoneOcrBulkOperation: WHEN: - The tracked task executes it THEN: - - OcrTemplate.DoesNotExist is raised instead of falling back to - template matching + - The task completes without falling back to template matching - Extraction never runs """ extract = mocker.patch( - "documents.bulk_edit.extract_zones", - return_value=None, + "documents.bulk_edit.run_zone_extraction", + return_value=[], ) - with pytest.raises(OcrTemplate.DoesNotExist): - execute_bulk_operation( - "run_zone_ocr", - [document.pk], - {"template": 999}, - admin_user.pk, - ) + select = mocker.patch("documents.bulk_edit.get_templates_for_document") + result = execute_bulk_operation( + "run_zone_ocr", + [document.pk], + {"template": 999}, + admin_user.pk, + ) + assert result == {"operation": "run_zone_ocr", "document_count": 1} + select.assert_not_called() extract.assert_not_called() diff --git a/src/documents/tests/test_tasks.py b/src/documents/tests/test_tasks.py index 6a7c5ae59b..7af705512e 100644 --- a/src/documents/tests/test_tasks.py +++ b/src/documents/tests/test_tasks.py @@ -11,7 +11,6 @@ from django.test import TestCase from django.test import override_settings from django.test.utils import CaptureQueriesContext from django.utils import timezone -from pytest_mock import MockerFixture from documents import tasks from documents.models import Correspondent @@ -21,15 +20,10 @@ from documents.models import Tag from documents.models import WorkflowAction from documents.sanity_checker import SanityCheckFailedException from documents.sanity_checker import SanityCheckMessages -from documents.tasks import extract_zones from documents.tests.helpers import dummy_preprocess -from documents.zone_ocr import ZoneOcrReason -from documents.zone_ocr import ZoneOcrResult from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import DocumentFactory -from paperless_testing.factories import OcrTemplateFactory -from paperless_testing.factories import UserFactory from paperless_testing.samples import SIMPLE_DIGITAL_PDF @@ -654,159 +648,3 @@ class TestClearExpiredSessions: assert set(Session.objects.values_list("session_key", flat=True)) == {"valid"} assert sum(query["sql"].startswith("DELETE FROM") for query in queries) == 2 - - -@pytest.mark.django_db -class TestExtractZonesTask: - """Tests for the extract_zones task.""" - - def test_task_runs_extraction(self, mocker: MockerFixture) -> None: - """ - GIVEN: - - A queued zone OCR task - WHEN: - - The task runs - THEN: - - Templates are selected for the requester and extraction runs on - the stored document - """ - extract = mocker.patch("documents.tasks.run_zone_extraction") - select = mocker.patch("documents.tasks.get_templates_for_document") - doc = DocumentFactory() - user = UserFactory() - - extract_zones(doc.id, requester=user) - - select.assert_called_once_with(doc, requester=user) - extract.assert_called_once_with(doc, templates=select.return_value) - - def test_task_runs_the_chosen_template(self, mocker: MockerFixture) -> None: - """ - GIVEN: - - A queued zone OCR task with a chosen template - WHEN: - - The task runs - THEN: - - Zone extraction runs with only that template - """ - extract = mocker.patch("documents.tasks.run_zone_extraction") - doc = DocumentFactory() - template = OcrTemplateFactory() - - extract_zones(doc.id, template.pk) - - extract.assert_called_once_with(doc, templates=[template]) - - @pytest.mark.parametrize( - ("reason", "returns_root"), - [ - pytest.param(None, True, id="written"), - pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"), - ], - ) - def test_task_reports_the_root_when_a_value_was_written( - self, - mocker: MockerFixture, - reason: ZoneOcrReason | None, - returns_root: bool, # noqa: FBT001 - ) -> None: - """ - GIVEN: - - A document version whose extraction does or does not write a value - WHEN: - - The task runs - THEN: - - The root document id is returned only when a value was written - """ - root = DocumentFactory() - DocumentFactory(root_document=root, version_index=1) - template = OcrTemplateFactory() - mocker.patch( - "documents.tasks.run_zone_extraction", - return_value=[ - ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason), - ], - ) - - result = extract_zones(root.pk, template.pk) - - assert result == (root.pk if returns_root else None) - - def test_task_runs_a_version_and_reports_the_root( - self, - mocker: MockerFixture, - ) -> None: - """ - GIVEN: - - A root document with two versions and a template - WHEN: - - The task runs for the older version id - THEN: - - Extraction runs on that version, not the newest - - The root document id is returned when a value was written - """ - root = DocumentFactory() - older = DocumentFactory(root_document=root, version_index=1) - DocumentFactory(root_document=root, version_index=2) - template = OcrTemplateFactory() - extract = mocker.patch( - "documents.tasks.run_zone_extraction", - return_value=[ - ZoneOcrResult(template="t", zone="z", custom_field="", reason=None), - ], - ) - - result = extract_zones(older.pk, template.pk) - - extract.assert_called_once_with(older, templates=[template]) - assert result == root.pk - - @pytest.mark.parametrize( - ("resolve_latest", "processes_root"), - [ - pytest.param(True, False, id="resolves-latest"), - pytest.param(False, True, id="exact-document"), - ], - ) - def test_task_resolves_latest_only_when_asked( - self, - mocker: MockerFixture, - resolve_latest: bool, # noqa: FBT001 - processes_root: bool, # noqa: FBT001 - ) -> None: - """ - GIVEN: - - A root document with a newer version and a template - WHEN: - - The task runs for the root id with resolve_latest on or off - THEN: - - The newest version is processed when resolve_latest is on - - The root document itself is processed when it is off - """ - root = DocumentFactory() - newest = DocumentFactory(root_document=root, version_index=1) - template = OcrTemplateFactory() - extract = mocker.patch("documents.tasks.run_zone_extraction", return_value=[]) - - extract_zones(root.pk, template.pk, resolve_latest=resolve_latest) - - extract.assert_called_once_with( - root if processes_root else newest, - templates=[template], - ) - - def test_task_skips_a_deleted_template(self, mocker: MockerFixture) -> None: - """ - GIVEN: - - A template deleted between queueing and running - WHEN: - - The task runs - THEN: - - Nothing is extracted - """ - extract = mocker.patch("documents.tasks.run_zone_extraction") - doc = DocumentFactory() - - extract_zones(doc.id, 999) - - extract.assert_not_called()