Fix: make bulk zone OCR survive a missing template or failing document

Inline extract_zones into run_zone_ocr, load documents and the template once, and process each root once. A failing document is logged and the task fails after the rest finish.
This commit is contained in:
stumpylog committed 2026-11-20 09:09:28 -08:00
1 parent a08918eb9c
commit ccbff94208
6 files changed
+346 -328

No files matched your search

+58 -15
View File
@@ -33,6 +33,7 @@ from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import OcrTemplate
from documents.models import PaperlessTask
from documents.models import StoragePath
from documents.models import Tag
@@ -41,13 +42,16 @@ from documents.permissions import set_permissions_for_objects
from documents.plugins.helpers import DocumentsStatusManager
from documents.tasks import bulk_update_documents
from documents.tasks import consume_file
from documents.tasks import extract_zones
from documents.tasks import remove_document_from_index
from documents.tasks import run_workflows_for_documents
from documents.tasks import update_document_content_maybe_archive_file
from documents.versioning import get_latest_version_for_root
from documents.versioning import get_root_document
from documents.versioning import latest_version
from documents.versioning import versions_newest_first
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import run_zone_extraction
from documents.zone_ocr import zone_results_written
if TYPE_CHECKING:
from collections.abc import Callable
@@ -443,7 +447,7 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
A root document with versions reprocesses its latest version, which is the
file whose content, archive and thumbnail are shown for it.
"""
latest_version = versions_newest_first(
newest_version_id = versions_newest_first(
Document.objects.filter(root_document=OuterRef("pk")),
).values("id")[:1]
source_ids = (
@@ -451,7 +455,7 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
.annotate(
source_id=Case(
When(root_document__isnull=False, then=F("id")),
default=Coalesce(Subquery(latest_version), F("id")),
default=Coalesce(Subquery(newest_version_id), F("id")),
),
)
.order_by()
@@ -1268,22 +1272,61 @@ def run_zone_ocr(
"""
Run zone OCR within the tracked bulk operation task, then finish once for
the root documents that had a value written. Root documents are run on
their newest version unless resolve_latest is False.
their newest version unless resolve_latest is False. Without a template,
every template matching the document that the user can view is run.
A failing document is logged and the rest still run; the task then fails.
"""
written_ids: list[int] = []
try:
for document_id in doc_ids:
written_id = extract_zones(
document_id=document_id,
template_id=template,
requester=user,
resolve_latest=resolve_latest,
selected_template: OcrTemplate | None = None
if template is not None:
selected_template = (
OcrTemplate.objects.filter(pk=template)
.prefetch_related("zones", "zones__custom_field")
.first()
)
if selected_template is None:
logger.warning(
"OCR template %s no longer exists, not running zone OCR",
template,
)
if written_id is not None:
written_ids.append(written_id)
return "OK"
documents = {d.pk: d for d in Document.objects.filter(pk__in=doc_ids)}
missing_ids = [pk for pk in dict.fromkeys(doc_ids) if pk not in documents]
if missing_ids:
logger.warning("Documents %s not found, skipping zone OCR", missing_ids)
targets: dict[int, Document] = {}
for document_id in doc_ids:
document = documents.get(document_id)
if document is None:
continue
if resolve_latest:
document = latest_version(document)
targets.setdefault(document.pk, document)
written_ids: list[int] = []
failed_ids: list[int] = []
try:
for document in targets.values():
try:
templates = (
[selected_template]
if selected_template is not None
else get_templates_for_document(document, requester=user)
)
results = run_zone_extraction(document, templates=templates)
if zone_results_written(results):
written_ids.append(get_root_document(document).pk)
except Exception:
logger.exception("Zone OCR failed for document %s", document.pk)
failed_ids.append(document.pk)
finally:
# Documents already written are finished even if a later one raises.
if written_ids:
bulk_update_documents(list(dict.fromkeys(written_ids)))
if failed_ids:
raise RuntimeError(
f"Zone OCR failed for {len(failed_ids)} of {len(targets)} documents",
)
return "OK"
+6 -2
View File
@@ -118,8 +118,12 @@ def execute_bulk_operation(
raise PermissionDenied("Insufficient permissions to run this workflow")
if operation == "run_zone_ocr" and parameters.get("template") is not None:
template = OcrTemplate.objects.get(pk=parameters["template"])
if not has_perms_owner_aware(user, "view_ocrtemplate", template):
template = OcrTemplate.objects.filter(pk=parameters["template"]).first()
if template is not None and not has_perms_owner_aware(
user,
"view_ocrtemplate",
template,
):
raise PermissionDenied("Insufficient permissions to run this OCR template")
actor_context = set_actor(user) if settings.AUDIT_LOG_ENABLED else nullcontext()
-51
View File
@@ -11,7 +11,6 @@ from tempfile import mkstemp
from celery import Task
from celery import shared_task
from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.contrib.sessions.models import Session
from django.db import models
@@ -49,7 +48,6 @@ from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentBarcode
from documents.models import DocumentType
from documents.models import OcrTemplate
from documents.models import PaperlessTask
from documents.models import ShareLink
from documents.models import ShareLinkBundle
@@ -72,12 +70,7 @@ from documents.utils import IterWrapper
from documents.utils import compute_checksum
from documents.utils import identity
from documents.versioning import annotate_effective_content
from documents.versioning import get_root_document
from documents.versioning import latest_version
from documents.workflows.utils import get_workflows_for_trigger
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import run_zone_extraction
from documents.zone_ocr import zone_results_written
from paperless.config import AIConfig
from paperless.config import BarcodeConfig
from paperless.config import RemoteOCRConfig
@@ -376,50 +369,6 @@ def _read_barcodes_for_reprocess(document: Document) -> list[StoredBarcode] | No
return None
@shared_task
def extract_zones(
document_id: int,
template_id: int | None = None,
requester: User | None = None,
*,
resolve_latest: bool = True,
) -> int | None:
"""
Run an OCR template against a stored document, or every template that
matches it when no template is given. Matched templates are limited to
those the requester can view.
A root document id is mapped to its newest version unless resolve_latest
is False, in which case exactly the given document is processed.
Returns the root document's id when any value was written, otherwise None.
The caller finishes the edit, for example with bulk_update_documents.
"""
document = Document.objects.get(id=document_id)
if resolve_latest:
document = latest_version(document)
if template_id is None:
templates = get_templates_for_document(document, requester=requester)
else:
try:
template = OcrTemplate.objects.prefetch_related(
"zones",
"zones__custom_field",
).get(id=template_id)
except OcrTemplate.DoesNotExist:
logger.warning(
"OCR template %s no longer exists, not running zone OCR",
template_id,
)
return None
templates = [template]
results = run_zone_extraction(document, templates=templates)
if zone_results_written(results):
return get_root_document(document).pk
return None
@shared_task
def update_document_content_maybe_archive_file(
document_id,
+242 -57
View File
@@ -26,6 +26,8 @@ from documents.models import Tag
from documents.pdf_ops import PageSpec
from documents.permissions import PermissionsSpec
from documents.permissions import set_permissions_for_objects
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
@@ -1936,17 +1938,34 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertIn("Error removing password from document", cm.output[0])
def _written_result() -> ZoneOcrResult:
return ZoneOcrResult(template="t", zone="z", custom_field="", reason=None)
def _unwritten_result() -> ZoneOcrResult:
return ZoneOcrResult(
template="t",
zone="z",
custom_field="",
reason=ZoneOcrReason.NO_VALUE,
)
@pytest.mark.django_db
class TestBulkEditRunZoneOcr:
"""Bulk run_zone_ocr runs zone extraction for each selected document."""
@pytest.fixture
def mock_extract_zones(self, mocker: MockerFixture) -> mock.MagicMock:
return mocker.patch("documents.bulk_edit.extract_zones", return_value=None)
def mock_extract(self, mocker: MockerFixture) -> mock.MagicMock:
return mocker.patch("documents.bulk_edit.run_zone_extraction", return_value=[])
def test_runs_every_selected_document_in_the_current_task(
@pytest.fixture
def mock_finish(self, mocker: MockerFixture) -> mock.MagicMock:
return mocker.patch("documents.bulk_edit.bulk_update_documents")
def test_selects_templates_for_the_requester(
self,
mock_extract_zones: mock.MagicMock,
mock_extract: mock.MagicMock,
mocker: MockerFixture,
) -> None:
"""
@@ -1955,102 +1974,157 @@ class TestBulkEditRunZoneOcr:
WHEN:
- run_zone_ocr is called for the documents without a template
THEN:
- Extraction runs inline for each document with the requester
- Nothing is queued
- Templates are selected for each document with the requester
- Extraction runs inline for each document
"""
select = mocker.patch("documents.bulk_edit.get_templates_for_document")
docs = DocumentFactory.create_batch(2)
user = UserFactory()
result = bulk_edit.run_zone_ocr([doc.id for doc in docs], user=user)
assert result == "OK"
assert mock_extract_zones.call_args_list == [
mocker.call(
document_id=doc.id,
template_id=None,
requester=user,
resolve_latest=True,
)
for doc in docs
assert select.call_args_list == [
mocker.call(doc, requester=user) for doc in docs
]
assert mock_extract.call_args_list == [
mocker.call(doc, templates=select.return_value) for doc in docs
]
mock_extract_zones.apply_async.assert_not_called()
def test_runs_the_chosen_template(
self,
mock_extract_zones: mock.MagicMock,
mock_extract: mock.MagicMock,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- A document and an OCR template
- Two documents and an OCR template
WHEN:
- run_zone_ocr is called for the document with the template
- run_zone_ocr is called for the documents with the template
THEN:
- Extraction runs inline with the template and no requester
- Nothing is queued
- Extraction runs for each document with only that template
- Template matching is not used
"""
doc = DocumentFactory()
select = mocker.patch("documents.bulk_edit.get_templates_for_document")
docs = DocumentFactory.create_batch(2)
template = OcrTemplateFactory()
bulk_edit.run_zone_ocr([doc.id], template=template.pk)
bulk_edit.run_zone_ocr([doc.id for doc in docs], template=template.pk)
mock_extract_zones.assert_called_once_with(
document_id=doc.id,
template_id=template.pk,
requester=None,
resolve_latest=True,
)
mock_extract_zones.apply_async.assert_not_called()
select.assert_not_called()
assert mock_extract.call_args_list == [
mocker.call(doc, templates=[template]) for doc in docs
]
def test_finishes_once_for_documents_with_a_write(
def test_deleted_template_runs_nothing(
self,
mock_extract_zones: mock.MagicMock,
mock_extract: mock.MagicMock,
mock_finish: mock.MagicMock,
) -> None:
"""
GIVEN:
- A chosen template that no longer exists
WHEN:
- run_zone_ocr is called
THEN:
- OK is returned without running extraction or finishing
"""
doc = DocumentFactory()
result = bulk_edit.run_zone_ocr([doc.pk], template=999)
assert result == "OK"
mock_extract.assert_not_called()
mock_finish.assert_not_called()
def test_missing_document_does_not_abort_the_others(
self,
mock_extract: mock.MagicMock,
mock_finish: mock.MagicMock,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- Three selected documents, the second of which had nothing written
and the third reporting the same root as the first
- A selection holding a nonexistent id and a trashed document
alongside a live document
WHEN:
- run_zone_ocr is called
THEN:
- bulk_update_documents runs once with the distinct written ids
- Only the live document is extracted
- The live document is finished
"""
live = DocumentFactory()
trashed = DocumentFactory()
trashed.delete()
template = OcrTemplateFactory()
mock_extract.return_value = [_written_result()]
bulk_edit.run_zone_ocr([999999, trashed.pk, live.pk], template=template.pk)
mock_extract.assert_called_once_with(live, templates=[template])
mock_finish.assert_called_once_with([live.pk])
def test_failing_document_does_not_abort_the_others(
self,
mock_extract: mock.MagicMock,
mock_finish: mock.MagicMock,
) -> None:
"""
GIVEN:
- Three selected documents, the second of which raises during extraction
WHEN:
- run_zone_ocr is called
THEN:
- The failure is logged and the remaining document still runs
- bulk_update_documents runs once with the written documents
- The task then fails, naming the number of failed documents
"""
docs = DocumentFactory.create_batch(3)
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
mock_extract_zones.side_effect = [docs[0].pk, None, docs[0].pk]
template = OcrTemplateFactory()
mock_extract.side_effect = [
[_written_result()],
RuntimeError("boom"),
[_written_result()],
]
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
with pytest.raises(RuntimeError, match="1 of 3 documents"):
bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk)
finish.assert_called_once_with([docs[0].pk])
assert mock_extract.call_count == 3
mock_finish.assert_called_once_with([docs[0].pk, docs[2].pk])
def test_finishes_written_documents_when_a_later_one_raises(
def test_finishes_each_root_once(
self,
mock_extract_zones: mock.MagicMock,
mocker: MockerFixture,
mock_extract: mock.MagicMock,
mock_finish: mock.MagicMock,
) -> None:
"""
GIVEN:
- Two selected documents, the second of which raises during extraction
- Two documents whose extraction writes, one being a version of the other
when not resolving the latest version
WHEN:
- run_zone_ocr is called
- run_zone_ocr is called without resolving the latest version
THEN:
- The exception propagates
- bulk_update_documents still runs with the first document's id
- bulk_update_documents runs once with the root id listed once
"""
docs = DocumentFactory.create_batch(2)
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
mock_extract_zones.side_effect = [docs[0].pk, RuntimeError("boom")]
root = DocumentFactory()
version = DocumentFactory(root_document=root, version_index=1)
template = OcrTemplateFactory()
mock_extract.return_value = [_written_result()]
with pytest.raises(RuntimeError, match="boom"):
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
bulk_edit.run_zone_ocr(
[root.pk, version.pk],
template=template.pk,
resolve_latest=False,
)
finish.assert_called_once_with([docs[0].pk])
assert mock_extract.call_count == 2
mock_finish.assert_called_once_with([root.pk])
def test_does_not_finish_when_nothing_was_written(
self,
mock_extract_zones: mock.MagicMock,
mocker: MockerFixture,
mock_extract: mock.MagicMock,
mock_finish: mock.MagicMock,
) -> None:
"""
GIVEN:
@@ -2061,11 +2135,122 @@ class TestBulkEditRunZoneOcr:
- bulk_update_documents is not run
"""
docs = DocumentFactory.create_batch(2)
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
template = OcrTemplateFactory()
mock_extract.return_value = [_unwritten_result()]
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk)
finish.assert_not_called()
mock_finish.assert_not_called()
def test_finishes_written_documents_when_the_run_is_interrupted(
self,
mock_extract: mock.MagicMock,
mock_finish: mock.MagicMock,
) -> None:
"""
GIVEN:
- Two selected documents, the second of which raises an exception
that is not an Exception subclass
WHEN:
- run_zone_ocr is called
THEN:
- The interruption propagates
- bulk_update_documents still runs with the first document's id
"""
docs = DocumentFactory.create_batch(2)
template = OcrTemplateFactory()
mock_extract.side_effect = [[_written_result()], KeyboardInterrupt()]
with pytest.raises(KeyboardInterrupt):
bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk)
mock_finish.assert_called_once_with([docs[0].pk])
@pytest.mark.parametrize(
("resolve_latest", "processes_root"),
[
pytest.param(True, False, id="resolves-latest"),
pytest.param(False, True, id="exact-document"),
],
)
def test_resolves_latest_only_when_asked(
self,
mock_extract: mock.MagicMock,
resolve_latest: bool, # noqa: FBT001
processes_root: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A root document with a newer version and a template
WHEN:
- run_zone_ocr runs for the root id with resolve_latest on or off
THEN:
- The newest version is processed when resolve_latest is on
- The root document itself is processed when it is off
"""
root = DocumentFactory()
newest = DocumentFactory(root_document=root, version_index=1)
template = OcrTemplateFactory()
bulk_edit.run_zone_ocr(
[root.pk],
template=template.pk,
resolve_latest=resolve_latest,
)
mock_extract.assert_called_once_with(
root if processes_root else newest,
templates=[template],
)
def test_root_and_its_version_are_processed_once(
self,
mock_extract: mock.MagicMock,
mock_finish: mock.MagicMock,
) -> None:
"""
GIVEN:
- A root document with a version, both listed in the selection
WHEN:
- run_zone_ocr runs with resolve_latest on
THEN:
- The newest version is extracted once
- The root is finished once
"""
root = DocumentFactory()
version = DocumentFactory(root_document=root, version_index=1)
template = OcrTemplateFactory()
mock_extract.return_value = [_written_result()]
bulk_edit.run_zone_ocr([root.pk, version.pk], template=template.pk)
mock_extract.assert_called_once_with(version, templates=[template])
mock_finish.assert_called_once_with([root.pk])
def test_a_version_reports_its_root(
self,
mock_extract: mock.MagicMock,
mock_finish: mock.MagicMock,
) -> None:
"""
GIVEN:
- A root with two versions and the older version selected
WHEN:
- run_zone_ocr runs without resolving the latest version
THEN:
- Extraction runs on that version, not the newest
- The root is finished
"""
root = DocumentFactory()
older = DocumentFactory(root_document=root, version_index=1)
DocumentFactory(root_document=root, version_index=2)
template = OcrTemplateFactory()
mock_extract.return_value = [_written_result()]
bulk_edit.run_zone_ocr([older.pk], template=template.pk, resolve_latest=False)
mock_extract.assert_called_once_with(older, templates=[template])
mock_finish.assert_called_once_with([root.pk])
@pytest.mark.django_db
+40 -41
View File
@@ -14,12 +14,12 @@ from rest_framework.test import APIClient
from documents.bulk_operations import execute_bulk_operation
from documents.models import Correspondent
from documents.models import Document
from documents.models import OcrTemplate
from documents.models import PaperlessTask
from documents.signals.handlers import before_task_publish_handler
from documents.signals.handlers import task_failure_handler
from documents.signals.handlers import task_postrun_handler
from documents.signals.handlers import task_revoked_handler
from documents.zone_ocr import ZoneOcrResult
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
from paperless_testing.factories import UserFactory
@@ -98,8 +98,8 @@ class TestRunZoneOcrBulkOperation:
template = OcrTemplateFactory()
enqueue = mocker.patch("documents.views.execute_bulk_operation.apply_async")
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
"documents.bulk_edit.run_zone_extraction",
return_value=[],
)
enqueue.return_value.id = "zone-ocr-task"
response = admin_client.post(
@@ -135,14 +135,13 @@ class TestRunZoneOcrBulkOperation:
WHEN:
- The tracked task executes it
THEN:
- Extraction is called synchronously with the requester
- No further extraction task is enqueued
- Extraction runs synchronously with only that template
- The result reports the operation and document count
"""
template = OcrTemplateFactory()
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
"documents.bulk_edit.run_zone_extraction",
return_value=[],
)
result = execute_bulk_operation(
"run_zone_ocr",
@@ -150,13 +149,7 @@ class TestRunZoneOcrBulkOperation:
{"template": template.pk},
admin_user.pk,
)
extract.assert_called_once_with(
document_id=document.pk,
template_id=template.pk,
requester=admin_user,
resolve_latest=True,
)
extract.apply_async.assert_not_called()
extract.assert_called_once_with(document, templates=[template])
assert result == {"operation": "run_zone_ocr", "document_count": 1}
def test_finishes_with_the_ids_that_were_written(
@@ -173,13 +166,21 @@ class TestRunZoneOcrBulkOperation:
THEN:
- bulk_update_documents runs once with that document id
"""
template = OcrTemplateFactory()
mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=document.pk,
"documents.bulk_edit.run_zone_extraction",
return_value=[
ZoneOcrResult(template="t", zone="z", custom_field="", reason=None),
],
)
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk)
execute_bulk_operation(
"run_zone_ocr",
[document.pk],
{"template": template.pk},
admin_user.pk,
)
finish.assert_called_once_with([document.pk])
@@ -195,20 +196,17 @@ class TestRunZoneOcrBulkOperation:
WHEN:
- The tracked task executes it
THEN:
- Extraction receives the requesting user so templates are limited to
those they can view
- Templates are selected for the requesting user so they are limited
to those they can view
"""
select = mocker.patch("documents.bulk_edit.get_templates_for_document")
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
"documents.bulk_edit.run_zone_extraction",
return_value=[],
)
execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk)
extract.assert_called_once_with(
document_id=document.pk,
template_id=None,
requester=admin_user,
resolve_latest=True,
)
select.assert_called_once_with(document, requester=admin_user)
extract.assert_called_once_with(document, templates=select.return_value)
@pytest.mark.parametrize(
"revoked",
@@ -259,14 +257,14 @@ class TestRunZoneOcrBulkOperation:
else:
User.objects.filter(pk=regular_user.pk).update(is_active=False)
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
"documents.bulk_edit.run_zone_extraction",
return_value=[],
)
with pytest.raises(PermissionDenied):
execute_bulk_operation(**enqueue.call_args.kwargs["kwargs"])
extract.assert_not_called()
def test_cannot_fall_back_to_matching_if_chosen_template_is_deleted(
def test_does_not_fall_back_to_matching_if_chosen_template_is_deleted(
self,
mocker: MockerFixture,
admin_user: User,
@@ -278,21 +276,22 @@ class TestRunZoneOcrBulkOperation:
WHEN:
- The tracked task executes it
THEN:
- OcrTemplate.DoesNotExist is raised instead of falling back to
template matching
- The task completes without falling back to template matching
- Extraction never runs
"""
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
"documents.bulk_edit.run_zone_extraction",
return_value=[],
)
with pytest.raises(OcrTemplate.DoesNotExist):
execute_bulk_operation(
"run_zone_ocr",
[document.pk],
{"template": 999},
admin_user.pk,
)
select = mocker.patch("documents.bulk_edit.get_templates_for_document")
result = execute_bulk_operation(
"run_zone_ocr",
[document.pk],
{"template": 999},
admin_user.pk,
)
assert result == {"operation": "run_zone_ocr", "document_count": 1}
select.assert_not_called()
extract.assert_not_called()
-162
View File
@@ -11,7 +11,6 @@ from django.test import TestCase
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
from django.utils import timezone
from pytest_mock import MockerFixture
from documents import tasks
from documents.models import Correspondent
@@ -21,15 +20,10 @@ from documents.models import Tag
from documents.models import WorkflowAction
from documents.sanity_checker import SanityCheckFailedException
from documents.sanity_checker import SanityCheckMessages
from documents.tasks import extract_zones
from documents.tests.helpers import dummy_preprocess
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
from paperless_testing.factories import UserFactory
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
@@ -654,159 +648,3 @@ class TestClearExpiredSessions:
assert set(Session.objects.values_list("session_key", flat=True)) == {"valid"}
assert sum(query["sql"].startswith("DELETE FROM") for query in queries) == 2
@pytest.mark.django_db
class TestExtractZonesTask:
"""Tests for the extract_zones task."""
def test_task_runs_extraction(self, mocker: MockerFixture) -> None:
"""
GIVEN:
- A queued zone OCR task
WHEN:
- The task runs
THEN:
- Templates are selected for the requester and extraction runs on
the stored document
"""
extract = mocker.patch("documents.tasks.run_zone_extraction")
select = mocker.patch("documents.tasks.get_templates_for_document")
doc = DocumentFactory()
user = UserFactory()
extract_zones(doc.id, requester=user)
select.assert_called_once_with(doc, requester=user)
extract.assert_called_once_with(doc, templates=select.return_value)
def test_task_runs_the_chosen_template(self, mocker: MockerFixture) -> None:
"""
GIVEN:
- A queued zone OCR task with a chosen template
WHEN:
- The task runs
THEN:
- Zone extraction runs with only that template
"""
extract = mocker.patch("documents.tasks.run_zone_extraction")
doc = DocumentFactory()
template = OcrTemplateFactory()
extract_zones(doc.id, template.pk)
extract.assert_called_once_with(doc, templates=[template])
@pytest.mark.parametrize(
("reason", "returns_root"),
[
pytest.param(None, True, id="written"),
pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"),
],
)
def test_task_reports_the_root_when_a_value_was_written(
self,
mocker: MockerFixture,
reason: ZoneOcrReason | None,
returns_root: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A document version whose extraction does or does not write a value
WHEN:
- The task runs
THEN:
- The root document id is returned only when a value was written
"""
root = DocumentFactory()
DocumentFactory(root_document=root, version_index=1)
template = OcrTemplateFactory()
mocker.patch(
"documents.tasks.run_zone_extraction",
return_value=[
ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason),
],
)
result = extract_zones(root.pk, template.pk)
assert result == (root.pk if returns_root else None)
def test_task_runs_a_version_and_reports_the_root(
self,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- A root document with two versions and a template
WHEN:
- The task runs for the older version id
THEN:
- Extraction runs on that version, not the newest
- The root document id is returned when a value was written
"""
root = DocumentFactory()
older = DocumentFactory(root_document=root, version_index=1)
DocumentFactory(root_document=root, version_index=2)
template = OcrTemplateFactory()
extract = mocker.patch(
"documents.tasks.run_zone_extraction",
return_value=[
ZoneOcrResult(template="t", zone="z", custom_field="", reason=None),
],
)
result = extract_zones(older.pk, template.pk)
extract.assert_called_once_with(older, templates=[template])
assert result == root.pk
@pytest.mark.parametrize(
("resolve_latest", "processes_root"),
[
pytest.param(True, False, id="resolves-latest"),
pytest.param(False, True, id="exact-document"),
],
)
def test_task_resolves_latest_only_when_asked(
self,
mocker: MockerFixture,
resolve_latest: bool, # noqa: FBT001
processes_root: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A root document with a newer version and a template
WHEN:
- The task runs for the root id with resolve_latest on or off
THEN:
- The newest version is processed when resolve_latest is on
- The root document itself is processed when it is off
"""
root = DocumentFactory()
newest = DocumentFactory(root_document=root, version_index=1)
template = OcrTemplateFactory()
extract = mocker.patch("documents.tasks.run_zone_extraction", return_value=[])
extract_zones(root.pk, template.pk, resolve_latest=resolve_latest)
extract.assert_called_once_with(
root if processes_root else newest,
templates=[template],
)
def test_task_skips_a_deleted_template(self, mocker: MockerFixture) -> None:
"""
GIVEN:
- A template deleted between queueing and running
WHEN:
- The task runs
THEN:
- Nothing is extracted
"""
extract = mocker.patch("documents.tasks.run_zone_extraction")
doc = DocumentFactory()
extract_zones(doc.id, 999)
extract.assert_not_called()