mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-21 04:52:40 +00:00
Fix: make bulk zone OCR survive a missing template or failing document
Inline extract_zones into run_zone_ocr, load documents and the template once, and process each root once. A failing document is logged and the task fails after the rest finish.
This commit is contained in:
1 parent
a08918eb9c
commit
ccbff94208
6 files changed
+346
-328
No files matched your search
+58
-15
@@ -33,6 +33,7 @@ from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import OcrTemplate
|
||||
from documents.models import PaperlessTask
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
@@ -41,13 +42,16 @@ from documents.permissions import set_permissions_for_objects
|
||||
from documents.plugins.helpers import DocumentsStatusManager
|
||||
from documents.tasks import bulk_update_documents
|
||||
from documents.tasks import consume_file
|
||||
from documents.tasks import extract_zones
|
||||
from documents.tasks import remove_document_from_index
|
||||
from documents.tasks import run_workflows_for_documents
|
||||
from documents.tasks import update_document_content_maybe_archive_file
|
||||
from documents.versioning import get_latest_version_for_root
|
||||
from documents.versioning import get_root_document
|
||||
from documents.versioning import latest_version
|
||||
from documents.versioning import versions_newest_first
|
||||
from documents.zone_ocr import get_templates_for_document
|
||||
from documents.zone_ocr import run_zone_extraction
|
||||
from documents.zone_ocr import zone_results_written
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
@@ -443,7 +447,7 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
|
||||
A root document with versions reprocesses its latest version, which is the
|
||||
file whose content, archive and thumbnail are shown for it.
|
||||
"""
|
||||
latest_version = versions_newest_first(
|
||||
newest_version_id = versions_newest_first(
|
||||
Document.objects.filter(root_document=OuterRef("pk")),
|
||||
).values("id")[:1]
|
||||
source_ids = (
|
||||
@@ -451,7 +455,7 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
|
||||
.annotate(
|
||||
source_id=Case(
|
||||
When(root_document__isnull=False, then=F("id")),
|
||||
default=Coalesce(Subquery(latest_version), F("id")),
|
||||
default=Coalesce(Subquery(newest_version_id), F("id")),
|
||||
),
|
||||
)
|
||||
.order_by()
|
||||
@@ -1268,22 +1272,61 @@ def run_zone_ocr(
|
||||
"""
|
||||
Run zone OCR within the tracked bulk operation task, then finish once for
|
||||
the root documents that had a value written. Root documents are run on
|
||||
their newest version unless resolve_latest is False.
|
||||
their newest version unless resolve_latest is False. Without a template,
|
||||
every template matching the document that the user can view is run.
|
||||
|
||||
A failing document is logged and the rest still run; the task then fails.
|
||||
"""
|
||||
written_ids: list[int] = []
|
||||
try:
|
||||
for document_id in doc_ids:
|
||||
written_id = extract_zones(
|
||||
document_id=document_id,
|
||||
template_id=template,
|
||||
requester=user,
|
||||
resolve_latest=resolve_latest,
|
||||
selected_template: OcrTemplate | None = None
|
||||
if template is not None:
|
||||
selected_template = (
|
||||
OcrTemplate.objects.filter(pk=template)
|
||||
.prefetch_related("zones", "zones__custom_field")
|
||||
.first()
|
||||
)
|
||||
if selected_template is None:
|
||||
logger.warning(
|
||||
"OCR template %s no longer exists, not running zone OCR",
|
||||
template,
|
||||
)
|
||||
if written_id is not None:
|
||||
written_ids.append(written_id)
|
||||
return "OK"
|
||||
|
||||
documents = {d.pk: d for d in Document.objects.filter(pk__in=doc_ids)}
|
||||
missing_ids = [pk for pk in dict.fromkeys(doc_ids) if pk not in documents]
|
||||
if missing_ids:
|
||||
logger.warning("Documents %s not found, skipping zone OCR", missing_ids)
|
||||
|
||||
targets: dict[int, Document] = {}
|
||||
for document_id in doc_ids:
|
||||
document = documents.get(document_id)
|
||||
if document is None:
|
||||
continue
|
||||
if resolve_latest:
|
||||
document = latest_version(document)
|
||||
targets.setdefault(document.pk, document)
|
||||
|
||||
written_ids: list[int] = []
|
||||
failed_ids: list[int] = []
|
||||
try:
|
||||
for document in targets.values():
|
||||
try:
|
||||
templates = (
|
||||
[selected_template]
|
||||
if selected_template is not None
|
||||
else get_templates_for_document(document, requester=user)
|
||||
)
|
||||
results = run_zone_extraction(document, templates=templates)
|
||||
if zone_results_written(results):
|
||||
written_ids.append(get_root_document(document).pk)
|
||||
except Exception:
|
||||
logger.exception("Zone OCR failed for document %s", document.pk)
|
||||
failed_ids.append(document.pk)
|
||||
finally:
|
||||
# Documents already written are finished even if a later one raises.
|
||||
if written_ids:
|
||||
bulk_update_documents(list(dict.fromkeys(written_ids)))
|
||||
|
||||
if failed_ids:
|
||||
raise RuntimeError(
|
||||
f"Zone OCR failed for {len(failed_ids)} of {len(targets)} documents",
|
||||
)
|
||||
return "OK"
|
||||
@@ -118,8 +118,12 @@ def execute_bulk_operation(
|
||||
raise PermissionDenied("Insufficient permissions to run this workflow")
|
||||
|
||||
if operation == "run_zone_ocr" and parameters.get("template") is not None:
|
||||
template = OcrTemplate.objects.get(pk=parameters["template"])
|
||||
if not has_perms_owner_aware(user, "view_ocrtemplate", template):
|
||||
template = OcrTemplate.objects.filter(pk=parameters["template"]).first()
|
||||
if template is not None and not has_perms_owner_aware(
|
||||
user,
|
||||
"view_ocrtemplate",
|
||||
template,
|
||||
):
|
||||
raise PermissionDenied("Insufficient permissions to run this OCR template")
|
||||
|
||||
actor_context = set_actor(user) if settings.AUDIT_LOG_ENABLED else nullcontext()
|
||||
|
||||
@@ -11,7 +11,6 @@ from tempfile import mkstemp
|
||||
from celery import Task
|
||||
from celery import shared_task
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import User
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.contrib.sessions.models import Session
|
||||
from django.db import models
|
||||
@@ -49,7 +48,6 @@ from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentBarcode
|
||||
from documents.models import DocumentType
|
||||
from documents.models import OcrTemplate
|
||||
from documents.models import PaperlessTask
|
||||
from documents.models import ShareLink
|
||||
from documents.models import ShareLinkBundle
|
||||
@@ -72,12 +70,7 @@ from documents.utils import IterWrapper
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import identity
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.versioning import get_root_document
|
||||
from documents.versioning import latest_version
|
||||
from documents.workflows.utils import get_workflows_for_trigger
|
||||
from documents.zone_ocr import get_templates_for_document
|
||||
from documents.zone_ocr import run_zone_extraction
|
||||
from documents.zone_ocr import zone_results_written
|
||||
from paperless.config import AIConfig
|
||||
from paperless.config import BarcodeConfig
|
||||
from paperless.config import RemoteOCRConfig
|
||||
@@ -376,50 +369,6 @@ def _read_barcodes_for_reprocess(document: Document) -> list[StoredBarcode] | No
|
||||
return None
|
||||
|
||||
|
||||
@shared_task
|
||||
def extract_zones(
|
||||
document_id: int,
|
||||
template_id: int | None = None,
|
||||
requester: User | None = None,
|
||||
*,
|
||||
resolve_latest: bool = True,
|
||||
) -> int | None:
|
||||
"""
|
||||
Run an OCR template against a stored document, or every template that
|
||||
matches it when no template is given. Matched templates are limited to
|
||||
those the requester can view.
|
||||
|
||||
A root document id is mapped to its newest version unless resolve_latest
|
||||
is False, in which case exactly the given document is processed.
|
||||
|
||||
Returns the root document's id when any value was written, otherwise None.
|
||||
The caller finishes the edit, for example with bulk_update_documents.
|
||||
"""
|
||||
document = Document.objects.get(id=document_id)
|
||||
if resolve_latest:
|
||||
document = latest_version(document)
|
||||
if template_id is None:
|
||||
templates = get_templates_for_document(document, requester=requester)
|
||||
else:
|
||||
try:
|
||||
template = OcrTemplate.objects.prefetch_related(
|
||||
"zones",
|
||||
"zones__custom_field",
|
||||
).get(id=template_id)
|
||||
except OcrTemplate.DoesNotExist:
|
||||
logger.warning(
|
||||
"OCR template %s no longer exists, not running zone OCR",
|
||||
template_id,
|
||||
)
|
||||
return None
|
||||
templates = [template]
|
||||
|
||||
results = run_zone_extraction(document, templates=templates)
|
||||
if zone_results_written(results):
|
||||
return get_root_document(document).pk
|
||||
return None
|
||||
|
||||
|
||||
@shared_task
|
||||
def update_document_content_maybe_archive_file(
|
||||
document_id,
|
||||
|
||||
@@ -26,6 +26,8 @@ from documents.models import Tag
|
||||
from documents.pdf_ops import PageSpec
|
||||
from documents.permissions import PermissionsSpec
|
||||
from documents.permissions import set_permissions_for_objects
|
||||
from documents.zone_ocr import ZoneOcrReason
|
||||
from documents.zone_ocr import ZoneOcrResult
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import OcrTemplateFactory
|
||||
@@ -1936,17 +1938,34 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
self.assertIn("Error removing password from document", cm.output[0])
|
||||
|
||||
|
||||
def _written_result() -> ZoneOcrResult:
|
||||
return ZoneOcrResult(template="t", zone="z", custom_field="", reason=None)
|
||||
|
||||
|
||||
def _unwritten_result() -> ZoneOcrResult:
|
||||
return ZoneOcrResult(
|
||||
template="t",
|
||||
zone="z",
|
||||
custom_field="",
|
||||
reason=ZoneOcrReason.NO_VALUE,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBulkEditRunZoneOcr:
|
||||
"""Bulk run_zone_ocr runs zone extraction for each selected document."""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_extract_zones(self, mocker: MockerFixture) -> mock.MagicMock:
|
||||
return mocker.patch("documents.bulk_edit.extract_zones", return_value=None)
|
||||
def mock_extract(self, mocker: MockerFixture) -> mock.MagicMock:
|
||||
return mocker.patch("documents.bulk_edit.run_zone_extraction", return_value=[])
|
||||
|
||||
def test_runs_every_selected_document_in_the_current_task(
|
||||
@pytest.fixture
|
||||
def mock_finish(self, mocker: MockerFixture) -> mock.MagicMock:
|
||||
return mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
|
||||
def test_selects_templates_for_the_requester(
|
||||
self,
|
||||
mock_extract_zones: mock.MagicMock,
|
||||
mock_extract: mock.MagicMock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
@@ -1955,102 +1974,157 @@ class TestBulkEditRunZoneOcr:
|
||||
WHEN:
|
||||
- run_zone_ocr is called for the documents without a template
|
||||
THEN:
|
||||
- Extraction runs inline for each document with the requester
|
||||
- Nothing is queued
|
||||
- Templates are selected for each document with the requester
|
||||
- Extraction runs inline for each document
|
||||
"""
|
||||
select = mocker.patch("documents.bulk_edit.get_templates_for_document")
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
|
||||
user = UserFactory()
|
||||
|
||||
result = bulk_edit.run_zone_ocr([doc.id for doc in docs], user=user)
|
||||
|
||||
assert result == "OK"
|
||||
assert mock_extract_zones.call_args_list == [
|
||||
mocker.call(
|
||||
document_id=doc.id,
|
||||
template_id=None,
|
||||
requester=user,
|
||||
resolve_latest=True,
|
||||
)
|
||||
for doc in docs
|
||||
assert select.call_args_list == [
|
||||
mocker.call(doc, requester=user) for doc in docs
|
||||
]
|
||||
assert mock_extract.call_args_list == [
|
||||
mocker.call(doc, templates=select.return_value) for doc in docs
|
||||
]
|
||||
mock_extract_zones.apply_async.assert_not_called()
|
||||
|
||||
def test_runs_the_chosen_template(
|
||||
self,
|
||||
mock_extract_zones: mock.MagicMock,
|
||||
mock_extract: mock.MagicMock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document and an OCR template
|
||||
- Two documents and an OCR template
|
||||
WHEN:
|
||||
- run_zone_ocr is called for the document with the template
|
||||
- run_zone_ocr is called for the documents with the template
|
||||
THEN:
|
||||
- Extraction runs inline with the template and no requester
|
||||
- Nothing is queued
|
||||
- Extraction runs for each document with only that template
|
||||
- Template matching is not used
|
||||
"""
|
||||
doc = DocumentFactory()
|
||||
select = mocker.patch("documents.bulk_edit.get_templates_for_document")
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
template = OcrTemplateFactory()
|
||||
|
||||
bulk_edit.run_zone_ocr([doc.id], template=template.pk)
|
||||
bulk_edit.run_zone_ocr([doc.id for doc in docs], template=template.pk)
|
||||
|
||||
mock_extract_zones.assert_called_once_with(
|
||||
document_id=doc.id,
|
||||
template_id=template.pk,
|
||||
requester=None,
|
||||
resolve_latest=True,
|
||||
)
|
||||
mock_extract_zones.apply_async.assert_not_called()
|
||||
select.assert_not_called()
|
||||
assert mock_extract.call_args_list == [
|
||||
mocker.call(doc, templates=[template]) for doc in docs
|
||||
]
|
||||
|
||||
def test_finishes_once_for_documents_with_a_write(
|
||||
def test_deleted_template_runs_nothing(
|
||||
self,
|
||||
mock_extract_zones: mock.MagicMock,
|
||||
mock_extract: mock.MagicMock,
|
||||
mock_finish: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A chosen template that no longer exists
|
||||
WHEN:
|
||||
- run_zone_ocr is called
|
||||
THEN:
|
||||
- OK is returned without running extraction or finishing
|
||||
"""
|
||||
doc = DocumentFactory()
|
||||
|
||||
result = bulk_edit.run_zone_ocr([doc.pk], template=999)
|
||||
|
||||
assert result == "OK"
|
||||
mock_extract.assert_not_called()
|
||||
mock_finish.assert_not_called()
|
||||
|
||||
def test_missing_document_does_not_abort_the_others(
|
||||
self,
|
||||
mock_extract: mock.MagicMock,
|
||||
mock_finish: mock.MagicMock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Three selected documents, the second of which had nothing written
|
||||
and the third reporting the same root as the first
|
||||
- A selection holding a nonexistent id and a trashed document
|
||||
alongside a live document
|
||||
WHEN:
|
||||
- run_zone_ocr is called
|
||||
THEN:
|
||||
- bulk_update_documents runs once with the distinct written ids
|
||||
- Only the live document is extracted
|
||||
- The live document is finished
|
||||
"""
|
||||
live = DocumentFactory()
|
||||
trashed = DocumentFactory()
|
||||
trashed.delete()
|
||||
template = OcrTemplateFactory()
|
||||
mock_extract.return_value = [_written_result()]
|
||||
|
||||
bulk_edit.run_zone_ocr([999999, trashed.pk, live.pk], template=template.pk)
|
||||
|
||||
mock_extract.assert_called_once_with(live, templates=[template])
|
||||
mock_finish.assert_called_once_with([live.pk])
|
||||
|
||||
def test_failing_document_does_not_abort_the_others(
|
||||
self,
|
||||
mock_extract: mock.MagicMock,
|
||||
mock_finish: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Three selected documents, the second of which raises during extraction
|
||||
WHEN:
|
||||
- run_zone_ocr is called
|
||||
THEN:
|
||||
- The failure is logged and the remaining document still runs
|
||||
- bulk_update_documents runs once with the written documents
|
||||
- The task then fails, naming the number of failed documents
|
||||
"""
|
||||
docs = DocumentFactory.create_batch(3)
|
||||
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
mock_extract_zones.side_effect = [docs[0].pk, None, docs[0].pk]
|
||||
template = OcrTemplateFactory()
|
||||
mock_extract.side_effect = [
|
||||
[_written_result()],
|
||||
RuntimeError("boom"),
|
||||
[_written_result()],
|
||||
]
|
||||
|
||||
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
|
||||
with pytest.raises(RuntimeError, match="1 of 3 documents"):
|
||||
bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk)
|
||||
|
||||
finish.assert_called_once_with([docs[0].pk])
|
||||
assert mock_extract.call_count == 3
|
||||
mock_finish.assert_called_once_with([docs[0].pk, docs[2].pk])
|
||||
|
||||
def test_finishes_written_documents_when_a_later_one_raises(
|
||||
def test_finishes_each_root_once(
|
||||
self,
|
||||
mock_extract_zones: mock.MagicMock,
|
||||
mocker: MockerFixture,
|
||||
mock_extract: mock.MagicMock,
|
||||
mock_finish: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Two selected documents, the second of which raises during extraction
|
||||
- Two documents whose extraction writes, one being a version of the other
|
||||
when not resolving the latest version
|
||||
WHEN:
|
||||
- run_zone_ocr is called
|
||||
- run_zone_ocr is called without resolving the latest version
|
||||
THEN:
|
||||
- The exception propagates
|
||||
- bulk_update_documents still runs with the first document's id
|
||||
- bulk_update_documents runs once with the root id listed once
|
||||
"""
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
mock_extract_zones.side_effect = [docs[0].pk, RuntimeError("boom")]
|
||||
root = DocumentFactory()
|
||||
version = DocumentFactory(root_document=root, version_index=1)
|
||||
template = OcrTemplateFactory()
|
||||
mock_extract.return_value = [_written_result()]
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
|
||||
bulk_edit.run_zone_ocr(
|
||||
[root.pk, version.pk],
|
||||
template=template.pk,
|
||||
resolve_latest=False,
|
||||
)
|
||||
|
||||
finish.assert_called_once_with([docs[0].pk])
|
||||
assert mock_extract.call_count == 2
|
||||
mock_finish.assert_called_once_with([root.pk])
|
||||
|
||||
def test_does_not_finish_when_nothing_was_written(
|
||||
self,
|
||||
mock_extract_zones: mock.MagicMock,
|
||||
mocker: MockerFixture,
|
||||
mock_extract: mock.MagicMock,
|
||||
mock_finish: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
@@ -2061,11 +2135,122 @@ class TestBulkEditRunZoneOcr:
|
||||
- bulk_update_documents is not run
|
||||
"""
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
template = OcrTemplateFactory()
|
||||
mock_extract.return_value = [_unwritten_result()]
|
||||
|
||||
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
|
||||
bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk)
|
||||
|
||||
finish.assert_not_called()
|
||||
mock_finish.assert_not_called()
|
||||
|
||||
def test_finishes_written_documents_when_the_run_is_interrupted(
|
||||
self,
|
||||
mock_extract: mock.MagicMock,
|
||||
mock_finish: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Two selected documents, the second of which raises an exception
|
||||
that is not an Exception subclass
|
||||
WHEN:
|
||||
- run_zone_ocr is called
|
||||
THEN:
|
||||
- The interruption propagates
|
||||
- bulk_update_documents still runs with the first document's id
|
||||
"""
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
template = OcrTemplateFactory()
|
||||
mock_extract.side_effect = [[_written_result()], KeyboardInterrupt()]
|
||||
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
bulk_edit.run_zone_ocr([d.pk for d in docs], template=template.pk)
|
||||
|
||||
mock_finish.assert_called_once_with([docs[0].pk])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("resolve_latest", "processes_root"),
|
||||
[
|
||||
pytest.param(True, False, id="resolves-latest"),
|
||||
pytest.param(False, True, id="exact-document"),
|
||||
],
|
||||
)
|
||||
def test_resolves_latest_only_when_asked(
|
||||
self,
|
||||
mock_extract: mock.MagicMock,
|
||||
resolve_latest: bool, # noqa: FBT001
|
||||
processes_root: bool, # noqa: FBT001
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a newer version and a template
|
||||
WHEN:
|
||||
- run_zone_ocr runs for the root id with resolve_latest on or off
|
||||
THEN:
|
||||
- The newest version is processed when resolve_latest is on
|
||||
- The root document itself is processed when it is off
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
newest = DocumentFactory(root_document=root, version_index=1)
|
||||
template = OcrTemplateFactory()
|
||||
|
||||
bulk_edit.run_zone_ocr(
|
||||
[root.pk],
|
||||
template=template.pk,
|
||||
resolve_latest=resolve_latest,
|
||||
)
|
||||
|
||||
mock_extract.assert_called_once_with(
|
||||
root if processes_root else newest,
|
||||
templates=[template],
|
||||
)
|
||||
|
||||
def test_root_and_its_version_are_processed_once(
|
||||
self,
|
||||
mock_extract: mock.MagicMock,
|
||||
mock_finish: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version, both listed in the selection
|
||||
WHEN:
|
||||
- run_zone_ocr runs with resolve_latest on
|
||||
THEN:
|
||||
- The newest version is extracted once
|
||||
- The root is finished once
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
version = DocumentFactory(root_document=root, version_index=1)
|
||||
template = OcrTemplateFactory()
|
||||
mock_extract.return_value = [_written_result()]
|
||||
|
||||
bulk_edit.run_zone_ocr([root.pk, version.pk], template=template.pk)
|
||||
|
||||
mock_extract.assert_called_once_with(version, templates=[template])
|
||||
mock_finish.assert_called_once_with([root.pk])
|
||||
|
||||
def test_a_version_reports_its_root(
|
||||
self,
|
||||
mock_extract: mock.MagicMock,
|
||||
mock_finish: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root with two versions and the older version selected
|
||||
WHEN:
|
||||
- run_zone_ocr runs without resolving the latest version
|
||||
THEN:
|
||||
- Extraction runs on that version, not the newest
|
||||
- The root is finished
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
older = DocumentFactory(root_document=root, version_index=1)
|
||||
DocumentFactory(root_document=root, version_index=2)
|
||||
template = OcrTemplateFactory()
|
||||
mock_extract.return_value = [_written_result()]
|
||||
|
||||
bulk_edit.run_zone_ocr([older.pk], template=template.pk, resolve_latest=False)
|
||||
|
||||
mock_extract.assert_called_once_with(older, templates=[template])
|
||||
mock_finish.assert_called_once_with([root.pk])
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
|
||||
@@ -14,12 +14,12 @@ from rest_framework.test import APIClient
|
||||
from documents.bulk_operations import execute_bulk_operation
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import OcrTemplate
|
||||
from documents.models import PaperlessTask
|
||||
from documents.signals.handlers import before_task_publish_handler
|
||||
from documents.signals.handlers import task_failure_handler
|
||||
from documents.signals.handlers import task_postrun_handler
|
||||
from documents.signals.handlers import task_revoked_handler
|
||||
from documents.zone_ocr import ZoneOcrResult
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import OcrTemplateFactory
|
||||
from paperless_testing.factories import UserFactory
|
||||
@@ -98,8 +98,8 @@ class TestRunZoneOcrBulkOperation:
|
||||
template = OcrTemplateFactory()
|
||||
enqueue = mocker.patch("documents.views.execute_bulk_operation.apply_async")
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
"documents.bulk_edit.run_zone_extraction",
|
||||
return_value=[],
|
||||
)
|
||||
enqueue.return_value.id = "zone-ocr-task"
|
||||
response = admin_client.post(
|
||||
@@ -135,14 +135,13 @@ class TestRunZoneOcrBulkOperation:
|
||||
WHEN:
|
||||
- The tracked task executes it
|
||||
THEN:
|
||||
- Extraction is called synchronously with the requester
|
||||
- No further extraction task is enqueued
|
||||
- Extraction runs synchronously with only that template
|
||||
- The result reports the operation and document count
|
||||
"""
|
||||
template = OcrTemplateFactory()
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
"documents.bulk_edit.run_zone_extraction",
|
||||
return_value=[],
|
||||
)
|
||||
result = execute_bulk_operation(
|
||||
"run_zone_ocr",
|
||||
@@ -150,13 +149,7 @@ class TestRunZoneOcrBulkOperation:
|
||||
{"template": template.pk},
|
||||
admin_user.pk,
|
||||
)
|
||||
extract.assert_called_once_with(
|
||||
document_id=document.pk,
|
||||
template_id=template.pk,
|
||||
requester=admin_user,
|
||||
resolve_latest=True,
|
||||
)
|
||||
extract.apply_async.assert_not_called()
|
||||
extract.assert_called_once_with(document, templates=[template])
|
||||
assert result == {"operation": "run_zone_ocr", "document_count": 1}
|
||||
|
||||
def test_finishes_with_the_ids_that_were_written(
|
||||
@@ -173,13 +166,21 @@ class TestRunZoneOcrBulkOperation:
|
||||
THEN:
|
||||
- bulk_update_documents runs once with that document id
|
||||
"""
|
||||
template = OcrTemplateFactory()
|
||||
mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=document.pk,
|
||||
"documents.bulk_edit.run_zone_extraction",
|
||||
return_value=[
|
||||
ZoneOcrResult(template="t", zone="z", custom_field="", reason=None),
|
||||
],
|
||||
)
|
||||
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
|
||||
execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk)
|
||||
execute_bulk_operation(
|
||||
"run_zone_ocr",
|
||||
[document.pk],
|
||||
{"template": template.pk},
|
||||
admin_user.pk,
|
||||
)
|
||||
|
||||
finish.assert_called_once_with([document.pk])
|
||||
|
||||
@@ -195,20 +196,17 @@ class TestRunZoneOcrBulkOperation:
|
||||
WHEN:
|
||||
- The tracked task executes it
|
||||
THEN:
|
||||
- Extraction receives the requesting user so templates are limited to
|
||||
those they can view
|
||||
- Templates are selected for the requesting user so they are limited
|
||||
to those they can view
|
||||
"""
|
||||
select = mocker.patch("documents.bulk_edit.get_templates_for_document")
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
"documents.bulk_edit.run_zone_extraction",
|
||||
return_value=[],
|
||||
)
|
||||
execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk)
|
||||
extract.assert_called_once_with(
|
||||
document_id=document.pk,
|
||||
template_id=None,
|
||||
requester=admin_user,
|
||||
resolve_latest=True,
|
||||
)
|
||||
select.assert_called_once_with(document, requester=admin_user)
|
||||
extract.assert_called_once_with(document, templates=select.return_value)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"revoked",
|
||||
@@ -259,14 +257,14 @@ class TestRunZoneOcrBulkOperation:
|
||||
else:
|
||||
User.objects.filter(pk=regular_user.pk).update(is_active=False)
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
"documents.bulk_edit.run_zone_extraction",
|
||||
return_value=[],
|
||||
)
|
||||
with pytest.raises(PermissionDenied):
|
||||
execute_bulk_operation(**enqueue.call_args.kwargs["kwargs"])
|
||||
extract.assert_not_called()
|
||||
|
||||
def test_cannot_fall_back_to_matching_if_chosen_template_is_deleted(
|
||||
def test_does_not_fall_back_to_matching_if_chosen_template_is_deleted(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
admin_user: User,
|
||||
@@ -278,21 +276,22 @@ class TestRunZoneOcrBulkOperation:
|
||||
WHEN:
|
||||
- The tracked task executes it
|
||||
THEN:
|
||||
- OcrTemplate.DoesNotExist is raised instead of falling back to
|
||||
template matching
|
||||
- The task completes without falling back to template matching
|
||||
- Extraction never runs
|
||||
"""
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
"documents.bulk_edit.run_zone_extraction",
|
||||
return_value=[],
|
||||
)
|
||||
with pytest.raises(OcrTemplate.DoesNotExist):
|
||||
execute_bulk_operation(
|
||||
"run_zone_ocr",
|
||||
[document.pk],
|
||||
{"template": 999},
|
||||
admin_user.pk,
|
||||
)
|
||||
select = mocker.patch("documents.bulk_edit.get_templates_for_document")
|
||||
result = execute_bulk_operation(
|
||||
"run_zone_ocr",
|
||||
[document.pk],
|
||||
{"template": 999},
|
||||
admin_user.pk,
|
||||
)
|
||||
assert result == {"operation": "run_zone_ocr", "document_count": 1}
|
||||
select.assert_not_called()
|
||||
extract.assert_not_called()
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ from django.test import TestCase
|
||||
from django.test import override_settings
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from django.utils import timezone
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from documents import tasks
|
||||
from documents.models import Correspondent
|
||||
@@ -21,15 +20,10 @@ from documents.models import Tag
|
||||
from documents.models import WorkflowAction
|
||||
from documents.sanity_checker import SanityCheckFailedException
|
||||
from documents.sanity_checker import SanityCheckMessages
|
||||
from documents.tasks import extract_zones
|
||||
from documents.tests.helpers import dummy_preprocess
|
||||
from documents.zone_ocr import ZoneOcrReason
|
||||
from documents.zone_ocr import ZoneOcrResult
|
||||
from paperless_testing.assertions import FileSystemAssertsMixin
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import OcrTemplateFactory
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
|
||||
|
||||
|
||||
@@ -654,159 +648,3 @@ class TestClearExpiredSessions:
|
||||
|
||||
assert set(Session.objects.values_list("session_key", flat=True)) == {"valid"}
|
||||
assert sum(query["sql"].startswith("DELETE FROM") for query in queries) == 2
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestExtractZonesTask:
|
||||
"""Tests for the extract_zones task."""
|
||||
|
||||
def test_task_runs_extraction(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A queued zone OCR task
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- Templates are selected for the requester and extraction runs on
|
||||
the stored document
|
||||
"""
|
||||
extract = mocker.patch("documents.tasks.run_zone_extraction")
|
||||
select = mocker.patch("documents.tasks.get_templates_for_document")
|
||||
doc = DocumentFactory()
|
||||
user = UserFactory()
|
||||
|
||||
extract_zones(doc.id, requester=user)
|
||||
|
||||
select.assert_called_once_with(doc, requester=user)
|
||||
extract.assert_called_once_with(doc, templates=select.return_value)
|
||||
|
||||
def test_task_runs_the_chosen_template(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A queued zone OCR task with a chosen template
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- Zone extraction runs with only that template
|
||||
"""
|
||||
extract = mocker.patch("documents.tasks.run_zone_extraction")
|
||||
doc = DocumentFactory()
|
||||
template = OcrTemplateFactory()
|
||||
|
||||
extract_zones(doc.id, template.pk)
|
||||
|
||||
extract.assert_called_once_with(doc, templates=[template])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("reason", "returns_root"),
|
||||
[
|
||||
pytest.param(None, True, id="written"),
|
||||
pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"),
|
||||
],
|
||||
)
|
||||
def test_task_reports_the_root_when_a_value_was_written(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
reason: ZoneOcrReason | None,
|
||||
returns_root: bool, # noqa: FBT001
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document version whose extraction does or does not write a value
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- The root document id is returned only when a value was written
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
DocumentFactory(root_document=root, version_index=1)
|
||||
template = OcrTemplateFactory()
|
||||
mocker.patch(
|
||||
"documents.tasks.run_zone_extraction",
|
||||
return_value=[
|
||||
ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason),
|
||||
],
|
||||
)
|
||||
|
||||
result = extract_zones(root.pk, template.pk)
|
||||
|
||||
assert result == (root.pk if returns_root else None)
|
||||
|
||||
def test_task_runs_a_version_and_reports_the_root(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with two versions and a template
|
||||
WHEN:
|
||||
- The task runs for the older version id
|
||||
THEN:
|
||||
- Extraction runs on that version, not the newest
|
||||
- The root document id is returned when a value was written
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
older = DocumentFactory(root_document=root, version_index=1)
|
||||
DocumentFactory(root_document=root, version_index=2)
|
||||
template = OcrTemplateFactory()
|
||||
extract = mocker.patch(
|
||||
"documents.tasks.run_zone_extraction",
|
||||
return_value=[
|
||||
ZoneOcrResult(template="t", zone="z", custom_field="", reason=None),
|
||||
],
|
||||
)
|
||||
|
||||
result = extract_zones(older.pk, template.pk)
|
||||
|
||||
extract.assert_called_once_with(older, templates=[template])
|
||||
assert result == root.pk
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("resolve_latest", "processes_root"),
|
||||
[
|
||||
pytest.param(True, False, id="resolves-latest"),
|
||||
pytest.param(False, True, id="exact-document"),
|
||||
],
|
||||
)
|
||||
def test_task_resolves_latest_only_when_asked(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
resolve_latest: bool, # noqa: FBT001
|
||||
processes_root: bool, # noqa: FBT001
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a newer version and a template
|
||||
WHEN:
|
||||
- The task runs for the root id with resolve_latest on or off
|
||||
THEN:
|
||||
- The newest version is processed when resolve_latest is on
|
||||
- The root document itself is processed when it is off
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
newest = DocumentFactory(root_document=root, version_index=1)
|
||||
template = OcrTemplateFactory()
|
||||
extract = mocker.patch("documents.tasks.run_zone_extraction", return_value=[])
|
||||
|
||||
extract_zones(root.pk, template.pk, resolve_latest=resolve_latest)
|
||||
|
||||
extract.assert_called_once_with(
|
||||
root if processes_root else newest,
|
||||
templates=[template],
|
||||
)
|
||||
|
||||
def test_task_skips_a_deleted_template(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A template deleted between queueing and running
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- Nothing is extracted
|
||||
"""
|
||||
extract = mocker.patch("documents.tasks.run_zone_extraction")
|
||||
doc = DocumentFactory()
|
||||
|
||||
extract_zones(doc.id, 999)
|
||||
|
||||
extract.assert_not_called()
|
||||
Reference in new issue
Block a user