Fix: queue single document zone OCR runs instead of running them in the request

Rendering and OCR ran inside the HTTP request and could block other
requests for minutes. The action now validates, then queues the bulk
zone OCR operation for the selected file and returns its task id.
This commit is contained in:
stumpylog committed 2026-11-19 15:25:48 -08:00
1 parent a6b81b2d64
commit c5fdba73a1
11 files changed
+379 -483

No files matched your search

+4 -1
View File
@@ -1263,10 +1263,12 @@ def run_zone_ocr(
*,
template: int | None = None,
user: User | None = None,
resolve_latest: bool = True,
) -> Literal["OK"]:
"""
Run zone OCR within the tracked bulk operation task, then finish once for
the root documents that had a value written.
the root documents that had a value written. Root documents are run on
their newest version unless resolve_latest is False.
"""
written_ids: list[int] = []
try:
@@ -1275,6 +1277,7 @@ def run_zone_ocr(
document_id=document_id,
template_id=template,
requester=user,
resolve_latest=resolve_latest,
)
if written_id is not None:
written_ids.append(written_id)
+11 -30
View File
@@ -99,9 +99,6 @@ from documents.validators import uri_validator
from documents.validators import url_validator
from documents.versioning import has_prefetched_effective_content
from documents.versioning import sort_versions_newest_first
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from documents.zone_ocr import ZoneOcrRunStatus
from documents.zone_ocr import ZoneOcrSkipReason
from documents.zone_ocr import field_key
@@ -1819,6 +1816,7 @@ class RunZoneOcrSerializer(serializers.Serializer[Any]):
required=False,
allow_null=True,
)
from_webui = serializers.BooleanField(required=False, default=False)
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
@@ -1833,34 +1831,18 @@ class RunZoneOcrSerializer(serializers.Serializer[Any]):
).prefetch_related("zones", "zones__custom_field")
class ZoneOcrResultSerializer(serializers.Serializer[ZoneOcrResult]):
"""The outcome of one zone, mirroring the ZoneOcrResult dataclass."""
template = serializers.CharField(read_only=True)
zone = serializers.CharField(read_only=True)
custom_field = serializers.CharField(read_only=True)
value = serializers.JSONField(
read_only=True,
allow_null=True,
help_text=(
"The value written to the target. A string, number, boolean or ISO "
"date string, depending on the target, or null when nothing was written."
),
)
reason = serializers.ChoiceField(
choices=[reason.value for reason in ZoneOcrReason],
read_only=True,
allow_null=True,
help_text="Why no value was written, or null when one was.",
)
class RunZoneOcrResponseSerializer(serializers.Serializer[dict[str, Any]]):
"""The response of running zone OCR on one document."""
"""The response of queueing zone OCR on one document."""
status = serializers.ChoiceField(
choices=[status.value for status in ZoneOcrRunStatus],
result = serializers.ChoiceField(
choices=["OK", "skipped"],
read_only=True,
help_text="OK when the run was queued, skipped when it was not.",
)
task_id = serializers.CharField(
read_only=True,
required=False,
help_text="The queued task. Absent when the run was skipped.",
)
reason = serializers.ChoiceField(
choices=[reason.value for reason in ZoneOcrSkipReason],
@@ -1868,7 +1850,6 @@ class RunZoneOcrResponseSerializer(serializers.Serializer[dict[str, Any]]):
required=False,
help_text="Present only when the run was skipped.",
)
results = ZoneOcrResultSerializer(many=True, read_only=True)
class OcrErrorSerializer(serializers.Serializer[dict[str, Any]]):
@@ -1878,7 +1859,7 @@ class OcrErrorSerializer(serializers.Serializer[dict[str, Any]]):
class RunZoneOcrDocumentsSerializer(DocumentSelectionSerializer, RunZoneOcrSerializer):
from_webui = serializers.BooleanField(required=False, default=False)
pass
class BulkEditSerializer(
+8 -1
View File
@@ -381,16 +381,23 @@ def extract_zones(
document_id: int,
template_id: int | None = None,
requester: User | None = None,
*,
resolve_latest: bool = True,
) -> int | None:
"""
Run an OCR template against a stored document, or every template that
matches it when no template is given. Matched templates are limited to
those the requester can view.
A root document id is mapped to its newest version unless resolve_latest
is False, in which case exactly the given document is processed.
Returns the root document's id when any value was written, otherwise None.
The caller finishes the edit, for example with bulk_update_documents.
"""
document = latest_version(Document.objects.get(id=document_id))
document = Document.objects.get(id=document_id)
if resolve_latest:
document = latest_version(document)
if template_id is None:
templates = get_templates_for_document(document, requester=requester)
else:
+207 -326
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import functools
import subprocess
from io import BytesIO
from types import SimpleNamespace
from typing import TYPE_CHECKING
from typing import Any
@@ -15,8 +16,8 @@ from PIL import Image
from rest_framework import status
from documents import bulk_edit
from documents.bulk_operations import execute_bulk_operation
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import MatchingModel
from documents.models import OcrTemplate
@@ -25,10 +26,7 @@ from documents.models import PaperlessTask
from documents.parsers import ParseError
from documents.search import get_backend
from documents.signals import document_consumption_finished
from documents.tasks import bulk_update_documents
from documents.zone_ocr import ZoneExtraction
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from documents.zone_ocr import apply_zone_plans
from documents.zone_ocr import plan_zone_extraction
from paperless_testing.factories import CustomFieldFactory
@@ -57,7 +55,8 @@ CONTENT = "Northwind Power & Light statement"
ENDPOINT = "/api/ocr_templates/"
RUN_ZONE_OCR_URL = "/api/documents/{pk}/run-zone-ocr/"
PREVIEW_ZONE_URL = f"{ENDPOINT}test-zone/"
SKIPPED_NO_TEMPLATE = {"status": "skipped", "reason": "no_template", "results": []}
SKIPPED_NO_TEMPLATE = {"result": "skipped", "reason": "no_template"}
TASK_ID = "zone-ocr-task"
def template_payload(**overrides: Any) -> dict[str, Any]:
@@ -149,14 +148,11 @@ def title_template(
@pytest.fixture
def mock_extract(mocker: MockerFixture) -> MagicMock:
return mocker.patch("documents.zone_ocr.run_zone_extraction")
@pytest.fixture
def mock_finish(mocker: MockerFixture) -> MagicMock:
def mock_enqueue(mocker: MockerFixture) -> MagicMock:
"""Replace queueing of the bulk operation task, returning a fixed task id."""
return mocker.patch(
"documents.views.bulk_edit.bulk_update_documents.apply_async",
"documents.views.execute_bulk_operation.apply_async",
return_value=SimpleNamespace(id=TASK_ID),
)
@@ -251,8 +247,8 @@ class TestZoneOcrIndexing:
- A document (optionally with an uploaded version) indexed for search
- An OCR template with a title zone and a custom field zone
WHEN:
- Zone OCR runs through ingestion, the single-document API or the
bulk task
- Zone OCR runs through ingestion, the single-document API (queued
as a bulk operation and run inline) or the bulk task
THEN:
- The root document title and custom field are updated
- The search index reflects the new title and custom field value
@@ -318,10 +314,14 @@ class TestZoneOcrIndexing:
backend.add_or_update(document)
assert backend.search_ids("title:before", user=None) == [document.pk]
# The single-document finish is a queued task, run it inline here.
# The single-document run is a queued bulk operation, run it inline here.
def run_inline(kwargs: dict[str, Any], **_: Any) -> SimpleNamespace:
execute_bulk_operation(**kwargs)
return SimpleNamespace(id="inline")
mocker.patch(
"documents.views.bulk_edit.bulk_update_documents.apply_async",
side_effect=lambda kwargs, **_: bulk_update_documents(**kwargs),
"documents.views.execute_bulk_operation.apply_async",
side_effect=run_inline,
)
with django_capture_on_commit_callbacks(execute=True):
if entry_point == "ingestion":
@@ -1827,9 +1827,9 @@ class TestOcrTemplateDelete:
@pytest.mark.django_db
@pytest.mark.usefixtures("paperless_dirs", "mock_finish")
@pytest.mark.usefixtures("paperless_dirs")
class TestRunZoneOcr:
"""Running zone OCR on documents through the run-zone-ocr endpoint."""
"""Queueing zone OCR on one document through the run-zone-ocr endpoint."""
@pytest.mark.parametrize(
"template_kwargs_list",
@@ -1848,7 +1848,7 @@ class TestRunZoneOcr:
def test_without_enabled_matching_template_is_skipped(
self,
admin_client: APIClient,
mock_extract: MagicMock,
mock_enqueue: MagicMock,
template_kwargs_list: list[dict[str, Any]],
) -> None:
"""
@@ -1856,9 +1856,9 @@ class TestRunZoneOcr:
- A document and either no templates, only a matching disabled
template, or only a template that does not match the document
WHEN:
- Zone OCR is run on the document
- Zone OCR is run on the document without choosing a template
THEN:
- The run is skipped with reason no_template and nothing is extracted
- The run is skipped with reason no_template and nothing is queued
"""
document = DocumentFactory(content=CONTENT)
for template_kwargs in template_kwargs_list:
@@ -1868,12 +1868,13 @@ class TestRunZoneOcr:
assert response.status_code == status.HTTP_200_OK
assert response.json() == SKIPPED_NO_TEMPLATE
mock_extract.assert_not_called()
mock_enqueue.assert_not_called()
def test_applies_a_chosen_template(
def test_queues_a_chosen_template(
self,
admin_client: APIClient,
mock_extract: MagicMock,
admin_user: User,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
@@ -1881,7 +1882,10 @@ class TestRunZoneOcr:
WHEN:
- Zone OCR is run on the document with that template chosen
THEN:
- The template is applied anyway, since it was chosen explicitly
- The bulk zone OCR operation is queued for the document and the
template, as the requesting user, even though the template does
not match
- The task id is returned
"""
document = DocumentFactory(content=CONTENT, mime_type="application/pdf")
document.source_path.write_bytes(b"test PDF")
@@ -1898,107 +1902,137 @@ class TestRunZoneOcr:
)
assert response.status_code == status.HTTP_200_OK
assert response.json()["status"] == "completed"
mock_extract.assert_called_once_with(document, templates=[template])
assert response.json() == {"result": "OK", "task_id": TASK_ID}
mock_enqueue.assert_called_once_with(
kwargs={
"operation": "run_zone_ocr",
"document_ids": [document.pk],
"parameters": {"template": template.pk, "resolve_latest": False},
"user_id": admin_user.pk,
"trigger_source": PaperlessTask.TriggerSource.API_UPLOAD,
},
headers={"trigger_source": PaperlessTask.TriggerSource.API_UPLOAD},
)
@pytest.mark.parametrize(
("reason", "finishes"),
("from_webui", "trigger_source"),
[
pytest.param(None, True, id="written"),
pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"),
pytest.param(True, PaperlessTask.TriggerSource.WEB_UI, id="web-ui"),
pytest.param(False, PaperlessTask.TriggerSource.API_UPLOAD, id="api"),
],
)
def test_finishes_the_edit_only_when_a_value_was_written(
def test_queues_every_matching_template_when_none_is_chosen(
self,
admin_client: APIClient,
mock_extract: MagicMock,
mock_finish: MagicMock,
reason: ZoneOcrReason | None,
finishes: bool, # noqa: FBT001
mock_enqueue: MagicMock,
from_webui: bool, # noqa: FBT001
trigger_source: PaperlessTask.TriggerSource,
) -> None:
"""
GIVEN:
- A document with a version and a template whose run does or does
not write a value
- A document with a matching template and a stored file
WHEN:
- Zone OCR is run on the version through the detail endpoint
- Zone OCR is run on the document without choosing a template,
with or without the web UI flag
THEN:
- The shared document update task is queued once for the root
document only when a value was written
- No per-document search reindex is requested directly
- The bulk operation is queued with no template, so the task
picks the matching templates, and with exact document processing
- The trigger source follows the web UI flag
"""
document = DocumentFactory(content=CONTENT)
version = DocumentFactory(
root_document=document,
version_index=1,
content=CONTENT,
)
version.source_path.write_bytes(b"test PDF")
template = OcrTemplateFactory(name="Northwind", match="Northwind")
mock_extract.return_value = [
ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason),
]
document = DocumentFactory(content=CONTENT, mime_type="application/pdf")
document.source_path.write_bytes(b"test PDF")
OcrTemplateFactory(name="Northwind", match="Northwind")
response = admin_client.post(
f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={version.pk}",
{"template": template.pk},
RUN_ZONE_OCR_URL.format(pk=document.pk),
{"from_webui": from_webui},
format="json",
)
assert response.status_code == status.HTTP_200_OK
if finishes:
mock_finish.assert_called_once_with(
kwargs={"document_ids": [document.pk]},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
else:
mock_finish.assert_not_called()
kwargs = mock_enqueue.call_args.kwargs
assert kwargs["kwargs"]["document_ids"] == [document.pk]
assert kwargs["kwargs"]["parameters"] == {"resolve_latest": False}
assert kwargs["kwargs"]["trigger_source"] == trigger_source
assert kwargs["headers"] == {"trigger_source": trigger_source}
@pytest.mark.parametrize(
"version",
("version", "queued"),
[
pytest.param("root", id="root-version"),
pytest.param("older", id="older-version"),
pytest.param("older", True, id="requested-older-version"),
pytest.param(None, False, id="latest-version-by-default"),
],
)
def test_matches_the_requested_file_version(
def test_matches_and_queues_the_resolved_file_version(
self,
admin_client: APIClient,
mock_extract: MagicMock,
version: str,
mock_enqueue: MagicMock,
version: str | None,
queued: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A document with two versions, the newest not matching the template
- A document with an older version matching the template and a
newest version that does not
WHEN:
- Zone OCR is run for the root document or an older version
- Zone OCR is run for the older version, or without a version
THEN:
- The template is applied to the requested file document
- Templates are matched against the resolved file
- The older version is queued by its own id when requested
- Without a version the latest does not match, so the run is skipped
"""
mock_extract.return_value = []
document = DocumentFactory(content=CONTENT)
document = DocumentFactory(content="Contoso")
older = DocumentFactory(
root_document=document,
version_index=1,
content=CONTENT,
)
DocumentFactory(root_document=document, version_index=2, content="Contoso")
template = OcrTemplateFactory(
name="Northwind",
match="Northwind",
source_width=100,
source_height=100,
)
file_document = {"root": document, "older": older}[version]
file_document.source_path.write_bytes(b"test PDF")
older.source_path.write_bytes(b"test PDF")
OcrTemplateFactory(name="Northwind", match="Northwind")
url = RUN_ZONE_OCR_URL.format(pk=document.pk)
if version is not None:
url = f"{url}?version={older.pk}"
response = admin_client.post(url)
assert response.status_code == status.HTTP_200_OK
if queued:
assert response.json() == {"result": "OK", "task_id": TASK_ID}
assert mock_enqueue.call_args.kwargs["kwargs"]["document_ids"] == [
older.pk,
]
else:
assert response.json() == SKIPPED_NO_TEMPLATE
mock_enqueue.assert_not_called()
def test_queues_the_root_file_exactly_when_requested(
self,
admin_client: APIClient,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
- A document with a newer version and a stored original file
WHEN:
- Zone OCR is run with the root document id as the version
THEN:
- The root id is queued with resolve_latest off, so the original
file is processed instead of the newest version
"""
document = DocumentFactory(content=CONTENT)
DocumentFactory(root_document=document, version_index=1, content="Contoso")
document.source_path.write_bytes(b"test PDF")
OcrTemplateFactory(name="Northwind", match="Northwind")
response = admin_client.post(
f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={file_document.pk}",
f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={document.pk}",
)
assert response.status_code == status.HTTP_200_OK
assert response.json()["status"] == "completed"
mock_extract.assert_called_once_with(file_document, templates=[template])
queued = mock_enqueue.call_args.kwargs["kwargs"]
assert queued["document_ids"] == [document.pk]
assert queued["parameters"] == {"resolve_latest": False}
@pytest.mark.parametrize(
"version",
@@ -2011,7 +2045,7 @@ class TestRunZoneOcr:
def test_rejects_invalid_or_unavailable_versions(
self,
admin_client: APIClient,
mock_extract: MagicMock,
mock_enqueue: MagicMock,
version: str,
) -> None:
"""
@@ -2020,7 +2054,7 @@ class TestRunZoneOcr:
WHEN:
- Zone OCR is run with an invalid or unavailable version
THEN:
- 404 is returned and nothing is extracted
- 404 is returned and nothing is queued
"""
document = DocumentFactory()
unrelated = DocumentFactory()
@@ -2040,16 +2074,20 @@ class TestRunZoneOcr:
)
assert response.status_code == status.HTTP_404_NOT_FOUND
mock_extract.assert_not_called()
mock_enqueue.assert_not_called()
def test_rejects_an_unknown_template(self, admin_client: APIClient) -> None:
def test_rejects_an_unknown_template(
self,
admin_client: APIClient,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
- A document
WHEN:
- Zone OCR is run with a template id that does not exist
THEN:
- 400 is returned
- 400 is returned and nothing is queued
"""
document = DocumentFactory(content=CONTENT)
@@ -2060,24 +2098,30 @@ class TestRunZoneOcr:
)
assert response.status_code == status.HTTP_400_BAD_REQUEST
mock_enqueue.assert_not_called()
def test_missing_document_is_an_error(self, admin_client: APIClient) -> None:
def test_missing_document_is_an_error(
self,
admin_client: APIClient,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
- No document with the requested id
WHEN:
- Zone OCR is run on it
THEN:
- 404 is returned
- 404 is returned and nothing is queued
"""
response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=99999))
assert response.status_code == status.HTTP_404_NOT_FOUND
mock_enqueue.assert_not_called()
def test_missing_file_is_an_error(
self,
admin_client: APIClient,
mock_extract: MagicMock,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
@@ -2085,7 +2129,7 @@ class TestRunZoneOcr:
WHEN:
- Zone OCR is run on the document
THEN:
- 404 is returned with a file not found error and nothing is extracted
- 404 is returned with a file not found error and nothing is queued
"""
document = DocumentFactory(content=CONTENT)
OcrTemplateFactory(
@@ -2097,171 +2141,34 @@ class TestRunZoneOcr:
assert response.status_code == status.HTTP_404_NOT_FOUND
assert response.json() == {"error": "Document file not found"}
mock_extract.assert_not_called()
mock_enqueue.assert_not_called()
def test_reports_completed_results(
def test_queueing_failure_is_a_bad_request(
self,
admin_client: APIClient,
mock_zone_ocr_run: MagicMock,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
- A document and a matching template with a title zone
- A document with a matching template and a stored file
- A task queue that raises when the task is queued
WHEN:
- Zone OCR is run on the document
THEN:
- The completed result lists the extracted title value
- 400 is returned
"""
document = DocumentFactory(
content=CONTENT,
mime_type="application/pdf",
)
write_blank_png(document.source_path)
mock_zone_ocr_run.return_value.stdout = "Extracted title"
template = OcrTemplateFactory(
name="Template",
match="Northwind",
)
OcrTemplateZoneFactory(
template=template,
name="Title zone",
height=100,
)
document = DocumentFactory(content=CONTENT, mime_type="application/pdf")
document.source_path.write_bytes(b"test PDF")
OcrTemplateFactory(name="Northwind", match="Northwind")
mock_enqueue.side_effect = RuntimeError("broker down")
response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk))
assert response.status_code == status.HTTP_200_OK
assert response.json() == {
"status": "completed",
"results": [
{
"template": template.name,
"zone": "Title zone",
"custom_field": "Title",
"value": "Extracted title",
"reason": None,
},
],
}
mock_zone_ocr_run.assert_called_once()
def test_does_not_report_an_old_title_when_extraction_is_empty(
self,
admin_client: APIClient,
mock_zone_ocr_run: MagicMock,
) -> None:
"""
GIVEN:
- A document with a title and a template whose title zone extracts
nothing
WHEN:
- Zone OCR is run on the document
THEN:
- The result has reason no_value and no value
- The document title is unchanged
"""
document = DocumentFactory(title="Existing title", content=CONTENT)
write_blank_png(document.source_path)
template = OcrTemplateFactory(
name="Template",
match="Northwind",
source_width=100,
source_height=100,
)
OcrTemplateZoneFactory(
template=template,
name="Title",
width=20,
height=20,
)
mock_zone_ocr_run.return_value.stdout = ""
response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk))
assert response.status_code == status.HTTP_200_OK
assert response.json()["results"][0]["reason"] == ZoneOcrReason.NO_VALUE
assert response.json()["results"][0]["value"] is None
document.refresh_from_db()
assert document.title == "Existing title"
def test_reports_all_target_types_and_missing_values(
self,
admin_client: APIClient,
mock_zone_ocr_run: MagicMock,
text_field: CustomField,
date_field: CustomField,
) -> None:
"""
GIVEN:
- A document with an existing custom field value and ASN
- A template with zones for every target type, some extracting nothing
WHEN:
- Zone OCR is run with that template chosen
THEN:
- Each zone reports the value for its target, or None when missing
"""
document = DocumentFactory(content=CONTENT, archive_serial_number=123)
write_blank_png(document.source_path)
mock_zone_ocr_run.side_effect = [
subprocess.CompletedProcess(args=[], returncode=0, stdout=value, stderr="")
for value in [
"New invoice",
"",
"orphan",
"New title",
"456",
"2026-04-13",
]
]
template = OcrTemplateFactory(
name="Test",
source_width=100,
source_height=100,
)
CustomFieldInstance.objects.create(
document=document,
field=text_field,
value_text="INV-123",
)
for name, target, field in [
("Present", "custom_field", text_field),
("Missing", "custom_field", date_field),
("Unassigned", "custom_field", None),
("Title", "title", None),
("ASN", "asn", None),
("Created", "created", None),
]:
OcrTemplateZoneFactory(
template=template,
name=name,
target=target,
custom_field=field,
width=10,
height=10,
)
response = admin_client.post(
RUN_ZONE_OCR_URL.format(pk=document.pk),
{"template": template.pk},
format="json",
)
assert response.status_code == status.HTTP_200_OK
values = {
result["zone"]: result["value"] for result in response.json()["results"]
}
assert values == {
"Present": "New invoice",
"Missing": None,
"Unassigned": None,
"Title": "New title",
"ASN": 456,
"Created": "2026-04-13",
}
assert response.status_code == status.HTTP_400_BAD_REQUEST
@pytest.mark.django_db
@pytest.mark.usefixtures("paperless_dirs", "mock_finish")
@pytest.mark.usefixtures("paperless_dirs")
class TestRunZoneOcrPermissions:
"""Permission checks for the run-zone-ocr endpoint."""
@@ -2269,7 +2176,7 @@ class TestRunZoneOcrPermissions:
self,
regular_user: User,
user_client: APIClient,
mock_extract: MagicMock,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
@@ -2279,7 +2186,7 @@ class TestRunZoneOcrPermissions:
- The editor runs zone OCR choosing that template
THEN:
- 400 is returned, naming the template field, and nothing is
extracted
queued
"""
grant_global(regular_user, "view_document", "change_document")
document = DocumentFactory(content=CONTENT, owner=regular_user)
@@ -2296,13 +2203,13 @@ class TestRunZoneOcrPermissions:
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert "template" in response.json()
mock_extract.assert_not_called()
mock_enqueue.assert_not_called()
def test_requires_change_permission_on_the_document(
self,
regular_user: User,
user_client: APIClient,
mock_extract: MagicMock,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
@@ -2311,7 +2218,7 @@ class TestRunZoneOcrPermissions:
WHEN:
- The user runs zone OCR on the document
THEN:
- 403 is returned and nothing is extracted
- 403 is returned and nothing is queued
"""
owner = UserFactory()
grant_global(regular_user, "view_document", "change_document")
@@ -2320,13 +2227,48 @@ class TestRunZoneOcrPermissions:
response = user_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk))
assert response.status_code == status.HTTP_403_FORBIDDEN
mock_extract.assert_not_called()
mock_enqueue.assert_not_called()
def test_a_version_path_id_is_authorized_through_its_root(
self,
regular_user: User,
user_client: APIClient,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
- A root document owned by another user and shared for editing
with the user, and one of its versions
WHEN:
- The user runs zone OCR using the version id in the path
THEN:
- The request is authorized through the root and the version is
queued
"""
owner = UserFactory()
grant_global(regular_user, "view_document", "change_document")
root = DocumentFactory(content=CONTENT, owner=owner)
version = DocumentFactory(
root_document=root,
version_index=1,
content=CONTENT,
)
version.source_path.write_bytes(b"test PDF")
grant_object(regular_user, root, "view_document", "change_document")
OcrTemplateFactory(name="Northwind", match="Northwind", owner=None)
response = user_client.post(RUN_ZONE_OCR_URL.format(pk=version.pk))
assert response.status_code == status.HTTP_200_OK
assert mock_enqueue.call_args.kwargs["kwargs"]["document_ids"] == [
version.pk,
]
def test_requires_global_change_permission(
self,
regular_user: User,
user_client: APIClient,
mock_extract: MagicMock,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
@@ -2335,7 +2277,7 @@ class TestRunZoneOcrPermissions:
WHEN:
- The user runs zone OCR on the document
THEN:
- 403 is returned and nothing is extracted
- 403 is returned and nothing is queued
"""
grant_global(regular_user, "view_document", "add_document")
document = DocumentFactory(content=CONTENT, owner=regular_user)
@@ -2343,13 +2285,13 @@ class TestRunZoneOcrPermissions:
response = user_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk))
assert response.status_code == status.HTTP_403_FORBIDDEN
mock_extract.assert_not_called()
mock_enqueue.assert_not_called()
def test_does_not_require_document_upload_permission(
self,
regular_user: User,
user_client: APIClient,
mock_extract: MagicMock,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
@@ -2358,7 +2300,7 @@ class TestRunZoneOcrPermissions:
WHEN:
- The user runs zone OCR on the document
THEN:
- The request succeeds and extraction runs
- The request succeeds and the run is queued as that user
"""
grant_global(regular_user, "change_document")
document = DocumentFactory(owner=regular_user, content=CONTENT)
@@ -2374,13 +2316,14 @@ class TestRunZoneOcrPermissions:
response = user_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk))
assert response.status_code == status.HTTP_200_OK
mock_extract.assert_called_once()
mock_enqueue.assert_called_once()
assert mock_enqueue.call_args.kwargs["kwargs"]["user_id"] == regular_user.pk
def test_skips_templates_private_to_the_requester(
self,
admin_client: APIClient,
admin_user: User,
mock_extract: MagicMock,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
@@ -2389,7 +2332,7 @@ class TestRunZoneOcrPermissions:
WHEN:
- The requester runs zone OCR on the document
THEN:
- The run is skipped with reason no_template and nothing is extracted
- The run is skipped with reason no_template and nothing is queued
"""
owner = UserFactory()
document = DocumentFactory(content=CONTENT, owner=owner)
@@ -2405,96 +2348,34 @@ class TestRunZoneOcrPermissions:
assert response.status_code == status.HTTP_200_OK
assert response.json() == SKIPPED_NO_TEMPLATE
mock_extract.assert_not_called()
mock_enqueue.assert_not_called()
def test_reports_only_templates_visible_to_the_document_owner(
self,
admin_client: APIClient,
admin_user: User,
mock_zone_ocr_run: MagicMock,
) -> None:
"""
GIVEN:
- A document owned by another user
- Templates owned by the document owner, unowned, shared with the
document owner, and private to the requester
WHEN:
- The requester runs zone OCR on the document
THEN:
- Only templates visible to the document owner run and are reported
"""
owner = UserFactory()
document = DocumentFactory(content=CONTENT, owner=owner)
write_blank_png(document.source_path)
mock_zone_ocr_run.return_value.stdout = "extracted"
for name, template_owner in [
("Owned", owner),
("Unowned", None),
("Shared", admin_user),
("Private", admin_user),
]:
template = title_template(
name,
template_owner,
source_width=100,
source_height=100,
)
if name == "Shared":
grant_object(owner, template, "view_ocrtemplate")
response = admin_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk))
assert response.status_code == status.HTTP_200_OK
assert sorted(result["template"] for result in response.json()["results"]) == [
"Owned",
"Shared",
"Unowned",
]
assert mock_zone_ocr_run.call_count == 3
def test_hides_the_owners_private_templates_from_an_editor(
def test_skips_when_only_the_owners_private_templates_match(
self,
regular_user: User,
user_client: APIClient,
mock_zone_ocr_run: MagicMock,
mock_enqueue: MagicMock,
) -> None:
"""
GIVEN:
- A document shared for editing with another user
- Templates private to the owner, unowned, and shared with both
- A matching template private to the document owner
WHEN:
- The editor runs zone OCR without choosing a template
THEN:
- Only templates the editor can also view run and are reported
- The run is skipped with reason no_template and nothing is queued
"""
owner = UserFactory()
grant_global(regular_user, "view_document", "change_document")
document = DocumentFactory(content=CONTENT, owner=owner)
grant_object(regular_user, document, "view_document", "change_document")
write_blank_png(document.source_path)
mock_zone_ocr_run.return_value.stdout = "extracted"
for name, template_owner in [
("Owner private", owner),
("Unowned", None),
("Shared", owner),
]:
template = title_template(
name,
template_owner,
source_width=100,
source_height=100,
)
if name == "Shared":
grant_object(regular_user, template, "view_ocrtemplate")
title_template("Owner private", owner, source_width=100, source_height=100)
response = user_client.post(RUN_ZONE_OCR_URL.format(pk=document.pk))
assert response.status_code == status.HTTP_200_OK
assert sorted(result["template"] for result in response.json()["results"]) == [
"Shared",
"Unowned",
]
assert mock_zone_ocr_run.call_count == 2
assert response.json() == SKIPPED_NO_TEMPLATE
mock_enqueue.assert_not_called()
@pytest.mark.django_db
+7 -1
View File
@@ -1966,7 +1966,12 @@ class TestBulkEditRunZoneOcr:
assert result == "OK"
assert mock_extract_zones.call_args_list == [
mocker.call(document_id=doc.id, template_id=None, requester=user)
mocker.call(
document_id=doc.id,
template_id=None,
requester=user,
resolve_latest=True,
)
for doc in docs
]
mock_extract_zones.apply_async.assert_not_called()
@@ -1993,6 +1998,7 @@ class TestBulkEditRunZoneOcr:
document_id=doc.id,
template_id=template.pk,
requester=None,
resolve_latest=True,
)
mock_extract_zones.apply_async.assert_not_called()
@@ -154,6 +154,7 @@ class TestRunZoneOcrBulkOperation:
document_id=document.pk,
template_id=template.pk,
requester=admin_user,
resolve_latest=True,
)
extract.apply_async.assert_not_called()
assert result == {"operation": "run_zone_ocr", "document_count": 1}
@@ -206,6 +207,7 @@ class TestRunZoneOcrBulkOperation:
document_id=document.pk,
template_id=None,
requester=admin_user,
resolve_latest=True,
)
@pytest.mark.parametrize(
+63
View File
@@ -732,6 +732,69 @@ class TestExtractZonesTask:
assert result == (root.pk if returns_root else None)
def test_task_runs_a_version_and_reports_the_root(
self,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- A root document with two versions and a template
WHEN:
- The task runs for the older version id
THEN:
- Extraction runs on that version, not the newest
- The root document id is returned when a value was written
"""
root = DocumentFactory()
older = DocumentFactory(root_document=root, version_index=1)
DocumentFactory(root_document=root, version_index=2)
template = OcrTemplateFactory()
extract = mocker.patch(
"documents.tasks.run_zone_extraction",
return_value=[
ZoneOcrResult(template="t", zone="z", custom_field="", reason=None),
],
)
result = extract_zones(older.pk, template.pk)
extract.assert_called_once_with(older, templates=[template])
assert result == root.pk
@pytest.mark.parametrize(
("resolve_latest", "processes_root"),
[
pytest.param(True, False, id="resolves-latest"),
pytest.param(False, True, id="exact-document"),
],
)
def test_task_resolves_latest_only_when_asked(
self,
mocker: MockerFixture,
resolve_latest: bool, # noqa: FBT001
processes_root: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A root document with a newer version and a template
WHEN:
- The task runs for the root id with resolve_latest on or off
THEN:
- The newest version is processed when resolve_latest is on
- The root document itself is processed when it is off
"""
root = DocumentFactory()
newest = DocumentFactory(root_document=root, version_index=1)
template = OcrTemplateFactory()
extract = mocker.patch("documents.tasks.run_zone_extraction", return_value=[])
extract_zones(root.pk, template.pk, resolve_latest=resolve_latest)
extract.assert_called_once_with(
root if processes_root else newest,
templates=[template],
)
def test_task_skips_a_deleted_template(self, mocker: MockerFixture) -> None:
"""
GIVEN:
+4 -67
View File
@@ -5,7 +5,6 @@ from __future__ import annotations
import shutil
import subprocess
from collections import Counter
from dataclasses import asdict
from datetime import date
from decimal import Decimal
from pathlib import Path
@@ -27,14 +26,10 @@ from documents.models import OcrTemplateZone
from documents.parsers import ParseError
from documents.serialisers import OcrTemplateTestResultSerializer
from documents.serialisers import RunZoneOcrResponseSerializer
from documents.serialisers import ZoneOcrResultSerializer
from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH
from documents.zone_ocr import ZoneExtraction
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from documents.zone_ocr import ZoneOcrRunStatus
from documents.zone_ocr import ZoneOcrSkipReason
from documents.zone_ocr import ZoneValue
from documents.zone_ocr import _apply_transform
from documents.zone_ocr import _combine_field_value
from documents.zone_ocr import _convert_value
@@ -3959,82 +3954,24 @@ class TestRealZoneOcr:
assert "This is a test document" in document.title
class TestZoneOcrResultSerializer:
@pytest.mark.parametrize(
"value",
[
pytest.param("text", id="string"),
pytest.param(42, id="int"),
pytest.param(1.5, id="float"),
pytest.param(True, id="bool"),
pytest.param(date(2025, 12, 25), id="date"),
pytest.param(None, id="none"),
],
)
@pytest.mark.parametrize(
"reason",
[pytest.param(None, id="no-reason")]
+ [pytest.param(reason, id=reason.value) for reason in ZoneOcrReason],
)
def test_wire_format_matches_dataclass(
self,
value: ZoneValue | None,
reason: ZoneOcrReason | None,
) -> None:
"""
GIVEN:
- A zone result with any value type and any reason
WHEN:
- It is serialized with the result serializer and rendered as JSON
THEN:
- The JSON equals the rendering of the dataclass as a dict
"""
result = ZoneOcrResult(
template="Template",
zone="Zone",
custom_field="Title",
value=value,
reason=reason,
)
assert JSONRenderer().render(
ZoneOcrResultSerializer(result).data,
) == JSONRenderer().render(asdict(result))
class TestRunZoneOcrResponseSerializer:
@pytest.mark.parametrize(
"body",
[
pytest.param(
{
"status": ZoneOcrRunStatus.SKIPPED,
"reason": ZoneOcrSkipReason.NO_TEMPLATE,
"results": [],
},
{"result": "skipped", "reason": ZoneOcrSkipReason.NO_TEMPLATE},
id="skipped",
),
pytest.param(
{
"status": ZoneOcrRunStatus.COMPLETED,
"results": [
{
"template": "Template",
"zone": "Zone",
"custom_field": "Title",
"value": "text",
"reason": None,
},
],
},
id="completed",
{"result": "OK", "task_id": "abc-123"},
id="queued",
),
],
)
def test_renders_body_unchanged(self, body: dict[str, Any]) -> None:
"""
GIVEN:
- A response body with the run status and results fields
- A response body for a skipped or a queued run
WHEN:
- The body is rendered through the response serializer as JSON
THEN:
+69 -48
View File
@@ -248,7 +248,6 @@ from documents.serialisers import UiSettingsViewSerializer
from documents.serialisers import WorkflowActionSerializer
from documents.serialisers import WorkflowSerializer
from documents.serialisers import WorkflowTriggerSerializer
from documents.serialisers import ZoneOcrResultSerializer
from documents.signals import document_updated
from documents.tasks import build_share_link_bundle
from documents.tasks import consume_file
@@ -2300,8 +2299,11 @@ class DocumentViewSet(
)
def run_zone_ocr(self, request: Request, pk: str | None = None) -> HttpResponse:
"""
Run zone-based OCR extraction on this document, with the given template
or with every template that matches it.
Queue zone-based OCR extraction on this document, with the given
template or with every template that matches it. The file version
given by the version parameter is run when it is not the root
document. Responds with the task id, or skipped when no template
applies.
"""
serializer = RunZoneOcrSerializer(
data=request.data,
@@ -2320,11 +2322,8 @@ class DocumentViewSet(
) or not has_perms_owner_aware(user, "change_document", document):
return HttpResponseForbidden("Insufficient permissions")
from documents.zone_ocr import ZoneOcrRunStatus
from documents.zone_ocr import ZoneOcrSkipReason
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import run_zone_extraction
from documents.zone_ocr import zone_results_written
resolution = resolve_effective_document(document, request)
if resolution.error == VersionResolutionError.INVALID:
@@ -2333,17 +2332,14 @@ class DocumentViewSet(
if file_document is None:
raise Http404
templates = (
[template]
if template is not None
else get_templates_for_document(file_document, requester=user)
)
if not templates:
if template is None and not get_templates_for_document(
file_document,
requester=user,
):
return Response(
{
"status": ZoneOcrRunStatus.SKIPPED,
"result": "skipped",
"reason": ZoneOcrSkipReason.NO_TEMPLATE,
"results": [],
},
)
@@ -2354,18 +2350,22 @@ class DocumentViewSet(
status=status.HTTP_404_NOT_FOUND,
)
results = run_zone_extraction(file_document, templates=templates)
if zone_results_written(results):
bulk_edit.bulk_update_documents.apply_async(
kwargs={"document_ids": [get_root_document(file_document).pk]},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
parameters: dict[str, Any] = {"resolve_latest": False}
if template is not None:
parameters["template"] = template.pk
try:
return queue_bulk_operation(
user,
bulk_edit.run_zone_ocr,
[file_document.pk],
parameters,
from_webui=serializer.validated_data["from_webui"],
)
except Exception as e:
logger.warning(f"An error occurred performing document zone OCR: {e!s}")
return HttpResponseBadRequest(
"Error performing document zone OCR, check logs for more detail.",
)
return Response(
{
"status": ZoneOcrRunStatus.COMPLETED,
"results": ZoneOcrResultSerializer(results, many=True).data,
},
)
@extend_schema(
operation_id="documents_delete_version",
@@ -3101,35 +3101,56 @@ class DocumentSelectionMixin:
return list(filtered_documents.values_list("pk", flat=True))
def queue_bulk_operation(
user: User,
method: Callable[..., Any],
documents: list[int],
parameters: dict[str, Any],
*,
from_webui: bool,
) -> Response:
"""
Validate the parameters against the bulk edit method, queue it as a tracked
bulk operation task and return the task id.
"""
if "update_callback" in parameters:
raise ValueError("Unsupported bulk operation parameter: update_callback")
trigger_source = (
PaperlessTask.TriggerSource.WEB_UI
if from_webui
else PaperlessTask.TriggerSource.API_UPLOAD
)
final_parameters = parameters.copy()
if method.__name__ in USER_OPERATIONS:
final_parameters["user"] = user
if method.__name__ in TRIGGER_SOURCE_OPERATIONS:
final_parameters["trigger_source"] = trigger_source
signature(method).bind(documents, **final_parameters)
task = execute_bulk_operation.apply_async(
kwargs={
"operation": method.__name__,
"document_ids": documents,
"parameters": parameters,
"user_id": user.pk,
"trigger_source": trigger_source,
},
headers={"trigger_source": trigger_source},
)
return Response({"result": "OK", "task_id": task.id})
class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
permission_classes = (IsAuthenticated,)
parser_classes = (parsers.JSONParser,)
def _queue_bulk_operation(self, method, documents, parameters, from_webui):
if "update_callback" in parameters:
raise ValueError("Unsupported bulk operation parameter: update_callback")
trigger_source = (
PaperlessTask.TriggerSource.WEB_UI
if from_webui
else PaperlessTask.TriggerSource.API_UPLOAD
return queue_bulk_operation(
cast("User", self.request.user),
method,
documents,
parameters,
from_webui=from_webui,
)
final_parameters = parameters.copy()
if method.__name__ in USER_OPERATIONS:
final_parameters["user"] = self.request.user
if method.__name__ in TRIGGER_SOURCE_OPERATIONS:
final_parameters["trigger_source"] = trigger_source
signature(method).bind(documents, **final_parameters)
task = execute_bulk_operation.apply_async(
kwargs={
"operation": method.__name__,
"document_ids": documents,
"parameters": parameters,
"user_id": self.request.user.pk,
"trigger_source": trigger_source,
},
headers={"trigger_source": trigger_source},
)
return Response({"result": "OK", "task_id": task.id})
def _has_document_permissions(
self,
+1 -8
View File
@@ -125,15 +125,8 @@ class ZoneOcrReason(StrEnum):
TEMPLATE_FAILED = "template_failed"
class ZoneOcrRunStatus(StrEnum):
"""The overall outcome of a zone OCR run on one document."""
COMPLETED = "completed"
SKIPPED = "skipped"
class ZoneOcrSkipReason(StrEnum):
"""Why a zone OCR run on one document was skipped."""
"""Why a zone OCR run on one document was not queued."""
NO_TEMPLATE = "no_template"