Fix: finish manual and bulk zone OCR runs like other bulk edits

Runs reindexed each document separately, never sent document_updated
and did not bump modified for custom field writes. They now finish
through bulk_update_documents for the documents that got a value.
This commit is contained in:
stumpylog committed 2026-11-19 14:18:30 -08:00
1 parent 0bca50588d
commit 54be9dc9c5
9 files changed
+368 -45

No files matched your search

+18 -3
View File
@@ -1264,8 +1264,23 @@ def run_zone_ocr(
template: int | None = None,
user: User | None = None,
) -> Literal["OK"]:
"""Run zone OCR within the tracked bulk operation task."""
for document_id in doc_ids:
extract_zones(document_id=document_id, template_id=template, requester=user)
"""
Run zone OCR within the tracked bulk operation task, then finish once for
the root documents that had a value written.
"""
written_ids: list[int] = []
try:
for document_id in doc_ids:
written_id = extract_zones(
document_id=document_id,
template_id=template,
requester=user,
)
if written_id is not None:
written_ids.append(written_id)
finally:
# Documents already written are finished even if a later one raises.
if written_ids:
bulk_update_documents(list(dict.fromkeys(written_ids)))
return "OK"
+11 -3
View File
@@ -72,10 +72,12 @@ from documents.utils import IterWrapper
from documents.utils import compute_checksum
from documents.utils import identity
from documents.versioning import annotate_effective_content
from documents.versioning import get_root_document
from documents.versioning import latest_version
from documents.workflows.utils import get_workflows_for_trigger
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import run_zone_extraction
from documents.zone_ocr import zone_results_written
from paperless.config import AIConfig
from paperless.config import BarcodeConfig
from paperless.config import RemoteOCRConfig
@@ -379,11 +381,14 @@ def extract_zones(
document_id: int,
template_id: int | None = None,
requester: User | None = None,
) -> None:
) -> int | None:
"""
Run an OCR template against a stored document, or every template that
matches it when no template is given. Matched templates are limited to
those the requester can view.
Returns the root document's id when any value was written, otherwise None.
The caller finishes the edit, for example with bulk_update_documents.
"""
document = latest_version(Document.objects.get(id=document_id))
if template_id is None:
@@ -399,10 +404,13 @@ def extract_zones(
"OCR template %s no longer exists, not running zone OCR",
template_id,
)
return
return None
templates = [template]
run_zone_extraction(document, templates=templates, reindex=True)
results = run_zone_extraction(document, templates=templates)
if zone_results_written(results):
return get_root_document(document).pk
return None
@shared_task
+74 -13
View File
@@ -14,18 +14,21 @@ from django.utils import timezone
from PIL import Image
from rest_framework import status
from documents import bulk_edit
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import MatchingModel
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.models import PaperlessTask
from documents.parsers import ParseError
from documents.search import get_backend
from documents.signals import document_consumption_finished
from documents.tasks import extract_zones
from documents.tasks import bulk_update_documents
from documents.zone_ocr import ZoneExtraction
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from paperless_testing.factories import CustomFieldFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
@@ -148,6 +151,13 @@ def mock_extract(mocker: MockerFixture) -> MagicMock:
return mocker.patch("documents.zone_ocr.run_zone_extraction")
@pytest.fixture
def mock_finish(mocker: MockerFixture) -> MagicMock:
return mocker.patch(
"documents.views.bulk_edit.bulk_update_documents.apply_async",
)
@pytest.fixture
def mock_preview(mocker: MockerFixture) -> MagicMock:
return mocker.patch("documents.zone_ocr.extract_zone_preview")
@@ -232,6 +242,7 @@ class TestZoneOcrIndexing:
admin_client: APIClient,
django_capture_on_commit_callbacks: DjangoCaptureOnCommitCallbacks,
mock_zone_ocr_run: MagicMock,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
@@ -305,6 +316,11 @@ class TestZoneOcrIndexing:
backend.add_or_update(document)
assert backend.search_ids("title:before", user=None) == [document.pk]
# The single-document finish is a queued task, run it inline here.
mocker.patch(
"documents.views.bulk_edit.bulk_update_documents.apply_async",
side_effect=lambda kwargs, **_: bulk_update_documents(**kwargs),
)
with django_capture_on_commit_callbacks(execute=True):
if entry_point == "ingestion":
document_consumption_finished.send(sender=None, document=file_document)
@@ -314,7 +330,7 @@ class TestZoneOcrIndexing:
)
assert response.status_code == status.HTTP_200_OK
else:
extract_zones(document.pk, template.pk)
bulk_edit.run_zone_ocr([document.pk], template=template.pk)
document.refresh_from_db()
assert document.title == "After OCR"
@@ -1801,7 +1817,7 @@ class TestOcrTemplateDelete:
@pytest.mark.django_db
@pytest.mark.usefixtures("paperless_dirs")
@pytest.mark.usefixtures("paperless_dirs", "mock_finish")
class TestRunZoneOcr:
"""Running zone OCR on documents through the run-zone-ocr endpoint."""
@@ -1873,11 +1889,60 @@ class TestRunZoneOcr:
assert response.status_code == status.HTTP_200_OK
assert response.json()["status"] == "completed"
mock_extract.assert_called_once_with(
document,
templates=[template],
reindex=True,
mock_extract.assert_called_once_with(document, templates=[template])
@pytest.mark.parametrize(
("reason", "finishes"),
[
pytest.param(None, True, id="written"),
pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"),
],
)
def test_finishes_the_edit_only_when_a_value_was_written(
self,
admin_client: APIClient,
mock_extract: MagicMock,
mock_finish: MagicMock,
reason: ZoneOcrReason | None,
finishes: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A document with a version and a template whose run does or does
not write a value
WHEN:
- Zone OCR is run on the version through the detail endpoint
THEN:
- The shared document update task is queued once for the root
document only when a value was written
- No per-document search reindex is requested directly
"""
document = DocumentFactory(content=CONTENT)
version = DocumentFactory(
root_document=document,
version_index=1,
content=CONTENT,
)
version.source_path.write_bytes(b"test PDF")
template = OcrTemplateFactory(name="Northwind", match="Northwind")
mock_extract.return_value = [
ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason),
]
response = admin_client.post(
f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={version.pk}",
{"template": template.pk},
format="json",
)
assert response.status_code == status.HTTP_200_OK
if finishes:
mock_finish.assert_called_once_with(
kwargs={"document_ids": [document.pk]},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
else:
mock_finish.assert_not_called()
@pytest.mark.parametrize(
"version",
@@ -1923,11 +1988,7 @@ class TestRunZoneOcr:
assert response.status_code == status.HTTP_200_OK
assert response.json()["status"] == "completed"
mock_extract.assert_called_once_with(
file_document,
templates=[template],
reindex=True,
)
mock_extract.assert_called_once_with(file_document, templates=[template])
@pytest.mark.parametrize(
"version",
@@ -2190,7 +2251,7 @@ class TestRunZoneOcr:
@pytest.mark.django_db
@pytest.mark.usefixtures("paperless_dirs")
@pytest.mark.usefixtures("paperless_dirs", "mock_finish")
class TestRunZoneOcrPermissions:
"""Permission checks for the run-zone-ocr endpoint."""
+66 -1
View File
@@ -1942,7 +1942,7 @@ class TestBulkEditRunZoneOcr:
@pytest.fixture
def mock_extract_zones(self, mocker: MockerFixture) -> mock.MagicMock:
return mocker.patch("documents.bulk_edit.extract_zones")
return mocker.patch("documents.bulk_edit.extract_zones", return_value=None)
def test_runs_every_selected_document_in_the_current_task(
self,
@@ -1996,6 +1996,71 @@ class TestBulkEditRunZoneOcr:
)
mock_extract_zones.apply_async.assert_not_called()
def test_finishes_once_for_documents_with_a_write(
self,
mock_extract_zones: mock.MagicMock,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- Three selected documents, the second of which had nothing written
and the third reporting the same root as the first
WHEN:
- run_zone_ocr is called
THEN:
- bulk_update_documents runs once with the distinct written ids
"""
docs = DocumentFactory.create_batch(3)
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
mock_extract_zones.side_effect = [docs[0].pk, None, docs[0].pk]
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
finish.assert_called_once_with([docs[0].pk])
def test_finishes_written_documents_when_a_later_one_raises(
self,
mock_extract_zones: mock.MagicMock,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- Two selected documents, the second of which raises during extraction
WHEN:
- run_zone_ocr is called
THEN:
- The exception propagates
- bulk_update_documents still runs with the first document's id
"""
docs = DocumentFactory.create_batch(2)
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
mock_extract_zones.side_effect = [docs[0].pk, RuntimeError("boom")]
with pytest.raises(RuntimeError, match="boom"):
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
finish.assert_called_once_with([docs[0].pk])
def test_does_not_finish_when_nothing_was_written(
self,
mock_extract_zones: mock.MagicMock,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- Selected documents with no successful write
WHEN:
- run_zone_ocr is called
THEN:
- bulk_update_documents is not run
"""
docs = DocumentFactory.create_batch(2)
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
finish.assert_not_called()
@pytest.mark.django_db
class TestBulkEditReprocess:
+44 -5
View File
@@ -97,7 +97,10 @@ class TestRunZoneOcrBulkOperation:
"""
template = OcrTemplateFactory()
enqueue = mocker.patch("documents.views.execute_bulk_operation.apply_async")
extract = mocker.patch("documents.bulk_edit.extract_zones")
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
)
enqueue.return_value.id = "zone-ocr-task"
response = admin_client.post(
RUN_ZONE_OCR_BULK_URL,
@@ -137,7 +140,10 @@ class TestRunZoneOcrBulkOperation:
- The result reports the operation and document count
"""
template = OcrTemplateFactory()
extract = mocker.patch("documents.bulk_edit.extract_zones")
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
)
result = execute_bulk_operation(
"run_zone_ocr",
[document.pk],
@@ -152,6 +158,30 @@ class TestRunZoneOcrBulkOperation:
extract.apply_async.assert_not_called()
assert result == {"operation": "run_zone_ocr", "document_count": 1}
def test_finishes_with_the_ids_that_were_written(
self,
mocker: MockerFixture,
admin_user: User,
document: Document,
) -> None:
"""
GIVEN:
- A queued zone OCR run whose extraction reports a written document
WHEN:
- The tracked task executes it
THEN:
- bulk_update_documents runs once with that document id
"""
mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=document.pk,
)
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk)
finish.assert_called_once_with([document.pk])
def test_without_template_selects_for_the_requester(
self,
mocker: MockerFixture,
@@ -167,7 +197,10 @@ class TestRunZoneOcrBulkOperation:
- Extraction receives the requesting user so templates are limited to
those they can view
"""
extract = mocker.patch("documents.bulk_edit.extract_zones")
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
)
execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk)
extract.assert_called_once_with(
document_id=document.pk,
@@ -223,7 +256,10 @@ class TestRunZoneOcrBulkOperation:
template.save(update_fields=["owner"])
else:
User.objects.filter(pk=regular_user.pk).update(is_active=False)
extract = mocker.patch("documents.bulk_edit.extract_zones")
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
)
with pytest.raises(PermissionDenied):
execute_bulk_operation(**enqueue.call_args.kwargs["kwargs"])
extract.assert_not_called()
@@ -244,7 +280,10 @@ class TestRunZoneOcrBulkOperation:
template matching
- Extraction never runs
"""
extract = mocker.patch("documents.bulk_edit.extract_zones")
extract = mocker.patch(
"documents.bulk_edit.extract_zones",
return_value=None,
)
with pytest.raises(OcrTemplate.DoesNotExist):
execute_bulk_operation(
"run_zone_ocr",
+39 -6
View File
@@ -23,6 +23,8 @@ from documents.sanity_checker import SanityCheckFailedException
from documents.sanity_checker import SanityCheckMessages
from documents.tasks import extract_zones
from documents.tests.helpers import dummy_preprocess
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
@@ -676,11 +678,7 @@ class TestExtractZonesTask:
extract_zones(doc.id, requester=user)
select.assert_called_once_with(doc, requester=user)
extract.assert_called_once_with(
doc,
templates=select.return_value,
reindex=True,
)
extract.assert_called_once_with(doc, templates=select.return_value)
def test_task_runs_the_chosen_template(self, mocker: MockerFixture) -> None:
"""
@@ -697,7 +695,42 @@ class TestExtractZonesTask:
extract_zones(doc.id, template.pk)
extract.assert_called_once_with(doc, templates=[template], reindex=True)
extract.assert_called_once_with(doc, templates=[template])
@pytest.mark.parametrize(
("reason", "returns_root"),
[
pytest.param(None, True, id="written"),
pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"),
],
)
def test_task_reports_the_root_when_a_value_was_written(
self,
mocker: MockerFixture,
reason: ZoneOcrReason | None,
returns_root: bool, # noqa: FBT001
) -> None:
"""
GIVEN:
- A document version whose extraction does or does not write a value
WHEN:
- The task runs
THEN:
- The root document id is returned only when a value was written
"""
root = DocumentFactory()
DocumentFactory(root_document=root, version_index=1)
template = OcrTemplateFactory()
mocker.patch(
"documents.tasks.run_zone_extraction",
return_value=[
ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason),
],
)
result = extract_zones(root.pk, template.pk)
assert result == (root.pk if returns_root else None)
def test_task_skips_a_deleted_template(self, mocker: MockerFixture) -> None:
"""
+99
View File
@@ -66,6 +66,7 @@ if TYPE_CHECKING:
from unittest.mock import MagicMock
from django.contrib.auth.models import User
from pytest_django.fixtures import DjangoCaptureOnCommitCallbacks
from pytest_django.fixtures import SettingsWrapper
from pytest_mock import MockerFixture
@@ -781,6 +782,104 @@ class TestSavedZoneExtraction:
document.refresh_from_db()
assert document.title == "Second"
@pytest.mark.parametrize(
"target",
[
pytest.param("title", id="builtin-field"),
pytest.param("custom_field", id="custom-field-only"),
],
)
def test_successful_write_bumps_modified_of_the_root(
self,
mock_zone_ocr_run: MagicMock,
page: Path,
document: Document,
template: OcrTemplate,
target: str,
) -> None:
"""
GIVEN:
- A document and a template with a zone for a built-in field or
only for a custom field
WHEN:
- Zone extraction writes the value
THEN:
- The root document's modified time is later than before the run
"""
field = CustomFieldFactory(name="Code", data_type="string")
if target == "title":
make_zone(template, "Title")
else:
make_zone(template, "Code", "custom_field", custom_field=field)
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
[],
0,
stdout="value",
)
before = Document.objects.get(pk=document.pk).modified
results = run_zone_extraction(document, page, templates=[template])
assert [result.reason for result in results] == [None]
assert Document.objects.get(pk=document.pk).modified > before
def test_without_a_write_leaves_modified_untouched(
self,
mock_zone_ocr_run: MagicMock,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- A template whose only zone reads no text
WHEN:
- Zone extraction runs
THEN:
- Nothing is written and modified is unchanged
"""
make_zone(template, "Title")
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
[],
0,
stdout="",
)
before = Document.objects.get(pk=document.pk).modified
results = run_zone_extraction(document, page, templates=[template])
assert [result.reason for result in results] == [ZoneOcrReason.NO_VALUE]
assert Document.objects.get(pk=document.pk).modified == before
def test_successful_write_does_not_reindex_per_document(
self,
django_capture_on_commit_callbacks: DjangoCaptureOnCommitCallbacks,
mock_zone_ocr_run: MagicMock,
page: Path,
document: Document,
template: OcrTemplate,
) -> None:
"""
GIVEN:
- A template that writes the title
WHEN:
- Zone extraction runs
THEN:
- No on-commit callbacks are registered; finishing is left to the
caller
"""
make_zone(template, "Title")
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
[],
0,
stdout="value",
)
with django_capture_on_commit_callbacks() as callbacks:
run_zone_extraction(document, page, templates=[template])
assert callbacks == []
def test_document_run_renders_each_page_once_across_templates(
self,
mocker: MockerFixture,
+7 -1
View File
@@ -2324,6 +2324,7 @@ class DocumentViewSet(
from documents.zone_ocr import ZoneOcrSkipReason
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import run_zone_extraction
from documents.zone_ocr import zone_results_written
resolution = resolve_effective_document(document, request)
if resolution.error == VersionResolutionError.INVALID:
@@ -2353,7 +2354,12 @@ class DocumentViewSet(
status=status.HTTP_404_NOT_FOUND,
)
results = run_zone_extraction(file_document, templates=templates, reindex=True)
results = run_zone_extraction(file_document, templates=templates)
if zone_results_written(results):
bulk_edit.bulk_update_documents.apply_async(
kwargs={"document_ids": [get_root_document(file_document).pk]},
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
)
return Response(
{
"status": ZoneOcrRunStatus.COMPLETED,
+10 -13
View File
@@ -14,7 +14,6 @@ from decimal import Decimal
from decimal import InvalidOperation
from enum import StrEnum
from functools import lru_cache
from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING
from typing import NewType
@@ -36,6 +35,7 @@ from django.core.exceptions import ValidationError
from django.db import transaction
from django.db.models import F
from django.db.models import Field
from django.utils import timezone
from PIL import Image
from documents.matching import matches
@@ -48,7 +48,6 @@ from documents.parsers import ParseError
from documents.parsers import rasterize_pdf_page_to_png
from documents.permissions import permitted_object_ids
from documents.regex import safe_regex_fullmatch
from documents.signals.handlers import add_to_index
from documents.validators import monetary_validator
from documents.versioning import get_root_document
from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES
@@ -220,14 +219,13 @@ def run_zone_extraction(
document: Document,
original_file: Path | None = None,
templates: list[OcrTemplate] | None = None,
*,
reindex: bool = False,
) -> list[ZoneOcrResult]:
"""
Extract zones from the selected file version and update its root document.
Run all matching templates when no templates are given.
Manual runs can request a search refresh after successful writes commit.
The root document's modified time is bumped when any value was written.
Callers finish the edit (index, signals) themselves; see zone_results_written.
"""
if templates is None:
templates = get_templates_for_document(document)
@@ -274,17 +272,16 @@ def run_zone_extraction(
_zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED)
for zone in zones
)
if reindex and any(result.reason is None for result in results):
# queue a reindex of the document after the transaction commits
root_document = get_root_document(document)
transaction.on_commit(partial(_reindex_document, root_document.pk))
if zone_results_written(results):
Document.objects.filter(pk=get_root_document(document).pk).update(
modified=timezone.now(),
)
return results
def _reindex_document(document_id: int) -> None:
document = Document.objects.filter(pk=document_id).first()
if document is not None:
add_to_index(sender=None, document=document)
def zone_results_written(results: list[ZoneOcrResult]) -> bool:
"""Whether at least one zone's value was written to the root document."""
return any(result.reason is None for result in results)
def _resolve_doc_path(