mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-20 04:26:53 +00:00
Fix: finish manual and bulk zone OCR runs like other bulk edits
Runs reindexed each document separately, never sent document_updated and did not bump modified for custom field writes. They now finish through bulk_update_documents for the documents that got a value.
This commit is contained in:
1 parent
0bca50588d
commit
54be9dc9c5
9 files changed
+368
-45
No files matched your search
@@ -1264,8 +1264,23 @@ def run_zone_ocr(
|
||||
template: int | None = None,
|
||||
user: User | None = None,
|
||||
) -> Literal["OK"]:
|
||||
"""Run zone OCR within the tracked bulk operation task."""
|
||||
for document_id in doc_ids:
|
||||
extract_zones(document_id=document_id, template_id=template, requester=user)
|
||||
"""
|
||||
Run zone OCR within the tracked bulk operation task, then finish once for
|
||||
the root documents that had a value written.
|
||||
"""
|
||||
written_ids: list[int] = []
|
||||
try:
|
||||
for document_id in doc_ids:
|
||||
written_id = extract_zones(
|
||||
document_id=document_id,
|
||||
template_id=template,
|
||||
requester=user,
|
||||
)
|
||||
if written_id is not None:
|
||||
written_ids.append(written_id)
|
||||
finally:
|
||||
# Documents already written are finished even if a later one raises.
|
||||
if written_ids:
|
||||
bulk_update_documents(list(dict.fromkeys(written_ids)))
|
||||
|
||||
return "OK"
|
||||
+11
-3
@@ -72,10 +72,12 @@ from documents.utils import IterWrapper
|
||||
from documents.utils import compute_checksum
|
||||
from documents.utils import identity
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.versioning import get_root_document
|
||||
from documents.versioning import latest_version
|
||||
from documents.workflows.utils import get_workflows_for_trigger
|
||||
from documents.zone_ocr import get_templates_for_document
|
||||
from documents.zone_ocr import run_zone_extraction
|
||||
from documents.zone_ocr import zone_results_written
|
||||
from paperless.config import AIConfig
|
||||
from paperless.config import BarcodeConfig
|
||||
from paperless.config import RemoteOCRConfig
|
||||
@@ -379,11 +381,14 @@ def extract_zones(
|
||||
document_id: int,
|
||||
template_id: int | None = None,
|
||||
requester: User | None = None,
|
||||
) -> None:
|
||||
) -> int | None:
|
||||
"""
|
||||
Run an OCR template against a stored document, or every template that
|
||||
matches it when no template is given. Matched templates are limited to
|
||||
those the requester can view.
|
||||
|
||||
Returns the root document's id when any value was written, otherwise None.
|
||||
The caller finishes the edit, for example with bulk_update_documents.
|
||||
"""
|
||||
document = latest_version(Document.objects.get(id=document_id))
|
||||
if template_id is None:
|
||||
@@ -399,10 +404,13 @@ def extract_zones(
|
||||
"OCR template %s no longer exists, not running zone OCR",
|
||||
template_id,
|
||||
)
|
||||
return
|
||||
return None
|
||||
templates = [template]
|
||||
|
||||
run_zone_extraction(document, templates=templates, reindex=True)
|
||||
results = run_zone_extraction(document, templates=templates)
|
||||
if zone_results_written(results):
|
||||
return get_root_document(document).pk
|
||||
return None
|
||||
|
||||
|
||||
@shared_task
|
||||
|
||||
@@ -14,18 +14,21 @@ from django.utils import timezone
|
||||
from PIL import Image
|
||||
from rest_framework import status
|
||||
|
||||
from documents import bulk_edit
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import MatchingModel
|
||||
from documents.models import OcrTemplate
|
||||
from documents.models import OcrTemplateZone
|
||||
from documents.models import PaperlessTask
|
||||
from documents.parsers import ParseError
|
||||
from documents.search import get_backend
|
||||
from documents.signals import document_consumption_finished
|
||||
from documents.tasks import extract_zones
|
||||
from documents.tasks import bulk_update_documents
|
||||
from documents.zone_ocr import ZoneExtraction
|
||||
from documents.zone_ocr import ZoneOcrReason
|
||||
from documents.zone_ocr import ZoneOcrResult
|
||||
from paperless_testing.factories import CustomFieldFactory
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import OcrTemplateFactory
|
||||
@@ -148,6 +151,13 @@ def mock_extract(mocker: MockerFixture) -> MagicMock:
|
||||
return mocker.patch("documents.zone_ocr.run_zone_extraction")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_finish(mocker: MockerFixture) -> MagicMock:
|
||||
return mocker.patch(
|
||||
"documents.views.bulk_edit.bulk_update_documents.apply_async",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_preview(mocker: MockerFixture) -> MagicMock:
|
||||
return mocker.patch("documents.zone_ocr.extract_zone_preview")
|
||||
@@ -232,6 +242,7 @@ class TestZoneOcrIndexing:
|
||||
admin_client: APIClient,
|
||||
django_capture_on_commit_callbacks: DjangoCaptureOnCommitCallbacks,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
@@ -305,6 +316,11 @@ class TestZoneOcrIndexing:
|
||||
backend.add_or_update(document)
|
||||
assert backend.search_ids("title:before", user=None) == [document.pk]
|
||||
|
||||
# The single-document finish is a queued task, run it inline here.
|
||||
mocker.patch(
|
||||
"documents.views.bulk_edit.bulk_update_documents.apply_async",
|
||||
side_effect=lambda kwargs, **_: bulk_update_documents(**kwargs),
|
||||
)
|
||||
with django_capture_on_commit_callbacks(execute=True):
|
||||
if entry_point == "ingestion":
|
||||
document_consumption_finished.send(sender=None, document=file_document)
|
||||
@@ -314,7 +330,7 @@ class TestZoneOcrIndexing:
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
else:
|
||||
extract_zones(document.pk, template.pk)
|
||||
bulk_edit.run_zone_ocr([document.pk], template=template.pk)
|
||||
|
||||
document.refresh_from_db()
|
||||
assert document.title == "After OCR"
|
||||
@@ -1801,7 +1817,7 @@ class TestOcrTemplateDelete:
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.usefixtures("paperless_dirs")
|
||||
@pytest.mark.usefixtures("paperless_dirs", "mock_finish")
|
||||
class TestRunZoneOcr:
|
||||
"""Running zone OCR on documents through the run-zone-ocr endpoint."""
|
||||
|
||||
@@ -1873,11 +1889,60 @@ class TestRunZoneOcr:
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["status"] == "completed"
|
||||
mock_extract.assert_called_once_with(
|
||||
document,
|
||||
templates=[template],
|
||||
reindex=True,
|
||||
mock_extract.assert_called_once_with(document, templates=[template])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("reason", "finishes"),
|
||||
[
|
||||
pytest.param(None, True, id="written"),
|
||||
pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"),
|
||||
],
|
||||
)
|
||||
def test_finishes_the_edit_only_when_a_value_was_written(
|
||||
self,
|
||||
admin_client: APIClient,
|
||||
mock_extract: MagicMock,
|
||||
mock_finish: MagicMock,
|
||||
reason: ZoneOcrReason | None,
|
||||
finishes: bool, # noqa: FBT001
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document with a version and a template whose run does or does
|
||||
not write a value
|
||||
WHEN:
|
||||
- Zone OCR is run on the version through the detail endpoint
|
||||
THEN:
|
||||
- The shared document update task is queued once for the root
|
||||
document only when a value was written
|
||||
- No per-document search reindex is requested directly
|
||||
"""
|
||||
document = DocumentFactory(content=CONTENT)
|
||||
version = DocumentFactory(
|
||||
root_document=document,
|
||||
version_index=1,
|
||||
content=CONTENT,
|
||||
)
|
||||
version.source_path.write_bytes(b"test PDF")
|
||||
template = OcrTemplateFactory(name="Northwind", match="Northwind")
|
||||
mock_extract.return_value = [
|
||||
ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason),
|
||||
]
|
||||
|
||||
response = admin_client.post(
|
||||
f"{RUN_ZONE_OCR_URL.format(pk=document.pk)}?version={version.pk}",
|
||||
{"template": template.pk},
|
||||
format="json",
|
||||
)
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
if finishes:
|
||||
mock_finish.assert_called_once_with(
|
||||
kwargs={"document_ids": [document.pk]},
|
||||
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
|
||||
)
|
||||
else:
|
||||
mock_finish.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"version",
|
||||
@@ -1923,11 +1988,7 @@ class TestRunZoneOcr:
|
||||
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
assert response.json()["status"] == "completed"
|
||||
mock_extract.assert_called_once_with(
|
||||
file_document,
|
||||
templates=[template],
|
||||
reindex=True,
|
||||
)
|
||||
mock_extract.assert_called_once_with(file_document, templates=[template])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"version",
|
||||
@@ -2190,7 +2251,7 @@ class TestRunZoneOcr:
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.usefixtures("paperless_dirs")
|
||||
@pytest.mark.usefixtures("paperless_dirs", "mock_finish")
|
||||
class TestRunZoneOcrPermissions:
|
||||
"""Permission checks for the run-zone-ocr endpoint."""
|
||||
|
||||
|
||||
@@ -1942,7 +1942,7 @@ class TestBulkEditRunZoneOcr:
|
||||
|
||||
@pytest.fixture
|
||||
def mock_extract_zones(self, mocker: MockerFixture) -> mock.MagicMock:
|
||||
return mocker.patch("documents.bulk_edit.extract_zones")
|
||||
return mocker.patch("documents.bulk_edit.extract_zones", return_value=None)
|
||||
|
||||
def test_runs_every_selected_document_in_the_current_task(
|
||||
self,
|
||||
@@ -1996,6 +1996,71 @@ class TestBulkEditRunZoneOcr:
|
||||
)
|
||||
mock_extract_zones.apply_async.assert_not_called()
|
||||
|
||||
def test_finishes_once_for_documents_with_a_write(
|
||||
self,
|
||||
mock_extract_zones: mock.MagicMock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Three selected documents, the second of which had nothing written
|
||||
and the third reporting the same root as the first
|
||||
WHEN:
|
||||
- run_zone_ocr is called
|
||||
THEN:
|
||||
- bulk_update_documents runs once with the distinct written ids
|
||||
"""
|
||||
docs = DocumentFactory.create_batch(3)
|
||||
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
mock_extract_zones.side_effect = [docs[0].pk, None, docs[0].pk]
|
||||
|
||||
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
|
||||
|
||||
finish.assert_called_once_with([docs[0].pk])
|
||||
|
||||
def test_finishes_written_documents_when_a_later_one_raises(
|
||||
self,
|
||||
mock_extract_zones: mock.MagicMock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Two selected documents, the second of which raises during extraction
|
||||
WHEN:
|
||||
- run_zone_ocr is called
|
||||
THEN:
|
||||
- The exception propagates
|
||||
- bulk_update_documents still runs with the first document's id
|
||||
"""
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
mock_extract_zones.side_effect = [docs[0].pk, RuntimeError("boom")]
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
|
||||
|
||||
finish.assert_called_once_with([docs[0].pk])
|
||||
|
||||
def test_does_not_finish_when_nothing_was_written(
|
||||
self,
|
||||
mock_extract_zones: mock.MagicMock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Selected documents with no successful write
|
||||
WHEN:
|
||||
- run_zone_ocr is called
|
||||
THEN:
|
||||
- bulk_update_documents is not run
|
||||
"""
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
|
||||
bulk_edit.run_zone_ocr([doc.pk for doc in docs])
|
||||
|
||||
finish.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBulkEditReprocess:
|
||||
|
||||
@@ -97,7 +97,10 @@ class TestRunZoneOcrBulkOperation:
|
||||
"""
|
||||
template = OcrTemplateFactory()
|
||||
enqueue = mocker.patch("documents.views.execute_bulk_operation.apply_async")
|
||||
extract = mocker.patch("documents.bulk_edit.extract_zones")
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
)
|
||||
enqueue.return_value.id = "zone-ocr-task"
|
||||
response = admin_client.post(
|
||||
RUN_ZONE_OCR_BULK_URL,
|
||||
@@ -137,7 +140,10 @@ class TestRunZoneOcrBulkOperation:
|
||||
- The result reports the operation and document count
|
||||
"""
|
||||
template = OcrTemplateFactory()
|
||||
extract = mocker.patch("documents.bulk_edit.extract_zones")
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
)
|
||||
result = execute_bulk_operation(
|
||||
"run_zone_ocr",
|
||||
[document.pk],
|
||||
@@ -152,6 +158,30 @@ class TestRunZoneOcrBulkOperation:
|
||||
extract.apply_async.assert_not_called()
|
||||
assert result == {"operation": "run_zone_ocr", "document_count": 1}
|
||||
|
||||
def test_finishes_with_the_ids_that_were_written(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
admin_user: User,
|
||||
document: Document,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A queued zone OCR run whose extraction reports a written document
|
||||
WHEN:
|
||||
- The tracked task executes it
|
||||
THEN:
|
||||
- bulk_update_documents runs once with that document id
|
||||
"""
|
||||
mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=document.pk,
|
||||
)
|
||||
finish = mocker.patch("documents.bulk_edit.bulk_update_documents")
|
||||
|
||||
execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk)
|
||||
|
||||
finish.assert_called_once_with([document.pk])
|
||||
|
||||
def test_without_template_selects_for_the_requester(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
@@ -167,7 +197,10 @@ class TestRunZoneOcrBulkOperation:
|
||||
- Extraction receives the requesting user so templates are limited to
|
||||
those they can view
|
||||
"""
|
||||
extract = mocker.patch("documents.bulk_edit.extract_zones")
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
)
|
||||
execute_bulk_operation("run_zone_ocr", [document.pk], {}, admin_user.pk)
|
||||
extract.assert_called_once_with(
|
||||
document_id=document.pk,
|
||||
@@ -223,7 +256,10 @@ class TestRunZoneOcrBulkOperation:
|
||||
template.save(update_fields=["owner"])
|
||||
else:
|
||||
User.objects.filter(pk=regular_user.pk).update(is_active=False)
|
||||
extract = mocker.patch("documents.bulk_edit.extract_zones")
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
)
|
||||
with pytest.raises(PermissionDenied):
|
||||
execute_bulk_operation(**enqueue.call_args.kwargs["kwargs"])
|
||||
extract.assert_not_called()
|
||||
@@ -244,7 +280,10 @@ class TestRunZoneOcrBulkOperation:
|
||||
template matching
|
||||
- Extraction never runs
|
||||
"""
|
||||
extract = mocker.patch("documents.bulk_edit.extract_zones")
|
||||
extract = mocker.patch(
|
||||
"documents.bulk_edit.extract_zones",
|
||||
return_value=None,
|
||||
)
|
||||
with pytest.raises(OcrTemplate.DoesNotExist):
|
||||
execute_bulk_operation(
|
||||
"run_zone_ocr",
|
||||
|
||||
@@ -23,6 +23,8 @@ from documents.sanity_checker import SanityCheckFailedException
|
||||
from documents.sanity_checker import SanityCheckMessages
|
||||
from documents.tasks import extract_zones
|
||||
from documents.tests.helpers import dummy_preprocess
|
||||
from documents.zone_ocr import ZoneOcrReason
|
||||
from documents.zone_ocr import ZoneOcrResult
|
||||
from paperless_testing.assertions import FileSystemAssertsMixin
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
@@ -676,11 +678,7 @@ class TestExtractZonesTask:
|
||||
extract_zones(doc.id, requester=user)
|
||||
|
||||
select.assert_called_once_with(doc, requester=user)
|
||||
extract.assert_called_once_with(
|
||||
doc,
|
||||
templates=select.return_value,
|
||||
reindex=True,
|
||||
)
|
||||
extract.assert_called_once_with(doc, templates=select.return_value)
|
||||
|
||||
def test_task_runs_the_chosen_template(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
@@ -697,7 +695,42 @@ class TestExtractZonesTask:
|
||||
|
||||
extract_zones(doc.id, template.pk)
|
||||
|
||||
extract.assert_called_once_with(doc, templates=[template], reindex=True)
|
||||
extract.assert_called_once_with(doc, templates=[template])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("reason", "returns_root"),
|
||||
[
|
||||
pytest.param(None, True, id="written"),
|
||||
pytest.param(ZoneOcrReason.NO_VALUE, False, id="nothing-written"),
|
||||
],
|
||||
)
|
||||
def test_task_reports_the_root_when_a_value_was_written(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
reason: ZoneOcrReason | None,
|
||||
returns_root: bool, # noqa: FBT001
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document version whose extraction does or does not write a value
|
||||
WHEN:
|
||||
- The task runs
|
||||
THEN:
|
||||
- The root document id is returned only when a value was written
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
DocumentFactory(root_document=root, version_index=1)
|
||||
template = OcrTemplateFactory()
|
||||
mocker.patch(
|
||||
"documents.tasks.run_zone_extraction",
|
||||
return_value=[
|
||||
ZoneOcrResult(template="t", zone="z", custom_field="", reason=reason),
|
||||
],
|
||||
)
|
||||
|
||||
result = extract_zones(root.pk, template.pk)
|
||||
|
||||
assert result == (root.pk if returns_root else None)
|
||||
|
||||
def test_task_skips_a_deleted_template(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
|
||||
@@ -66,6 +66,7 @@ if TYPE_CHECKING:
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from pytest_django.fixtures import DjangoCaptureOnCommitCallbacks
|
||||
from pytest_django.fixtures import SettingsWrapper
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
@@ -781,6 +782,104 @@ class TestSavedZoneExtraction:
|
||||
document.refresh_from_db()
|
||||
assert document.title == "Second"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
pytest.param("title", id="builtin-field"),
|
||||
pytest.param("custom_field", id="custom-field-only"),
|
||||
],
|
||||
)
|
||||
def test_successful_write_bumps_modified_of_the_root(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
page: Path,
|
||||
document: Document,
|
||||
template: OcrTemplate,
|
||||
target: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document and a template with a zone for a built-in field or
|
||||
only for a custom field
|
||||
WHEN:
|
||||
- Zone extraction writes the value
|
||||
THEN:
|
||||
- The root document's modified time is later than before the run
|
||||
"""
|
||||
field = CustomFieldFactory(name="Code", data_type="string")
|
||||
if target == "title":
|
||||
make_zone(template, "Title")
|
||||
else:
|
||||
make_zone(template, "Code", "custom_field", custom_field=field)
|
||||
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
|
||||
[],
|
||||
0,
|
||||
stdout="value",
|
||||
)
|
||||
before = Document.objects.get(pk=document.pk).modified
|
||||
|
||||
results = run_zone_extraction(document, page, templates=[template])
|
||||
|
||||
assert [result.reason for result in results] == [None]
|
||||
assert Document.objects.get(pk=document.pk).modified > before
|
||||
|
||||
def test_without_a_write_leaves_modified_untouched(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
page: Path,
|
||||
document: Document,
|
||||
template: OcrTemplate,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A template whose only zone reads no text
|
||||
WHEN:
|
||||
- Zone extraction runs
|
||||
THEN:
|
||||
- Nothing is written and modified is unchanged
|
||||
"""
|
||||
make_zone(template, "Title")
|
||||
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
|
||||
[],
|
||||
0,
|
||||
stdout="",
|
||||
)
|
||||
before = Document.objects.get(pk=document.pk).modified
|
||||
|
||||
results = run_zone_extraction(document, page, templates=[template])
|
||||
|
||||
assert [result.reason for result in results] == [ZoneOcrReason.NO_VALUE]
|
||||
assert Document.objects.get(pk=document.pk).modified == before
|
||||
|
||||
def test_successful_write_does_not_reindex_per_document(
|
||||
self,
|
||||
django_capture_on_commit_callbacks: DjangoCaptureOnCommitCallbacks,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
page: Path,
|
||||
document: Document,
|
||||
template: OcrTemplate,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A template that writes the title
|
||||
WHEN:
|
||||
- Zone extraction runs
|
||||
THEN:
|
||||
- No on-commit callbacks are registered; finishing is left to the
|
||||
caller
|
||||
"""
|
||||
make_zone(template, "Title")
|
||||
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
|
||||
[],
|
||||
0,
|
||||
stdout="value",
|
||||
)
|
||||
|
||||
with django_capture_on_commit_callbacks() as callbacks:
|
||||
run_zone_extraction(document, page, templates=[template])
|
||||
|
||||
assert callbacks == []
|
||||
|
||||
def test_document_run_renders_each_page_once_across_templates(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
|
||||
@@ -2324,6 +2324,7 @@ class DocumentViewSet(
|
||||
from documents.zone_ocr import ZoneOcrSkipReason
|
||||
from documents.zone_ocr import get_templates_for_document
|
||||
from documents.zone_ocr import run_zone_extraction
|
||||
from documents.zone_ocr import zone_results_written
|
||||
|
||||
resolution = resolve_effective_document(document, request)
|
||||
if resolution.error == VersionResolutionError.INVALID:
|
||||
@@ -2353,7 +2354,12 @@ class DocumentViewSet(
|
||||
status=status.HTTP_404_NOT_FOUND,
|
||||
)
|
||||
|
||||
results = run_zone_extraction(file_document, templates=templates, reindex=True)
|
||||
results = run_zone_extraction(file_document, templates=templates)
|
||||
if zone_results_written(results):
|
||||
bulk_edit.bulk_update_documents.apply_async(
|
||||
kwargs={"document_ids": [get_root_document(file_document).pk]},
|
||||
headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
|
||||
)
|
||||
return Response(
|
||||
{
|
||||
"status": ZoneOcrRunStatus.COMPLETED,
|
||||
|
||||
+10
-13
@@ -14,7 +14,6 @@ from decimal import Decimal
|
||||
from decimal import InvalidOperation
|
||||
from enum import StrEnum
|
||||
from functools import lru_cache
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import NewType
|
||||
@@ -36,6 +35,7 @@ from django.core.exceptions import ValidationError
|
||||
from django.db import transaction
|
||||
from django.db.models import F
|
||||
from django.db.models import Field
|
||||
from django.utils import timezone
|
||||
from PIL import Image
|
||||
|
||||
from documents.matching import matches
|
||||
@@ -48,7 +48,6 @@ from documents.parsers import ParseError
|
||||
from documents.parsers import rasterize_pdf_page_to_png
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.regex import safe_regex_fullmatch
|
||||
from documents.signals.handlers import add_to_index
|
||||
from documents.validators import monetary_validator
|
||||
from documents.versioning import get_root_document
|
||||
from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES
|
||||
@@ -220,14 +219,13 @@ def run_zone_extraction(
|
||||
document: Document,
|
||||
original_file: Path | None = None,
|
||||
templates: list[OcrTemplate] | None = None,
|
||||
*,
|
||||
reindex: bool = False,
|
||||
) -> list[ZoneOcrResult]:
|
||||
"""
|
||||
Extract zones from the selected file version and update its root document.
|
||||
Run all matching templates when no templates are given.
|
||||
|
||||
Manual runs can request a search refresh after successful writes commit.
|
||||
The root document's modified time is bumped when any value was written.
|
||||
Callers finish the edit (index, signals) themselves; see zone_results_written.
|
||||
"""
|
||||
if templates is None:
|
||||
templates = get_templates_for_document(document)
|
||||
@@ -274,17 +272,16 @@ def run_zone_extraction(
|
||||
_zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED)
|
||||
for zone in zones
|
||||
)
|
||||
if reindex and any(result.reason is None for result in results):
|
||||
# queue a reindex of the document after the transaction commits
|
||||
root_document = get_root_document(document)
|
||||
transaction.on_commit(partial(_reindex_document, root_document.pk))
|
||||
if zone_results_written(results):
|
||||
Document.objects.filter(pk=get_root_document(document).pk).update(
|
||||
modified=timezone.now(),
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def _reindex_document(document_id: int) -> None:
|
||||
document = Document.objects.filter(pk=document_id).first()
|
||||
if document is not None:
|
||||
add_to_index(sender=None, document=document)
|
||||
def zone_results_written(results: list[ZoneOcrResult]) -> bool:
|
||||
"""Whether at least one zone's value was written to the root document."""
|
||||
return any(result.reason is None for result in results)
|
||||
|
||||
|
||||
def _resolve_doc_path(
|
||||
|
||||
Reference in new issue
Block a user