Fix: read zones before the consume transaction and write them before workflows

Zone OCR ran inside the consume transaction, read the working copy instead
of the archive, and ran after the document added workflows. Zones are now
read from the archive before the transaction and written just before
those workflows, keeping values supplied at consume time.
This commit is contained in:
stumpylog committed 2026-11-19 15:09:41 -08:00
1 parent 54be9dc9c5
commit a6b81b2d64
9 files changed
+1112 -164

No files matched your search

+14 -5
View File
@@ -872,10 +872,18 @@ is left empty, the values are joined with spaces in zone order.
### When templates run
When a document is added, every enabled template that matches its content runs automatically. A template is
only used for a document if the document's owner can view the template, so users' templates apply to their own
documents, and unowned templates apply to everyone's. Templates run in name order, so if two templates fill the
same field, the later one wins.
When a document is added, every enabled template that matches its content runs automatically, reading the
zones from the archived file, or from the original PDF or image when there is no archived file. A template is
only used if the document's owner can view it, so users' templates apply to their own documents, and unowned
templates apply to everyone's. The owner here is the one set by the upload, the mail rule or a consumption
workflow. A document without an owner at that point, for example from the consume folder or a mail rule
without an owner, considers every enabled template. An owner assigned later by a "Document Added" workflow
does not change which templates ran. Templates run in name order, so if two templates fill the same field, the
later one wins.
Zone OCR keeps values supplied when the document is consumed: an uploaded title, ASN, created date or custom
field value, a mail rule's title, a barcode ASN, and values assigned by a consumption workflow. Zones fill the
remaining fields before "Document Added" workflows run, so those workflows can use or overwrite the values.
Turn off **Enabled** to pause a template without deleting it. Disabled templates never run automatically, but
can still be run manually.
@@ -897,7 +905,8 @@ the latest version of each document and are processed in the background.
!!! warning
Zone OCR overwrites the fields it fills. A zone that finds no value leaves its field unchanged.
Running zone OCR manually or in bulk overwrites the fields it fills. A zone that finds no value leaves its
field unchanged.
Managing OCR templates requires the OCR template permissions. Running zone OCR on documents requires
permission to change them, and choosing a template requires permission to view it.
-2
View File
@@ -15,7 +15,6 @@ class DocumentsConfig(AppConfig):
from documents.signals.handlers import add_to_index
from documents.signals.handlers import run_workflows_added
from documents.signals.handlers import run_workflows_updated
from documents.signals.handlers import run_zone_ocr_extraction
from documents.signals.handlers import send_websocket_document_updated
from documents.signals.handlers import set_correspondent
from documents.signals.handlers import set_document_type
@@ -28,7 +27,6 @@ class DocumentsConfig(AppConfig):
document_consumption_finished.connect(set_tags)
document_consumption_finished.connect(set_storage_path)
document_consumption_finished.connect(run_workflows_added)
document_consumption_finished.connect(run_zone_ocr_extraction)
document_consumption_finished.connect(add_to_index)
document_consumption_finished.connect(add_or_update_document_in_llm_index)
+124
View File
@@ -34,6 +34,7 @@ from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentBarcode
from documents.models import DocumentType
from documents.models import OcrTemplateZone
from documents.models import StoragePath
from documents.models import Tag
from documents.models import WorkflowTrigger
@@ -56,6 +57,13 @@ from documents.utils import compute_checksum
from documents.utils import copy_basic_file_stats
from documents.utils import copy_file_with_basic_stats
from documents.utils import run_subprocess
from documents.versioning import get_root_document
from documents.zone_ocr import FieldKey
from documents.zone_ocr import TemplatePlan
from documents.zone_ocr import apply_zone_plans
from documents.zone_ocr import field_key
from documents.zone_ocr import get_templates_for_content
from documents.zone_ocr import plan_zone_extraction
from paperless.config import BarcodeConfig
from paperless.config import OcrConfig
from paperless.config import RemoteOCRConfig
@@ -588,6 +596,16 @@ class ConsumerPlugin(
exception=e,
)
# Zone OCR renders and reads pages, so it runs here, before the
# transaction opens. Its values are written once the document
# is stored.
zone_plans = self._plan_zone_ocr(
text=text,
archive_path=archive_path,
mime_type=mime_type,
page_count=page_count,
)
# Prepare the document classifier.
# TODO: I don't really like to do this here, but this way we avoid
@@ -672,6 +690,11 @@ class ConsumerPlugin(
mime_type=mime_type,
)
# Before the consumption finished signal, so DOCUMENT_ADDED
# workflows and the other handlers of that signal see the
# zone values.
self._apply_zone_ocr(document, zone_plans)
# If we get here, it was successful. Proceed with post-consume
# hooks. If they fail, nothing will get changed.
@@ -986,6 +1009,107 @@ class ConsumerPlugin(
self._store_barcodes(document)
def _plan_zone_ocr(
self,
*,
text: str | None,
archive_path: Path | None,
mime_type: str,
page_count: int | None,
) -> list[TemplatePlan]:
"""
Select the OCR templates matching the parsed text and compute their zone
values, without writing anything. Zones are read from the archive, the
file the template editor draws on, or from the working copy of a PDF or
image without one. Any error is logged and yields no zone values.
"""
try:
if self.input_doc.root_document_id is not None:
root_doc = Document.objects.select_related("owner").get(
pk=self.input_doc.root_document_id,
)
owner = get_root_document(root_doc).owner
elif self.metadata.owner_id is not None:
owner = User.objects.get(pk=self.metadata.owner_id)
else:
owner = None
templates = get_templates_for_content(
text or "",
owner,
title=self.filename,
)
if not templates:
return []
if archive_path is not None and Path(archive_path).is_file():
doc_path: Path | None = Path(archive_path)
elif mime_type == "application/pdf" or mime_type.startswith("image/"):
doc_path = self.working_copy
else:
doc_path = None
return plan_zone_extraction(
doc_path,
page_count,
templates,
label=self.filename,
)
except Exception:
self.log.exception(
f"Zone OCR failed for {self.filename}, continuing without zone values",
)
return []
def _apply_zone_ocr(self, document: Document, plans: list[TemplatePlan]) -> None:
"""
Write the planned zone values to the stored document, or to the root of a
new version. For a new document, a value supplied explicitly in the
metadata overrides is kept rather than replaced by a zone.
"""
if not plans:
return
try:
protected = (
frozenset()
if self.input_doc.root_document_id is not None
else self._explicit_zone_targets()
)
root_doc = get_root_document(document)
results = apply_zone_plans(
root_doc,
plans,
label=f"document {root_doc.pk}",
protected=protected,
)
except Exception:
self.log.exception(
f"Zone OCR could not write values for {self.filename}",
)
return
for result in results:
self.log.debug(
f"Zone OCR: template '{result.template}' zone '{result.zone}' "
f"for {result.custom_field}: "
f"{result.reason if result.reason is not None else 'written'}",
)
def _explicit_zone_targets(self) -> frozenset[FieldKey]:
"""The zone targets given a value by the metadata overrides.
A custom field listed without a value is not a supplied value.
"""
targets = set()
if self.metadata.title is not None:
targets.add(field_key(OcrTemplateZone.TargetType.TITLE, None))
if self.metadata.asn is not None:
targets.add(field_key(OcrTemplateZone.TargetType.ASN, None))
if self.metadata.created is not None:
targets.add(field_key(OcrTemplateZone.TargetType.CREATED, None))
for field_id, value in (self.metadata.custom_fields or {}).items():
if value is not None:
targets.add(
field_key(OcrTemplateZone.TargetType.CUSTOM_FIELD, int(field_id)),
)
return frozenset(targets)
def _read_version_barcodes(self, mime_type: str, work_dir: Path) -> None:
barcode_settings = BarcodeConfig()
if not barcode_settings.barcode_store_values:
-18
View File
@@ -1478,24 +1478,6 @@ def close_connection_pool_on_worker_init(**kwargs) -> None:
conn.close_pool()
def run_zone_ocr_extraction(
sender: Any,
document: Document,
original_file: Path | None = None,
**kwargs: Any,
) -> None:
"""Apply matching templates without failing consumption if extraction fails."""
try:
from documents.zone_ocr import run_zone_extraction
run_zone_extraction(document, Path(original_file) if original_file else None)
except Exception:
logger.exception(
"Zone OCR extraction failed for document %s",
document.pk,
)
@worker_process_shutdown.connect
def close_connection_pool_on_worker_shutdown(**kwargs) -> None: # pragma: no cover
"""
@@ -29,6 +29,8 @@ from documents.tasks import bulk_update_documents
from documents.zone_ocr import ZoneExtraction
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrResult
from documents.zone_ocr import apply_zone_plans
from documents.zone_ocr import plan_zone_extraction
from paperless_testing.factories import CustomFieldFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
@@ -323,6 +325,14 @@ class TestZoneOcrIndexing:
)
with django_capture_on_commit_callbacks(execute=True):
if entry_point == "ingestion":
# The consumer writes the planned values, then sends the signal.
plans = plan_zone_extraction(
file_document.source_path,
file_document.page_count,
[template],
label="ingestion",
)
apply_zone_plans(document, plans, label="ingestion")
document_consumption_finished.send(sender=None, document=file_document)
elif entry_point == "single":
response = admin_client.post(
@@ -0,0 +1,676 @@
"""Tests for zone OCR during consumption."""
from __future__ import annotations
import json
import logging
import shutil
import tempfile
from datetime import date
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
from typing import Self
import pytest
from django.conf import settings
from django.db import transaction
from PIL import Image
from documents import consumer as consumer_module
from documents.consumer import ConsumerPlugin
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from paperless_testing.factories import CustomFieldFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import OcrTemplateFactory
from paperless_testing.factories import OcrTemplateZoneFactory
from paperless_testing.factories import TagFactory
from paperless_testing.factories import UserFactory
from paperless_testing.fakes.progress import FakeProgressManager
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
if TYPE_CHECKING:
from collections.abc import Callable
from unittest.mock import MagicMock
from pytest_django.fixtures import SettingsWrapper
from pytest_mock import MockerFixture
from documents.zone_ocr import TemplatePlan
from paperless_testing.dirs import PaperlessDirs
CONTENT = "Northwind Power & Light statement"
# The text each zone's fake OCR returns, by zone name.
ZONE_TEXT = {
"Title": "Zone title",
"ASN": "ASN 777",
"Created": "2024-03-05",
"Code": "INV42",
}
MIME_TYPES = {
".pdf": "application/pdf",
".png": "image/png",
".eml": "message/rfc822",
}
class ZoneTestParser:
"""A parser returning CONTENT, with a PDF archive when one is requested."""
name = "zone-test-parser"
version = "0.1"
author = "test"
url = "test"
@classmethod
def supported_mime_types(cls) -> dict[str, str]:
return {mime: suffix for suffix, mime in MIME_TYPES.items()}
@classmethod
def score(cls, mime_type: str, filename: str, path: Path | None = None) -> int:
return 0
@property
def can_produce_archive(self) -> bool:
return True
@property
def requires_pdf_rendition(self) -> bool:
return False
def __enter__(self) -> Self:
self._tmpdir = Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
self._archive: Path | None = None
self._thumb = self._tmpdir / "thumb.webp"
self._thumb.write_bytes(b"thumbnail")
return self
def __exit__(self, *args: object) -> None:
shutil.rmtree(self._tmpdir, ignore_errors=True)
def configure(self, context: object) -> None:
"""The test parser ignores its context."""
def parse(
self,
document_path: Path,
mime_type: str,
*,
produce_archive: bool = True,
) -> None:
if produce_archive:
self._archive = self._tmpdir / "archive.pdf"
shutil.copy(SIMPLE_DIGITAL_PDF, self._archive)
def get_text(self) -> str:
return CONTENT
def get_date(self) -> None:
return None
def get_archive_path(self) -> Path | None:
return self._archive
def get_thumbnail(self, document_path: Path, mime_type: str) -> Path:
return self._thumb
def get_page_count(self, document_path: Path, mime_type: str) -> int:
return 1
def extract_metadata(self, document_path: Path, mime_type: str) -> list[Any]:
return []
@pytest.fixture
def page_image(tmp_path: Path) -> Path:
page = tmp_path / "page.png"
with Image.new("RGB", (200, 200), "white") as image:
image.save(page)
return page
@pytest.fixture
def mock_render(mocker: MockerFixture, page_image: Path) -> MagicMock:
"""Fake page rendering, recording whether a transaction was open."""
render = mocker.patch("documents.zone_ocr._render_pages")
render.in_transaction = []
def record(*args: Any, **kwargs: Any) -> dict[int, Path]:
render.in_transaction.append(transaction.get_connection().in_atomic_block)
return {0: page_image}
render.side_effect = record
return render
@pytest.fixture
def mock_ocr(mocker: MockerFixture) -> MagicMock:
"""Fake zone OCR returning ZONE_TEXT, recording whether a transaction was open."""
ocr = mocker.patch("documents.zone_ocr._ocr_text")
ocr.in_transaction = []
def read(cropped: Image.Image, zone: OcrTemplateZone, tmp_dir: Path) -> str:
ocr.in_transaction.append(transaction.get_connection().in_atomic_block)
return ZONE_TEXT[zone.name]
ocr.side_effect = read
return ocr
@pytest.fixture
def consume(
paperless_dirs: PaperlessDirs,
mocker: MockerFixture,
) -> Callable[..., Document]:
"""Consume a file of the given suffix through the ConsumerPlugin."""
registry = mocker.patch("documents.consumer.get_parser_registry")
registry.return_value.get_parser_for_file.return_value = ZoneTestParser
mocker.patch(
"documents.consumer.magic.from_file",
side_effect=lambda path, *, mime=False: MIME_TYPES[Path(path).suffix],
)
def run(
suffix: str = ".pdf",
overrides: DocumentMetadataOverrides | None = None,
root_document_id: int | None = None,
) -> Document:
source = paperless_dirs.scratch_dir / f"upload{suffix}"
if suffix == ".pdf":
shutil.copy(SIMPLE_DIGITAL_PDF, source)
else:
source.write_bytes(f"{suffix} {source.name}".encode())
plugin = ConsumerPlugin(
ConsumableDocument(
DocumentSource.ApiUpload,
original_file=source,
root_document_id=root_document_id,
),
overrides or DocumentMetadataOverrides(),
FakeProgressManager(source.name, None),
paperless_dirs.scratch_dir,
"task-id",
)
plugin.setup()
try:
result = plugin.run()
finally:
plugin.cleanup()
return Document.objects.get(pk=result["document_id"])
return run
def make_template(
name: str = "Invoice",
targets: tuple[str, ...] = ("Title",),
custom_field: CustomField | None = None,
**kwargs: Any,
) -> OcrTemplate:
"""A template matching CONTENT with one zone per named target."""
template = OcrTemplateFactory(
name=name,
match="Northwind",
source_width=100,
source_height=100,
**kwargs,
)
zone_targets = {
"Title": OcrTemplateZone.TargetType.TITLE,
"ASN": OcrTemplateZone.TargetType.ASN,
"Created": OcrTemplateZone.TargetType.CREATED,
"Code": OcrTemplateZone.TargetType.CUSTOM_FIELD,
}
for order, zone_name in enumerate(targets):
OcrTemplateZoneFactory(
template=template,
name=zone_name,
target=zone_targets[zone_name],
custom_field=custom_field if zone_name == "Code" else None,
y=order * 20,
width=50,
height=20,
order=order,
)
return template
def custom_field_value(document: Document, field: CustomField) -> Any:
instance = CustomFieldInstance.objects.filter(
document=document,
field=field,
).first()
return instance.value if instance is not None else None
@pytest.mark.django_db(transaction=True)
@pytest.mark.usefixtures("mock_ocr")
class TestConsumerZoneOcrTransaction:
"""Zone OCR runs outside the consumer's database transaction."""
def test_ocr_runs_before_the_transaction_opens(
self,
consume: Callable[..., Document],
mock_render: MagicMock,
mock_ocr: MagicMock,
) -> None:
"""
GIVEN:
- A template matching the consumed document's text
WHEN:
- The document is consumed
THEN:
- Pages are rendered and zones are read with no transaction open
- The zone value is written to the document
"""
make_template()
document = consume()
assert mock_render.in_transaction == [False]
assert mock_ocr.in_transaction == [False]
assert document.title == "Zone title"
@pytest.mark.django_db
@pytest.mark.usefixtures("mock_ocr")
class TestConsumerZoneOcr:
"""Zone values are computed before and written during consumption."""
@pytest.fixture
def code_field(self) -> CustomField:
return CustomFieldFactory(
name="Invoice code",
data_type=CustomField.FieldDataType.STRING,
)
@pytest.mark.parametrize(
("suffix", "generation", "expected_file"),
[
pytest.param(".pdf", "always", "archive.pdf", id="pdf-archive"),
pytest.param(".eml", "always", "archive.pdf", id="mail-archive"),
pytest.param(".pdf", "never", "upload.pdf", id="pdf-working-copy"),
pytest.param(".png", "never", "upload.png", id="image-working-copy"),
],
)
def test_zones_are_read_from_the_archive_or_working_copy(
self,
consume: Callable[..., Document],
mock_render: MagicMock,
settings: SettingsWrapper,
suffix: str,
generation: str,
expected_file: str,
) -> None:
"""
GIVEN:
- A template matching the consumed document's text
WHEN:
- A document is consumed with or without an archive
THEN:
- Zones are read from the archive when there is one, otherwise
from the working copy of the PDF or image
"""
settings.ARCHIVE_FILE_GENERATION = generation
make_template()
document = consume(suffix)
mock_render.assert_called_once()
assert mock_render.call_args.args[0].name == expected_file
assert document.title == "Zone title"
def test_mail_without_archive_is_not_rendered(
self,
consume: Callable[..., Document],
mock_render: MagicMock,
settings: SettingsWrapper,
) -> None:
"""
GIVEN:
- A template matching the consumed document's text
WHEN:
- An email is consumed without an archive
THEN:
- No page is rendered from the email file
- The document is consumed without zone values
"""
settings.ARCHIVE_FILE_GENERATION = "never"
make_template()
document = consume(".eml")
mock_render.assert_not_called()
assert document.title == "upload"
@pytest.mark.usefixtures("mock_render")
def test_values_are_written_before_document_added_workflows(
self,
consume: Callable[..., Document],
code_field: CustomField,
) -> None:
"""
GIVEN:
- A template writing a custom field
- A DOCUMENT_ADDED workflow filtering on that custom field's value
and assigning a tag
WHEN:
- A matching document is consumed
THEN:
- The workflow sees the zone value and assigns its tag
"""
make_template(targets=("Code",), custom_field=code_field)
tag = TagFactory(name="Zone matched")
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
filter_custom_field_query=json.dumps(
["AND", [[code_field.pk, "exact", "INV42"]]],
),
)
action = WorkflowAction.objects.create()
action.assign_tags.add(tag)
workflow = Workflow.objects.create(name="On zone value", order=0)
workflow.triggers.add(trigger)
workflow.actions.add(action)
document = consume()
assert custom_field_value(document, code_field) == "INV42"
assert list(document.tags.all()) == [tag]
@pytest.mark.parametrize(
"overridden",
[
pytest.param("title", id="title"),
pytest.param("asn", id="asn"),
pytest.param("created", id="created"),
pytest.param("custom_field", id="custom-field"),
],
)
@pytest.mark.usefixtures("mock_render")
def test_explicit_values_are_not_overwritten(
self,
consume: Callable[..., Document],
code_field: CustomField,
overridden: str,
) -> None:
"""
GIVEN:
- A template with title, ASN, created and custom field zones
- Metadata overrides giving one of those fields a value
WHEN:
- A matching document is consumed
THEN:
- The supplied value is kept
- Every other zone value is written
"""
make_template(
targets=("Title", "ASN", "Created", "Code"),
custom_field=code_field,
)
explicit = {
"title": "Upload title",
"asn": 123,
"created": date(2020, 1, 2),
"custom_field": "Given",
}
overrides = {
"title": DocumentMetadataOverrides(title="Upload title"),
"asn": DocumentMetadataOverrides(asn=123),
"created": DocumentMetadataOverrides(created=date(2020, 1, 2)),
"custom_field": DocumentMetadataOverrides(
custom_fields={code_field.pk: "Given"},
),
}[overridden]
expected = {
"title": "Zone title",
"asn": 777,
"created": date(2024, 3, 5),
"custom_field": "INV42",
} | {overridden: explicit[overridden]}
document = consume(overrides=overrides)
assert {
"title": document.title,
"asn": document.archive_serial_number,
"created": document.created,
"custom_field": custom_field_value(document, code_field),
} == expected
@pytest.mark.usefixtures("mock_render")
def test_custom_field_listed_without_a_value_is_filled(
self,
consume: Callable[..., Document],
code_field: CustomField,
) -> None:
"""
GIVEN:
- A template with a custom field zone
- Metadata overrides listing that custom field without a value
WHEN:
- A matching document is consumed
THEN:
- The zone value is written to the custom field
"""
make_template(targets=("Code",), custom_field=code_field)
document = consume(
overrides=DocumentMetadataOverrides(custom_fields={code_field.pk: None}),
)
assert custom_field_value(document, code_field) == "INV42"
@pytest.mark.usefixtures("mock_render")
def test_custom_field_deleted_before_writing_is_skipped(
self,
consume: Callable[..., Document],
code_field: CustomField,
mocker: MockerFixture,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A template with title and custom field zones
- The custom field deleted after the zone values are planned
WHEN:
- A matching document is consumed
THEN:
- The document is stored with the title
- The custom field zone is reported as unassigned
"""
make_template(targets=("Title", "Code"), custom_field=code_field)
plan = consumer_module.plan_zone_extraction
def plan_then_delete(*args: Any, **kwargs: Any) -> list[TemplatePlan]:
plans = plan(*args, **kwargs)
CustomField.objects.filter(pk=code_field.pk).delete()
return plans
mocker.patch(
"documents.consumer.plan_zone_extraction",
side_effect=plan_then_delete,
)
caplog.set_level(logging.DEBUG, logger="paperless.consumer")
document = consume()
assert document.title == "Zone title"
assert not CustomFieldInstance.objects.filter(document=document).exists()
assert "zone 'Code' for Invoice code: unassigned" in caplog.text
@pytest.mark.parametrize(
("with_owner", "expected_title", "expected_code"),
[
pytest.param(True, "Zone title", None, id="owner-sees-own-templates"),
pytest.param(False, "Zone title", "INV42", id="no-owner-sees-all"),
],
)
@pytest.mark.usefixtures("mock_render")
def test_templates_are_selected_by_the_metadata_owner(
self,
consume: Callable[..., Document],
code_field: CustomField,
with_owner: bool, # noqa: FBT001
expected_title: str,
expected_code: str | None,
) -> None:
"""
GIVEN:
- A template owned by the upload's owner writing the title
- A private template of another user writing a custom field
WHEN:
- A matching document is consumed with or without an owner
THEN:
- With an owner, only the templates that owner can view apply
- Without an owner, every template applies
"""
owner = UserFactory()
make_template("Mine", owner=owner)
make_template(
"Private",
targets=("Code",),
custom_field=code_field,
owner=UserFactory(),
)
document = consume(
overrides=DocumentMetadataOverrides(
owner_id=owner.pk if with_owner else None,
),
)
assert document.title == expected_title
assert custom_field_value(document, code_field) == expected_code
@pytest.mark.usefixtures("mock_render")
def test_duplicate_asn_is_checked_when_writing(
self,
consume: Callable[..., Document],
) -> None:
"""
GIVEN:
- A template with title and ASN zones
- Another document already using the zone's ASN
WHEN:
- A matching document is consumed
THEN:
- The ASN is not written
- The title from the same template is written
"""
DocumentFactory(archive_serial_number=777)
make_template(targets=("Title", "ASN"))
document = consume()
assert document.archive_serial_number is None
assert document.title == "Zone title"
@pytest.mark.parametrize(
"failing",
[
pytest.param(
"documents.consumer.get_templates_for_content",
id="template-selection",
),
pytest.param("documents.consumer.plan_zone_extraction", id="extraction"),
],
)
@pytest.mark.usefixtures("mock_render")
def test_read_phase_failure_does_not_fail_consumption(
self,
consume: Callable[..., Document],
mocker: MockerFixture,
caplog: pytest.LogCaptureFixture,
failing: str,
) -> None:
"""
GIVEN:
- A template matching the consumed document's text
- Template selection or zone extraction raising an error
WHEN:
- The document is consumed
THEN:
- The document is stored without zone values
- The failure is logged
"""
make_template()
mocker.patch(failing, side_effect=RuntimeError("zone OCR broke"))
document = consume()
assert document.title == "upload"
assert "Zone OCR failed for upload.pdf" in caplog.text
@pytest.mark.usefixtures("mock_render")
def test_write_failure_does_not_fail_consumption(
self,
consume: Callable[..., Document],
mocker: MockerFixture,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A template matching the consumed document's text
- Writing the template's values raising an error
WHEN:
- The document is consumed
THEN:
- The document is stored without the template's values
- The failed template is logged
"""
make_template()
mocker.patch(
"documents.zone_ocr._apply_field_updates",
side_effect=RuntimeError("write failed"),
)
document = consume()
assert document.title == "upload"
assert "error processing template 'Invoice'" in caplog.text
@pytest.mark.usefixtures("mock_render")
def test_version_values_are_written_to_the_root(
self,
consume: Callable[..., Document],
code_field: CustomField,
) -> None:
"""
GIVEN:
- A root document whose owner can view a template writing the title
and a custom field
- A private template of another user
WHEN:
- A new version of the root document is consumed
THEN:
- The zone values are written to the root document
- The version keeps the root's previous title and gets no custom field
- The other user's template does not apply
"""
owner = UserFactory()
root = DocumentFactory(title="Root title", owner=owner, content=CONTENT)
make_template(targets=("Title", "Code"), custom_field=code_field, owner=owner)
make_template("Private", targets=("ASN",), owner=UserFactory())
version = consume(root_document_id=root.pk)
root.refresh_from_db()
assert version.root_document_id == root.pk
assert root.title == "Zone title"
assert custom_field_value(root, code_field) == "INV42"
assert root.archive_serial_number is None
assert version.title == "Root title"
assert custom_field_value(version, code_field) is None
-67
View File
@@ -13,81 +13,14 @@ from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.models import PaperlessTask
from documents.signals.handlers import before_task_publish_handler
from documents.signals.handlers import run_zone_ocr_extraction
from documents.signals.handlers import task_failure_handler
from documents.signals.handlers import task_postrun_handler
from documents.signals.handlers import task_prerun_handler
from documents.signals.handlers import task_revoked_handler
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import PaperlessTaskFactory
from paperless_testing.factories import UserFactory
@pytest.mark.django_db
class TestRunZoneOcrExtraction:
"""Zone OCR extraction triggered by the consumption finished signal."""
@pytest.mark.parametrize(
"original_file",
[
pytest.param(None, id="no-original-file"),
pytest.param("/tmp/original.pdf", id="str-path"),
pytest.param(Path("/tmp/original.pdf"), id="path-object"),
],
)
def test_consumption_runs_zone_extraction(
self,
original_file: str | Path | None,
mocker: pytest_mock.MockerFixture,
) -> None:
"""
GIVEN:
- A consumed document and an original file given as None, a str, or a Path
WHEN:
- The consumption finished handler runs zone OCR extraction
THEN:
- Zone extraction is called once with the document and the
original file normalised to a Path (or None)
"""
extract = mocker.patch("documents.zone_ocr.run_zone_extraction")
document = DocumentFactory()
run_zone_ocr_extraction(
sender=None,
document=document,
original_file=original_file,
)
extract.assert_called_once_with(
document,
Path(original_file) if original_file else None,
)
def test_zone_extraction_failure_does_not_fail_consumption(
self,
mocker: pytest_mock.MockerFixture,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A consumed document whose zone extraction raises an error
WHEN:
- The consumption finished handler runs zone OCR extraction
THEN:
- The error is not propagated
- The failure is logged with the document id
"""
mocker.patch(
"documents.zone_ocr.run_zone_extraction",
side_effect=RuntimeError("OCR failed"),
)
document = DocumentFactory()
run_zone_ocr_extraction(sender=None, document=document)
assert f"Zone OCR extraction failed for document {document.pk}" in caplog.text
@pytest.fixture
def consume_input_doc():
doc = mock.MagicMock(spec=ConsumableDocument)
+34
View File
@@ -52,6 +52,7 @@ from documents.zone_ocr import _render_pages
from documents.zone_ocr import _resolve_doc_path
from documents.zone_ocr import _zone_language
from documents.zone_ocr import extract_zone_preview
from documents.zone_ocr import get_templates_for_content
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import run_zone_extraction
from paperless_testing.factories import CustomFieldFactory
@@ -1707,6 +1708,39 @@ class TestTemplateVisibility:
get_templates_for_document(document, requester=other),
) == Counter([unowned, private])
@pytest.mark.parametrize(
("has_owner", "expected_names"),
[
pytest.param(True, {"Mine", "Unowned"}, id="owner"),
pytest.param(False, {"Mine", "Unowned", "Private"}, id="no-owner"),
],
)
@pytest.mark.usefixtures("mine", "unowned", "private")
def test_content_selection_follows_the_owner_rule(
self,
owner: User,
has_owner: bool, # noqa: FBT001
expected_names: set[str],
) -> None:
"""
GIVEN:
- Templates owned by two users and an unowned template, all
matching "Northwind", and one matching only other text
WHEN:
- Templates are selected for unsaved text, with or without an owner
THEN:
- Only matching templates apply
- An owner sees owned and unowned templates, no owner sees all
"""
make_visible_template("Other sender", match="Contoso")
templates = get_templates_for_content(
"Northwind Power & Light statement",
owner if has_owner else None,
)
assert {template.name for template in templates} == expected_names
@pytest.mark.usefixtures("private")
def test_versions_use_the_root_documents_owner(
self,
+254 -72
View File
@@ -57,6 +57,7 @@ if TYPE_CHECKING:
from typing import Any
from django.contrib.auth.models import User
from django.db.models import QuerySet
logger = logging.getLogger("paperless.zone_ocr")
@@ -146,6 +147,25 @@ class ZoneOcrResult:
reason: ZoneOcrReason | None = None
@dataclass(kw_only=True, slots=True)
class PlannedUpdate:
"""A field update and the accepted zone results it was combined from."""
key: FieldKey
update: FieldUpdate
results: list[ZoneOcrResult]
@dataclass(kw_only=True, slots=True)
class TemplatePlan:
"""A template's zone results and the field updates they produce, unwritten."""
template: OcrTemplate
zones: list[OcrTemplateZone]
results: list[ZoneOcrResult]
updates: list[PlannedUpdate]
# Result labels for zones that target a built-in field.
BUILTIN_TARGET_LABELS: dict[OcrTemplateZone.TargetType, str] = {
OcrTemplateZone.TargetType.TITLE: "Title",
@@ -187,20 +207,7 @@ def get_templates_for_document(
view, so a user editing someone else's document never runs or sees that
owner's private templates.
"""
owner = get_root_document(document).owner
templates = OcrTemplate.objects.filter(enabled=True)
if owner is not None:
templates = templates.filter(
id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"),
)
if requester is not None:
templates = templates.filter(
id__in=permitted_object_ids(requester, OcrTemplate, "view_ocrtemplate"),
)
templates = templates.order_by("name", "pk").prefetch_related(
"zones",
"zones__custom_field",
)
templates = _visible_templates(get_root_document(document).owner, requester)
match_document = document
if document.root_document_id is None and templates:
# The original file can be selected even when newer versions exist.
@@ -215,6 +222,43 @@ def get_templates_for_document(
return [template for template in templates if matches(template, match_document)]
def get_templates_for_content(
content: str,
owner: User | None,
*,
title: str = "",
) -> list[OcrTemplate]:
"""
The enabled templates that match the given text and that the owner can view,
for a document that is not saved yet. A missing owner considers every
template, as get_templates_for_document does. The title only labels the
matching log messages.
"""
templates = _visible_templates(owner, None)
match_document = Document(title=title, content=content)
return [template for template in templates if matches(template, match_document)]
def _visible_templates(
owner: User | None,
requester: User | None,
) -> QuerySet[OcrTemplate]:
"""The enabled templates both the owner and the requester can view, if given."""
templates = OcrTemplate.objects.filter(enabled=True)
if owner is not None:
templates = templates.filter(
id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"),
)
if requester is not None:
templates = templates.filter(
id__in=permitted_object_ids(requester, OcrTemplate, "view_ocrtemplate"),
)
return templates.order_by("name", "pk").prefetch_related(
"zones",
"zones__custom_field",
)
def run_zone_extraction(
document: Document,
original_file: Path | None = None,
@@ -232,46 +276,14 @@ def run_zone_extraction(
if not templates:
return []
doc_path = _resolve_doc_path(document, original_file)
if doc_path is None:
logger.warning(
"Zone OCR: no accessible file for document %d",
document.pk,
)
return [
_zone_result(template, zone, reason=ZoneOcrReason.MISSING_FILE)
for template in templates
for zone in template.zones.all()
]
template_zones = [(template, list(template.zones.all())) for template in templates]
pages = {
_resolve_page_idx(zone.page, document.page_count)
for _, zones in template_zones
for zone in zones
}
if not pages:
return []
results = []
with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir:
tmp_path = Path(tmp_dir)
page_images = _render_pages(doc_path, pages, tmp_path, document.page_count)
for template, zones in template_zones:
try:
results.extend(
_process_template(document, page_images, template, zones, tmp_path),
)
except Exception:
# One failed template must not prevent the remaining templates from running.
logger.exception(
"Zone OCR: error processing template '%s' for document %d",
template.name,
document.pk,
)
results.extend(
_zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED)
for zone in zones
)
label = f"document {document.pk}"
plans = plan_zone_extraction(
_resolve_doc_path(document, original_file),
document.page_count,
templates,
label=label,
)
results = apply_zone_plans(get_root_document(document), plans, label=label)
if zone_results_written(results):
Document.objects.filter(pk=get_root_document(document).pk).update(
modified=timezone.now(),
@@ -279,6 +291,124 @@ def run_zone_extraction(
return results
def plan_zone_extraction(
doc_path: Path | None,
page_count: int | None,
templates: list[OcrTemplate],
*,
label: str,
) -> list[TemplatePlan]:
"""
Render the zones' pages from doc_path and compute each template's field
updates without writing anything. A template that fails yields only
TEMPLATE_FAILED results, and a missing file yields MISSING_FILE results for
every zone. The label names the document in log messages.
"""
template_zones = [(template, list(template.zones.all())) for template in templates]
if doc_path is None:
logger.warning("Zone OCR: no accessible file for %s", label)
return [
_failed_plan(template, zones, ZoneOcrReason.MISSING_FILE)
for template, zones in template_zones
]
pages = {
_resolve_page_idx(zone.page, page_count)
for _, zones in template_zones
for zone in zones
}
if not pages:
return []
plans = []
with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir:
tmp_path = Path(tmp_dir)
page_images = _render_pages(doc_path, pages, tmp_path, page_count)
for template, zones in template_zones:
try:
plans.append(
_plan_template(page_count, page_images, template, zones, tmp_path),
)
except Exception:
# One failed template must not prevent the remaining templates
# from running.
logger.exception(
"Zone OCR: error processing template '%s' for %s",
template.name,
label,
)
plans.append(
_failed_plan(template, zones, ZoneOcrReason.TEMPLATE_FAILED),
)
return plans
def apply_zone_plans(
root_document: Document,
plans: list[TemplatePlan],
*,
label: str,
protected: frozenset[FieldKey] = frozenset(),
) -> list[ZoneOcrResult]:
"""
Write each template's planned updates to the root document, each template
in its own atomic block, and return the zone results. Fields whose key is in
protected are not written, and the results of every zone targeting them are
left out. A template whose writes fail gets TEMPLATE_FAILED results and the
others still apply.
"""
results: list[ZoneOcrResult] = []
for plan in plans:
for planned in plan.updates:
if planned.key in protected:
logger.info(
"Zone OCR: template '%s' kept the supplied %s of %s",
plan.template.name,
planned.update.field_name,
label,
)
updates = [planned for planned in plan.updates if planned.key not in protected]
try:
if updates:
_apply_template_updates(root_document, updates)
plan_results = plan.results
except Exception:
logger.exception(
"Zone OCR: error processing template '%s' for %s",
plan.template.name,
label,
)
plan_results = _failed_plan(
plan.template,
plan.zones,
ZoneOcrReason.TEMPLATE_FAILED,
).results
# Each zone has exactly one result, in zone order.
results.extend(
result
for zone, result in zip(plan.zones, plan_results, strict=True)
if _zone_key(zone) not in protected
)
return results
def _zone_key(zone: OcrTemplateZone) -> FieldKey:
"""The combine_formats key of the field a zone fills."""
return field_key(OcrTemplateZone.TargetType(zone.target), zone.custom_field_id)
def _failed_plan(
template: OcrTemplate,
zones: list[OcrTemplateZone],
reason: ZoneOcrReason,
) -> TemplatePlan:
return TemplatePlan(
template=template,
zones=zones,
results=[_zone_result(template, zone, reason=reason) for zone in zones],
updates=[],
)
def zone_results_written(results: list[ZoneOcrResult]) -> bool:
"""Whether at least one zone's value was written to the root document."""
return any(result.reason is None for result in results)
@@ -288,7 +418,7 @@ def _resolve_doc_path(
document: Document,
original_file: Path | None,
) -> Path | None:
"""Prefer the archive, then the source, then the ingestion signal's file."""
"""Prefer the archive, then the source, then the given file."""
candidates: list[Path | None] = []
if document.has_archive_version:
candidates.append(document.archive_path)
@@ -316,7 +446,21 @@ def _process_template(
zones: list[OcrTemplateZone],
tmp_path: Path,
) -> list[ZoneOcrResult]:
"""Extract and combine zone values before applying a template's updates."""
"""Extract and combine zone values, then apply a template's updates."""
plan = _plan_template(document.page_count, page_images, template, zones, tmp_path)
if plan.updates:
_apply_template_updates(get_root_document(document), plan.updates)
return plan.results
def _plan_template(
page_count: int | None,
page_images: dict[int, Path],
template: OcrTemplate,
zones: list[OcrTemplateZone],
tmp_path: Path,
) -> TemplatePlan:
"""Extract and combine zone values into a template's updates, unwritten."""
# Each zone's result and its accepted text, by target field.
grouped: dict[
FieldKey,
@@ -324,7 +468,7 @@ def _process_template(
] = {}
results = []
for zone in zones:
page_idx = _resolve_page_idx(zone.page, document.page_count)
page_idx = _resolve_page_idx(zone.page, page_count)
extracted = None
reason: ZoneOcrReason | None = ZoneOcrReason.MISSING_PAGE
if page_idx in page_images:
@@ -351,16 +495,12 @@ def _process_template(
reason = ZoneOcrReason.REGEX_MISMATCH
result = _zone_result(template, zone, reason=reason)
results.append(result)
key = field_key(
OcrTemplateZone.TargetType(zone.target),
zone.custom_field_id,
)
key = _zone_key(zone)
grouped.setdefault(key, []).append(
(zone, result, extracted if reason is None else None),
)
updates = []
root_document = get_root_document(document)
combine_formats: dict[FieldKey, str] = template.combine_formats or {}
for key, field_zones in grouped.items():
accepted = [result for _, result, _ in field_zones if result.reason is None]
@@ -381,15 +521,8 @@ def _process_template(
)
target_zone = field_zones[0][0]
update = _prepare_field_update(target_zone, value) if value else None
if update is not None and _asn_in_use(update, root_document):
# Saving would violate the unique ASN and roll back every other
# write of this template, so only this field is dropped.
for result in accepted:
result.value = None
result.reason = ZoneOcrReason.DUPLICATE_ASN
continue
if update is not None:
updates.append(update)
updates.append(PlannedUpdate(key=key, update=update, results=accepted))
for result in accepted:
result.value = update.value if update else None
if update is None:
@@ -398,8 +531,57 @@ def _process_template(
if key == UNASSIGNED_FIELD_KEY
else ZoneOcrReason.INVALID_VALUE
)
_apply_field_updates(root_document, updates)
return results
return TemplatePlan(
template=template,
zones=zones,
results=results,
updates=updates,
)
def _apply_template_updates(
root_document: Document,
updates: list[PlannedUpdate],
) -> None:
"""Write one template's updates to the root document in one atomic block.
The ASN duplicate check and the custom field existence check run here, at
write time, so they see the database as it is when the values are saved.
"""
field_ids = {
planned.update.custom_field.pk
for planned in updates
if planned.update.custom_field is not None
}
with transaction.atomic():
existing_field_ids = (
set(
CustomField.objects.filter(pk__in=field_ids).values_list(
"pk",
flat=True,
),
)
if field_ids
else set()
)
writable = []
for planned in updates:
custom_field = planned.update.custom_field
if custom_field is not None and custom_field.pk not in existing_field_ids:
# The field was deleted after the values were planned.
for result in planned.results:
result.value = None
result.reason = ZoneOcrReason.UNASSIGNED
continue
if _asn_in_use(planned.update, root_document):
# Saving would violate the unique ASN and roll back every other
# write of this template, so only this field is dropped.
for result in planned.results:
result.value = None
result.reason = ZoneOcrReason.DUPLICATE_ASN
continue
writable.append(planned.update)
_apply_field_updates(root_document, writable)
def _asn_in_use(update: FieldUpdate, root_document: Document) -> bool: