Fix: read zones before the consume transaction and write them before workflows

Zone OCR ran inside the consume transaction, read the working copy instead
of the archive, and ran after the document added workflows. Zones are now
read from the archive before the transaction and written just before
those workflows, keeping values supplied at consume time.
This commit is contained in:
stumpylog committed 2026-11-19 15:09:41 -08:00
1 parent 54be9dc9c5
commit a6b81b2d64
9 files changed
+1112 -164

No files matched your search

+254 -72
View File
@@ -57,6 +57,7 @@ if TYPE_CHECKING:
from typing import Any
from django.contrib.auth.models import User
from django.db.models import QuerySet
logger = logging.getLogger("paperless.zone_ocr")
@@ -146,6 +147,25 @@ class ZoneOcrResult:
reason: ZoneOcrReason | None = None
@dataclass(kw_only=True, slots=True)
class PlannedUpdate:
"""A field update and the accepted zone results it was combined from."""
key: FieldKey
update: FieldUpdate
results: list[ZoneOcrResult]
@dataclass(kw_only=True, slots=True)
class TemplatePlan:
"""A template's zone results and the field updates they produce, unwritten."""
template: OcrTemplate
zones: list[OcrTemplateZone]
results: list[ZoneOcrResult]
updates: list[PlannedUpdate]
# Result labels for zones that target a built-in field.
BUILTIN_TARGET_LABELS: dict[OcrTemplateZone.TargetType, str] = {
OcrTemplateZone.TargetType.TITLE: "Title",
@@ -187,20 +207,7 @@ def get_templates_for_document(
view, so a user editing someone else's document never runs or sees that
owner's private templates.
"""
owner = get_root_document(document).owner
templates = OcrTemplate.objects.filter(enabled=True)
if owner is not None:
templates = templates.filter(
id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"),
)
if requester is not None:
templates = templates.filter(
id__in=permitted_object_ids(requester, OcrTemplate, "view_ocrtemplate"),
)
templates = templates.order_by("name", "pk").prefetch_related(
"zones",
"zones__custom_field",
)
templates = _visible_templates(get_root_document(document).owner, requester)
match_document = document
if document.root_document_id is None and templates:
# The original file can be selected even when newer versions exist.
@@ -215,6 +222,43 @@ def get_templates_for_document(
return [template for template in templates if matches(template, match_document)]
def get_templates_for_content(
content: str,
owner: User | None,
*,
title: str = "",
) -> list[OcrTemplate]:
"""
The enabled templates that match the given text and that the owner can view,
for a document that is not saved yet. A missing owner considers every
template, as get_templates_for_document does. The title only labels the
matching log messages.
"""
templates = _visible_templates(owner, None)
match_document = Document(title=title, content=content)
return [template for template in templates if matches(template, match_document)]
def _visible_templates(
owner: User | None,
requester: User | None,
) -> QuerySet[OcrTemplate]:
"""The enabled templates both the owner and the requester can view, if given."""
templates = OcrTemplate.objects.filter(enabled=True)
if owner is not None:
templates = templates.filter(
id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"),
)
if requester is not None:
templates = templates.filter(
id__in=permitted_object_ids(requester, OcrTemplate, "view_ocrtemplate"),
)
return templates.order_by("name", "pk").prefetch_related(
"zones",
"zones__custom_field",
)
def run_zone_extraction(
document: Document,
original_file: Path | None = None,
@@ -232,46 +276,14 @@ def run_zone_extraction(
if not templates:
return []
doc_path = _resolve_doc_path(document, original_file)
if doc_path is None:
logger.warning(
"Zone OCR: no accessible file for document %d",
document.pk,
)
return [
_zone_result(template, zone, reason=ZoneOcrReason.MISSING_FILE)
for template in templates
for zone in template.zones.all()
]
template_zones = [(template, list(template.zones.all())) for template in templates]
pages = {
_resolve_page_idx(zone.page, document.page_count)
for _, zones in template_zones
for zone in zones
}
if not pages:
return []
results = []
with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir:
tmp_path = Path(tmp_dir)
page_images = _render_pages(doc_path, pages, tmp_path, document.page_count)
for template, zones in template_zones:
try:
results.extend(
_process_template(document, page_images, template, zones, tmp_path),
)
except Exception:
# One failed template must not prevent the remaining templates from running.
logger.exception(
"Zone OCR: error processing template '%s' for document %d",
template.name,
document.pk,
)
results.extend(
_zone_result(template, zone, reason=ZoneOcrReason.TEMPLATE_FAILED)
for zone in zones
)
label = f"document {document.pk}"
plans = plan_zone_extraction(
_resolve_doc_path(document, original_file),
document.page_count,
templates,
label=label,
)
results = apply_zone_plans(get_root_document(document), plans, label=label)
if zone_results_written(results):
Document.objects.filter(pk=get_root_document(document).pk).update(
modified=timezone.now(),
@@ -279,6 +291,124 @@ def run_zone_extraction(
return results
def plan_zone_extraction(
doc_path: Path | None,
page_count: int | None,
templates: list[OcrTemplate],
*,
label: str,
) -> list[TemplatePlan]:
"""
Render the zones' pages from doc_path and compute each template's field
updates without writing anything. A template that fails yields only
TEMPLATE_FAILED results, and a missing file yields MISSING_FILE results for
every zone. The label names the document in log messages.
"""
template_zones = [(template, list(template.zones.all())) for template in templates]
if doc_path is None:
logger.warning("Zone OCR: no accessible file for %s", label)
return [
_failed_plan(template, zones, ZoneOcrReason.MISSING_FILE)
for template, zones in template_zones
]
pages = {
_resolve_page_idx(zone.page, page_count)
for _, zones in template_zones
for zone in zones
}
if not pages:
return []
plans = []
with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir:
tmp_path = Path(tmp_dir)
page_images = _render_pages(doc_path, pages, tmp_path, page_count)
for template, zones in template_zones:
try:
plans.append(
_plan_template(page_count, page_images, template, zones, tmp_path),
)
except Exception:
# One failed template must not prevent the remaining templates
# from running.
logger.exception(
"Zone OCR: error processing template '%s' for %s",
template.name,
label,
)
plans.append(
_failed_plan(template, zones, ZoneOcrReason.TEMPLATE_FAILED),
)
return plans
def apply_zone_plans(
root_document: Document,
plans: list[TemplatePlan],
*,
label: str,
protected: frozenset[FieldKey] = frozenset(),
) -> list[ZoneOcrResult]:
"""
Write each template's planned updates to the root document, each template
in its own atomic block, and return the zone results. Fields whose key is in
protected are not written, and the results of every zone targeting them are
left out. A template whose writes fail gets TEMPLATE_FAILED results and the
others still apply.
"""
results: list[ZoneOcrResult] = []
for plan in plans:
for planned in plan.updates:
if planned.key in protected:
logger.info(
"Zone OCR: template '%s' kept the supplied %s of %s",
plan.template.name,
planned.update.field_name,
label,
)
updates = [planned for planned in plan.updates if planned.key not in protected]
try:
if updates:
_apply_template_updates(root_document, updates)
plan_results = plan.results
except Exception:
logger.exception(
"Zone OCR: error processing template '%s' for %s",
plan.template.name,
label,
)
plan_results = _failed_plan(
plan.template,
plan.zones,
ZoneOcrReason.TEMPLATE_FAILED,
).results
# Each zone has exactly one result, in zone order.
results.extend(
result
for zone, result in zip(plan.zones, plan_results, strict=True)
if _zone_key(zone) not in protected
)
return results
def _zone_key(zone: OcrTemplateZone) -> FieldKey:
"""The combine_formats key of the field a zone fills."""
return field_key(OcrTemplateZone.TargetType(zone.target), zone.custom_field_id)
def _failed_plan(
template: OcrTemplate,
zones: list[OcrTemplateZone],
reason: ZoneOcrReason,
) -> TemplatePlan:
return TemplatePlan(
template=template,
zones=zones,
results=[_zone_result(template, zone, reason=reason) for zone in zones],
updates=[],
)
def zone_results_written(results: list[ZoneOcrResult]) -> bool:
"""Whether at least one zone's value was written to the root document."""
return any(result.reason is None for result in results)
@@ -288,7 +418,7 @@ def _resolve_doc_path(
document: Document,
original_file: Path | None,
) -> Path | None:
"""Prefer the archive, then the source, then the ingestion signal's file."""
"""Prefer the archive, then the source, then the given file."""
candidates: list[Path | None] = []
if document.has_archive_version:
candidates.append(document.archive_path)
@@ -316,7 +446,21 @@ def _process_template(
zones: list[OcrTemplateZone],
tmp_path: Path,
) -> list[ZoneOcrResult]:
"""Extract and combine zone values before applying a template's updates."""
"""Extract and combine zone values, then apply a template's updates."""
plan = _plan_template(document.page_count, page_images, template, zones, tmp_path)
if plan.updates:
_apply_template_updates(get_root_document(document), plan.updates)
return plan.results
def _plan_template(
page_count: int | None,
page_images: dict[int, Path],
template: OcrTemplate,
zones: list[OcrTemplateZone],
tmp_path: Path,
) -> TemplatePlan:
"""Extract and combine zone values into a template's updates, unwritten."""
# Each zone's result and its accepted text, by target field.
grouped: dict[
FieldKey,
@@ -324,7 +468,7 @@ def _process_template(
] = {}
results = []
for zone in zones:
page_idx = _resolve_page_idx(zone.page, document.page_count)
page_idx = _resolve_page_idx(zone.page, page_count)
extracted = None
reason: ZoneOcrReason | None = ZoneOcrReason.MISSING_PAGE
if page_idx in page_images:
@@ -351,16 +495,12 @@ def _process_template(
reason = ZoneOcrReason.REGEX_MISMATCH
result = _zone_result(template, zone, reason=reason)
results.append(result)
key = field_key(
OcrTemplateZone.TargetType(zone.target),
zone.custom_field_id,
)
key = _zone_key(zone)
grouped.setdefault(key, []).append(
(zone, result, extracted if reason is None else None),
)
updates = []
root_document = get_root_document(document)
combine_formats: dict[FieldKey, str] = template.combine_formats or {}
for key, field_zones in grouped.items():
accepted = [result for _, result, _ in field_zones if result.reason is None]
@@ -381,15 +521,8 @@ def _process_template(
)
target_zone = field_zones[0][0]
update = _prepare_field_update(target_zone, value) if value else None
if update is not None and _asn_in_use(update, root_document):
# Saving would violate the unique ASN and roll back every other
# write of this template, so only this field is dropped.
for result in accepted:
result.value = None
result.reason = ZoneOcrReason.DUPLICATE_ASN
continue
if update is not None:
updates.append(update)
updates.append(PlannedUpdate(key=key, update=update, results=accepted))
for result in accepted:
result.value = update.value if update else None
if update is None:
@@ -398,8 +531,57 @@ def _process_template(
if key == UNASSIGNED_FIELD_KEY
else ZoneOcrReason.INVALID_VALUE
)
_apply_field_updates(root_document, updates)
return results
return TemplatePlan(
template=template,
zones=zones,
results=results,
updates=updates,
)
def _apply_template_updates(
root_document: Document,
updates: list[PlannedUpdate],
) -> None:
"""Write one template's updates to the root document in one atomic block.
The ASN duplicate check and the custom field existence check run here, at
write time, so they see the database as it is when the values are saved.
"""
field_ids = {
planned.update.custom_field.pk
for planned in updates
if planned.update.custom_field is not None
}
with transaction.atomic():
existing_field_ids = (
set(
CustomField.objects.filter(pk__in=field_ids).values_list(
"pk",
flat=True,
),
)
if field_ids
else set()
)
writable = []
for planned in updates:
custom_field = planned.update.custom_field
if custom_field is not None and custom_field.pk not in existing_field_ids:
# The field was deleted after the values were planned.
for result in planned.results:
result.value = None
result.reason = ZoneOcrReason.UNASSIGNED
continue
if _asn_in_use(planned.update, root_document):
# Saving would violate the unique ASN and roll back every other
# write of this template, so only this field is dropped.
for result in planned.results:
result.value = None
result.reason = ZoneOcrReason.DUPLICATE_ASN
continue
writable.append(planned.update)
_apply_field_updates(root_document, writable)
def _asn_in_use(update: FieldUpdate, root_document: Document) -> bool: