Chore: resolve the zone OCR source file in one place

This commit is contained in:
stumpylog committed 2026-11-20 09:34:24 -08:00
1 parent 36de14d2c5
commit 5182b719c5
4 files changed
+184 -121

No files matched your search

@@ -3163,3 +3163,25 @@ class TestZoneEditorVersions:
assert response.status_code == status.HTTP_200_OK
assert mock_preview.call_args.args[0] == newer.archive_path
assert mock_preview.call_args.kwargs["is_image"] is False
def test_page_image_falls_back_to_the_source_when_the_archive_is_missing(
self,
preview_client: APIClient,
versioned_document: SimpleNamespace,
) -> None:
"""
GIVEN:
- An image document that has an archive filename but no archive file
WHEN:
- A page image is requested
THEN:
- The source image is served
"""
Document.objects.filter(pk=versioned_document.newer.pk).update(
archive_filename="missing.pdf",
)
response = preview_client.get(preview_url(versioned_document.root))
assert response.status_code == status.HTTP_200_OK
assert response.content == b"newer"
+122 -87
View File
@@ -30,6 +30,7 @@ from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH
from documents.zone_ocr import ZoneExtraction
from documents.zone_ocr import ZoneOcrReason
from documents.zone_ocr import ZoneOcrSkipReason
from documents.zone_ocr import ZoneSourceFile
from documents.zone_ocr import _apply_transform
from documents.zone_ocr import _combine_field_value
from documents.zone_ocr import _convert_value
@@ -43,11 +44,11 @@ from documents.zone_ocr import _parse_number
from documents.zone_ocr import _prepare_field_update
from documents.zone_ocr import _process_template
from documents.zone_ocr import _render_pages
from documents.zone_ocr import _resolve_doc_path
from documents.zone_ocr import _zone_language
from documents.zone_ocr import extract_zone_preview
from documents.zone_ocr import get_templates_for_content
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import resolve_zone_source
from documents.zone_ocr import run_zone_extraction
from paperless_testing.factories import CustomFieldFactory
from paperless_testing.factories import DocumentFactory
@@ -510,17 +511,16 @@ class TestZoneExtraction:
class TestSavedZoneExtraction:
"""Tests for extracting zones from a saved template and writing the results."""
@pytest.fixture
def page(self, paperless_dirs: PaperlessDirs) -> Path:
page = paperless_dirs.scratch_dir / "page.png"
with Image.new("RGB", (200, 100), "white") as image:
image.save(page)
return page
@pytest.fixture
def document(self) -> Document:
return DocumentFactory(title="Original", mime_type="image/png")
@pytest.fixture
def page(self, document: Document) -> Path:
with Image.new("RGB", (200, 100), "white") as image:
image.save(document.source_path, format="PNG")
return document.source_path
@pytest.fixture
def template(self) -> OcrTemplate:
return make_template("Account")
@@ -782,7 +782,7 @@ class TestSavedZoneExtraction:
for value in ["First", "Second", "rejected"]
]
run_zone_extraction(document, page, templates=templates)
run_zone_extraction(document, templates=templates)
document.refresh_from_db()
assert document.title == "Second"
@@ -823,7 +823,7 @@ class TestSavedZoneExtraction:
)
before = Document.objects.get(pk=document.pk).modified
results = run_zone_extraction(document, page, templates=[template])
results = run_zone_extraction(document, templates=[template])
assert [result.reason for result in results] == [None]
assert Document.objects.get(pk=document.pk).modified > before
@@ -851,7 +851,7 @@ class TestSavedZoneExtraction:
)
before = Document.objects.get(pk=document.pk).modified
results = run_zone_extraction(document, page, templates=[template])
results = run_zone_extraction(document, templates=[template])
assert [result.reason for result in results] == [ZoneOcrReason.NO_VALUE]
assert Document.objects.get(pk=document.pk).modified == before
@@ -881,7 +881,7 @@ class TestSavedZoneExtraction:
)
with django_capture_on_commit_callbacks() as callbacks:
run_zone_extraction(document, page, templates=[template])
run_zone_extraction(document, templates=[template])
assert callbacks == []
@@ -908,8 +908,7 @@ class TestSavedZoneExtraction:
make_zone(template, "Last", page=-1)
second = make_template("Second")
make_zone(second, "Last again", page=2)
pdf = paperless_dirs.scratch_dir / "document.pdf"
pdf.write_bytes(b"%PDF-1.4")
document.source_path.write_bytes(b"%PDF-1.4")
rendered = []
def render(in_path: Path, out_path: Path, **kwargs: Any) -> None:
@@ -927,7 +926,7 @@ class TestSavedZoneExtraction:
stdout="extracted",
)
results = run_zone_extraction(document, pdf, templates=[template, second])
results = run_zone_extraction(document, templates=[template, second])
assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2]
assert [result.reason for result in results] == [None, None, None]
@@ -1018,7 +1017,7 @@ class TestSavedZoneExtraction:
"""
make_zone(template, "Title")
results = run_zone_extraction(document, None, templates=[template])
results = run_zone_extraction(document, templates=[template])
assert len(results) == 1
assert results[0].reason == ZoneOcrReason.MISSING_FILE
@@ -1131,7 +1130,7 @@ class TestSavedZoneExtraction:
for value in ["Rolled back", "42", "Applied"]
]
results = run_zone_extraction(document, page, templates=[template, second])
results = run_zone_extraction(document, templates=[template, second])
assert [result.reason for result in results] == [
ZoneOcrReason.TEMPLATE_FAILED,
@@ -1165,7 +1164,7 @@ class TestSavedZoneExtraction:
make_zone(second, "Second")
mock_zone_ocr_run.side_effect = ocr_outputs("Ignored", "Applied")
results = run_zone_extraction(document, page, templates=[template, second])
results = run_zone_extraction(document, templates=[template, second])
assert [result.reason for result in results] == [
ZoneOcrReason.TEMPLATE_FAILED,
@@ -1894,7 +1893,7 @@ class TestTemplateVisibility:
stderr="",
)
results = run_zone_extraction(document, None)
results = run_zone_extraction(document)
assert Counter(result.template for result in results) == Counter(
[mine.name, unowned.name],
@@ -3585,86 +3584,124 @@ class TestRenderPages:
mock_rasterize.assert_not_called()
class TestResolveDocPath:
"""Tests for _resolve_doc_path."""
class TestResolveZoneSource:
"""Tests for resolve_zone_source."""
@pytest.fixture
def document(self, mocker: MockerFixture) -> MagicMock:
doc = mocker.MagicMock()
doc.has_archive_version = False
doc.source_path = Path("/nonexistent/source.pdf")
return doc
def source(self, tmp_path: Path) -> Path:
path = tmp_path / "source.png"
path.touch()
return path
def test_returns_none_when_no_files_exist(self, document: MagicMock) -> None:
@pytest.fixture
def archive(self, tmp_path: Path) -> Path:
path = tmp_path / "archive.pdf"
path.touch()
return path
def test_returns_none_when_no_files_exist(self, mocker: MockerFixture) -> None:
"""
GIVEN:
- A document with no archive version and a missing source file
WHEN:
- The document path is resolved without an original file
- The zone source is resolved
THEN:
- None is returned
"""
assert _resolve_doc_path(document, None) is None
document = mocker.MagicMock(
has_archive_version=False,
source_path=Path("/nonexistent/source.pdf"),
)
def test_prefers_archive_and_falls_back_to_source(
assert resolve_zone_source(document) is None
def test_prefers_archive_and_never_treats_it_as_an_image(
self,
mocker: MockerFixture,
tmp_path: Path,
source: Path,
archive: Path,
) -> None:
"""
GIVEN:
- A document with existing archive and source files
- A document with an image original and an existing archive
WHEN:
- The path is resolved, then the archive file is removed and it is
resolved again
- The zone source is resolved
THEN:
- The archive path is returned first, then the source path
- The archive is returned and is not an image
"""
source = tmp_path / "source.pdf"
archive = tmp_path / "archive.pdf"
source.touch()
archive.touch()
document = mocker.MagicMock(
has_archive_version=True,
archive_path=archive,
source_path=source,
mime_type="image/png",
)
assert _resolve_doc_path(document, None) == archive
assert resolve_zone_source(document) == ZoneSourceFile(
path=archive,
is_image=False,
)
@pytest.mark.parametrize(
("mime_type", "expected_is_image"),
[
pytest.param("image/png", True, id="image"),
pytest.param("application/pdf", False, id="pdf"),
],
)
def test_uses_source_with_image_flag_from_mime_type(
self,
mocker: MockerFixture,
source: Path,
mime_type: str,
*,
expected_is_image: bool,
) -> None:
"""
GIVEN:
- A document without an archive version
WHEN:
- The zone source is resolved
THEN:
- The source file is returned, flagged as an image only for image
mime types
"""
document = mocker.MagicMock(
has_archive_version=False,
source_path=source,
mime_type=mime_type,
)
assert resolve_zone_source(document) == ZoneSourceFile(
path=source,
is_image=expected_is_image,
)
def test_falls_back_to_source_when_archive_is_missing_on_disk(
self,
mocker: MockerFixture,
source: Path,
archive: Path,
) -> None:
"""
GIVEN:
- A document flagged as having an archive whose file was removed
WHEN:
- The zone source is resolved
THEN:
- The source file is returned
"""
archive.unlink()
assert _resolve_doc_path(document, None) == source
document = mocker.MagicMock(
has_archive_version=True,
archive_path=archive,
source_path=source,
mime_type="image/png",
)
def test_returns_original_file_as_fallback(
self,
document: MagicMock,
tmp_path: Path,
) -> None:
"""
GIVEN:
- A document whose own files are missing and an existing original file
WHEN:
- The document path is resolved with the original file
THEN:
- The original file path is returned
"""
original = tmp_path / "original.pdf"
original.touch()
assert _resolve_doc_path(document, original) == original
def test_returns_none_for_missing_original_file(
self,
document: MagicMock,
tmp_path: Path,
) -> None:
"""
GIVEN:
- A document whose own files are missing and an original file path
that does not exist
WHEN:
- The document path is resolved with the original file
THEN:
- None is returned
"""
assert _resolve_doc_path(document, tmp_path / "missing.pdf") is None
assert resolve_zone_source(document) == ZoneSourceFile(
path=source,
is_image=True,
)
@pytest.mark.django_db
@@ -3681,11 +3718,10 @@ class TestRunZoneExtraction:
)
@pytest.fixture
def source(self, paperless_dirs: PaperlessDirs) -> Path:
source = paperless_dirs.scratch_dir / "document.png"
def source(self, document: Document) -> Path:
with Image.new("RGB", (200, 200), "white") as image:
image.save(source)
return source
image.save(document.source_path, format="PNG")
return document.source_path
@pytest.fixture(autouse=True)
def ocr_reads_text(self, mock_zone_ocr_run: MagicMock) -> None:
@@ -3779,7 +3815,7 @@ class TestRunZoneExtraction:
"""
template_with_zone(template_name, **template_kwargs)
results = run_zone_extraction(document, source)
results = run_zone_extraction(document)
assert results == []
mock_zone_ocr_run.assert_not_called()
@@ -3805,7 +3841,7 @@ class TestRunZoneExtraction:
match="test",
)
results = run_zone_extraction(document, source)
results = run_zone_extraction(document)
assert results == []
mock_zone_ocr_run.assert_not_called()
@@ -3841,14 +3877,14 @@ class TestRunZoneExtraction:
template_with_zone("First", enabled=True)
template_with_zone("Second", enabled=True)
results = run_zone_extraction(document, source)
results = run_zone_extraction(document)
assert Counter((r.value, r.reason) for r in results) == Counter(
[(None, ZoneOcrReason.TEMPLATE_FAILED), ("extracted", None)],
)
assert {r.template for r in results} == {"First", "Second"}
def test_handles_none_original_file(
def test_reports_missing_file_when_document_has_no_file(
self,
mock_zone_ocr_run: MagicMock,
template_with_zone: Callable[..., OcrTemplate],
@@ -3856,8 +3892,7 @@ class TestRunZoneExtraction:
) -> None:
"""
GIVEN:
- A matching template, a document with no accessible file and no
original file
- A matching template and a document with no accessible file
WHEN:
- Zone extraction runs
THEN:
@@ -3866,7 +3901,7 @@ class TestRunZoneExtraction:
template = template_with_zone("Matching", enabled=True)
zone = template.zones.get()
results = run_zone_extraction(document, None)
results = run_zone_extraction(document)
assert [(r.template, r.zone, r.reason) for r in results] == [
("Matching", zone.name, ZoneOcrReason.MISSING_FILE),
@@ -3901,7 +3936,7 @@ class TestRunZoneExtraction:
for name in template_names:
template_with_zone(name, enabled=True)
results = run_zone_extraction(document, source)
results = run_zone_extraction(document)
assert sorted((r.template, r.reason) for r in results) == [
(name, None) for name in sorted(template_names)
@@ -3950,7 +3985,7 @@ class TestRealZoneOcr:
)
shutil.copy(simple_digital_pdf_file, document.source_path)
results = run_zone_extraction(document, None)
results = run_zone_extraction(document)
assert [result.reason for result in results] == [None]
document.refresh_from_db()
@@ -3978,7 +4013,7 @@ class TestRealZoneOcr:
document.source_path,
)
results = run_zone_extraction(document, None)
results = run_zone_extraction(document)
assert [result.reason for result in results] == [None]
document.refresh_from_db()
+13 -11
View File
@@ -2324,6 +2324,7 @@ class DocumentViewSet(
from documents.zone_ocr import ZoneOcrSkipReason
from documents.zone_ocr import get_templates_for_document
from documents.zone_ocr import resolve_zone_source
resolution = resolve_effective_document(document, request)
if resolution.error == VersionResolutionError.INVALID:
@@ -2343,8 +2344,7 @@ class DocumentViewSet(
},
)
doc_path = file_document.archive_path or file_document.source_path
if not doc_path.is_file():
if resolve_zone_source(file_document) is None:
return Response(
{"error": "Document file not found"},
status=status.HTTP_404_NOT_FOUND,
@@ -5896,6 +5896,8 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem
page: str,
) -> HttpResponse:
"""Render a page for the template editor's zone overlay."""
from documents.zone_ocr import resolve_zone_source
document: Document = get_object_or_404(
Document.objects.select_related("root_document__owner"),
pk=doc_id,
@@ -5913,12 +5915,12 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem
f"Page {page_num} out of range (document has {file_doc.page_count} pages)",
)
archive_path = file_doc.archive_path
doc_path = archive_path or file_doc.source_path
if not doc_path.is_file():
source = resolve_zone_source(file_doc)
if source is None:
raise Http404("Document file not found")
doc_path = source.path
if archive_path is None and file_doc.mime_type.startswith("image/"):
if source.is_image:
return HttpResponse(
doc_path.read_bytes(),
content_type=file_doc.mime_type,
@@ -5961,6 +5963,7 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem
def test_zone(self, request: Request) -> HttpResponse:
"""Extract an ad-hoc zone without changing the document."""
from documents.zone_ocr import extract_zone_preview
from documents.zone_ocr import resolve_zone_source
serializer = OcrTemplateTestSerializer(
data=request.data,
@@ -5971,8 +5974,8 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem
file_doc = self._resolve_file_document(document, request)
doc_path = file_doc.archive_path or file_doc.source_path
if not doc_path.is_file():
source = resolve_zone_source(file_doc)
if source is None:
return Response(
{"error": "Document file not found"},
status=status.HTTP_404_NOT_FOUND,
@@ -5981,13 +5984,12 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem
zone = OcrTemplateZone(**serializer.validated_data["zone"])
result = extract_zone_preview(
doc_path,
source.path,
zone,
zone.zone_source_width or 0,
zone.zone_source_height or 0,
file_doc.page_count,
is_image=file_doc.archive_path is None
and file_doc.mime_type.startswith("image/"),
is_image=source.is_image,
)
regex_match = None
+27 -23
View File
@@ -109,6 +109,30 @@ class FieldUpdate:
custom_field: CustomField | None = None
@dataclass(frozen=True, kw_only=True, slots=True)
class ZoneSourceFile:
path: Path
is_image: bool
def resolve_zone_source(document: Document) -> ZoneSourceFile | None:
"""
Pick the stored file zone OCR reads: the archive PDF when it exists on disk,
else the source file when it exists, else None.
"""
if document.has_archive_version:
archive_path = document.archive_path
if archive_path is not None and archive_path.is_file():
return ZoneSourceFile(path=archive_path, is_image=False)
source_path = document.source_path
if source_path.is_file():
return ZoneSourceFile(
path=source_path,
is_image=document.mime_type.startswith("image/"),
)
return None
class ZoneOcrReason(StrEnum):
"""Why a zone produced no value for its target."""
@@ -253,7 +277,6 @@ def _visible_templates(
def run_zone_extraction(
document: Document,
original_file: Path | None = None,
templates: list[OcrTemplate] | None = None,
) -> list[ZoneOcrResult]:
"""
@@ -269,14 +292,13 @@ def run_zone_extraction(
return []
label = f"document {document.pk}"
doc_path = _resolve_doc_path(document, original_file)
is_archive = doc_path is not None and doc_path == document.archive_path
source = resolve_zone_source(document)
plans = plan_zone_extraction(
doc_path,
source.path if source else None,
document.page_count,
templates,
label=label,
is_image=not is_archive and document.mime_type.startswith("image/"),
is_image=source.is_image if source else False,
)
results = apply_zone_plans(get_root_document(document), plans, label=label)
if zone_results_written(results):
@@ -417,24 +439,6 @@ def zone_results_written(results: list[ZoneOcrResult]) -> bool:
return any(result.reason is None for result in results)
def _resolve_doc_path(
document: Document,
original_file: Path | None,
) -> Path | None:
"""Prefer the archive, then the source, then the given file."""
candidates: list[Path | None] = []
if document.has_archive_version:
candidates.append(document.archive_path)
candidates.append(document.source_path)
if original_file is not None:
candidates.append(original_file)
for path in candidates:
if path is not None and Path(path).is_file():
return Path(path)
return None
def _resolve_page_idx(page_value: int | None, page_count: int | None) -> int:
"""Resolve a 1-based page or -1 (last page), defaulting to the first page."""
if page_value == -1: