From 5182b719c56fec47f18546235f2929da9f26a583 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Fri, 20 Nov 2026 09:34:24 -0800 Subject: [PATCH] Chore: resolve the zone OCR source file in one place --- src/documents/tests/test_api_ocr_templates.py | 22 ++ src/documents/tests/test_zone_ocr.py | 209 ++++++++++-------- src/documents/views.py | 24 +- src/documents/zone_ocr.py | 50 +++-- 4 files changed, 184 insertions(+), 121 deletions(-) diff --git a/src/documents/tests/test_api_ocr_templates.py b/src/documents/tests/test_api_ocr_templates.py index ad0081eed9..cc9ba7e01e 100644 --- a/src/documents/tests/test_api_ocr_templates.py +++ b/src/documents/tests/test_api_ocr_templates.py @@ -3163,3 +3163,25 @@ class TestZoneEditorVersions: assert response.status_code == status.HTTP_200_OK assert mock_preview.call_args.args[0] == newer.archive_path assert mock_preview.call_args.kwargs["is_image"] is False + + def test_page_image_falls_back_to_the_source_when_the_archive_is_missing( + self, + preview_client: APIClient, + versioned_document: SimpleNamespace, + ) -> None: + """ + GIVEN: + - An image document that has an archive filename but no archive file + WHEN: + - A page image is requested + THEN: + - The source image is served + """ + Document.objects.filter(pk=versioned_document.newer.pk).update( + archive_filename="missing.pdf", + ) + + response = preview_client.get(preview_url(versioned_document.root)) + + assert response.status_code == status.HTTP_200_OK + assert response.content == b"newer" diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index b1de64f14f..44b608fa21 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -30,6 +30,7 @@ from documents.zone_ocr import MAX_PARSED_TEXT_LENGTH from documents.zone_ocr import ZoneExtraction from documents.zone_ocr import ZoneOcrReason from documents.zone_ocr import ZoneOcrSkipReason +from documents.zone_ocr import ZoneSourceFile from documents.zone_ocr import _apply_transform from documents.zone_ocr import _combine_field_value from documents.zone_ocr import _convert_value @@ -43,11 +44,11 @@ from documents.zone_ocr import _parse_number from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages -from documents.zone_ocr import _resolve_doc_path from documents.zone_ocr import _zone_language from documents.zone_ocr import extract_zone_preview from documents.zone_ocr import get_templates_for_content from documents.zone_ocr import get_templates_for_document +from documents.zone_ocr import resolve_zone_source from documents.zone_ocr import run_zone_extraction from paperless_testing.factories import CustomFieldFactory from paperless_testing.factories import DocumentFactory @@ -510,17 +511,16 @@ class TestZoneExtraction: class TestSavedZoneExtraction: """Tests for extracting zones from a saved template and writing the results.""" - @pytest.fixture - def page(self, paperless_dirs: PaperlessDirs) -> Path: - page = paperless_dirs.scratch_dir / "page.png" - with Image.new("RGB", (200, 100), "white") as image: - image.save(page) - return page - @pytest.fixture def document(self) -> Document: return DocumentFactory(title="Original", mime_type="image/png") + @pytest.fixture + def page(self, document: Document) -> Path: + with Image.new("RGB", (200, 100), "white") as image: + image.save(document.source_path, format="PNG") + return document.source_path + @pytest.fixture def template(self) -> OcrTemplate: return make_template("Account") @@ -782,7 +782,7 @@ class TestSavedZoneExtraction: for value in ["First", "Second", "rejected"] ] - run_zone_extraction(document, page, templates=templates) + run_zone_extraction(document, templates=templates) document.refresh_from_db() assert document.title == "Second" @@ -823,7 +823,7 @@ class TestSavedZoneExtraction: ) before = Document.objects.get(pk=document.pk).modified - results = run_zone_extraction(document, page, templates=[template]) + results = run_zone_extraction(document, templates=[template]) assert [result.reason for result in results] == [None] assert Document.objects.get(pk=document.pk).modified > before @@ -851,7 +851,7 @@ class TestSavedZoneExtraction: ) before = Document.objects.get(pk=document.pk).modified - results = run_zone_extraction(document, page, templates=[template]) + results = run_zone_extraction(document, templates=[template]) assert [result.reason for result in results] == [ZoneOcrReason.NO_VALUE] assert Document.objects.get(pk=document.pk).modified == before @@ -881,7 +881,7 @@ class TestSavedZoneExtraction: ) with django_capture_on_commit_callbacks() as callbacks: - run_zone_extraction(document, page, templates=[template]) + run_zone_extraction(document, templates=[template]) assert callbacks == [] @@ -908,8 +908,7 @@ class TestSavedZoneExtraction: make_zone(template, "Last", page=-1) second = make_template("Second") make_zone(second, "Last again", page=2) - pdf = paperless_dirs.scratch_dir / "document.pdf" - pdf.write_bytes(b"%PDF-1.4") + document.source_path.write_bytes(b"%PDF-1.4") rendered = [] def render(in_path: Path, out_path: Path, **kwargs: Any) -> None: @@ -927,7 +926,7 @@ class TestSavedZoneExtraction: stdout="extracted", ) - results = run_zone_extraction(document, pdf, templates=[template, second]) + results = run_zone_extraction(document, templates=[template, second]) assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2] assert [result.reason for result in results] == [None, None, None] @@ -1018,7 +1017,7 @@ class TestSavedZoneExtraction: """ make_zone(template, "Title") - results = run_zone_extraction(document, None, templates=[template]) + results = run_zone_extraction(document, templates=[template]) assert len(results) == 1 assert results[0].reason == ZoneOcrReason.MISSING_FILE @@ -1131,7 +1130,7 @@ class TestSavedZoneExtraction: for value in ["Rolled back", "42", "Applied"] ] - results = run_zone_extraction(document, page, templates=[template, second]) + results = run_zone_extraction(document, templates=[template, second]) assert [result.reason for result in results] == [ ZoneOcrReason.TEMPLATE_FAILED, @@ -1165,7 +1164,7 @@ class TestSavedZoneExtraction: make_zone(second, "Second") mock_zone_ocr_run.side_effect = ocr_outputs("Ignored", "Applied") - results = run_zone_extraction(document, page, templates=[template, second]) + results = run_zone_extraction(document, templates=[template, second]) assert [result.reason for result in results] == [ ZoneOcrReason.TEMPLATE_FAILED, @@ -1894,7 +1893,7 @@ class TestTemplateVisibility: stderr="", ) - results = run_zone_extraction(document, None) + results = run_zone_extraction(document) assert Counter(result.template for result in results) == Counter( [mine.name, unowned.name], @@ -3585,86 +3584,124 @@ class TestRenderPages: mock_rasterize.assert_not_called() -class TestResolveDocPath: - """Tests for _resolve_doc_path.""" +class TestResolveZoneSource: + """Tests for resolve_zone_source.""" @pytest.fixture - def document(self, mocker: MockerFixture) -> MagicMock: - doc = mocker.MagicMock() - doc.has_archive_version = False - doc.source_path = Path("/nonexistent/source.pdf") - return doc + def source(self, tmp_path: Path) -> Path: + path = tmp_path / "source.png" + path.touch() + return path - def test_returns_none_when_no_files_exist(self, document: MagicMock) -> None: + @pytest.fixture + def archive(self, tmp_path: Path) -> Path: + path = tmp_path / "archive.pdf" + path.touch() + return path + + def test_returns_none_when_no_files_exist(self, mocker: MockerFixture) -> None: """ GIVEN: - A document with no archive version and a missing source file WHEN: - - The document path is resolved without an original file + - The zone source is resolved THEN: - None is returned """ - assert _resolve_doc_path(document, None) is None + document = mocker.MagicMock( + has_archive_version=False, + source_path=Path("/nonexistent/source.pdf"), + ) - def test_prefers_archive_and_falls_back_to_source( + assert resolve_zone_source(document) is None + + def test_prefers_archive_and_never_treats_it_as_an_image( self, mocker: MockerFixture, - tmp_path: Path, + source: Path, + archive: Path, ) -> None: """ GIVEN: - - A document with existing archive and source files + - A document with an image original and an existing archive WHEN: - - The path is resolved, then the archive file is removed and it is - resolved again + - The zone source is resolved THEN: - - The archive path is returned first, then the source path + - The archive is returned and is not an image """ - source = tmp_path / "source.pdf" - archive = tmp_path / "archive.pdf" - source.touch() - archive.touch() document = mocker.MagicMock( has_archive_version=True, archive_path=archive, source_path=source, + mime_type="image/png", ) - assert _resolve_doc_path(document, None) == archive + + assert resolve_zone_source(document) == ZoneSourceFile( + path=archive, + is_image=False, + ) + + @pytest.mark.parametrize( + ("mime_type", "expected_is_image"), + [ + pytest.param("image/png", True, id="image"), + pytest.param("application/pdf", False, id="pdf"), + ], + ) + def test_uses_source_with_image_flag_from_mime_type( + self, + mocker: MockerFixture, + source: Path, + mime_type: str, + *, + expected_is_image: bool, + ) -> None: + """ + GIVEN: + - A document without an archive version + WHEN: + - The zone source is resolved + THEN: + - The source file is returned, flagged as an image only for image + mime types + """ + document = mocker.MagicMock( + has_archive_version=False, + source_path=source, + mime_type=mime_type, + ) + + assert resolve_zone_source(document) == ZoneSourceFile( + path=source, + is_image=expected_is_image, + ) + + def test_falls_back_to_source_when_archive_is_missing_on_disk( + self, + mocker: MockerFixture, + source: Path, + archive: Path, + ) -> None: + """ + GIVEN: + - A document flagged as having an archive whose file was removed + WHEN: + - The zone source is resolved + THEN: + - The source file is returned + """ archive.unlink() - assert _resolve_doc_path(document, None) == source + document = mocker.MagicMock( + has_archive_version=True, + archive_path=archive, + source_path=source, + mime_type="image/png", + ) - def test_returns_original_file_as_fallback( - self, - document: MagicMock, - tmp_path: Path, - ) -> None: - """ - GIVEN: - - A document whose own files are missing and an existing original file - WHEN: - - The document path is resolved with the original file - THEN: - - The original file path is returned - """ - original = tmp_path / "original.pdf" - original.touch() - assert _resolve_doc_path(document, original) == original - - def test_returns_none_for_missing_original_file( - self, - document: MagicMock, - tmp_path: Path, - ) -> None: - """ - GIVEN: - - A document whose own files are missing and an original file path - that does not exist - WHEN: - - The document path is resolved with the original file - THEN: - - None is returned - """ - assert _resolve_doc_path(document, tmp_path / "missing.pdf") is None + assert resolve_zone_source(document) == ZoneSourceFile( + path=source, + is_image=True, + ) @pytest.mark.django_db @@ -3681,11 +3718,10 @@ class TestRunZoneExtraction: ) @pytest.fixture - def source(self, paperless_dirs: PaperlessDirs) -> Path: - source = paperless_dirs.scratch_dir / "document.png" + def source(self, document: Document) -> Path: with Image.new("RGB", (200, 200), "white") as image: - image.save(source) - return source + image.save(document.source_path, format="PNG") + return document.source_path @pytest.fixture(autouse=True) def ocr_reads_text(self, mock_zone_ocr_run: MagicMock) -> None: @@ -3779,7 +3815,7 @@ class TestRunZoneExtraction: """ template_with_zone(template_name, **template_kwargs) - results = run_zone_extraction(document, source) + results = run_zone_extraction(document) assert results == [] mock_zone_ocr_run.assert_not_called() @@ -3805,7 +3841,7 @@ class TestRunZoneExtraction: match="test", ) - results = run_zone_extraction(document, source) + results = run_zone_extraction(document) assert results == [] mock_zone_ocr_run.assert_not_called() @@ -3841,14 +3877,14 @@ class TestRunZoneExtraction: template_with_zone("First", enabled=True) template_with_zone("Second", enabled=True) - results = run_zone_extraction(document, source) + results = run_zone_extraction(document) assert Counter((r.value, r.reason) for r in results) == Counter( [(None, ZoneOcrReason.TEMPLATE_FAILED), ("extracted", None)], ) assert {r.template for r in results} == {"First", "Second"} - def test_handles_none_original_file( + def test_reports_missing_file_when_document_has_no_file( self, mock_zone_ocr_run: MagicMock, template_with_zone: Callable[..., OcrTemplate], @@ -3856,8 +3892,7 @@ class TestRunZoneExtraction: ) -> None: """ GIVEN: - - A matching template, a document with no accessible file and no - original file + - A matching template and a document with no accessible file WHEN: - Zone extraction runs THEN: @@ -3866,7 +3901,7 @@ class TestRunZoneExtraction: template = template_with_zone("Matching", enabled=True) zone = template.zones.get() - results = run_zone_extraction(document, None) + results = run_zone_extraction(document) assert [(r.template, r.zone, r.reason) for r in results] == [ ("Matching", zone.name, ZoneOcrReason.MISSING_FILE), @@ -3901,7 +3936,7 @@ class TestRunZoneExtraction: for name in template_names: template_with_zone(name, enabled=True) - results = run_zone_extraction(document, source) + results = run_zone_extraction(document) assert sorted((r.template, r.reason) for r in results) == [ (name, None) for name in sorted(template_names) @@ -3950,7 +3985,7 @@ class TestRealZoneOcr: ) shutil.copy(simple_digital_pdf_file, document.source_path) - results = run_zone_extraction(document, None) + results = run_zone_extraction(document) assert [result.reason for result in results] == [None] document.refresh_from_db() @@ -3978,7 +4013,7 @@ class TestRealZoneOcr: document.source_path, ) - results = run_zone_extraction(document, None) + results = run_zone_extraction(document) assert [result.reason for result in results] == [None] document.refresh_from_db() diff --git a/src/documents/views.py b/src/documents/views.py index c73b22de45..cd40338fc0 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -2324,6 +2324,7 @@ class DocumentViewSet( from documents.zone_ocr import ZoneOcrSkipReason from documents.zone_ocr import get_templates_for_document + from documents.zone_ocr import resolve_zone_source resolution = resolve_effective_document(document, request) if resolution.error == VersionResolutionError.INVALID: @@ -2343,8 +2344,7 @@ class DocumentViewSet( }, ) - doc_path = file_document.archive_path or file_document.source_path - if not doc_path.is_file(): + if resolve_zone_source(file_document) is None: return Response( {"error": "Document file not found"}, status=status.HTTP_404_NOT_FOUND, @@ -5896,6 +5896,8 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem page: str, ) -> HttpResponse: """Render a page for the template editor's zone overlay.""" + from documents.zone_ocr import resolve_zone_source + document: Document = get_object_or_404( Document.objects.select_related("root_document__owner"), pk=doc_id, @@ -5913,12 +5915,12 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem f"Page {page_num} out of range (document has {file_doc.page_count} pages)", ) - archive_path = file_doc.archive_path - doc_path = archive_path or file_doc.source_path - if not doc_path.is_file(): + source = resolve_zone_source(file_doc) + if source is None: raise Http404("Document file not found") + doc_path = source.path - if archive_path is None and file_doc.mime_type.startswith("image/"): + if source.is_image: return HttpResponse( doc_path.read_bytes(), content_type=file_doc.mime_type, @@ -5961,6 +5963,7 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem def test_zone(self, request: Request) -> HttpResponse: """Extract an ad-hoc zone without changing the document.""" from documents.zone_ocr import extract_zone_preview + from documents.zone_ocr import resolve_zone_source serializer = OcrTemplateTestSerializer( data=request.data, @@ -5971,8 +5974,8 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem file_doc = self._resolve_file_document(document, request) - doc_path = file_doc.archive_path or file_doc.source_path - if not doc_path.is_file(): + source = resolve_zone_source(file_doc) + if source is None: return Response( {"error": "Document file not found"}, status=status.HTTP_404_NOT_FOUND, @@ -5981,13 +5984,12 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem zone = OcrTemplateZone(**serializer.validated_data["zone"]) result = extract_zone_preview( - doc_path, + source.path, zone, zone.zone_source_width or 0, zone.zone_source_height or 0, file_doc.page_count, - is_image=file_doc.archive_path is None - and file_doc.mime_type.startswith("image/"), + is_image=source.is_image, ) regex_match = None diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index ad24364477..42663b4327 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -109,6 +109,30 @@ class FieldUpdate: custom_field: CustomField | None = None +@dataclass(frozen=True, kw_only=True, slots=True) +class ZoneSourceFile: + path: Path + is_image: bool + + +def resolve_zone_source(document: Document) -> ZoneSourceFile | None: + """ + Pick the stored file zone OCR reads: the archive PDF when it exists on disk, + else the source file when it exists, else None. + """ + if document.has_archive_version: + archive_path = document.archive_path + if archive_path is not None and archive_path.is_file(): + return ZoneSourceFile(path=archive_path, is_image=False) + source_path = document.source_path + if source_path.is_file(): + return ZoneSourceFile( + path=source_path, + is_image=document.mime_type.startswith("image/"), + ) + return None + + class ZoneOcrReason(StrEnum): """Why a zone produced no value for its target.""" @@ -253,7 +277,6 @@ def _visible_templates( def run_zone_extraction( document: Document, - original_file: Path | None = None, templates: list[OcrTemplate] | None = None, ) -> list[ZoneOcrResult]: """ @@ -269,14 +292,13 @@ def run_zone_extraction( return [] label = f"document {document.pk}" - doc_path = _resolve_doc_path(document, original_file) - is_archive = doc_path is not None and doc_path == document.archive_path + source = resolve_zone_source(document) plans = plan_zone_extraction( - doc_path, + source.path if source else None, document.page_count, templates, label=label, - is_image=not is_archive and document.mime_type.startswith("image/"), + is_image=source.is_image if source else False, ) results = apply_zone_plans(get_root_document(document), plans, label=label) if zone_results_written(results): @@ -417,24 +439,6 @@ def zone_results_written(results: list[ZoneOcrResult]) -> bool: return any(result.reason is None for result in results) -def _resolve_doc_path( - document: Document, - original_file: Path | None, -) -> Path | None: - """Prefer the archive, then the source, then the given file.""" - candidates: list[Path | None] = [] - if document.has_archive_version: - candidates.append(document.archive_path) - candidates.append(document.source_path) - if original_file is not None: - candidates.append(original_file) - - for path in candidates: - if path is not None and Path(path).is_file(): - return Path(path) - return None - - def _resolve_page_idx(page_value: int | None, page_count: int | None) -> int: """Resolve a 1-based page or -1 (last page), defaulting to the first page.""" if page_value == -1: