diff --git a/docs/usage.md b/docs/usage.md index 226a189025..988ae91392 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -850,7 +850,7 @@ layout. ### Zone settings - **Page**: the page the zone is on. Turn on **Last page** for zones that are always on the final page, - regardless of how many pages a document has. + regardless of how many pages a document has. If the number of pages cannot be determined, the zone is skipped. - **OCR language**: the Tesseract languages used to read the zone. Languages other than the defaults must be installed, see [`PAPERLESS_OCR_LANGUAGES`](configuration.md#PAPERLESS_OCR_LANGUAGES). - **Transform**: post-processing applied to the text, such as stripping whitespace, keeping only numbers, diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index 0a198c37dd..6791a6a761 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -44,6 +44,7 @@ from documents.zone_ocr import _parse_number from documents.zone_ocr import _prepare_field_update from documents.zone_ocr import _process_template from documents.zone_ocr import _render_pages +from documents.zone_ocr import _resolve_page_idx from documents.zone_ocr import _zone_language from documents.zone_ocr import extract_zone_preview from documents.zone_ocr import get_templates_for_content @@ -3704,6 +3705,38 @@ class TestResolveZoneSource: ) +class TestResolvePageIdx: + """Tests for _resolve_page_idx.""" + + @pytest.mark.parametrize( + ("page_value", "page_count", "expected"), + [ + pytest.param(None, 4, 0, id="blank-first-page"), + pytest.param(1, 4, 0, id="page-1"), + pytest.param(3, 4, 2, id="page-3"), + pytest.param(-1, 4, 3, id="last-page-with-count"), + pytest.param(-1, None, None, id="last-page-count-unknown"), + pytest.param(-1, 0, None, id="last-page-count-zero"), + ], + ) + def test_resolves_page_to_index( + self, + page_value: int | None, + page_count: int | None, + expected: int | None, + ) -> None: + """ + GIVEN: + - A zone page value and a document page count + WHEN: + - The page is resolved to a 0-based index + THEN: + - Pages are 1-based, blank is the first page, and the last page + without a known page count cannot be located + """ + assert _resolve_page_idx(page_value, page_count) == expected + + @pytest.mark.django_db @pytest.mark.usefixtures("paperless_dirs") class TestRunZoneExtraction: @@ -3884,6 +3917,63 @@ class TestRunZoneExtraction: ) assert {r.template for r in results} == {"First", "Second"} + def test_last_page_zone_without_page_count_is_missing_page( + self, + mock_zone_ocr_run: MagicMock, + document: Document, + source: Path, + ) -> None: + """ + GIVEN: + - A matching template with a last-page zone and a PDF document whose + page count is unknown and cannot be read from the file + WHEN: + - Zone extraction runs + THEN: + - The zone is reported as missing_page, no text is read and the + document is not changed + """ + document.page_count = None + document.mime_type = "application/pdf" + document.save() + document.source_path.write_bytes(b"not a pdf") + template = OcrTemplateFactory(name="Matching", match="test") + zone = make_zone(template, "Last", page=-1) + + results = run_zone_extraction(document, templates=[template]) + + assert [(r.template, r.zone, r.reason) for r in results] == [ + ("Matching", zone.name, ZoneOcrReason.MISSING_PAGE), + ] + mock_zone_ocr_run.assert_not_called() + document.refresh_from_db() + assert document.title == "Test" + + def test_last_page_zone_on_an_image_without_page_count_reads_the_image( + self, + document: Document, + source: Path, + ) -> None: + """ + GIVEN: + - A matching template with a last-page title zone and an image + document whose page count is unknown + WHEN: + - Zone extraction runs + THEN: + - The image is read as its only page and the title is written + """ + document.page_count = None + document.save() + template = OcrTemplateFactory(name="Matching", match="test") + make_zone(template, "Last", page=-1) + + results = run_zone_extraction(document, templates=[template]) + + assert [r.reason for r in results] == [None] + document.refresh_from_db() + assert document.title == "extracted" + def test_reports_missing_file_when_document_has_no_file( self, mock_zone_ocr_run: MagicMock, diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index cf56293de6..ce67087879 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -47,6 +47,7 @@ from documents.permissions import permitted_object_ids from documents.regex import safe_regex_fullmatch from documents.validators import monetary_validator from documents.versioning import get_root_document +from paperless.parsers.utils import get_page_count_for_pdf from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES if TYPE_CHECKING: @@ -329,13 +330,15 @@ def plan_zone_extraction( for template, zones in template_zones ] + if not any(zones for _, zones in template_zones): + return [] + page_count = _effective_page_count(doc_path, page_count, is_image=is_image) pages = { - _resolve_page_idx(zone.page, page_count) + page_idx for _, zones in template_zones for zone in zones + if (page_idx := _resolve_page_idx(zone.page, page_count)) is not None } - if not pages: - return [] plans = [] settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: @@ -438,13 +441,30 @@ def zone_results_written(results: list[ZoneOcrResult]) -> bool: return any(result.reason is None for result in results) -def _resolve_page_idx(page_value: int | None, page_count: int | None) -> int: +def _effective_page_count( + doc_path: Path, + page_count: int | None, + *, + is_image: bool, +) -> int | None: + """The stored page count, else 1 for an image, else the pages in the PDF.""" + if page_count: + return page_count + if is_image: + return 1 + return get_page_count_for_pdf(doc_path) + + +def _resolve_page_idx(page_value: int | None, page_count: int | None) -> int | None: """ Resolve a zone's 1-based page or -1 (last page) to a 0-based index, defaulting - to the first page. Without a page count, the last page resolves to the first. + to the first page. The last page without a known page count resolves to None, + meaning the zone cannot be located. """ if page_value == -1: - return max((page_count or 1) - 1, 0) + if not page_count: + return None + return page_count - 1 return max((page_value or 1) - 1, 0) @@ -480,7 +500,7 @@ def _plan_template( page_idx = _resolve_page_idx(zone.page, page_count) extracted = None reason: ZoneOcrReason | None = ZoneOcrReason.MISSING_PAGE - if page_idx in page_images: + if page_idx is not None and page_idx in page_images: extraction = _extract_zone( page_images[page_idx], zone, @@ -852,18 +872,19 @@ def extract_zone_preview( is_image: bool, ) -> ZoneExtraction: """Extract a zone without writing its value to the document.""" + page_count = _effective_page_count(doc_path, page_count, is_image=is_image) page_idx = _resolve_page_idx(zone.page, page_count) settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: tmp_path = Path(tmp_dir) page_images = _render_pages( doc_path, - {page_idx}, + set() if page_idx is None else {page_idx}, tmp_path, page_count, is_image=is_image, ) - if page_idx not in page_images: + if page_idx is None or page_idx not in page_images: return ZoneExtraction() return _extract_zone(