diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 48983bb232..4d744c2d10 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -90,18 +90,31 @@ def rasterize_pdf_page_to_png( out_path: Path, *, dpi: float, + page: int = 1, + timeout: float | None = None, logging_group=None, ) -> None: """ - Rasterizes the first page of a PDF to a PNG with pdftoppm. + Rasterizes one page of a PDF to a PNG with pdftoppm. + + Args: + in_path: The PDF to render. + out_path: Where to write the PNG. Exactly this one file is written. + dpi: Resolution, which may be fractional. + page: The 1-based page number to render, the first page by default. + timeout: Seconds to allow pdftoppm to run, no limit by default. + logging_group: Group attached to the debug log record. + + Raises: + ParseError: pdftoppm failed, timed out or could not render the page. """ args = [ "pdftoppm", - # Only the first page is needed, first and last page are both 1 + # Only the requested page is needed, first and last page are the same "-f", - "1", + str(page), "-l", - "1", + str(page), # Resolution in DPI, which may be fractional "-r", str(dpi), @@ -121,9 +134,11 @@ def rasterize_pdf_page_to_png( logger.debug("Execute: " + " ".join(args), extra={"group": logging_group}) try: - run_subprocess(args, logger=logger) + run_subprocess(args, logger=logger, timeout=timeout) except subprocess.CalledProcessError as e: raise ParseError(f"pdftoppm failed at {args}") from e + except subprocess.TimeoutExpired as e: + raise ParseError(f"pdftoppm timed out after {timeout}s at {args}") from e except Exception as e: # pragma: no cover raise ParseError("Unknown error running pdftoppm") from e diff --git a/src/documents/tests/test_api_ocr_templates.py b/src/documents/tests/test_api_ocr_templates.py index 0ca620a96f..e10d0b585b 100644 --- a/src/documents/tests/test_api_ocr_templates.py +++ b/src/documents/tests/test_api_ocr_templates.py @@ -5,7 +5,6 @@ from __future__ import annotations import functools import subprocess from io import BytesIO -from pathlib import Path from typing import TYPE_CHECKING from typing import Any @@ -21,6 +20,7 @@ from documents.models import Document from documents.models import MatchingModel from documents.models import OcrTemplate from documents.models import OcrTemplateZone +from documents.parsers import ParseError from documents.search import get_backend from documents.signals import document_consumption_finished from documents.tasks import extract_zones @@ -34,6 +34,7 @@ from paperless_testing.permissions import grant_object if TYPE_CHECKING: from collections.abc import Callable + from pathlib import Path from unittest.mock import MagicMock from django.contrib.auth.models import User @@ -152,7 +153,7 @@ def mock_preview(mocker: MockerFixture) -> MagicMock: @pytest.fixture def mock_render(mocker: MockerFixture) -> MagicMock: - return mocker.patch("documents.views.subprocess.run") + return mocker.patch("documents.views.rasterize_pdf_page_to_png") def detail_url(pk: int) -> str: @@ -1851,7 +1852,7 @@ class TestDocumentPageImage: WHEN: - Its page image is requested THEN: - - The original file is served and no render subprocess runs + - The original file is served and no page is rendered """ document = DocumentFactory(owner=preview_user, mime_type="image/png") document.source_path.write_bytes(b"image contents") @@ -1907,12 +1908,12 @@ class TestDocumentPageImage: WHEN: - The second page image is requested THEN: - - pdftoppm is invoked for page 2 with the expected arguments + - The renderer is invoked for page 2 at 150 DPI with a 30 second timeout - The rendered page is served and the temporary directory removed """ - def render_page(command: list[str], **kwargs: Any) -> None: - Path(command[-1] + "-2.png").write_bytes(b"rendered page") + def render_page(in_path: Path, out_path: Path, **kwargs: Any) -> None: + out_path.write_bytes(b"rendered page") mock_render.side_effect = render_page @@ -1921,61 +1922,34 @@ class TestDocumentPageImage: assert response.status_code == status.HTTP_200_OK assert response["Content-Type"] == "image/png" assert response.content == b"rendered page" - command = mock_render.call_args.args[0] - assert command[:9] == [ - "pdftoppm", - "-png", - "-r", - "150", - "-f", - "2", - "-l", - "2", - str(preview_document.source_path), - ] - assert mock_render.call_args.kwargs == { - "check": True, - "capture_output": True, - "timeout": 30, - } - assert not Path(command[-1]).parent.exists() + mock_render.assert_called_once() + in_path, out_path = mock_render.call_args.args + assert in_path == preview_document.source_path + assert mock_render.call_args.kwargs == {"dpi": 150, "page": 2, "timeout": 30} + assert not out_path.parent.exists() - @pytest.mark.parametrize( - "error", - [ - pytest.param( - subprocess.CalledProcessError( - 1, - ["pdftoppm"], - stderr=b"sensitive server path", - ), - id="called-process-error", - ), - pytest.param(subprocess.TimeoutExpired(["pdftoppm"], 30), id="timeout"), - pytest.param(FileNotFoundError("pdftoppm"), id="missing-binary"), - ], - ) - def test_render_failures_are_controlled_and_do_not_expose_paths( + def test_render_failure_is_controlled_and_does_not_expose_paths( self, preview_client: APIClient, preview_document: Document, mock_render: MagicMock, - error: Exception, ) -> None: """ GIVEN: - - A PDF document and a render subprocess that fails + - A PDF document whose page render fails WHEN: - Its page image is requested THEN: - - 404 is returned without leaking the subprocess error output + - 404 is returned without leaking the renderer's error message """ - mock_render.side_effect = error + mock_render.side_effect = ParseError( + "pdftoppm failed at /sensitive/server/path", + ) response = preview_client.get(preview_url(preview_document)) assert response.status_code == status.HTTP_404_NOT_FOUND - assert "sensitive server path" not in str(response.content) + assert "sensitive" not in str(response.content) @pytest.mark.usefixtures("mock_render") def test_missing_render_output_returns_not_found( @@ -1985,7 +1959,7 @@ class TestDocumentPageImage: ) -> None: """ GIVEN: - - A PDF document and a render subprocess that writes no output + - A PDF document whose page render writes no output WHEN: - Its page image is requested THEN: diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py index cf27994c10..bfb7eb4716 100644 --- a/src/documents/tests/test_parsers.py +++ b/src/documents/tests/test_parsers.py @@ -421,6 +421,101 @@ class TestRasterizePdfPageToPng: with pytest.raises(ParseError): rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72) + def test_renders_the_requested_page(self, tmp_path: Path) -> None: + """ + GIVEN: + - A two page PDF with a 144x72 first page and a 300x300 second page + WHEN: + - The second page is rasterized at 72 DPI + THEN: + - Only out_path is written, sized like the second page + """ + pdf_path = self._write_pdf(tmp_path / "in.pdf") + out_dir = tmp_path / "out" + out_dir.mkdir() + out_path = out_dir / "page2.png" + + rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72, page=2, timeout=30) + + assert list(out_dir.iterdir()) == [out_path] + with Image.open(out_path) as im: + assert im.size == (300, 300) + + def test_page_out_of_range_raises_parse_error(self, tmp_path: Path) -> None: + """ + GIVEN: + - A two page PDF + WHEN: + - A page past the end is rasterized + THEN: + - A ParseError is raised and nothing is written + """ + pdf_path = self._write_pdf(tmp_path / "in.pdf") + out_path = tmp_path / "page3.png" + + with pytest.raises(ParseError): + rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72, page=3) + + assert not out_path.exists() + + def test_arguments_and_timeout_are_passed_to_pdftoppm( + self, + tmp_path: Path, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - A subprocess runner that succeeds + WHEN: + - Page 4 is rasterized with a timeout + THEN: + - pdftoppm renders only that page and the timeout reaches the runner + """ + run = mocker.patch("documents.parsers.run_subprocess") + + rasterize_pdf_page_to_png( + tmp_path / "in.pdf", + tmp_path / "out.png", + dpi=150, + page=4, + timeout=12, + ) + + args = run.call_args.args[0] + assert args[args.index("-f") + 1] == "4" + assert args[args.index("-l") + 1] == "4" + assert args[args.index("-r") + 1] == "150" + assert "-singlefile" in args + assert "-cropbox" in args + assert args[-1] == str(tmp_path / "out") + assert run.call_args.kwargs["timeout"] == 12 + + def test_timeout_raises_parse_error( + self, + tmp_path: Path, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - A subprocess runner that times out + WHEN: + - A page is rasterized + THEN: + - A ParseError is raised + """ + mocker.patch( + "documents.parsers.run_subprocess", + side_effect=subprocess.TimeoutExpired(["pdftoppm"], 5), + ) + + with pytest.raises(ParseError, match="timed out"): + rasterize_pdf_page_to_png( + tmp_path / "in.pdf", + tmp_path / "out.png", + dpi=72, + timeout=5, + ) + class TestEncodeThumbnailWebp: @pytest.mark.parametrize( diff --git a/src/documents/tests/test_zone_ocr.py b/src/documents/tests/test_zone_ocr.py index 92090ec73d..912d4a5437 100644 --- a/src/documents/tests/test_zone_ocr.py +++ b/src/documents/tests/test_zone_ocr.py @@ -20,6 +20,7 @@ from documents.models import Document from documents.models import MatchingModel from documents.models import OcrTemplate from documents.models import OcrTemplateZone +from documents.parsers import ParseError from documents.zone_ocr import _apply_transform from documents.zone_ocr import _combine_field_value from documents.zone_ocr import _convert_value @@ -291,7 +292,6 @@ class TestZoneExtraction: subprocess.CalledProcessError(1, "tesseract"), id="called-process-error-without-stderr", ), - pytest.param(FileNotFoundError("tesseract"), id="reader-not-found"), ], ) def test_reader_failures_return_no_value( @@ -304,7 +304,7 @@ class TestZoneExtraction: ) -> None: """ GIVEN: - - A reader that fails with a timeout, process error or missing binary + - A reader that fails with a timeout or process error WHEN: - The zone is extracted THEN: @@ -700,6 +700,7 @@ class TestSavedZoneExtraction: def test_document_run_renders_each_page_once_across_templates( self, + mocker: MockerFixture, mock_zone_ocr_run: MagicMock, paperless_dirs: PaperlessDirs, document: Document, @@ -723,28 +724,24 @@ class TestSavedZoneExtraction: pdf.write_bytes(b"%PDF-1.4") rendered = [] - def execute(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - if args[0] == "pdftoppm": - output = Path(args[-1]).with_suffix(".png") - with Image.new("RGB", (100, 50), "white") as image: - image.save(output) - rendered.append(output) - return subprocess.CompletedProcess(args, 0) - return subprocess.CompletedProcess(args, 0, stdout="extracted") + def render(in_path: Path, out_path: Path, **kwargs: Any) -> None: + with Image.new("RGB", (100, 50), "white") as image: + image.save(out_path) + rendered.append(out_path) - mock_zone_ocr_run.side_effect = execute + mock_rasterize = mocker.patch( + "documents.zone_ocr.rasterize_pdf_page_to_png", + side_effect=render, + ) + mock_zone_ocr_run.return_value = subprocess.CompletedProcess( + [], + 0, + stdout="extracted", + ) results = run_zone_extraction(document, pdf, templates=[template, second]) - render_calls = [ - call - for call in mock_zone_ocr_run.call_args_list - if call.args[0][0] == "pdftoppm" - ] - assert len(render_calls) == 2 - assert [ - call.args[0][call.args[0].index("-f") + 1] for call in render_calls - ] == ["1", "2"] + assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2] assert [result.reason for result in results] == [None, None, None] assert not any(path.exists() for path in rendered) @@ -2822,11 +2819,21 @@ class TestRenderPages: pdf.write_bytes(b"%PDF-1.4") return pdf + @pytest.fixture + def mock_rasterize(self, mocker: MockerFixture) -> MagicMock: + def render(in_path: Path, out_path: Path, **kwargs: Any) -> None: + out_path.write_bytes(b"rendered") + + return mocker.patch( + "documents.zone_ocr.rasterize_pdf_page_to_png", + side_effect=render, + ) + def test_renders_requested_pages_in_order_and_skips_out_of_range( self, pdf: Path, tmp_path: Path, - mock_zone_ocr_run: MagicMock, + mock_rasterize: MagicMock, ) -> None: """ GIVEN: @@ -2834,30 +2841,22 @@ class TestRenderPages: WHEN: - The pages are rendered THEN: - - Only the in range pages are rendered, in order, with the expected - renderer arguments + - Only the in range pages are rendered, in order, as 1-based pages + at 300 DPI with a 60 second timeout """ - - def render(args: list[str], **kwargs: Any) -> None: - Path(args[-1]).with_suffix(".png").write_bytes(b"rendered") - - mock_zone_ocr_run.side_effect = render pages = _render_pages(pdf, {-1, 0, 2, 3}, tmp_path, 3) - assert list(pages) == [0, 2] - assert mock_zone_ocr_run.call_count == 2 - for page, call in zip([1, 3], mock_zone_ocr_run.call_args_list, strict=True): - args = call.args[0] - assert "-singlefile" in args - assert args[args.index("-f") + 1] == str(page) - assert args[args.index("-l") + 1] == str(page) - assert call.kwargs["timeout"] == 60 - assert call.kwargs["check"] - @pytest.mark.usefixtures("mock_zone_ocr_run") + assert list(pages) == [0, 2] + assert mock_rasterize.call_count == 2 + for page, call in zip([1, 3], mock_rasterize.call_args_list, strict=True): + assert call.args[0] == pdf + assert call.kwargs == {"dpi": 300, "page": page, "timeout": 60} + def test_missing_render_output_is_not_a_page( self, pdf: Path, tmp_path: Path, + mock_rasterize: MagicMock, ) -> None: """ GIVEN: @@ -2867,19 +2866,17 @@ class TestRenderPages: THEN: - No pages are returned """ + mock_rasterize.side_effect = None + assert _render_pages(pdf, {0}, tmp_path, None) == {} @pytest.mark.parametrize( "error", [ - pytest.param(subprocess.TimeoutExpired("pdftoppm", 60), id="timeout"), + pytest.param(ParseError("pdftoppm timed out after 60s"), id="timeout"), pytest.param( - subprocess.CalledProcessError(1, "pdftoppm", stderr=b"failure"), - id="called-process-error-with-stderr", - ), - pytest.param( - subprocess.CalledProcessError(1, "pdftoppm"), - id="called-process-error-without-stderr", + ParseError("pdftoppm failed at ['pdftoppm']"), + id="render-failure", ), ], ) @@ -2887,7 +2884,7 @@ class TestRenderPages: self, pdf: Path, tmp_path: Path, - mock_zone_ocr_run: MagicMock, + mock_rasterize: MagicMock, error: Exception, ) -> None: """ @@ -2899,36 +2896,19 @@ class TestRenderPages: - The failing page is skipped and the other page is still rendered """ - def render(args: list[str], **kwargs: Any) -> None: - if args[args.index("-f") + 1] == "1": + def render(in_path: Path, out_path: Path, *, page: int, **kwargs: Any) -> None: + if page == 1: raise error - Path(args[-1]).with_suffix(".png").write_bytes(b"rendered") + out_path.write_bytes(b"rendered") + + mock_rasterize.side_effect = render - mock_zone_ocr_run.side_effect = render assert list(_render_pages(pdf, {0, 1}, tmp_path, 2)) == [1] - def test_missing_renderer_stops_without_retrying_every_page( - self, - pdf: Path, - tmp_path: Path, - mock_zone_ocr_run: MagicMock, - ) -> None: - """ - GIVEN: - - A renderer binary that is not installed - WHEN: - - Two pages are rendered - THEN: - - No pages are returned and the renderer is only attempted once - """ - mock_zone_ocr_run.side_effect = FileNotFoundError("pdftoppm") - assert _render_pages(pdf, {0, 1}, tmp_path, 2) == {} - mock_zone_ocr_run.assert_called_once() - def test_image_is_used_directly_only_for_first_page( self, tmp_path: Path, - mock_zone_ocr_run: MagicMock, + mock_rasterize: MagicMock, ) -> None: """ GIVEN: @@ -2944,7 +2924,7 @@ class TestRenderPages: image.save(image_path) assert _render_pages(image_path, {0}, tmp_path, 1) == {0: image_path} assert _render_pages(image_path, {1}, tmp_path, 1) == {} - mock_zone_ocr_run.assert_not_called() + mock_rasterize.assert_not_called() class TestDetectMime: diff --git a/src/documents/utils.py b/src/documents/utils.py index 4d0b5811c9..8aca566b96 100644 --- a/src/documents/utils.py +++ b/src/documents/utils.py @@ -164,14 +164,23 @@ def run_subprocess( check_exit_code: bool = True, log_stdout: bool = True, log_stderr: bool = True, + timeout: float | None = None, ) -> CompletedProcess: """ - Runs a subprocess and logs its output, checking return code if requested + Runs a subprocess and logs its output, checking return code if requested. + + A timeout, in seconds, raises subprocess.TimeoutExpired once exceeded. """ proc_name = arguments[0] - completed_proc = run(args=arguments, env=env, capture_output=True, check=False) + completed_proc = run( + args=arguments, + env=env, + capture_output=True, + check=False, + timeout=timeout, + ) if logger: logger.info(f"{proc_name} exited {completed_proc.returncode}") diff --git a/src/documents/views.py b/src/documents/views.py index 34c04cb457..56bbc61df6 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -3,7 +3,6 @@ import logging import os import platform import re -import subprocess import tempfile import zipfile from collections import Counter @@ -176,6 +175,8 @@ from documents.models import UiSettings from documents.models import Workflow from documents.models import WorkflowAction from documents.models import WorkflowTrigger +from documents.parsers import ParseError +from documents.parsers import rasterize_pdf_page_to_png from documents.permissions import AcknowledgeTasksPermissions from documents.permissions import PaperlessAdminPermissions from documents.permissions import PaperlessNotePermissions @@ -5871,26 +5872,16 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem return HttpResponse(doc_path.read_bytes(), content_type=document.mime_type) with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir: - output_prefix = Path(tmp_dir) / "page" + rendered = Path(tmp_dir) / "page.png" try: - subprocess.run( - [ - "pdftoppm", - "-png", - "-r", - "150", - "-f", - str(page_num + 1), - "-l", - str(page_num + 1), - str(doc_path), - str(output_prefix), - ], - check=True, - capture_output=True, + rasterize_pdf_page_to_png( + doc_path, + rendered, + dpi=150, + page=page_num + 1, timeout=30, ) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + except ParseError as error: logger.warning( "Failed to render page %s of document %s: %s", page_num, @@ -5898,15 +5889,11 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem error, ) raise Http404("Failed to render page") from error - except FileNotFoundError as error: - logger.warning("pdftoppm is not available") - raise Http404("Failed to render page") from error - rendered = sorted(Path(tmp_dir).glob("page-*.png")) - if not rendered: + if not rendered.is_file(): raise Http404("No rendered page found") - content = rendered[0].read_bytes() + content = rendered.read_bytes() return HttpResponse(content, content_type="image/png") diff --git a/src/documents/zone_ocr.py b/src/documents/zone_ocr.py index 214abcac2c..300a35538e 100644 --- a/src/documents/zone_ocr.py +++ b/src/documents/zone_ocr.py @@ -37,6 +37,8 @@ from documents.models import CustomFieldInstance from documents.models import Document from documents.models import OcrTemplate from documents.models import OcrTemplateZone +from documents.parsers import ParseError +from documents.parsers import rasterize_pdf_page_to_png from documents.permissions import permitted_object_ids from documents.regex import safe_regex_fullmatch from documents.signals.handlers import add_to_index @@ -400,41 +402,23 @@ def _render_pages( if actual_page < 0 or (page_count and actual_page >= page_count): continue - output_prefix = tmp_dir / f"page_{actual_page}" + rendered = tmp_dir / f"page_{actual_page}.png" try: - subprocess.run( - [ - "pdftoppm", - "-png", - "-singlefile", - "-r", - "300", - "-f", - str(actual_page + 1), # pdftoppm is 1-indexed - "-l", - str(actual_page + 1), - str(doc_path), - str(output_prefix), - ], - check=True, - capture_output=True, + rasterize_pdf_page_to_png( + doc_path, + rendered, + dpi=300, + page=actual_page + 1, # pdftoppm is 1-indexed timeout=60, ) - except subprocess.TimeoutExpired: - logger.error("Zone OCR: pdftoppm timed out for page %d", actual_page) - continue - except subprocess.CalledProcessError as e: + except ParseError as e: logger.exception( "Zone OCR: pdftoppm failed for page %d: %s", actual_page, - e.stderr.decode(errors="replace") if e.stderr else str(e), + e, ) continue - except FileNotFoundError: - logger.error("Zone OCR: pdftoppm not found") - return result - rendered = output_prefix.with_suffix(".png") if rendered.is_file(): result[actual_page] = rendered @@ -520,9 +504,6 @@ def _ocr_text(cropped: Image.Image, zone: OcrTemplateZone, tmp_dir: Path) -> str e.stderr[:200] if e.stderr else str(e), ) return None - except FileNotFoundError: - logger.error("Zone OCR: Tesseract not found") - return None def _extract_zone(