Chore: render zone OCR pages through the shared pdftoppm helper

pdftoppm was called in three places with different options. Missing-binary handling is dropped since pdftoppm and tesseract are required at startup.
This commit is contained in:
stumpylog committed 2026-11-18 14:51:40 -08:00
1 parent c1bb327f1f
commit 409539d4d1
7 files changed
+217 -176

No files matched your search

+20 -5
View File
@@ -90,18 +90,31 @@ def rasterize_pdf_page_to_png(
out_path: Path,
*,
dpi: float,
page: int = 1,
timeout: float | None = None,
logging_group=None,
) -> None:
"""
Rasterizes the first page of a PDF to a PNG with pdftoppm.
Rasterizes one page of a PDF to a PNG with pdftoppm.
Args:
in_path: The PDF to render.
out_path: Where to write the PNG. Exactly this one file is written.
dpi: Resolution, which may be fractional.
page: The 1-based page number to render, the first page by default.
timeout: Seconds to allow pdftoppm to run, no limit by default.
logging_group: Group attached to the debug log record.
Raises:
ParseError: pdftoppm failed, timed out or could not render the page.
"""
args = [
"pdftoppm",
# Only the first page is needed, first and last page are both 1
# Only the requested page is needed, first and last page are the same
"-f",
"1",
str(page),
"-l",
"1",
str(page),
# Resolution in DPI, which may be fractional
"-r",
str(dpi),
@@ -121,9 +134,11 @@ def rasterize_pdf_page_to_png(
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
try:
run_subprocess(args, logger=logger)
run_subprocess(args, logger=logger, timeout=timeout)
except subprocess.CalledProcessError as e:
raise ParseError(f"pdftoppm failed at {args}") from e
except subprocess.TimeoutExpired as e:
raise ParseError(f"pdftoppm timed out after {timeout}s at {args}") from e
except Exception as e: # pragma: no cover
raise ParseError("Unknown error running pdftoppm") from e
+20 -46
View File
@@ -5,7 +5,6 @@ from __future__ import annotations
import functools
import subprocess
from io import BytesIO
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
@@ -21,6 +20,7 @@ from documents.models import Document
from documents.models import MatchingModel
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.parsers import ParseError
from documents.search import get_backend
from documents.signals import document_consumption_finished
from documents.tasks import extract_zones
@@ -34,6 +34,7 @@ from paperless_testing.permissions import grant_object
if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path
from unittest.mock import MagicMock
from django.contrib.auth.models import User
@@ -152,7 +153,7 @@ def mock_preview(mocker: MockerFixture) -> MagicMock:
@pytest.fixture
def mock_render(mocker: MockerFixture) -> MagicMock:
return mocker.patch("documents.views.subprocess.run")
return mocker.patch("documents.views.rasterize_pdf_page_to_png")
def detail_url(pk: int) -> str:
@@ -1851,7 +1852,7 @@ class TestDocumentPageImage:
WHEN:
- Its page image is requested
THEN:
- The original file is served and no render subprocess runs
- The original file is served and no page is rendered
"""
document = DocumentFactory(owner=preview_user, mime_type="image/png")
document.source_path.write_bytes(b"image contents")
@@ -1907,12 +1908,12 @@ class TestDocumentPageImage:
WHEN:
- The second page image is requested
THEN:
- pdftoppm is invoked for page 2 with the expected arguments
- The renderer is invoked for page 2 at 150 DPI with a 30 second timeout
- The rendered page is served and the temporary directory removed
"""
def render_page(command: list[str], **kwargs: Any) -> None:
Path(command[-1] + "-2.png").write_bytes(b"rendered page")
def render_page(in_path: Path, out_path: Path, **kwargs: Any) -> None:
out_path.write_bytes(b"rendered page")
mock_render.side_effect = render_page
@@ -1921,61 +1922,34 @@ class TestDocumentPageImage:
assert response.status_code == status.HTTP_200_OK
assert response["Content-Type"] == "image/png"
assert response.content == b"rendered page"
command = mock_render.call_args.args[0]
assert command[:9] == [
"pdftoppm",
"-png",
"-r",
"150",
"-f",
"2",
"-l",
"2",
str(preview_document.source_path),
]
assert mock_render.call_args.kwargs == {
"check": True,
"capture_output": True,
"timeout": 30,
}
assert not Path(command[-1]).parent.exists()
mock_render.assert_called_once()
in_path, out_path = mock_render.call_args.args
assert in_path == preview_document.source_path
assert mock_render.call_args.kwargs == {"dpi": 150, "page": 2, "timeout": 30}
assert not out_path.parent.exists()
@pytest.mark.parametrize(
"error",
[
pytest.param(
subprocess.CalledProcessError(
1,
["pdftoppm"],
stderr=b"sensitive server path",
),
id="called-process-error",
),
pytest.param(subprocess.TimeoutExpired(["pdftoppm"], 30), id="timeout"),
pytest.param(FileNotFoundError("pdftoppm"), id="missing-binary"),
],
)
def test_render_failures_are_controlled_and_do_not_expose_paths(
def test_render_failure_is_controlled_and_does_not_expose_paths(
self,
preview_client: APIClient,
preview_document: Document,
mock_render: MagicMock,
error: Exception,
) -> None:
"""
GIVEN:
- A PDF document and a render subprocess that fails
- A PDF document whose page render fails
WHEN:
- Its page image is requested
THEN:
- 404 is returned without leaking the subprocess error output
- 404 is returned without leaking the renderer's error message
"""
mock_render.side_effect = error
mock_render.side_effect = ParseError(
"pdftoppm failed at /sensitive/server/path",
)
response = preview_client.get(preview_url(preview_document))
assert response.status_code == status.HTTP_404_NOT_FOUND
assert "sensitive server path" not in str(response.content)
assert "sensitive" not in str(response.content)
@pytest.mark.usefixtures("mock_render")
def test_missing_render_output_returns_not_found(
@@ -1985,7 +1959,7 @@ class TestDocumentPageImage:
) -> None:
"""
GIVEN:
- A PDF document and a render subprocess that writes no output
- A PDF document whose page render writes no output
WHEN:
- Its page image is requested
THEN:
+95
View File
@@ -421,6 +421,101 @@ class TestRasterizePdfPageToPng:
with pytest.raises(ParseError):
rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72)
def test_renders_the_requested_page(self, tmp_path: Path) -> None:
"""
GIVEN:
- A two page PDF with a 144x72 first page and a 300x300 second page
WHEN:
- The second page is rasterized at 72 DPI
THEN:
- Only out_path is written, sized like the second page
"""
pdf_path = self._write_pdf(tmp_path / "in.pdf")
out_dir = tmp_path / "out"
out_dir.mkdir()
out_path = out_dir / "page2.png"
rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72, page=2, timeout=30)
assert list(out_dir.iterdir()) == [out_path]
with Image.open(out_path) as im:
assert im.size == (300, 300)
def test_page_out_of_range_raises_parse_error(self, tmp_path: Path) -> None:
"""
GIVEN:
- A two page PDF
WHEN:
- A page past the end is rasterized
THEN:
- A ParseError is raised and nothing is written
"""
pdf_path = self._write_pdf(tmp_path / "in.pdf")
out_path = tmp_path / "page3.png"
with pytest.raises(ParseError):
rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72, page=3)
assert not out_path.exists()
def test_arguments_and_timeout_are_passed_to_pdftoppm(
self,
tmp_path: Path,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- A subprocess runner that succeeds
WHEN:
- Page 4 is rasterized with a timeout
THEN:
- pdftoppm renders only that page and the timeout reaches the runner
"""
run = mocker.patch("documents.parsers.run_subprocess")
rasterize_pdf_page_to_png(
tmp_path / "in.pdf",
tmp_path / "out.png",
dpi=150,
page=4,
timeout=12,
)
args = run.call_args.args[0]
assert args[args.index("-f") + 1] == "4"
assert args[args.index("-l") + 1] == "4"
assert args[args.index("-r") + 1] == "150"
assert "-singlefile" in args
assert "-cropbox" in args
assert args[-1] == str(tmp_path / "out")
assert run.call_args.kwargs["timeout"] == 12
def test_timeout_raises_parse_error(
self,
tmp_path: Path,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- A subprocess runner that times out
WHEN:
- A page is rasterized
THEN:
- A ParseError is raised
"""
mocker.patch(
"documents.parsers.run_subprocess",
side_effect=subprocess.TimeoutExpired(["pdftoppm"], 5),
)
with pytest.raises(ParseError, match="timed out"):
rasterize_pdf_page_to_png(
tmp_path / "in.pdf",
tmp_path / "out.png",
dpi=72,
timeout=5,
)
class TestEncodeThumbnailWebp:
@pytest.mark.parametrize(
+50 -70
View File
@@ -20,6 +20,7 @@ from documents.models import Document
from documents.models import MatchingModel
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.parsers import ParseError
from documents.zone_ocr import _apply_transform
from documents.zone_ocr import _combine_field_value
from documents.zone_ocr import _convert_value
@@ -291,7 +292,6 @@ class TestZoneExtraction:
subprocess.CalledProcessError(1, "tesseract"),
id="called-process-error-without-stderr",
),
pytest.param(FileNotFoundError("tesseract"), id="reader-not-found"),
],
)
def test_reader_failures_return_no_value(
@@ -304,7 +304,7 @@ class TestZoneExtraction:
) -> None:
"""
GIVEN:
- A reader that fails with a timeout, process error or missing binary
- A reader that fails with a timeout or process error
WHEN:
- The zone is extracted
THEN:
@@ -700,6 +700,7 @@ class TestSavedZoneExtraction:
def test_document_run_renders_each_page_once_across_templates(
self,
mocker: MockerFixture,
mock_zone_ocr_run: MagicMock,
paperless_dirs: PaperlessDirs,
document: Document,
@@ -723,28 +724,24 @@ class TestSavedZoneExtraction:
pdf.write_bytes(b"%PDF-1.4")
rendered = []
def execute(args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]:
if args[0] == "pdftoppm":
output = Path(args[-1]).with_suffix(".png")
with Image.new("RGB", (100, 50), "white") as image:
image.save(output)
rendered.append(output)
return subprocess.CompletedProcess(args, 0)
return subprocess.CompletedProcess(args, 0, stdout="extracted")
def render(in_path: Path, out_path: Path, **kwargs: Any) -> None:
with Image.new("RGB", (100, 50), "white") as image:
image.save(out_path)
rendered.append(out_path)
mock_zone_ocr_run.side_effect = execute
mock_rasterize = mocker.patch(
"documents.zone_ocr.rasterize_pdf_page_to_png",
side_effect=render,
)
mock_zone_ocr_run.return_value = subprocess.CompletedProcess(
[],
0,
stdout="extracted",
)
results = run_zone_extraction(document, pdf, templates=[template, second])
render_calls = [
call
for call in mock_zone_ocr_run.call_args_list
if call.args[0][0] == "pdftoppm"
]
assert len(render_calls) == 2
assert [
call.args[0][call.args[0].index("-f") + 1] for call in render_calls
] == ["1", "2"]
assert [call.kwargs["page"] for call in mock_rasterize.call_args_list] == [1, 2]
assert [result.reason for result in results] == [None, None, None]
assert not any(path.exists() for path in rendered)
@@ -2822,11 +2819,21 @@ class TestRenderPages:
pdf.write_bytes(b"%PDF-1.4")
return pdf
@pytest.fixture
def mock_rasterize(self, mocker: MockerFixture) -> MagicMock:
def render(in_path: Path, out_path: Path, **kwargs: Any) -> None:
out_path.write_bytes(b"rendered")
return mocker.patch(
"documents.zone_ocr.rasterize_pdf_page_to_png",
side_effect=render,
)
def test_renders_requested_pages_in_order_and_skips_out_of_range(
self,
pdf: Path,
tmp_path: Path,
mock_zone_ocr_run: MagicMock,
mock_rasterize: MagicMock,
) -> None:
"""
GIVEN:
@@ -2834,30 +2841,22 @@ class TestRenderPages:
WHEN:
- The pages are rendered
THEN:
- Only the in range pages are rendered, in order, with the expected
renderer arguments
- Only the in range pages are rendered, in order, as 1-based pages
at 300 DPI with a 60 second timeout
"""
def render(args: list[str], **kwargs: Any) -> None:
Path(args[-1]).with_suffix(".png").write_bytes(b"rendered")
mock_zone_ocr_run.side_effect = render
pages = _render_pages(pdf, {-1, 0, 2, 3}, tmp_path, 3)
assert list(pages) == [0, 2]
assert mock_zone_ocr_run.call_count == 2
for page, call in zip([1, 3], mock_zone_ocr_run.call_args_list, strict=True):
args = call.args[0]
assert "-singlefile" in args
assert args[args.index("-f") + 1] == str(page)
assert args[args.index("-l") + 1] == str(page)
assert call.kwargs["timeout"] == 60
assert call.kwargs["check"]
@pytest.mark.usefixtures("mock_zone_ocr_run")
assert list(pages) == [0, 2]
assert mock_rasterize.call_count == 2
for page, call in zip([1, 3], mock_rasterize.call_args_list, strict=True):
assert call.args[0] == pdf
assert call.kwargs == {"dpi": 300, "page": page, "timeout": 60}
def test_missing_render_output_is_not_a_page(
self,
pdf: Path,
tmp_path: Path,
mock_rasterize: MagicMock,
) -> None:
"""
GIVEN:
@@ -2867,19 +2866,17 @@ class TestRenderPages:
THEN:
- No pages are returned
"""
mock_rasterize.side_effect = None
assert _render_pages(pdf, {0}, tmp_path, None) == {}
@pytest.mark.parametrize(
"error",
[
pytest.param(subprocess.TimeoutExpired("pdftoppm", 60), id="timeout"),
pytest.param(ParseError("pdftoppm timed out after 60s"), id="timeout"),
pytest.param(
subprocess.CalledProcessError(1, "pdftoppm", stderr=b"failure"),
id="called-process-error-with-stderr",
),
pytest.param(
subprocess.CalledProcessError(1, "pdftoppm"),
id="called-process-error-without-stderr",
ParseError("pdftoppm failed at ['pdftoppm']"),
id="render-failure",
),
],
)
@@ -2887,7 +2884,7 @@ class TestRenderPages:
self,
pdf: Path,
tmp_path: Path,
mock_zone_ocr_run: MagicMock,
mock_rasterize: MagicMock,
error: Exception,
) -> None:
"""
@@ -2899,36 +2896,19 @@ class TestRenderPages:
- The failing page is skipped and the other page is still rendered
"""
def render(args: list[str], **kwargs: Any) -> None:
if args[args.index("-f") + 1] == "1":
def render(in_path: Path, out_path: Path, *, page: int, **kwargs: Any) -> None:
if page == 1:
raise error
Path(args[-1]).with_suffix(".png").write_bytes(b"rendered")
out_path.write_bytes(b"rendered")
mock_rasterize.side_effect = render
mock_zone_ocr_run.side_effect = render
assert list(_render_pages(pdf, {0, 1}, tmp_path, 2)) == [1]
def test_missing_renderer_stops_without_retrying_every_page(
self,
pdf: Path,
tmp_path: Path,
mock_zone_ocr_run: MagicMock,
) -> None:
"""
GIVEN:
- A renderer binary that is not installed
WHEN:
- Two pages are rendered
THEN:
- No pages are returned and the renderer is only attempted once
"""
mock_zone_ocr_run.side_effect = FileNotFoundError("pdftoppm")
assert _render_pages(pdf, {0, 1}, tmp_path, 2) == {}
mock_zone_ocr_run.assert_called_once()
def test_image_is_used_directly_only_for_first_page(
self,
tmp_path: Path,
mock_zone_ocr_run: MagicMock,
mock_rasterize: MagicMock,
) -> None:
"""
GIVEN:
@@ -2944,7 +2924,7 @@ class TestRenderPages:
image.save(image_path)
assert _render_pages(image_path, {0}, tmp_path, 1) == {0: image_path}
assert _render_pages(image_path, {1}, tmp_path, 1) == {}
mock_zone_ocr_run.assert_not_called()
mock_rasterize.assert_not_called()
class TestDetectMime:
+11 -2
View File
@@ -164,14 +164,23 @@ def run_subprocess(
check_exit_code: bool = True,
log_stdout: bool = True,
log_stderr: bool = True,
timeout: float | None = None,
) -> CompletedProcess:
"""
Runs a subprocess and logs its output, checking return code if requested
Runs a subprocess and logs its output, checking return code if requested.
A timeout, in seconds, raises subprocess.TimeoutExpired once exceeded.
"""
proc_name = arguments[0]
completed_proc = run(args=arguments, env=env, capture_output=True, check=False)
completed_proc = run(
args=arguments,
env=env,
capture_output=True,
check=False,
timeout=timeout,
)
if logger:
logger.info(f"{proc_name} exited {completed_proc.returncode}")
+11 -24
View File
@@ -3,7 +3,6 @@ import logging
import os
import platform
import re
import subprocess
import tempfile
import zipfile
from collections import Counter
@@ -176,6 +175,8 @@ from documents.models import UiSettings
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.parsers import ParseError
from documents.parsers import rasterize_pdf_page_to_png
from documents.permissions import AcknowledgeTasksPermissions
from documents.permissions import PaperlessAdminPermissions
from documents.permissions import PaperlessNotePermissions
@@ -5871,26 +5872,16 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem
return HttpResponse(doc_path.read_bytes(), content_type=document.mime_type)
with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir:
output_prefix = Path(tmp_dir) / "page"
rendered = Path(tmp_dir) / "page.png"
try:
subprocess.run(
[
"pdftoppm",
"-png",
"-r",
"150",
"-f",
str(page_num + 1),
"-l",
str(page_num + 1),
str(doc_path),
str(output_prefix),
],
check=True,
capture_output=True,
rasterize_pdf_page_to_png(
doc_path,
rendered,
dpi=150,
page=page_num + 1,
timeout=30,
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
except ParseError as error:
logger.warning(
"Failed to render page %s of document %s: %s",
page_num,
@@ -5898,15 +5889,11 @@ class OcrTemplateViewSet(BulkPermissionMixin, PassUserMixin, ModelViewSet[OcrTem
error,
)
raise Http404("Failed to render page") from error
except FileNotFoundError as error:
logger.warning("pdftoppm is not available")
raise Http404("Failed to render page") from error
rendered = sorted(Path(tmp_dir).glob("page-*.png"))
if not rendered:
if not rendered.is_file():
raise Http404("No rendered page found")
content = rendered[0].read_bytes()
content = rendered.read_bytes()
return HttpResponse(content, content_type="image/png")
+10 -29
View File
@@ -37,6 +37,8 @@ from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.parsers import ParseError
from documents.parsers import rasterize_pdf_page_to_png
from documents.permissions import permitted_object_ids
from documents.regex import safe_regex_fullmatch
from documents.signals.handlers import add_to_index
@@ -400,41 +402,23 @@ def _render_pages(
if actual_page < 0 or (page_count and actual_page >= page_count):
continue
output_prefix = tmp_dir / f"page_{actual_page}"
rendered = tmp_dir / f"page_{actual_page}.png"
try:
subprocess.run(
[
"pdftoppm",
"-png",
"-singlefile",
"-r",
"300",
"-f",
str(actual_page + 1), # pdftoppm is 1-indexed
"-l",
str(actual_page + 1),
str(doc_path),
str(output_prefix),
],
check=True,
capture_output=True,
rasterize_pdf_page_to_png(
doc_path,
rendered,
dpi=300,
page=actual_page + 1, # pdftoppm is 1-indexed
timeout=60,
)
except subprocess.TimeoutExpired:
logger.error("Zone OCR: pdftoppm timed out for page %d", actual_page)
continue
except subprocess.CalledProcessError as e:
except ParseError as e:
logger.exception(
"Zone OCR: pdftoppm failed for page %d: %s",
actual_page,
e.stderr.decode(errors="replace") if e.stderr else str(e),
e,
)
continue
except FileNotFoundError:
logger.error("Zone OCR: pdftoppm not found")
return result
rendered = output_prefix.with_suffix(".png")
if rendered.is_file():
result[actual_page] = rendered
@@ -520,9 +504,6 @@ def _ocr_text(cropped: Image.Image, zone: OcrTemplateZone, tmp_dir: Path) -> str
e.stderr[:200] if e.stderr else str(e),
)
return None
except FileNotFoundError:
logger.error("Zone OCR: Tesseract not found")
return None
def _extract_zone(