diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 1e0ba23b1..331168ba2 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -111,7 +111,7 @@ jobs: timeout-minutes: 12 uses: $/.github/actions/apt-install with: - packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils + packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils qpdf - name: Configure ImageMagick run: | sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml diff --git a/docs/configuration.md b/docs/configuration.md index 3257a815c..80aecaa60 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1315,29 +1315,17 @@ valid crontab(5) expression describing when to run. #### [`PAPERLESS_CONVERT_MEMORY_LIMIT=`](#PAPERLESS_CONVERT_MEMORY_LIMIT) {#PAPERLESS_CONVERT_MEMORY_LIMIT} -: On smaller systems, or even in the case of Very Large Documents, the -consumer may explode, complaining about how it's "unable to extend -pixel cache". In such cases, try setting this to a reasonably low -value, like 32. The default is to use whatever is necessary to do -everything without writing to disk, and units are in megabytes. +!!! warning - For more information on how to use this value, you should search the - web for "MAGICK_MEMORY_LIMIT". - - Defaults to 0, which disables the limit. + Deprecated and has no effect, since PDF thumbnails no longer use + ImageMagick. It will be removed in a future release. #### [`PAPERLESS_CONVERT_TMPDIR=`](#PAPERLESS_CONVERT_TMPDIR) {#PAPERLESS_CONVERT_TMPDIR} -: Similar to the memory limit, if you've got a small system and your -OS mounts /tmp as tmpfs, you should set this to a path that's on a -physical disk, like /home/your_user/tmp or something. ImageMagick -will use this as scratch space when crunching through very large -documents. +!!! warning - For more information on how to use this value, you should search the - web for "MAGICK_TMPDIR". - - Default is none, which disables the temporary directory. + Deprecated and has no effect, since PDF thumbnails no longer use + ImageMagick. It will be removed in a future release. #### [`PAPERLESS_APPS=`](#PAPERLESS_APPS) {#PAPERLESS_APPS} diff --git a/docs/setup.md b/docs/setup.md index e666cb58d..514cf424c 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -177,12 +177,12 @@ to a positive number to enable polling and disable native filesystem notificatio - `pkg-config` for mysqlclient (python dependency) - `fonts-liberation` for generating thumbnails for plain text files - - `imagemagick` >= 6 for PDF conversion + - `imagemagick` >= 6 for image alpha handling - `gnupg` for decrypting GPG-encrypted email - `libpq-dev` for PostgreSQL - `libmagic-dev` for mime type detection - `mariadb-client` for MariaDB compile time - - `poppler-utils` for barcode detection + - `poppler-utils` for thumbnail generation and barcode detection Use this list for your preferred package management: @@ -416,11 +416,8 @@ to a positive number to enable polling and disable native filesystem notificatio You may need to change the path in the files. Example: `ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO` -12. Configure ImageMagick to allow processing of PDF documents and disable - formats that Paperless-ngx does not use. Most distributions disable PDF - processing by default, since PDF documents can contain malware. If you - don't enable it, Paperless-ngx will fall back to Ghostscript for certain - steps such as thumbnail generation. +12. Harden ImageMagick by disabling formats that Paperless-ngx does not use. + PDF processing is not needed and should stay disabled. Configure the active ImageMagick policy file (commonly `/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and diff --git a/paperless.conf.example b/paperless.conf.example index e2cedf63c..740bc9429 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -50,8 +50,6 @@ PAPERLESS_SECRET_KEY=change-me #PAPERLESS_OCR_ROTATE_PAGES=true #PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0 #PAPERLESS_OCR_USER_ARGS={} -#PAPERLESS_CONVERT_MEMORY_LIMIT=0 -#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless # Software tweaks diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 69ee4e285..a0002cc5d 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -1,8 +1,8 @@ from __future__ import annotations import logging +import math import mimetypes -import os import shutil import subprocess import tempfile @@ -68,58 +68,6 @@ def get_supported_file_extensions() -> set[str]: return extensions -def run_convert( - input_file, - output_file, - *, - density=None, - scale=None, - alpha=None, - strip=False, - trim=False, - type=None, - depth=None, - auto_orient=False, - use_cropbox=False, - extra=None, - logging_group=None, -) -> None: - environment = os.environ.copy() - if settings.CONVERT_MEMORY_LIMIT: - # MAGICK_MEMORY_LIMIT sets the maximum amount of RAM the pixel cache can use. - # MAGICK_MAP_LIMIT sets the maximum amount of memory-mapped I/O allowed. - # - # For large-format documents ImageMagick will hit the RAM limit and - # immediately try to "map" the remaining data. If MAGICK_MAP_LIMIT isn't - # also set, the process may trigger an OOM kill because the default - # system/policy map limit is often too restrictive for these massive bitmaps. - environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT - environment["MAGICK_MAP_LIMIT"] = settings.CONVERT_MEMORY_LIMIT - if settings.CONVERT_TMPDIR: - environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR - - args = [settings.CONVERT_BINARY] - args += ["-density", str(density)] if density else [] - args += ["-scale", str(scale)] if scale else [] - args += ["-alpha", str(alpha)] if alpha else [] - args += ["-strip"] if strip else [] - args += ["-trim"] if trim else [] - args += ["-type", str(type)] if type else [] - args += ["-depth", str(depth)] if depth else [] - args += ["-auto-orient"] if auto_orient else [] - args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else [] - args += [str(input_file), str(output_file)] - - logger.debug("Execute: " + " ".join(args), extra={"group": logging_group}) - - try: - run_subprocess(args, environment, logger) - except subprocess.CalledProcessError as e: - raise ParseError(f"Convert failed at {args}") from e - except Exception as e: # pragma: no cover - raise ParseError("Unknown error running convert") from e - - def get_default_thumbnail() -> Path: """ Returns the path to a generic thumbnail @@ -127,46 +75,153 @@ def get_default_thumbnail() -> Path: return (Path(__file__).parent / "resources" / "document.webp").resolve() -def make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group=None) -> Path: - out_path: Path = Path(temp_dir) / "convert_gs.webp" +_THUMBNAIL_MAX_WIDTH = 500 +_THUMBNAIL_MAX_HEIGHT = 5000 +# Used only when the page geometry cannot be read +_THUMBNAIL_FALLBACK_DPI = 150 +# Matches the density the thumbnail was previously rendered at before scaling +_THUMBNAIL_MAX_DPI = 300 - # if convert fails, fall back to extracting - # the first PDF page as a PNG using Ghostscript - logger.warning( - "Thumbnail generation with ImageMagick failed, falling back " - "to ghostscript. Check your /etc/ImageMagick-x/policy.xml!", - extra={"group": logging_group}, - ) - # Ghostscript doesn't handle WebP outputs - gs_out_path: Path = Path(temp_dir) / "gs_out.png" - cmd = [settings.GS_BINARY, "-q", "-sDEVICE=pngalpha", "-o", gs_out_path, in_path] + +def rasterize_pdf_page_to_png( + in_path: Path, + out_path: Path, + *, + dpi: int, + logging_group=None, +) -> None: + """ + Rasterizes the first page of a PDF to a PNG with pdftoppm. + """ + # -singlefile drops the page number and -png appends ".png", so pass the + # path without its suffix + args = [ + "pdftoppm", + "-f", + "1", + "-l", + "1", + "-r", + str(dpi), + "-png", + "-singlefile", + "-cropbox", + str(in_path), + str(out_path.with_suffix("")), + ] + + logger.debug("Execute: " + " ".join(args), extra={"group": logging_group}) try: - try: - run_subprocess(cmd, logger=logger) - except subprocess.CalledProcessError as e: - raise ParseError(f"Thumbnail (gs) failed at {cmd}") from e - # then run convert on the output from gs to make WebP - run_convert( - density=300, - scale="500x5000>", - alpha="remove", - strip=True, - trim=False, - auto_orient=True, - input_file=gs_out_path, - output_file=out_path, - logging_group=logging_group, - ) + run_subprocess(args, logger=logger) + except subprocess.CalledProcessError as e: + raise ParseError(f"pdftoppm failed at {args}") from e + except Exception as e: # pragma: no cover + raise ParseError("Unknown error running pdftoppm") from e + +def encode_thumbnail_webp( + png_path: Path, + out_path: Path, +) -> None: + """ + Flattens alpha onto white, shrinks to fit and saves as WebP. + """ + from PIL import Image + + try: + with Image.open(png_path) as im: + if im.mode in ("RGBA", "LA"): + flattened = Image.new("RGB", im.size, (255, 255, 255)) + flattened.paste(im, mask=im.split()[-1]) + else: + flattened = im.convert("RGB") + + flattened.thumbnail((_THUMBNAIL_MAX_WIDTH, _THUMBNAIL_MAX_HEIGHT)) + flattened.save(out_path, format="WEBP") + except (OSError, Image.DecompressionBombError) as e: + raise ParseError(f"Unable to encode thumbnail from {png_path}") from e + + +def _compute_thumbnail_dpi(in_path: Path, logging_group=None) -> int: + """ + Computes the DPI which renders the first page of the PDF at or just above + the thumbnail size in one pass, never above 300. + """ + from paperless.parsers.utils import get_pdf_first_page_size_points + + size = get_pdf_first_page_size_points(in_path) + if size is None: + logger.debug( + "Could not read PDF page size, using fallback DPI", + extra={"group": logging_group}, + ) + return _THUMBNAIL_FALLBACK_DPI + + width_pts, height_pts = size + dpi_for_width = _THUMBNAIL_MAX_WIDTH * 72 / width_pts + dpi_for_height = _THUMBNAIL_MAX_HEIGHT * 72 / height_pts + # Round up so the render is never a few pixels short of the target; the + # shrink-only clamp in encode_thumbnail_webp trims the excess. + return max( + 1, + math.ceil(min(_THUMBNAIL_MAX_DPI, dpi_for_width, dpi_for_height)), + ) + + +def _render_pdf_thumbnail( + in_path: Path, + png_path: Path, + out_path: Path, + logging_group=None, +) -> None: + dpi = _compute_thumbnail_dpi(in_path, logging_group=logging_group) + rasterize_pdf_page_to_png( + in_path, + png_path, + dpi=dpi, + logging_group=logging_group, + ) + encode_thumbnail_webp(png_path, out_path) + + +def _repair_pdf_with_qpdf(in_path: Path, out_path: Path) -> None: + # qpdf exits 3 after a repair; --warning-exit-0 keeps that from failing + try: + shutil.copy(in_path, out_path) + run_subprocess( + ["qpdf", "--warning-exit-0", "--replace-input", str(out_path)], + logger=logger, + ) + except (subprocess.CalledProcessError, OSError) as e: + raise ParseError(f"qpdf repair failed for {in_path}") from e + + +def make_thumbnail_from_pdf_qpdf_fallback( + in_path: Path, + temp_dir: Path, + logging_group=None, +) -> Path: + png_path = temp_dir / "page1_repaired.png" + out_path = temp_dir / "convert_qpdf.webp" + repaired_path = temp_dir / "repaired.pdf" + + logger.warning( + "Thumbnail generation with pdftoppm failed, attempting qpdf repair and retry.", + extra={"group": logging_group}, + ) + + try: + _repair_pdf_with_qpdf(in_path, repaired_path) + _render_pdf_thumbnail(repaired_path, png_path, out_path, logging_group) return out_path except ParseError as e: - logger.error(f"Unable to make thumbnail with Ghostscript: {e}") + logger.error(f"Unable to make thumbnail after qpdf repair: {e}") # The caller might expect a generated thumbnail that can be moved, # so we need to copy it before it gets moved. # https://github.com/paperless-ngx/paperless-ngx/issues/3631 - default_thumbnail_path: Path = Path(temp_dir) / "document.webp" + default_thumbnail_path = temp_dir / "document.webp" copy_file_with_basic_stats(get_default_thumbnail(), default_thumbnail_path) return default_thumbnail_path @@ -175,25 +230,18 @@ def make_thumbnail_from_pdf(in_path: Path, temp_dir: Path, logging_group=None) - """ The thumbnail of a PDF is just a 500px wide image of the first page. """ + png_path: Path = temp_dir / "page1.png" out_path: Path = temp_dir / "convert.webp" - # Run convert to get a decent thumbnail try: - run_convert( - density=300, - scale="500x5000>", - alpha="remove", - strip=True, - trim=False, - auto_orient=True, - use_cropbox=True, - input_file=f"{in_path}[0]", - output_file=str(out_path), - logging_group=logging_group, - ) + _render_pdf_thumbnail(in_path, png_path, out_path, logging_group) except ParseError as e: - logger.error(f"Unable to make thumbnail with convert: {e}") - out_path = make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group) + logger.error(f"Unable to make thumbnail with pdftoppm: {e}") + out_path = make_thumbnail_from_pdf_qpdf_fallback( + in_path, + temp_dir, + logging_group, + ) return out_path diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py index 5f3893112..5ece78225 100644 --- a/src/documents/tests/test_parsers.py +++ b/src/documents/tests/test_parsers.py @@ -1,11 +1,22 @@ +import subprocess from collections.abc import Generator +from pathlib import Path +import pikepdf import pytest +from PIL import Image from pytest_django.fixtures import Settings +from pytest_mock import MockerFixture +from documents.parsers import ParseError +from documents.parsers import _compute_thumbnail_dpi +from documents.parsers import encode_thumbnail_webp from documents.parsers import get_default_file_extension +from documents.parsers import get_default_thumbnail from documents.parsers import get_supported_file_extensions from documents.parsers import is_file_ext_supported +from documents.parsers import make_thumbnail_from_pdf +from documents.parsers import rasterize_pdf_page_to_png from paperless.parsers.registry import get_parser_registry from paperless.parsers.registry import reset_parser_registry from paperless.parsers.tesseract import RasterisedDocumentParser @@ -125,3 +136,329 @@ class TestParserAvailability: assert is_file_ext_supported(".pdf") assert not is_file_ext_supported(".hsdfh") assert not is_file_ext_supported("") + + +class TestComputeThumbnailDpi: + @pytest.mark.parametrize( + ("size", "expected"), + [ + pytest.param((612.0, 792.0), 59, id="letter-width-bound"), + pytest.param((792.0, 612.0), 46, id="landscape-rounded-up"), + pytest.param((612.0, 100000.0), 4, id="tall-strip-height-bound"), + pytest.param((200.0, 300.0), 180, id="small-page-width-bound"), + pytest.param((72.0, 72.0), 300, id="tiny-page-capped-at-300"), + pytest.param((1000000.0, 1000000.0), 1, id="huge-page-minimum-one"), + pytest.param(None, 150, id="unreadable-geometry-fallback"), + ], + ) + def test_dpi_from_page_size( + self, + mocker: MockerFixture, + tmp_path: Path, + size: tuple[float, float] | None, + expected: int, + ) -> None: + """ + GIVEN: + - A first page of the given size, or unreadable geometry + WHEN: + - The thumbnail DPI is computed + THEN: + - The expected DPI is returned + """ + mocker.patch( + "paperless.parsers.utils.get_pdf_first_page_size_points", + return_value=size, + ) + assert _compute_thumbnail_dpi(tmp_path / "doc.pdf") == expected + + +class TestMakeThumbnailFromPdf: + @pytest.fixture + def work_dir(self, tmp_path: Path) -> Path: + path = tmp_path / "work" + path.mkdir() + return path + + @staticmethod + def _write_blank_pdf(path: Path, page_size: tuple[int, int] = (612, 792)) -> Path: + pdf = pikepdf.new() + pdf.add_blank_page(page_size=page_size) + pdf.save(path, object_stream_mode=pikepdf.ObjectStreamMode.disable) + return path + + @pytest.mark.parametrize( + ("page_size", "expected_width"), + [ + pytest.param((612, 792), 500, id="letter"), + pytest.param((792, 612), 500, id="landscape-letter"), + pytest.param((595, 842), 500, id="a4"), + pytest.param((200, 300), 500, id="small-page"), + pytest.param((72, 72), 300, id="tiny-page-capped"), + ], + ) + def test_thumbnail_width( + self, + tmp_path: Path, + work_dir: Path, + page_size: tuple[int, int], + expected_width: int, + ) -> None: + """ + GIVEN: + - A PDF whose first page has the given size in points + WHEN: + - A thumbnail is made from it + THEN: + - The thumbnail has the expected width + """ + pdf_path = self._write_blank_pdf(tmp_path / "in.pdf", page_size) + + thumb = make_thumbnail_from_pdf(pdf_path, work_dir) + + assert thumb == work_dir / "convert.webp" + with Image.open(thumb) as im: + assert im.format == "WEBP" + assert im.width == expected_width + + @classmethod + def _write_pdf_without_xref(cls, path: Path) -> Path: + """ + Cuts off the xref and trailer, which pdftoppm cannot recover from but qpdf can. + """ + cls._write_blank_pdf(path) + data = path.read_bytes() + path.write_bytes(data[: data.rindex(b"\nxref")]) + return path + + def test_qpdf_repair_produces_real_thumbnail( + self, + tmp_path: Path, + work_dir: Path, + ) -> None: + """ + GIVEN: + - A PDF with its xref table and trailer cut off + WHEN: + - A thumbnail is made from it + THEN: + - The thumbnail is rendered from a qpdf repaired copy + - The original file is unchanged + """ + pdf_path = self._write_pdf_without_xref(tmp_path / "broken.pdf") + original_bytes = pdf_path.read_bytes() + + with pytest.raises(ParseError): + rasterize_pdf_page_to_png(pdf_path, work_dir / "probe.png", dpi=50) + + thumb = make_thumbnail_from_pdf(pdf_path, work_dir) + + assert thumb == work_dir / "convert_qpdf.webp" + with Image.open(thumb) as im: + assert im.format == "WEBP" + assert im.width == 500 + assert pdf_path.read_bytes() == original_bytes + + @pytest.mark.parametrize( + "qpdf_error", + [ + pytest.param(subprocess.CalledProcessError(2, "qpdf"), id="qpdf-fails"), + pytest.param(None, id="repaired-still-unrenderable"), + ], + ) + def test_double_failure_uses_default_thumbnail( + self, + mocker: MockerFixture, + tmp_path: Path, + work_dir: Path, + qpdf_error: subprocess.CalledProcessError | None, + ) -> None: + """ + GIVEN: + - A PDF that cannot be rendered, even after qpdf repair + WHEN: + - A thumbnail is made from it + THEN: + - A copy of the default thumbnail is returned + """ + mocker.patch( + "documents.parsers.rasterize_pdf_page_to_png", + side_effect=ParseError("Does not compute."), + ) + if qpdf_error is not None: + mocker.patch("documents.parsers.run_subprocess", side_effect=qpdf_error) + pdf_path = self._write_blank_pdf(tmp_path / "in.pdf") + + thumb = make_thumbnail_from_pdf(pdf_path, work_dir) + + assert thumb == work_dir / "document.webp" + assert thumb.read_bytes() == get_default_thumbnail().read_bytes() + + +class TestRasterizePdfPageToPng: + @staticmethod + def _write_pdf( + path: Path, + *, + crop_box: tuple[float, float, float, float] | None = None, + rotate: int | None = None, + ) -> Path: + pdf = pikepdf.new() + pdf.add_blank_page(page_size=(144, 72)) + pdf.add_blank_page(page_size=(300, 300)) + page = pdf.pages[0] + if crop_box is not None: + page.obj.CropBox = pikepdf.Array(crop_box) + if rotate is not None: + page.obj.Rotate = rotate + pdf.save(path) + return path + + @pytest.mark.parametrize( + ("crop_box", "rotate", "expected_size"), + [ + pytest.param(None, None, (144, 72), id="plain"), + pytest.param(None, 90, (72, 144), id="rotated-90"), + pytest.param((0, 0, 72, 36), None, (72, 36), id="crop-box"), + ], + ) + def test_renders_first_page_to_exact_path( + self, + tmp_path: Path, + crop_box: tuple[float, float, float, float] | None, + rotate: int | None, + expected_size: tuple[int, int], + ) -> None: + """ + GIVEN: + - A two page PDF, first page optionally cropped or rotated + WHEN: + - The first page is rasterized at 72 DPI + THEN: + - Only out_path is written, sized to the first page's crop and rotation + """ + pdf_path = self._write_pdf( + tmp_path / "in.pdf", + crop_box=crop_box, + rotate=rotate, + ) + out_dir = tmp_path / "out" + out_dir.mkdir() + out_path = out_dir / "page1.png" + + rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72) + + assert list(out_dir.iterdir()) == [out_path] + with Image.open(out_path) as im: + assert im.format == "PNG" + assert im.size == expected_size + + def test_failure_raises_parse_error(self, tmp_path: Path) -> None: + """ + GIVEN: + - A file that is not a PDF + WHEN: + - Rasterization is attempted + THEN: + - A ParseError is raised + """ + bad = tmp_path / "bad.pdf" + bad.write_bytes(b"not a pdf") + + with pytest.raises(ParseError): + rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72) + + +class TestEncodeThumbnailWebp: + @pytest.mark.parametrize( + ("mode", "color"), + [ + pytest.param("RGBA", (0, 0, 0, 0), id="rgba"), + pytest.param("LA", (0, 0), id="la"), + ], + ) + def test_alpha_flattened_onto_white( + self, + tmp_path: Path, + mode: str, + color: tuple[int, ...], + ) -> None: + """ + GIVEN: + - A fully transparent PNG with an alpha channel + WHEN: + - It is encoded as a thumbnail + THEN: + - The WebP output is RGB with the transparency flattened to white + """ + png_path = tmp_path / "in.png" + Image.new(mode, (20, 10), color).save(png_path) + out_path = tmp_path / "out.webp" + + encode_thumbnail_webp(png_path, out_path) + + with Image.open(out_path) as im: + assert im.format == "WEBP" + assert im.mode == "RGB" + assert im.size == (20, 10) + red, green, blue = im.getpixel((10, 5)) + assert min(red, green, blue) >= 250 + + @pytest.mark.parametrize( + ("in_size", "expected_size"), + [ + pytest.param((1000, 2000), (500, 1000), id="too-wide-shrunk"), + pytest.param((100, 10000), (50, 5000), id="too-tall-shrunk"), + pytest.param((100, 200), (100, 200), id="small-not-enlarged"), + ], + ) + def test_size_clamped_and_downsampled( + self, + tmp_path: Path, + in_size: tuple[int, int], + expected_size: tuple[int, int], + ) -> None: + """ + GIVEN: + - A rendered image + WHEN: + - It is encoded as a thumbnail + THEN: + - It is fit within 500x5000 and never enlarged + """ + png_path = tmp_path / "in.png" + Image.new("RGB", in_size, (255, 255, 255)).save(png_path) + out_path = tmp_path / "out.webp" + + encode_thumbnail_webp(png_path, out_path) + + with Image.open(out_path) as im: + assert im.size == expected_size + + @pytest.mark.parametrize( + "error", + [ + pytest.param(OSError("broken image"), id="os-error"), + pytest.param(Image.DecompressionBombError("too large"), id="bomb"), + ], + ) + def test_decode_failure_raises_parse_error( + self, + tmp_path: Path, + mocker: MockerFixture, + error: Exception, + ) -> None: + """ + GIVEN: + - Opening the rendered image fails + WHEN: + - It is encoded as a thumbnail + THEN: + - A ParseError is raised + """ + png_path = tmp_path / "in.png" + Image.new("RGB", (10, 10)).save(png_path) + mocker.patch("PIL.Image.open", side_effect=error) + + with pytest.raises(ParseError): + encode_thumbnail_webp(png_path, tmp_path / "out.webp") diff --git a/src/paperless/parsers/utils.py b/src/paperless/parsers/utils.py index 9fe1d4908..f9e299d21 100644 --- a/src/paperless/parsers/utils.py +++ b/src/paperless/parsers/utils.py @@ -265,6 +265,52 @@ def get_page_count_for_pdf( return None +def get_pdf_first_page_size_points( + path: Path, + log: logging.Logger | None = None, +) -> tuple[float, float] | None: + """Return the first page's (width, height) in PDF points, post-rotation. + + Uses the CropBox (MediaBox if absent), which must match pdftoppm's + ``-cropbox`` or the computed DPI targets the wrong box. + + Swaps width and height for 90/270 rotation. ``page.rotation`` resolves + inherited and negative ``/Rotate`` values, a raw lookup does not. + + Parameters + ---------- + path: + Absolute path to the PDF file. + log: + Logger for warnings. Falls back to the module-level logger when omitted. + + Returns + ------- + tuple[float, float] | None + ``(width_points, height_points)``, or ``None`` if the file cannot be + opened, has no pages, or the page box is degenerate. + """ + import pikepdf + + _log = log or logger + try: + with pikepdf.Pdf.open(path) as pdf: + if len(pdf.pages) == 0: + return None + page = pdf.pages[0] + llx, lly, urx, ury = (float(v) for v in page.cropbox) + width = abs(urx - llx) + height = abs(ury - lly) + if width <= 0 or height <= 0: + return None + if page.rotation in (90, 270): + width, height = height, width + return width, height + except Exception as e: + _log.warning("Could not determine PDF page size for %s: %s", path, e) + return None + + def extract_pdf_metadata( document_path: Path, log: logging.Logger | None = None, diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py index 7ca3bb66e..7a5ce30b9 100644 --- a/src/paperless/settings/__init__.py +++ b/src/paperless/settings/__init__.py @@ -986,8 +986,6 @@ GNUPG_HOME = os.getenv("HOME", "/tmp") # Convert is part of the ImageMagick package CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert") -CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR") -CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT") GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs") diff --git a/src/paperless/tests/parsers/test_tesseract_parser.py b/src/paperless/tests/parsers/test_tesseract_parser.py index bb6a85031..f057ad086 100644 --- a/src/paperless/tests/parsers/test_tesseract_parser.py +++ b/src/paperless/tests/parsers/test_tesseract_parser.py @@ -15,9 +15,10 @@ from typing import TYPE_CHECKING import pytest from ocrmypdf import SubprocessOutputError +from PIL import Image from documents.parsers import ParseError -from documents.parsers import run_convert +from documents.parsers import rasterize_pdf_page_to_png from paperless.models import ModeChoices from paperless.parsers import ParserProtocol from paperless.parsers.tesseract import RasterisedDocumentParser @@ -280,24 +281,71 @@ class TestGetThumbnail: ) assert thumb.is_file() - def test_thumbnail_fallback_on_convert_error( + @pytest.mark.parametrize( + ("filename", "expected_height"), + [ + pytest.param("simple-digital.pdf", 647, id="portrait-letter"), + pytest.param("rotated.pdf", 386, id="landscape"), + ], + ) + def test_thumbnail_is_correct_format_and_size( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + filename: str, + expected_height: int, + ) -> None: + """ + GIVEN: + - A portrait or landscape PDF + WHEN: + - A thumbnail is generated + THEN: + - A 500px wide WebP keeping the page's aspect ratio + """ + thumb = tesseract_parser.get_thumbnail( + tesseract_samples_dir / filename, + "application/pdf", + ) + with Image.open(thumb) as im: + assert im.format == "WEBP" + assert im.width == 500 + assert im.height == pytest.approx(expected_height, abs=2) + + def test_thumbnail_fallback_on_pdftoppm_error( self, mocker: MockerFixture, tesseract_parser: RasterisedDocumentParser, tesseract_samples_dir: Path, ) -> None: - def _raise_on_pdf(input_file, output_file, **kwargs) -> None: - if ".pdf" in str(input_file): + """ + GIVEN: + - Rasterizing the original PDF fails + WHEN: + - A thumbnail is generated + THEN: + - The PDF is repaired with qpdf and a real thumbnail is rendered + """ + original = tesseract_samples_dir / "simple-digital.pdf" + + def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None: + if in_path == original: raise ParseError("Does not compute.") - run_convert(input_file=input_file, output_file=output_file, **kwargs) + rasterize_pdf_page_to_png(in_path, out_path, **kwargs) - mocker.patch("documents.parsers.run_convert", side_effect=_raise_on_pdf) - - thumb = tesseract_parser.get_thumbnail( - tesseract_samples_dir / "simple-digital.pdf", - "application/pdf", + rasterize = mocker.patch( + "documents.parsers.rasterize_pdf_page_to_png", + side_effect=_fail_on_original, ) + + thumb = tesseract_parser.get_thumbnail(original, "application/pdf") + + assert rasterize.call_count == 2 assert thumb.is_file() + assert thumb.name == "convert_qpdf.webp" + with Image.open(thumb) as im: + assert im.format == "WEBP" + assert im.width == 500 def test_thumbnail_encrypted_pdf( self, diff --git a/src/paperless/tests/test_parser_utils.py b/src/paperless/tests/test_parser_utils.py index f0a63ea44..4cd67297d 100644 --- a/src/paperless/tests/test_parser_utils.py +++ b/src/paperless/tests/test_parser_utils.py @@ -6,8 +6,10 @@ import codecs from pathlib import Path from typing import TYPE_CHECKING +import pikepdf import pytest +from paperless.parsers.utils import get_pdf_first_page_size_points from paperless.parsers.utils import is_tagged_pdf from paperless.parsers.utils import pdf_born_digital_text from paperless.parsers.utils import post_process_text @@ -70,6 +72,149 @@ class TestIsTaggedPdf: assert is_tagged_pdf(bad) is False +class TestGetPdfFirstPageSizePoints: + @staticmethod + def _write_pdf( + path: Path, + *, + media_box: tuple[float, float, float, float] = (0, 0, 600, 800), + crop_box: tuple[float, float, float, float] | None = None, + page_rotate: int | None = None, + inherited_rotate: int | None = None, + ) -> Path: + pdf = pikepdf.new() + pdf.add_blank_page(page_size=(media_box[2], media_box[3])) + page = pdf.pages[0] + page.obj.MediaBox = pikepdf.Array(media_box) + if crop_box is not None: + page.obj.CropBox = pikepdf.Array(crop_box) + if page_rotate is not None: + page.obj.Rotate = page_rotate + if inherited_rotate is not None: + pdf.Root.Pages.Rotate = inherited_rotate + pdf.save(path) + return path + + def test_letter_sample(self) -> None: + """ + GIVEN: + - A US Letter sample PDF with no CropBox and no rotation + WHEN: + - The first page size is requested + THEN: + - The MediaBox size in points is returned + """ + assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") == ( + 612.0, + 792.0, + ) + + @pytest.mark.parametrize( + ("rotate", "expected"), + [ + pytest.param(0, (600.0, 800.0), id="rotate-0"), + pytest.param(90, (800.0, 600.0), id="rotate-90"), + pytest.param(180, (600.0, 800.0), id="rotate-180"), + pytest.param(270, (800.0, 600.0), id="rotate-270"), + pytest.param(-90, (800.0, 600.0), id="rotate-negative-90"), + ], + ) + def test_page_rotation_swaps_dimensions( + self, + tmp_path: Path, + rotate: int, + expected: tuple[float, float], + ) -> None: + """ + GIVEN: + - A portrait PDF page with /Rotate set directly on the page + WHEN: + - The first page size is requested + THEN: + - Width and height are swapped for quarter-turn rotations only + """ + pdf_path = self._write_pdf(tmp_path / "rotated.pdf", page_rotate=rotate) + assert get_pdf_first_page_size_points(pdf_path) == expected + + def test_inherited_rotation_swaps_dimensions(self, tmp_path: Path) -> None: + """ + GIVEN: + - A page inheriting /Rotate 90 from the /Pages node + WHEN: + - The first page size is requested + THEN: + - Width and height are swapped + """ + pdf_path = self._write_pdf(tmp_path / "inherited.pdf", inherited_rotate=90) + assert get_pdf_first_page_size_points(pdf_path) == (800.0, 600.0) + + def test_crop_box_preferred_over_media_box(self, tmp_path: Path) -> None: + """ + GIVEN: + - A PDF page with a CropBox smaller than its MediaBox + WHEN: + - The first page size is requested + THEN: + - The CropBox size is returned + """ + pdf_path = self._write_pdf( + tmp_path / "cropped.pdf", + crop_box=(50, 100, 350, 500), + ) + assert get_pdf_first_page_size_points(pdf_path) == (300.0, 400.0) + + def test_degenerate_box_returns_none(self, tmp_path: Path) -> None: + """ + GIVEN: + - A PDF page whose box has zero width + WHEN: + - The first page size is requested + THEN: + - None is returned + """ + pdf_path = self._write_pdf( + tmp_path / "degenerate.pdf", + media_box=(0, 0, 600, 800), + crop_box=(100, 0, 100, 800), + ) + assert get_pdf_first_page_size_points(pdf_path) is None + + def test_nonexistent_path_returns_none(self) -> None: + """ + GIVEN: + - A path that does not exist + WHEN: + - The first page size is requested + THEN: + - None is returned and nothing is raised + """ + assert get_pdf_first_page_size_points(Path("/nonexistent/file.pdf")) is None + + def test_corrupt_pdf_returns_none(self, tmp_path: Path) -> None: + """ + GIVEN: + - A file that is not a PDF + WHEN: + - The first page size is requested + THEN: + - None is returned and nothing is raised + """ + bad = tmp_path / "bad.pdf" + bad.write_bytes(b"not a pdf") + assert get_pdf_first_page_size_points(bad) is None + + def test_encrypted_pdf_returns_none(self) -> None: + """ + GIVEN: + - A password protected PDF + WHEN: + - The first page size is requested + THEN: + - None is returned and nothing is raised + """ + assert get_pdf_first_page_size_points(SAMPLES / "encrypted.pdf") is None + + class TestPostProcessText: @pytest.mark.parametrize( ("source", "expected"),