From d4826f50c99216cd72105de15f0e6dd0fa19d04b Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:25:52 -0700 Subject: [PATCH 1/5] Fix: prevent doc permissions queries on schema view (#14503) --- src/documents/views.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/documents/views.py b/src/documents/views.py index a7e4350e8..0edae28d6 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -498,6 +498,11 @@ class BulkPermissionMixin: """ context = super().get_serializer_context() + if getattr(self, "swagger_fake_view", False): + # Schema generation only needs the serializer shape; without a + # paginated page we'd otherwise scan the entire queryset below. + return context + if getattr(self, "action", None) != "list": # Batching only pays off across a page of objects; for single-object # actions (retrieve, update, ...) the per-object fallback in From f730513e79da9b13a3a0b44cb6360457d29a6bd3 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:27:06 +0000 Subject: [PATCH 2/5] Auto translate strings --- src/locale/en_US/LC_MESSAGES/django.po | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index 8356bf749..3f930263f 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-10-19 15:40+0000\n" +"POT-Creation-Date: 2026-10-20 05:26+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -1653,7 +1653,7 @@ msgid "workflow runs" msgstr "" #: documents/serialisers.py:515 documents/serialisers.py:872 -#: documents/serialisers.py:2902 documents/views.py:363 documents/views.py:2768 +#: documents/serialisers.py:2902 documents/views.py:363 documents/views.py:2773 #: paperless_mail/serialisers.py:156 msgid "Insufficient permissions." msgstr "" @@ -1694,7 +1694,7 @@ msgstr "" msgid "Duplicate document identifiers are not allowed." msgstr "" -#: documents/serialisers.py:2988 documents/views.py:4816 +#: documents/serialisers.py:2988 documents/views.py:4821 #, python-format msgid "Documents not found: %(ids)s" msgstr "" @@ -1949,36 +1949,36 @@ msgstr "" msgid "Specify only one of text, title_search, query, or more_like_id." msgstr "" -#: documents/views.py:356 documents/views.py:2765 +#: documents/views.py:356 documents/views.py:2770 msgid "Invalid more_like_id" msgstr "" -#: documents/views.py:1703 +#: documents/views.py:1708 msgid "Invalid AI configuration." msgstr "" -#: documents/views.py:1714 +#: documents/views.py:1719 msgid "AI backend request timed out." msgstr "" -#: documents/views.py:1726 +#: documents/views.py:1731 msgid "AI backend rejected the request. Check logs for details." msgstr "" -#: documents/views.py:4832 +#: documents/views.py:4837 #, python-format msgid "Insufficient permissions to share document %(id)s." msgstr "" -#: documents/views.py:4879 +#: documents/views.py:4884 msgid "Bundle is already being processed." msgstr "" -#: documents/views.py:4945 +#: documents/views.py:4950 msgid "The share link bundle is still being prepared. Please try again later." msgstr "" -#: documents/views.py:4959 +#: documents/views.py:4964 msgid "The share link bundle is unavailable." msgstr "" From a05afd654ca7643dcd18e59bb491ebef6dbb435a Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:22:42 -0700 Subject: [PATCH 3/5] Feature: Generate PDF thumbnails via poppler utils (#14420) * Chore: Generate PDF thumbnails with pdftoppm and Pillow instead of ImageMagick Replace the ImageMagick convert based PDF thumbnail path with pdftoppm and Pillow. Keeps thumbnails at 500px wide for small and odd sized pages, covers the qpdf repair and double failure paths, removes the unused run_convert helper, hardens thumbnail encoding, deprecates PAPERLESS_CONVERT_MEMORY_LIMIT and PAPERLESS_CONVERT_TMPDIR, and corrects the related docs and CI. Co-Authored-By: Claude Sonnet 5.5 * Chore: Supersample the PDF thumbnail render 2x Render the PDF page at twice the target size and downsample, with edge case handling for small and odd sized pages, simplified helpers and tests, and trimmed comments. Co-Authored-By: Claude Sonnet 5.5 * Fix: use a fractional thumbnail DPI so oversized pages render a bounded raster * Deprecate the unused PAPERLESS_GS_BINARY setting too * Warn at startup when pdftoppm is missing * Resample PDF thumbnails once instead of twice with LANCZOS, still never enlarging * Read PDF page size from pdfinfo so it matches pdftoppm * Mark the thumbnail constants Final * Comment each pdftoppm option --- .github/workflows/ci-backend.yml | 2 +- docs/configuration.md | 29 +- docs/setup.md | 11 +- paperless.conf.example | 3 - src/documents/parsers.py | 272 +++++++----- src/documents/tests/test_parsers.py | 395 ++++++++++++++++++ src/paperless/checks.py | 2 +- src/paperless/parsers/utils.py | 66 +++ src/paperless/settings/__init__.py | 4 - .../tests/parsers/test_tesseract_parser.py | 68 ++- src/paperless/tests/test_checks.py | 14 + src/paperless/tests/test_parser_utils.py | 193 +++++++++ 12 files changed, 916 insertions(+), 143 deletions(-) diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 1e0ba23b1..331168ba2 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -111,7 +111,7 @@ jobs: timeout-minutes: 12 uses: $/.github/actions/apt-install with: - packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils + packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils qpdf - name: Configure ImageMagick run: | sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml diff --git a/docs/configuration.md b/docs/configuration.md index 3257a815c..a6f8afa89 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1315,29 +1315,17 @@ valid crontab(5) expression describing when to run. #### [`PAPERLESS_CONVERT_MEMORY_LIMIT=`](#PAPERLESS_CONVERT_MEMORY_LIMIT) {#PAPERLESS_CONVERT_MEMORY_LIMIT} -: On smaller systems, or even in the case of Very Large Documents, the -consumer may explode, complaining about how it's "unable to extend -pixel cache". In such cases, try setting this to a reasonably low -value, like 32. The default is to use whatever is necessary to do -everything without writing to disk, and units are in megabytes. +!!! warning - For more information on how to use this value, you should search the - web for "MAGICK_MEMORY_LIMIT". - - Defaults to 0, which disables the limit. + Deprecated and has no effect, since PDF thumbnails no longer use + ImageMagick. It will be removed in a future release. #### [`PAPERLESS_CONVERT_TMPDIR=`](#PAPERLESS_CONVERT_TMPDIR) {#PAPERLESS_CONVERT_TMPDIR} -: Similar to the memory limit, if you've got a small system and your -OS mounts /tmp as tmpfs, you should set this to a path that's on a -physical disk, like /home/your_user/tmp or something. ImageMagick -will use this as scratch space when crunching through very large -documents. +!!! warning - For more information on how to use this value, you should search the - web for "MAGICK_TMPDIR". - - Default is none, which disables the temporary directory. + Deprecated and has no effect, since PDF thumbnails no longer use + ImageMagick. It will be removed in a future release. #### [`PAPERLESS_APPS=`](#PAPERLESS_APPS) {#PAPERLESS_APPS} @@ -1889,7 +1877,10 @@ the literal path for that program. #### [`PAPERLESS_GS_BINARY=`](#PAPERLESS_GS_BINARY) {#PAPERLESS_GS_BINARY} -: Defaults to "gs". +!!! warning + + Deprecated and has no effect, since nothing in Paperless-ngx calls + Ghostscript by this path. It will be removed in a future release. ## Docker-specific options {#docker} diff --git a/docs/setup.md b/docs/setup.md index e666cb58d..514cf424c 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -177,12 +177,12 @@ to a positive number to enable polling and disable native filesystem notificatio - `pkg-config` for mysqlclient (python dependency) - `fonts-liberation` for generating thumbnails for plain text files - - `imagemagick` >= 6 for PDF conversion + - `imagemagick` >= 6 for image alpha handling - `gnupg` for decrypting GPG-encrypted email - `libpq-dev` for PostgreSQL - `libmagic-dev` for mime type detection - `mariadb-client` for MariaDB compile time - - `poppler-utils` for barcode detection + - `poppler-utils` for thumbnail generation and barcode detection Use this list for your preferred package management: @@ -416,11 +416,8 @@ to a positive number to enable polling and disable native filesystem notificatio You may need to change the path in the files. Example: `ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO` -12. Configure ImageMagick to allow processing of PDF documents and disable - formats that Paperless-ngx does not use. Most distributions disable PDF - processing by default, since PDF documents can contain malware. If you - don't enable it, Paperless-ngx will fall back to Ghostscript for certain - steps such as thumbnail generation. +12. Harden ImageMagick by disabling formats that Paperless-ngx does not use. + PDF processing is not needed and should stay disabled. Configure the active ImageMagick policy file (commonly `/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and diff --git a/paperless.conf.example b/paperless.conf.example index e2cedf63c..7e6960c82 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -50,8 +50,6 @@ PAPERLESS_SECRET_KEY=change-me #PAPERLESS_OCR_ROTATE_PAGES=true #PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0 #PAPERLESS_OCR_USER_ARGS={} -#PAPERLESS_CONVERT_MEMORY_LIMIT=0 -#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless # Software tweaks @@ -91,4 +89,3 @@ PAPERLESS_SECRET_KEY=change-me # Binaries #PAPERLESS_CONVERT_BINARY=/usr/bin/convert -#PAPERLESS_GS_BINARY=/usr/bin/gs diff --git a/src/documents/parsers.py b/src/documents/parsers.py index 69ee4e285..48983bb23 100644 --- a/src/documents/parsers.py +++ b/src/documents/parsers.py @@ -2,12 +2,12 @@ from __future__ import annotations import logging import mimetypes -import os import shutil import subprocess import tempfile from pathlib import Path from typing import TYPE_CHECKING +from typing import Final from django.conf import settings @@ -68,58 +68,6 @@ def get_supported_file_extensions() -> set[str]: return extensions -def run_convert( - input_file, - output_file, - *, - density=None, - scale=None, - alpha=None, - strip=False, - trim=False, - type=None, - depth=None, - auto_orient=False, - use_cropbox=False, - extra=None, - logging_group=None, -) -> None: - environment = os.environ.copy() - if settings.CONVERT_MEMORY_LIMIT: - # MAGICK_MEMORY_LIMIT sets the maximum amount of RAM the pixel cache can use. - # MAGICK_MAP_LIMIT sets the maximum amount of memory-mapped I/O allowed. - # - # For large-format documents ImageMagick will hit the RAM limit and - # immediately try to "map" the remaining data. If MAGICK_MAP_LIMIT isn't - # also set, the process may trigger an OOM kill because the default - # system/policy map limit is often too restrictive for these massive bitmaps. - environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT - environment["MAGICK_MAP_LIMIT"] = settings.CONVERT_MEMORY_LIMIT - if settings.CONVERT_TMPDIR: - environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR - - args = [settings.CONVERT_BINARY] - args += ["-density", str(density)] if density else [] - args += ["-scale", str(scale)] if scale else [] - args += ["-alpha", str(alpha)] if alpha else [] - args += ["-strip"] if strip else [] - args += ["-trim"] if trim else [] - args += ["-type", str(type)] if type else [] - args += ["-depth", str(depth)] if depth else [] - args += ["-auto-orient"] if auto_orient else [] - args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else [] - args += [str(input_file), str(output_file)] - - logger.debug("Execute: " + " ".join(args), extra={"group": logging_group}) - - try: - run_subprocess(args, environment, logger) - except subprocess.CalledProcessError as e: - raise ParseError(f"Convert failed at {args}") from e - except Exception as e: # pragma: no cover - raise ParseError("Unknown error running convert") from e - - def get_default_thumbnail() -> Path: """ Returns the path to a generic thumbnail @@ -127,46 +75,181 @@ def get_default_thumbnail() -> Path: return (Path(__file__).parent / "resources" / "document.webp").resolve() -def make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group=None) -> Path: - out_path: Path = Path(temp_dir) / "convert_gs.webp" +_THUMBNAIL_MAX_WIDTH: Final[int] = 500 +_THUMBNAIL_MAX_HEIGHT: Final[int] = 5000 +# Used only when the page geometry cannot be read +_THUMBNAIL_FALLBACK_DPI: Final[int] = 150 +# Applied before supersampling, so tiny pages are not enlarged +_THUMBNAIL_MAX_DPI: Final[int] = 300 +# Rendering at a multiple and downsampling keeps text crisper +_THUMBNAIL_SUPERSAMPLE: Final[int] = 2 - # if convert fails, fall back to extracting - # the first PDF page as a PNG using Ghostscript - logger.warning( - "Thumbnail generation with ImageMagick failed, falling back " - "to ghostscript. Check your /etc/ImageMagick-x/policy.xml!", - extra={"group": logging_group}, - ) - # Ghostscript doesn't handle WebP outputs - gs_out_path: Path = Path(temp_dir) / "gs_out.png" - cmd = [settings.GS_BINARY, "-q", "-sDEVICE=pngalpha", "-o", gs_out_path, in_path] + +def rasterize_pdf_page_to_png( + in_path: Path, + out_path: Path, + *, + dpi: float, + logging_group=None, +) -> None: + """ + Rasterizes the first page of a PDF to a PNG with pdftoppm. + """ + args = [ + "pdftoppm", + # Only the first page is needed, first and last page are both 1 + "-f", + "1", + "-l", + "1", + # Resolution in DPI, which may be fractional + "-r", + str(dpi), + # Write a PNG + "-png", + # Write exactly one file with no page number in its name. -png appends + # ".png" to the output root, so the path is passed without its suffix + "-singlefile", + # Render the CropBox instead of the MediaBox, as a viewer would. This + # must be the same box get_pdf_first_page_size_points measures, or the + # DPI is computed for the wrong size + "-cropbox", + str(in_path), + str(out_path.with_suffix("")), + ] + + logger.debug("Execute: " + " ".join(args), extra={"group": logging_group}) try: - try: - run_subprocess(cmd, logger=logger) - except subprocess.CalledProcessError as e: - raise ParseError(f"Thumbnail (gs) failed at {cmd}") from e - # then run convert on the output from gs to make WebP - run_convert( - density=300, - scale="500x5000>", - alpha="remove", - strip=True, - trim=False, - auto_orient=True, - input_file=gs_out_path, - output_file=out_path, - logging_group=logging_group, - ) + run_subprocess(args, logger=logger) + except subprocess.CalledProcessError as e: + raise ParseError(f"pdftoppm failed at {args}") from e + except Exception as e: # pragma: no cover + raise ParseError("Unknown error running pdftoppm") from e + +def encode_thumbnail_webp( + png_path: Path, + out_path: Path, + *, + supersample: int = 1, +) -> None: + """ + Flattens alpha onto white, undoes supersampling, shrinks to fit and saves as WebP. + """ + from PIL import Image + + try: + with Image.open(png_path) as im: + if im.mode in ("RGBA", "LA"): + flattened = Image.new("RGB", im.size, (255, 255, 255)) + flattened.paste(im, mask=im.split()[-1]) + else: + flattened = im.convert("RGB") + + # One resize avoids a second, softer pass: undo the supersample and + # clamp to the bounds together, never enlarging + scale = min( + 1 / supersample, + _THUMBNAIL_MAX_WIDTH / flattened.width, + _THUMBNAIL_MAX_HEIGHT / flattened.height, + ) + if scale < 1: + flattened = flattened.resize( + ( + max(1, round(flattened.width * scale)), + max(1, round(flattened.height * scale)), + ), + Image.Resampling.LANCZOS, + ) + + flattened.save(out_path, format="WEBP") + except (OSError, Image.DecompressionBombError) as e: + raise ParseError(f"Unable to encode thumbnail from {png_path}") from e + + +def _compute_thumbnail_dpi(in_path: Path, logging_group=None) -> tuple[float, int]: + """ + Returns (dpi, supersample). Unknown geometry is not supersampled, since + the render size cannot be bounded. + + The DPI is left fractional rather than floored at 1, so an oversized page + still renders at about the thumbnail size instead of an unbounded raster. + pdftoppm rounds pixel dimensions up, so the render is never short of the + target; the shrink-only clamp in encode_thumbnail_webp trims any excess. + """ + from paperless.parsers.utils import get_pdf_first_page_size_points + + size = get_pdf_first_page_size_points(in_path) + if size is None: + logger.debug( + "Could not read PDF page size, using fallback DPI", + extra={"group": logging_group}, + ) + return _THUMBNAIL_FALLBACK_DPI, 1 + + width_pts, height_pts = size + dpi_for_width = _THUMBNAIL_MAX_WIDTH * 72 / width_pts + dpi_for_height = _THUMBNAIL_MAX_HEIGHT * 72 / height_pts + return ( + min(_THUMBNAIL_MAX_DPI, dpi_for_width, dpi_for_height), + _THUMBNAIL_SUPERSAMPLE, + ) + + +def _render_pdf_thumbnail( + in_path: Path, + png_path: Path, + out_path: Path, + logging_group=None, +) -> None: + dpi, supersample = _compute_thumbnail_dpi(in_path, logging_group=logging_group) + rasterize_pdf_page_to_png( + in_path, + png_path, + dpi=dpi * supersample, + logging_group=logging_group, + ) + encode_thumbnail_webp(png_path, out_path, supersample=supersample) + + +def _repair_pdf_with_qpdf(in_path: Path, out_path: Path) -> None: + # qpdf exits 3 after a repair; --warning-exit-0 keeps that from failing + try: + shutil.copy(in_path, out_path) + run_subprocess( + ["qpdf", "--warning-exit-0", "--replace-input", str(out_path)], + logger=logger, + ) + except (subprocess.CalledProcessError, OSError) as e: + raise ParseError(f"qpdf repair failed for {in_path}") from e + + +def make_thumbnail_from_pdf_qpdf_fallback( + in_path: Path, + temp_dir: Path, + logging_group=None, +) -> Path: + png_path = temp_dir / "page1_repaired.png" + out_path = temp_dir / "convert_qpdf.webp" + repaired_path = temp_dir / "repaired.pdf" + + logger.warning( + "Thumbnail generation with pdftoppm failed, attempting qpdf repair and retry.", + extra={"group": logging_group}, + ) + + try: + _repair_pdf_with_qpdf(in_path, repaired_path) + _render_pdf_thumbnail(repaired_path, png_path, out_path, logging_group) return out_path except ParseError as e: - logger.error(f"Unable to make thumbnail with Ghostscript: {e}") + logger.error(f"Unable to make thumbnail after qpdf repair: {e}") # The caller might expect a generated thumbnail that can be moved, # so we need to copy it before it gets moved. # https://github.com/paperless-ngx/paperless-ngx/issues/3631 - default_thumbnail_path: Path = Path(temp_dir) / "document.webp" + default_thumbnail_path = temp_dir / "document.webp" copy_file_with_basic_stats(get_default_thumbnail(), default_thumbnail_path) return default_thumbnail_path @@ -175,25 +258,18 @@ def make_thumbnail_from_pdf(in_path: Path, temp_dir: Path, logging_group=None) - """ The thumbnail of a PDF is just a 500px wide image of the first page. """ + png_path: Path = temp_dir / "page1.png" out_path: Path = temp_dir / "convert.webp" - # Run convert to get a decent thumbnail try: - run_convert( - density=300, - scale="500x5000>", - alpha="remove", - strip=True, - trim=False, - auto_orient=True, - use_cropbox=True, - input_file=f"{in_path}[0]", - output_file=str(out_path), - logging_group=logging_group, - ) + _render_pdf_thumbnail(in_path, png_path, out_path, logging_group) except ParseError as e: - logger.error(f"Unable to make thumbnail with convert: {e}") - out_path = make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group) + logger.error(f"Unable to make thumbnail with pdftoppm: {e}") + out_path = make_thumbnail_from_pdf_qpdf_fallback( + in_path, + temp_dir, + logging_group, + ) return out_path diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py index 5f3893112..cf27994c1 100644 --- a/src/documents/tests/test_parsers.py +++ b/src/documents/tests/test_parsers.py @@ -1,11 +1,22 @@ +import subprocess from collections.abc import Generator +from pathlib import Path +import pikepdf import pytest +from PIL import Image from pytest_django.fixtures import Settings +from pytest_mock import MockerFixture +from documents.parsers import ParseError +from documents.parsers import _compute_thumbnail_dpi +from documents.parsers import encode_thumbnail_webp from documents.parsers import get_default_file_extension +from documents.parsers import get_default_thumbnail from documents.parsers import get_supported_file_extensions from documents.parsers import is_file_ext_supported +from documents.parsers import make_thumbnail_from_pdf +from documents.parsers import rasterize_pdf_page_to_png from paperless.parsers.registry import get_parser_registry from paperless.parsers.registry import reset_parser_registry from paperless.parsers.tesseract import RasterisedDocumentParser @@ -125,3 +136,387 @@ class TestParserAvailability: assert is_file_ext_supported(".pdf") assert not is_file_ext_supported(".hsdfh") assert not is_file_ext_supported("") + + +class TestComputeThumbnailDpi: + @pytest.mark.parametrize( + ("size", "expected"), + [ + pytest.param((612.0, 792.0), (500 * 72 / 612, 2), id="letter-width-bound"), + pytest.param( + (792.0, 612.0), + (500 * 72 / 792, 2), + id="landscape-width-bound", + ), + pytest.param( + (612.0, 100000.0), + (5000 * 72 / 100000, 2), + id="tall-strip-height-bound", + ), + pytest.param((200.0, 300.0), (180, 2), id="small-page-width-bound"), + pytest.param((72.0, 72.0), (300, 2), id="tiny-page-capped-at-300"), + pytest.param( + (1000000.0, 1000000.0), + (500 * 72 / 1000000, 2), + id="huge-page-fractional", + ), + pytest.param(None, (150, 1), id="unreadable-geometry-fallback"), + ], + ) + def test_dpi_from_page_size( + self, + mocker: MockerFixture, + tmp_path: Path, + size: tuple[float, float] | None, + expected: tuple[float, int], + ) -> None: + """ + GIVEN: + - A first page of the given size, or unreadable geometry + WHEN: + - The thumbnail DPI is computed + THEN: + - The expected DPI and supersample factor are returned + """ + mocker.patch( + "paperless.parsers.utils.get_pdf_first_page_size_points", + return_value=size, + ) + assert _compute_thumbnail_dpi(tmp_path / "doc.pdf") == pytest.approx(expected) + + +class TestMakeThumbnailFromPdf: + @pytest.fixture + def work_dir(self, tmp_path: Path) -> Path: + path = tmp_path / "work" + path.mkdir() + return path + + @staticmethod + def _write_blank_pdf(path: Path, page_size: tuple[int, int] = (612, 792)) -> Path: + pdf = pikepdf.new() + pdf.add_blank_page(page_size=page_size) + pdf.save(path, object_stream_mode=pikepdf.ObjectStreamMode.disable) + return path + + @pytest.mark.parametrize( + ("size", "expected_dpi", "expected_supersample"), + [ + pytest.param( + (612.0, 792.0), + 2 * 500 * 72 / 612, + 2, + id="known-geometry-2x", + ), + pytest.param(None, 150, 1, id="unreadable-geometry-plain-fallback"), + ], + ) + def test_render_dpi_requested( + self, + mocker: MockerFixture, + tmp_path: Path, + work_dir: Path, + size: tuple[float, float] | None, + expected_dpi: float, + expected_supersample: int, + ) -> None: + """ + GIVEN: + - Readable or unreadable page geometry + WHEN: + - A thumbnail is made + THEN: + - Rasterize and encode get the matching DPI and supersample factor + """ + mocker.patch( + "paperless.parsers.utils.get_pdf_first_page_size_points", + return_value=size, + ) + rasterize = mocker.patch("documents.parsers.rasterize_pdf_page_to_png") + encode = mocker.patch("documents.parsers.encode_thumbnail_webp") + + make_thumbnail_from_pdf(tmp_path / "in.pdf", work_dir) + + assert rasterize.call_args.kwargs["dpi"] == pytest.approx(expected_dpi) + assert encode.call_args.kwargs["supersample"] == expected_supersample + + @pytest.mark.parametrize( + ("page_size", "expected_width"), + [ + pytest.param((612, 792), 500, id="letter"), + pytest.param((792, 612), 500, id="landscape-letter"), + pytest.param((595, 842), 500, id="a4"), + pytest.param((200, 300), 500, id="small-page"), + pytest.param((72, 72), 300, id="tiny-page-capped"), + ], + ) + def test_thumbnail_width( + self, + tmp_path: Path, + work_dir: Path, + page_size: tuple[int, int], + expected_width: int, + ) -> None: + """ + GIVEN: + - A PDF whose first page has the given size in points + WHEN: + - A thumbnail is made from it + THEN: + - The thumbnail has the expected width + """ + pdf_path = self._write_blank_pdf(tmp_path / "in.pdf", page_size) + + thumb = make_thumbnail_from_pdf(pdf_path, work_dir) + + assert thumb == work_dir / "convert.webp" + with Image.open(thumb) as im: + assert im.format == "WEBP" + assert im.width == expected_width + + @classmethod + def _write_pdf_without_xref(cls, path: Path) -> Path: + """ + Cuts off the xref and trailer, which pdftoppm cannot recover from but qpdf can. + """ + cls._write_blank_pdf(path) + data = path.read_bytes() + path.write_bytes(data[: data.rindex(b"\nxref")]) + return path + + def test_qpdf_repair_produces_real_thumbnail( + self, + tmp_path: Path, + work_dir: Path, + ) -> None: + """ + GIVEN: + - A PDF with its xref table and trailer cut off + WHEN: + - A thumbnail is made from it + THEN: + - The thumbnail is rendered from a qpdf repaired copy + - The original file is unchanged + """ + pdf_path = self._write_pdf_without_xref(tmp_path / "broken.pdf") + original_bytes = pdf_path.read_bytes() + + with pytest.raises(ParseError): + rasterize_pdf_page_to_png(pdf_path, work_dir / "probe.png", dpi=50) + + thumb = make_thumbnail_from_pdf(pdf_path, work_dir) + + assert thumb == work_dir / "convert_qpdf.webp" + with Image.open(thumb) as im: + assert im.format == "WEBP" + assert im.width == 500 + assert pdf_path.read_bytes() == original_bytes + + @pytest.mark.parametrize( + "qpdf_error", + [ + pytest.param(subprocess.CalledProcessError(2, "qpdf"), id="qpdf-fails"), + pytest.param(None, id="repaired-still-unrenderable"), + ], + ) + def test_double_failure_uses_default_thumbnail( + self, + mocker: MockerFixture, + tmp_path: Path, + work_dir: Path, + qpdf_error: subprocess.CalledProcessError | None, + ) -> None: + """ + GIVEN: + - A PDF that cannot be rendered, even after qpdf repair + WHEN: + - A thumbnail is made from it + THEN: + - A copy of the default thumbnail is returned + """ + mocker.patch( + "documents.parsers.rasterize_pdf_page_to_png", + side_effect=ParseError("Does not compute."), + ) + if qpdf_error is not None: + mocker.patch("documents.parsers.run_subprocess", side_effect=qpdf_error) + pdf_path = self._write_blank_pdf(tmp_path / "in.pdf") + + thumb = make_thumbnail_from_pdf(pdf_path, work_dir) + + assert thumb == work_dir / "document.webp" + assert thumb.read_bytes() == get_default_thumbnail().read_bytes() + + +class TestRasterizePdfPageToPng: + @staticmethod + def _write_pdf( + path: Path, + *, + crop_box: tuple[float, float, float, float] | None = None, + rotate: int | None = None, + ) -> Path: + pdf = pikepdf.new() + pdf.add_blank_page(page_size=(144, 72)) + pdf.add_blank_page(page_size=(300, 300)) + page = pdf.pages[0] + if crop_box is not None: + page.obj.CropBox = pikepdf.Array(crop_box) + if rotate is not None: + page.obj.Rotate = rotate + pdf.save(path) + return path + + @pytest.mark.parametrize( + ("crop_box", "rotate", "expected_size"), + [ + pytest.param(None, None, (144, 72), id="plain"), + pytest.param(None, 90, (72, 144), id="rotated-90"), + pytest.param((0, 0, 72, 36), None, (72, 36), id="crop-box"), + ], + ) + def test_renders_first_page_to_exact_path( + self, + tmp_path: Path, + crop_box: tuple[float, float, float, float] | None, + rotate: int | None, + expected_size: tuple[int, int], + ) -> None: + """ + GIVEN: + - A two page PDF, first page optionally cropped or rotated + WHEN: + - The first page is rasterized at 72 DPI + THEN: + - Only out_path is written, sized to the first page's crop and rotation + """ + pdf_path = self._write_pdf( + tmp_path / "in.pdf", + crop_box=crop_box, + rotate=rotate, + ) + out_dir = tmp_path / "out" + out_dir.mkdir() + out_path = out_dir / "page1.png" + + rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72) + + assert list(out_dir.iterdir()) == [out_path] + with Image.open(out_path) as im: + assert im.format == "PNG" + assert im.size == expected_size + + def test_failure_raises_parse_error(self, tmp_path: Path) -> None: + """ + GIVEN: + - A file that is not a PDF + WHEN: + - Rasterization is attempted + THEN: + - A ParseError is raised + """ + bad = tmp_path / "bad.pdf" + bad.write_bytes(b"not a pdf") + + with pytest.raises(ParseError): + rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72) + + +class TestEncodeThumbnailWebp: + @pytest.mark.parametrize( + ("mode", "color"), + [ + pytest.param("RGBA", (0, 0, 0, 0), id="rgba"), + pytest.param("LA", (0, 0), id="la"), + ], + ) + def test_alpha_flattened_onto_white( + self, + tmp_path: Path, + mode: str, + color: tuple[int, ...], + ) -> None: + """ + GIVEN: + - A fully transparent PNG with an alpha channel + WHEN: + - It is encoded as a thumbnail + THEN: + - The WebP output is RGB with the transparency flattened to white + """ + png_path = tmp_path / "in.png" + Image.new(mode, (20, 10), color).save(png_path) + out_path = tmp_path / "out.webp" + + encode_thumbnail_webp(png_path, out_path) + + with Image.open(out_path) as im: + assert im.format == "WEBP" + assert im.mode == "RGB" + assert im.size == (20, 10) + red, green, blue = im.getpixel((10, 5)) + assert min(red, green, blue) >= 250 + + @pytest.mark.parametrize( + ("in_size", "supersample", "expected_size"), + [ + pytest.param((1000, 2000), 1, (500, 1000), id="too-wide-shrunk"), + pytest.param((100, 10000), 1, (50, 5000), id="too-tall-shrunk"), + pytest.param((100, 200), 1, (100, 200), id="small-not-enlarged"), + pytest.param((1000, 1400), 2, (500, 700), id="2x-halved"), + pytest.param((1001, 1401), 2, (500, 700), id="2x-odd-rounded"), + pytest.param((600, 800), 2, (300, 400), id="2x-small-not-enlarged"), + pytest.param((900, 1200), 2, (450, 600), id="2x-below-clamp"), + ], + ) + def test_size_clamped_and_downsampled( + self, + tmp_path: Path, + in_size: tuple[int, int], + supersample: int, + expected_size: tuple[int, int], + ) -> None: + """ + GIVEN: + - A rendered image and its supersample factor + WHEN: + - It is encoded as a thumbnail with that factor + THEN: + - It is downsampled, fit within 500x5000 and never enlarged + """ + png_path = tmp_path / "in.png" + Image.new("RGB", in_size, (255, 255, 255)).save(png_path) + out_path = tmp_path / "out.webp" + + encode_thumbnail_webp(png_path, out_path, supersample=supersample) + + with Image.open(out_path) as im: + assert im.size == expected_size + + @pytest.mark.parametrize( + "error", + [ + pytest.param(OSError("broken image"), id="os-error"), + pytest.param(Image.DecompressionBombError("too large"), id="bomb"), + ], + ) + def test_decode_failure_raises_parse_error( + self, + tmp_path: Path, + mocker: MockerFixture, + error: Exception, + ) -> None: + """ + GIVEN: + - Opening the rendered image fails + WHEN: + - It is encoded as a thumbnail + THEN: + - A ParseError is raised + """ + png_path = tmp_path / "in.png" + Image.new("RGB", (10, 10)).save(png_path) + mocker.patch("PIL.Image.open", side_effect=error) + + with pytest.raises(ParseError): + encode_thumbnail_webp(png_path, tmp_path / "out.webp") diff --git a/src/paperless/checks.py b/src/paperless/checks.py index 16bbff55e..8c333eefc 100644 --- a/src/paperless/checks.py +++ b/src/paperless/checks.py @@ -82,7 +82,7 @@ def binaries_check(app_configs: Any, **kwargs: Any) -> list[Error]: error = "Paperless can't find {}. Without it, consumption is impossible." hint = "Either it's not in your ${PATH} or it's not installed." - binaries = (settings.CONVERT_BINARY, "tesseract", "gs") + binaries = (settings.CONVERT_BINARY, "tesseract", "gs", "pdftoppm", "pdfinfo") check_messages = [] for binary in binaries: diff --git a/src/paperless/parsers/utils.py b/src/paperless/parsers/utils.py index 9fe1d4908..b01e1678e 100644 --- a/src/paperless/parsers/utils.py +++ b/src/paperless/parsers/utils.py @@ -10,7 +10,9 @@ from __future__ import annotations import codecs import logging +import os import re +import subprocess import tempfile from pathlib import Path from typing import TYPE_CHECKING @@ -25,6 +27,15 @@ logger = logging.getLogger("paperless.parsers.utils") # Used by both the consumer (archive decision) and the tesseract parser (skip-OCR decision). PDF_TEXT_MIN_LENGTH: Final[int] = 50 +_PDFINFO_PAGE_SIZE: Final[re.Pattern[str]] = re.compile( + r"^Page\s+1 size:\s+([\d.eE+-]+) x ([\d.eE+-]+) pts", + re.MULTILINE, +) +_PDFINFO_PAGE_ROTATION: Final[re.Pattern[str]] = re.compile( + r"^Page\s+1 rot:\s+(\d+)", + re.MULTILINE, +) + def is_tagged_pdf( path: Path, @@ -265,6 +276,61 @@ def get_page_count_for_pdf( return None +def get_pdf_first_page_size_points( + path: Path, + log: logging.Logger | None = None, +) -> tuple[float, float] | None: + """Return the first page's (width, height) in PDF points, post-rotation. + + Asks ``pdfinfo`` rather than reading the page boxes directly, so the size + matches what ``pdftoppm -cropbox`` renders, including poppler's clipping of + the CropBox to the MediaBox. pdfinfo reports the box size before rotation, + so width and height are swapped for 90/270 rotation. + + Parameters + ---------- + path: + Absolute path to the PDF file. + log: + Logger for warnings. Falls back to the module-level logger when omitted. + + Returns + ------- + tuple[float, float] | None + ``(width_points, height_points)``, or ``None`` if pdfinfo fails, the + file has no pages, or the page size is degenerate. + """ + _log = log or logger + try: + proc = subprocess.run( + ["pdfinfo", "-f", "1", "-l", "1", str(path)], + capture_output=True, + check=True, + text=True, + errors="ignore", + # The labels parsed below are localized + env={**os.environ, "LC_ALL": "C"}, + ) + except (OSError, subprocess.CalledProcessError) as e: + _log.warning("Could not determine PDF page size for %s: %s", path, e) + return None + + size_match = _PDFINFO_PAGE_SIZE.search(proc.stdout) + if size_match is None: + _log.warning("Could not determine PDF page size for %s", path) + return None + + width = float(size_match.group(1)) + height = float(size_match.group(2)) + if width <= 0 or height <= 0: + return None + + rotation_match = _PDFINFO_PAGE_ROTATION.search(proc.stdout) + if rotation_match is not None and int(rotation_match.group(1)) in (90, 270): + width, height = height, width + return width, height + + def extract_pdf_metadata( document_path: Path, log: logging.Logger | None = None, diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py index 004b23feb..23e700461 100644 --- a/src/paperless/settings/__init__.py +++ b/src/paperless/settings/__init__.py @@ -978,10 +978,6 @@ GNUPG_HOME = os.getenv("HOME", "/tmp") # Convert is part of the ImageMagick package CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert") -CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR") -CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT") - -GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs") # Fallback layout for .eml consumption EMAIL_PARSE_DEFAULT_LAYOUT = get_int_from_env( diff --git a/src/paperless/tests/parsers/test_tesseract_parser.py b/src/paperless/tests/parsers/test_tesseract_parser.py index bb6a85031..f057ad086 100644 --- a/src/paperless/tests/parsers/test_tesseract_parser.py +++ b/src/paperless/tests/parsers/test_tesseract_parser.py @@ -15,9 +15,10 @@ from typing import TYPE_CHECKING import pytest from ocrmypdf import SubprocessOutputError +from PIL import Image from documents.parsers import ParseError -from documents.parsers import run_convert +from documents.parsers import rasterize_pdf_page_to_png from paperless.models import ModeChoices from paperless.parsers import ParserProtocol from paperless.parsers.tesseract import RasterisedDocumentParser @@ -280,24 +281,71 @@ class TestGetThumbnail: ) assert thumb.is_file() - def test_thumbnail_fallback_on_convert_error( + @pytest.mark.parametrize( + ("filename", "expected_height"), + [ + pytest.param("simple-digital.pdf", 647, id="portrait-letter"), + pytest.param("rotated.pdf", 386, id="landscape"), + ], + ) + def test_thumbnail_is_correct_format_and_size( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + filename: str, + expected_height: int, + ) -> None: + """ + GIVEN: + - A portrait or landscape PDF + WHEN: + - A thumbnail is generated + THEN: + - A 500px wide WebP keeping the page's aspect ratio + """ + thumb = tesseract_parser.get_thumbnail( + tesseract_samples_dir / filename, + "application/pdf", + ) + with Image.open(thumb) as im: + assert im.format == "WEBP" + assert im.width == 500 + assert im.height == pytest.approx(expected_height, abs=2) + + def test_thumbnail_fallback_on_pdftoppm_error( self, mocker: MockerFixture, tesseract_parser: RasterisedDocumentParser, tesseract_samples_dir: Path, ) -> None: - def _raise_on_pdf(input_file, output_file, **kwargs) -> None: - if ".pdf" in str(input_file): + """ + GIVEN: + - Rasterizing the original PDF fails + WHEN: + - A thumbnail is generated + THEN: + - The PDF is repaired with qpdf and a real thumbnail is rendered + """ + original = tesseract_samples_dir / "simple-digital.pdf" + + def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None: + if in_path == original: raise ParseError("Does not compute.") - run_convert(input_file=input_file, output_file=output_file, **kwargs) + rasterize_pdf_page_to_png(in_path, out_path, **kwargs) - mocker.patch("documents.parsers.run_convert", side_effect=_raise_on_pdf) - - thumb = tesseract_parser.get_thumbnail( - tesseract_samples_dir / "simple-digital.pdf", - "application/pdf", + rasterize = mocker.patch( + "documents.parsers.rasterize_pdf_page_to_png", + side_effect=_fail_on_original, ) + + thumb = tesseract_parser.get_thumbnail(original, "application/pdf") + + assert rasterize.call_count == 2 assert thumb.is_file() + assert thumb.name == "convert_qpdf.webp" + with Image.open(thumb) as im: + assert im.format == "WEBP" + assert im.width == 500 def test_thumbnail_encrypted_pdf( self, diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py index 4d8a5bfa6..0ea129adf 100644 --- a/src/paperless/tests/test_checks.py +++ b/src/paperless/tests/test_checks.py @@ -1,4 +1,5 @@ import os +import shutil from collections.abc import Callable from pathlib import Path from unittest import mock @@ -29,6 +30,19 @@ class TestChecks: settings.CONVERT_BINARY = "uuuhh" assert len(binaries_check(None)) == 1 + @pytest.mark.parametrize("binary", ["pdftoppm", "pdfinfo"]) + def test_binaries_fail_poppler(self, mocker: MockerFixture, binary: str) -> None: + real_which = shutil.which + mocker.patch( + "paperless.checks.shutil.which", + side_effect=lambda name: None if name == binary else real_which(name), + ) + + msgs = binaries_check(None) + + assert len(msgs) == 1 + assert binary in msgs[0].msg + @pytest.mark.usefixtures("paperless_dirs") def test_paths_check(self) -> None: assert paths_check(None) == [] diff --git a/src/paperless/tests/test_parser_utils.py b/src/paperless/tests/test_parser_utils.py index f0a63ea44..7bae1f78d 100644 --- a/src/paperless/tests/test_parser_utils.py +++ b/src/paperless/tests/test_parser_utils.py @@ -6,8 +6,10 @@ import codecs from pathlib import Path from typing import TYPE_CHECKING +import pikepdf import pytest +from paperless.parsers.utils import get_pdf_first_page_size_points from paperless.parsers.utils import is_tagged_pdf from paperless.parsers.utils import pdf_born_digital_text from paperless.parsers.utils import post_process_text @@ -70,6 +72,197 @@ class TestIsTaggedPdf: assert is_tagged_pdf(bad) is False +class TestGetPdfFirstPageSizePoints: + @staticmethod + def _write_pdf( + path: Path, + *, + media_box: tuple[float, float, float, float] = (0, 0, 600, 800), + crop_box: tuple[float, float, float, float] | None = None, + page_rotate: int | None = None, + inherited_rotate: int | None = None, + ) -> Path: + pdf = pikepdf.new() + pdf.add_blank_page(page_size=(media_box[2], media_box[3])) + page = pdf.pages[0] + page.obj.MediaBox = pikepdf.Array(media_box) + if crop_box is not None: + page.obj.CropBox = pikepdf.Array(crop_box) + if page_rotate is not None: + page.obj.Rotate = page_rotate + if inherited_rotate is not None: + pdf.Root.Pages.Rotate = inherited_rotate + pdf.save(path) + return path + + def test_letter_sample(self) -> None: + """ + GIVEN: + - A US Letter sample PDF with no CropBox and no rotation + WHEN: + - The first page size is requested + THEN: + - The MediaBox size in points is returned + """ + assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") == ( + 612.0, + 792.0, + ) + + @pytest.mark.parametrize( + ("rotate", "expected"), + [ + pytest.param(0, (600.0, 800.0), id="rotate-0"), + pytest.param(90, (800.0, 600.0), id="rotate-90"), + pytest.param(180, (600.0, 800.0), id="rotate-180"), + pytest.param(270, (800.0, 600.0), id="rotate-270"), + pytest.param(-90, (800.0, 600.0), id="rotate-negative-90"), + ], + ) + def test_page_rotation_swaps_dimensions( + self, + tmp_path: Path, + rotate: int, + expected: tuple[float, float], + ) -> None: + """ + GIVEN: + - A portrait PDF page with /Rotate set directly on the page + WHEN: + - The first page size is requested + THEN: + - Width and height are swapped for quarter-turn rotations only + """ + pdf_path = self._write_pdf(tmp_path / "rotated.pdf", page_rotate=rotate) + assert get_pdf_first_page_size_points(pdf_path) == expected + + def test_inherited_rotation_swaps_dimensions(self, tmp_path: Path) -> None: + """ + GIVEN: + - A page inheriting /Rotate 90 from the /Pages node + WHEN: + - The first page size is requested + THEN: + - Width and height are swapped + """ + pdf_path = self._write_pdf(tmp_path / "inherited.pdf", inherited_rotate=90) + assert get_pdf_first_page_size_points(pdf_path) == (800.0, 600.0) + + def test_crop_box_preferred_over_media_box(self, tmp_path: Path) -> None: + """ + GIVEN: + - A PDF page with a CropBox smaller than its MediaBox + WHEN: + - The first page size is requested + THEN: + - The CropBox size is returned + """ + pdf_path = self._write_pdf( + tmp_path / "cropped.pdf", + crop_box=(50, 100, 350, 500), + ) + assert get_pdf_first_page_size_points(pdf_path) == (300.0, 400.0) + + def test_crop_box_clipped_to_media_box(self, tmp_path: Path) -> None: + """ + GIVEN: + - A PDF page whose CropBox extends well past its MediaBox + WHEN: + - The first page size is requested + THEN: + - The size is the CropBox clipped to the MediaBox, as pdftoppm renders it + """ + pdf_path = self._write_pdf( + tmp_path / "oversized-crop.pdf", + media_box=(0, 0, 200, 200), + crop_box=(-200, -200, 600, 600), + ) + assert get_pdf_first_page_size_points(pdf_path) == (200.0, 200.0) + + def test_crop_box_with_rotation(self, tmp_path: Path) -> None: + """ + GIVEN: + - A rotated PDF page with a CropBox smaller than its MediaBox + WHEN: + - The first page size is requested + THEN: + - The CropBox size is returned with width and height swapped + """ + pdf_path = self._write_pdf( + tmp_path / "cropped-rotated.pdf", + media_box=(0, 0, 600, 300), + crop_box=(100, 50, 400, 250), + page_rotate=270, + ) + assert get_pdf_first_page_size_points(pdf_path) == (200.0, 300.0) + + def test_pdfinfo_not_installed_returns_none( + self, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: + - The pdfinfo binary cannot be executed + WHEN: + - The first page size is requested + THEN: + - None is returned and nothing is raised + """ + mocker.patch("subprocess.run", side_effect=FileNotFoundError("pdfinfo")) + assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") is None + + def test_degenerate_box_returns_none(self, tmp_path: Path) -> None: + """ + GIVEN: + - A PDF page whose box has zero width + WHEN: + - The first page size is requested + THEN: + - None is returned + """ + pdf_path = self._write_pdf( + tmp_path / "degenerate.pdf", + media_box=(0, 0, 600, 800), + crop_box=(100, 0, 100, 800), + ) + assert get_pdf_first_page_size_points(pdf_path) is None + + def test_nonexistent_path_returns_none(self) -> None: + """ + GIVEN: + - A path that does not exist + WHEN: + - The first page size is requested + THEN: + - None is returned and nothing is raised + """ + assert get_pdf_first_page_size_points(Path("/nonexistent/file.pdf")) is None + + def test_corrupt_pdf_returns_none(self, tmp_path: Path) -> None: + """ + GIVEN: + - A file that is not a PDF + WHEN: + - The first page size is requested + THEN: + - None is returned and nothing is raised + """ + bad = tmp_path / "bad.pdf" + bad.write_bytes(b"not a pdf") + assert get_pdf_first_page_size_points(bad) is None + + def test_encrypted_pdf_returns_none(self) -> None: + """ + GIVEN: + - A password protected PDF + WHEN: + - The first page size is requested + THEN: + - None is returned and nothing is raised + """ + assert get_pdf_first_page_size_points(SAMPLES / "encrypted.pdf") is None + + class TestPostProcessText: @pytest.mark.parametrize( ("source", "expected"), From c089bba32ce566c21cde2d253b849547e2840d47 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:22:43 -0700 Subject: [PATCH 4/5] Chore: Drop ImageMagik as a dependency (#14440) * Mark the thumbnail constants Final * Comment each pdftoppm option * Feature/Chore: Replaces all usages of ImageMagik's convert binary with a simple Pillow based helper This removes ImageMagik as a dependency entirly and all documentation is cleaned up from it. We no longer need to install it, have a policy for it to allow PDFs and no longer shell out to it either. * Fix: Save WebP images losslessly when removing the alpha channel Pillow's default WebP save is lossy, so a WebP with an alpha channel came back from remove_alpha_channel with slightly shifted pixel values before it reached OCR. Save WebP lossless; other formats are unchanged. Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --- .devcontainer/Dockerfile | 16 --- .github/workflows/ci-backend.yml | 5 +- Dockerfile | 3 - .../etc/ImageMagick-6/paperless-policy.xml | 101 ------------------ docs/configuration.md | 5 +- docs/setup.md | 10 +- paperless.conf.example | 4 - src/documents/converters.py | 13 +-- src/documents/tests/test_utils.py | 101 ++++++++++++++++++ src/documents/utils.py | 28 +++++ src/paperless/checks.py | 2 +- src/paperless/parsers/tesseract.py | 15 +-- src/paperless/settings/__init__.py | 3 - src/paperless/tests/test_checks.py | 14 ++- 14 files changed, 151 insertions(+), 169 deletions(-) delete mode 100644 docker/rootfs/etc/ImageMagick-6/paperless-policy.xml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c5bd4ec6c..93d0881d4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -39,7 +39,6 @@ ARG RUNTIME_PACKAGES="\ ghostscript \ gnupg \ icc-profiles-free \ - imagemagick \ # PostgreSQL postgresql-client \ # MySQL / MariaDB @@ -93,21 +92,6 @@ RUN set -eux \ https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \ && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb -# setup docker-specific things -# These change sometimes, but rarely -WORKDIR /usr/src/paperless/src/docker/ - -COPY [ \ - "docker/rootfs/etc/ImageMagick-6/paperless-policy.xml", \ - "./" \ -] - -RUN set -eux \ - && echo "Configuring ImageMagick" \ - && mkdir -p /etc/ImageMagick-6 \ - && mv paperless-policy.xml /etc/ImageMagick-6/policy.xml - - # Packages needed only for building a few quick Python # dependencies ARG BUILD_PACKAGES="\ diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 331168ba2..f24308639 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -111,10 +111,7 @@ jobs: timeout-minutes: 12 uses: $/.github/actions/apt-install with: - packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils qpdf - - name: Configure ImageMagick - run: | - sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml + packages: unpaper tesseract-ocr ghostscript poppler-utils qpdf - name: Install Python dependencies env: PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }} diff --git a/Dockerfile b/Dockerfile index 50a15aa81..83108bf79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -129,7 +129,6 @@ ARG RUNTIME_PACKAGES="\ ghostscript \ gnupg \ icc-profiles-free \ - imagemagick \ # PostgreSQL postgresql-client \ # MySQL / MariaDB @@ -167,8 +166,6 @@ RUN set -eux \ https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \ && echo "Installing jbig2enc" \ && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \ - && echo "Configuring imagemagick" \ - && cp /etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml \ && echo "Cleaning up image layer" \ && rm --force --verbose *.deb \ && rm --recursive --force --verbose /var/lib/apt/lists/* \ diff --git a/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml b/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml deleted file mode 100644 index 7943ca56e..000000000 --- a/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml +++ /dev/null @@ -1,101 +0,0 @@ - - - - - -]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/configuration.md b/docs/configuration.md index a6f8afa89..0a6ce1c3b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1873,7 +1873,10 @@ the literal path for that program. #### [`PAPERLESS_CONVERT_BINARY=`](#PAPERLESS_CONVERT_BINARY) {#PAPERLESS_CONVERT_BINARY} -: Defaults to "convert". +!!! warning + + Deprecated and has no effect, since Paperless-ngx no longer uses + ImageMagick. It will be removed in a future release. #### [`PAPERLESS_GS_BINARY=`](#PAPERLESS_GS_BINARY) {#PAPERLESS_GS_BINARY} diff --git a/docs/setup.md b/docs/setup.md index 514cf424c..d3905f50d 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -177,7 +177,6 @@ to a positive number to enable polling and disable native filesystem notificatio - `pkg-config` for mysqlclient (python dependency) - `fonts-liberation` for generating thumbnails for plain text files - - `imagemagick` >= 6 for image alpha handling - `gnupg` for decrypting GPG-encrypted email - `libpq-dev` for PostgreSQL - `libmagic-dev` for mime type detection @@ -187,7 +186,7 @@ to a positive number to enable polling and disable native filesystem notificatio Use this list for your preferred package management: ``` - python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils + python3 python3-pip python3-dev fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils ``` These dependencies are required for OCRmyPDF, which is used for text @@ -416,13 +415,6 @@ to a positive number to enable polling and disable native filesystem notificatio You may need to change the path in the files. Example: `ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO` -12. Harden ImageMagick by disabling formats that Paperless-ngx does not use. - PDF processing is not needed and should stay disabled. - - Configure the active ImageMagick policy file (commonly - `/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and - adjust similar to [the docker policy file](https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/refs/heads/main/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml). You should also include restrictions as noted there. - **Optional: Install the [jbig2enc](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html) encoder.** This will reduce the size of generated PDF documents. You'll most likely need to compile this yourself, because this software has been patented until around 2017 and binary packages are not available for most distributions. diff --git a/paperless.conf.example b/paperless.conf.example index 7e6960c82..80c0608de 100644 --- a/paperless.conf.example +++ b/paperless.conf.example @@ -85,7 +85,3 @@ PAPERLESS_SECRET_KEY=change-me #PAPERLESS_TIKA_ENABLED=false #PAPERLESS_TIKA_ENDPOINT=http://localhost:9998 #PAPERLESS_TIKA_GOTENBERG_ENDPOINT=http://localhost:3000 - -# Binaries - -#PAPERLESS_CONVERT_BINARY=/usr/bin/convert diff --git a/src/documents/converters.py b/src/documents/converters.py index fcd0a8857..76f518d7e 100644 --- a/src/documents/converters.py +++ b/src/documents/converters.py @@ -1,12 +1,11 @@ from pathlib import Path import img2pdf -from django.conf import settings from PIL import Image from documents.utils import copy_basic_file_stats from documents.utils import maybe_override_pixel_limit -from documents.utils import run_subprocess +from documents.utils import remove_alpha_channel def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path: @@ -27,15 +26,7 @@ def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path: # Note the save into the temp folder, so as not to trigger a new # consume scratch_image = target_directory / tiff_path.name - run_subprocess( - [ - settings.CONVERT_BINARY, - "-alpha", - "off", - tiff_path, - scratch_image, - ], - ) + remove_alpha_channel(tiff_path, scratch_image) else: # Not modifying the original, safe to use in place scratch_image = tiff_path diff --git a/src/documents/tests/test_utils.py b/src/documents/tests/test_utils.py index af45880ee..d34a8f8a7 100644 --- a/src/documents/tests/test_utils.py +++ b/src/documents/tests/test_utils.py @@ -1,6 +1,11 @@ +from pathlib import Path + +import pytest import pytest_mock +from PIL import Image from documents.utils import QuerySetStream +from documents.utils import remove_alpha_channel class TestQuerySetStream: @@ -33,3 +38,99 @@ class TestQuerySetStream: # the thing being verified here. mock_queryset.count.assert_called_with() mock_queryset.iterator.assert_called_once_with(chunk_size=1000) + + +class TestRemoveAlphaChannel: + @pytest.mark.parametrize( + ("image_format", "mode", "expected_mode", "color", "expected_pixel"), + [ + pytest.param( + "PNG", + "RGBA", + "RGB", + (10, 20, 30, 0), + (10, 20, 30), + id="png-rgba", + ), + pytest.param( + "TIFF", + "RGBA", + "RGB", + (10, 20, 30, 128), + (10, 20, 30), + id="tiff-rgba", + ), + pytest.param("PNG", "LA", "L", (40, 0), 40, id="png-la"), + ], + ) + def test_drops_alpha_and_keeps_format( + self, + tmp_path: Path, + image_format: str, + mode: str, + expected_mode: str, + color: tuple[int, ...], + expected_pixel: int | tuple[int, ...], + ) -> None: + """ + GIVEN: + - An image with an alpha channel + WHEN: + - The alpha channel is removed to a destination without an extension + THEN: + - The destination has the matching opaque mode, the same size and + format, and the color values are kept rather than composited + """ + source = tmp_path / "source" + Image.new(mode, (8, 6), color).save(source, format=image_format) + dest = tmp_path / "no-alpha" + + result = remove_alpha_channel(source, dest) + + assert result == dest + with Image.open(dest) as out: + assert out.mode == expected_mode + assert out.size == (8, 6) + assert out.format == image_format + assert out.getpixel((0, 0)) == expected_pixel + + def test_keeps_every_frame_of_a_multi_page_tiff(self, tmp_path: Path) -> None: + """ + GIVEN: + - A multi-page TIFF where every page has an alpha channel + WHEN: + - The alpha channel is removed + THEN: + - All pages are kept and none has an alpha channel + """ + source = tmp_path / "multi.tiff" + first = Image.new("RGBA", (8, 6), (255, 0, 0, 50)) + rest = [Image.new("RGBA", (8, 6), (0, 255, 0, 50)) for _ in range(2)] + first.save(source, save_all=True, append_images=rest) + dest = tmp_path / "multi-no-alpha.tiff" + + remove_alpha_channel(source, dest) + + with Image.open(dest) as out: + assert out.n_frames == 3 + for index in range(out.n_frames): + out.seek(index) + assert out.mode == "RGB" + + def test_keeps_dpi(self, tmp_path: Path) -> None: + """ + GIVEN: + - An alpha image with a DPI set + WHEN: + - The alpha channel is removed + THEN: + - The DPI is unchanged, since OCR scaling depends on it + """ + source = tmp_path / "source.png" + Image.new("RGBA", (8, 6)).save(source, dpi=(300, 300)) + dest = tmp_path / "dest.png" + + remove_alpha_channel(source, dest) + + with Image.open(dest) as out: + assert round(out.info["dpi"][0]) == 300 diff --git a/src/documents/utils.py b/src/documents/utils.py index d6e505aa4..4d0b5811c 100644 --- a/src/documents/utils.py +++ b/src/documents/utils.py @@ -9,11 +9,13 @@ from pathlib import Path from subprocess import CompletedProcess from subprocess import run from typing import TYPE_CHECKING +from typing import Any from typing import Generic from typing import TypeVar from django.conf import settings from PIL import Image +from PIL import ImageSequence if TYPE_CHECKING: from django.db.models import Model @@ -128,6 +130,32 @@ def maybe_override_pixel_limit() -> None: Image.MAX_IMAGE_PIXELS = pixel_count +def remove_alpha_channel(source: Path, dest: Path) -> Path: + """ + Writes a copy of the image with its alpha channel dropped, keeping the + format, every frame, the DPI and any ICC profile. + + The channel is dropped, not composited, so transparent pixels keep their + underlying color. Returns dest. + """ + with Image.open(source) as im: + image_format = im.format + opaque_mode = "L" if im.mode == "LA" else "RGB" + frames = [frame.convert(opaque_mode) for frame in ImageSequence.Iterator(im)] + save_kwargs: dict[str, Any] = {} + if "dpi" in im.info: + save_kwargs["dpi"] = im.info["dpi"] + if "icc_profile" in im.info: + save_kwargs["icc_profile"] = im.info["icc_profile"] + if image_format == "WEBP": + save_kwargs["lossless"] = True + if len(frames) > 1: + save_kwargs["save_all"] = True + save_kwargs["append_images"] = frames[1:] + frames[0].save(dest, format=image_format, **save_kwargs) + return dest + + def run_subprocess( arguments: list[str], env: dict[str, str] | None = None, diff --git a/src/paperless/checks.py b/src/paperless/checks.py index 8c333eefc..870f1bf63 100644 --- a/src/paperless/checks.py +++ b/src/paperless/checks.py @@ -82,7 +82,7 @@ def binaries_check(app_configs: Any, **kwargs: Any) -> list[Error]: error = "Paperless can't find {}. Without it, consumption is impossible." hint = "Either it's not in your ${PATH} or it's not installed." - binaries = (settings.CONVERT_BINARY, "tesseract", "gs", "pdftoppm", "pdfinfo") + binaries = ("tesseract", "gs", "pdftoppm", "pdfinfo") check_messages = [] for binary in binaries: diff --git a/src/paperless/parsers/tesseract.py b/src/paperless/parsers/tesseract.py index 1091dffda..5a3b77f88 100644 --- a/src/paperless/parsers/tesseract.py +++ b/src/paperless/parsers/tesseract.py @@ -19,7 +19,7 @@ from documents.parsers import ParseError from documents.parsers import make_thumbnail_from_pdf from documents.utils import copy_file_with_basic_stats from documents.utils import maybe_override_pixel_limit -from documents.utils import run_subprocess +from documents.utils import remove_alpha_channel from paperless.config import OcrConfig from paperless.models import CleanChoices from paperless.models import ModeChoices @@ -197,18 +197,7 @@ class RasterisedDocumentParser: return im.mode in ("RGBA", "LA") def remove_alpha(self, image_path: Path) -> Path: - no_alpha_image = Path(self.tempdir) / "image-no-alpha" - run_subprocess( - [ - settings.CONVERT_BINARY, - "-alpha", - "off", - str(image_path), - str(no_alpha_image), - ], - logger=self.log, - ) - return no_alpha_image + return remove_alpha_channel(image_path, Path(self.tempdir) / "image-no-alpha") def get_dpi(self, image: Path) -> int | None: try: diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py index 23e700461..c17aa1e38 100644 --- a/src/paperless/settings/__init__.py +++ b/src/paperless/settings/__init__.py @@ -976,9 +976,6 @@ MAX_IMAGE_PIXELS: Final[int | None] = get_int_from_env( # GNUPG needs a home directory for some reason GNUPG_HOME = os.getenv("HOME", "/tmp") -# Convert is part of the ImageMagick package -CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert") - # Fallback layout for .eml consumption EMAIL_PARSE_DEFAULT_LAYOUT = get_int_from_env( "PAPERLESS_EMAIL_PARSE_DEFAULT_LAYOUT", diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py index 0ea129adf..88ef36046 100644 --- a/src/paperless/tests/test_checks.py +++ b/src/paperless/tests/test_checks.py @@ -26,9 +26,17 @@ class TestChecks: def test_binaries(self) -> None: assert binaries_check(None) == [] - def test_binaries_fail(self, settings: Settings) -> None: - settings.CONVERT_BINARY = "uuuhh" - assert len(binaries_check(None)) == 1 + def test_binaries_fail(self, mocker: MockerFixture) -> None: + """ + GIVEN: + - None of tesseract, ghostscript, pdftoppm or pdfinfo can be found + WHEN: + - The binaries check runs + THEN: + - One warning is raised per missing binary + """ + mocker.patch("paperless.checks.shutil.which", return_value=None) + assert len(binaries_check(None)) == 4 @pytest.mark.parametrize("binary", ["pdftoppm", "pdfinfo"]) def test_binaries_fail_poppler(self, mocker: MockerFixture, binary: str) -> None: From 040ceb8503d5763ea2e5a90b55597b88cf13bf3a Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:16:05 -0700 Subject: [PATCH 5/5] Fix: standardize some dropdown sizing (#14510) --- .../app/components/chat/chat/chat.component.html | 4 ++-- .../dates-dropdown/dates-dropdown.component.html | 4 ++-- .../filterable-dropdown.component.html | 6 +++--- .../filterable-dropdown.component.scss | 2 +- .../toggleable-dropdown-button.component.html | 2 +- .../toggleable-dropdown-button.component.scss | 5 +++++ .../permissions-filter-dropdown.component.html | 14 +++++++------- .../document-detail/document-detail.component.html | 14 +++++++------- .../bulk-editor/bulk-editor.component.html | 14 +++++++------- src-ui/src/styles.scss | 1 + 10 files changed, 36 insertions(+), 30 deletions(-) diff --git a/src-ui/src/app/components/chat/chat/chat.component.html b/src-ui/src/app/components/chat/chat/chat.component.html index eb4f8cb01..c49864e74 100644 --- a/src-ui/src/app/components/chat/chat/chat.component.html +++ b/src-ui/src/app/components/chat/chat/chat.component.html @@ -6,7 +6,7 @@
- {{ownerExclusionFilterLabel}} + {{ownerExclusionFilterLabel}}
@if (userIsOwner && (requiresPassword || password)) { }
@@ -75,11 +75,11 @@
@if (emailEnabled) { }
diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html index df9d5c299..610ca9a96 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.html @@ -87,16 +87,16 @@
@@ -116,12 +116,12 @@
@if (permissionService.currentUserCan(PermissionAction.Add, PermissionType.ShareLinkBundle)) { } @if (permissionService.currentUserCan(PermissionAction.View, PermissionType.ShareLinkBundle)) { } @if ( @@ -133,7 +133,7 @@ } @if (emailEnabled) { }
diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index c135d343b..ab64a9cea 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -578,6 +578,7 @@ ul.pagination { } .dropdown-menu { + --bs-dropdown-font-size: var(--pngx-body-font-size); background-color: var(--bs-body-bg); .dropdown-divider {