amorfo77/paperless-ngx
1
0 Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-10-13 19:47:13 +00:00
Code Issues Projects Releases Wiki Activity

Feature/Chore: Replaces all usages of ImageMagik's convert binary with a simple Pillow based helper

Browse Source 
This removes ImageMagik as a dependency entirly and all documentation is cleaned up from it.
We no longer need to install it, have a policy for it to allow PDFs and no longer shell
out to it either.
This commit is contained in:
stumpylog committed 2026-10-13 10:24:44 -07:00
1 parent 4c5927a45f
commit ea08908879
14 files changed
+149 -166
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.devcontainer/Dockerfile
-16
View File
@@ -39,7 +39,6 @@ ARG RUNTIME_PACKAGES="\
ghostscript \
gnupg \
icc-profiles-free \
imagemagick \
# PostgreSQL
postgresql-client \
# MySQL / MariaDB
@@ -93,21 +92,6 @@ RUN set -eux \
https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb
# setup docker-specific things
# These change sometimes, but rarely
WORKDIR /usr/src/paperless/src/docker/
COPY [ \
"docker/rootfs/etc/ImageMagick-6/paperless-policy.xml", \
"./" \
]
RUN set -eux \
&& echo "Configuring ImageMagick" \
&& mkdir -p /etc/ImageMagick-6 \
&& mv paperless-policy.xml /etc/ImageMagick-6/policy.xml
# Packages needed only for building a few quick Python
# dependencies
ARG BUILD_PACKAGES="\
.github/workflows/ci-backend.yml
+1 -4
View File
@@ -111,10 +111,7 @@ jobs:
timeout-minutes: 12
uses: $/.github/actions/apt-install
with:
packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils qpdf
- name: Configure ImageMagick
run: |
sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml
packages: unpaper tesseract-ocr ghostscript poppler-utils qpdf
- name: Install Python dependencies
env:
PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}
Dockerfile
-3
View File
@@ -129,7 +129,6 @@ ARG RUNTIME_PACKAGES="\
ghostscript \
gnupg \
icc-profiles-free \
imagemagick \
# PostgreSQL
postgresql-client \
# MySQL / MariaDB
@@ -167,8 +166,6 @@ RUN set -eux \
https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& echo "Installing jbig2enc" \
&& dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& echo "Configuring imagemagick" \
&& cp /etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml \
&& echo "Cleaning up image layer" \
&& rm --force --verbose *.deb \
&& rm --recursive --force --verbose /var/lib/apt/lists/* \
docker/rootfs/etc/ImageMagick-6/paperless-policy.xml
-101
View File
@@ -1,101 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE policymap [
<!ELEMENT policymap (policy)+>
<!ATTLIST policymap xmlns CDATA #FIXED ''>
<!ELEMENT policy EMPTY>
<!ATTLIST policy xmlns CDATA #FIXED '' domain NMTOKEN #REQUIRED
name NMTOKEN #IMPLIED pattern CDATA #IMPLIED rights NMTOKEN #IMPLIED
stealth NMTOKEN #IMPLIED value CDATA #IMPLIED>
]>
<!--
Configure ImageMagick policies.
Domains include system, delegate, coder, filter, path, or resource.
Rights include none, read, write, execute and all. Use | to combine them,
for example: "read | write" to permit read from, or write to, a path.
Use a glob expression as a pattern.
Suppose we do not want users to process MPEG video images:
<policy domain="delegate" rights="none" pattern="mpeg:decode" />
Here we do not want users reading images from HTTP:
<policy domain="coder" rights="none" pattern="HTTP" />
The /repository file system is restricted to read only. We use a glob
expression to match all paths that start with /repository:
<policy domain="path" rights="read" pattern="/repository/*" />
Lets prevent users from executing any image filters:
<policy domain="filter" rights="none" pattern="*" />
Any large image is cached to disk rather than memory:
<policy domain="resource" name="area" value="1GP"/>
Define arguments for the memory, map, area, width, height and disk resources
with SI prefixes (.e.g 100MB). In addition, resource policies are maximums
for each instance of ImageMagick (e.g. policy memory limit 1GB, -limit 2GB
exceeds policy maximum so memory limit is 1GB).
Rules are processed in order. Here we want to restrict ImageMagick to only
read or write a small subset of proven web-safe image types:
<policy domain="delegate" rights="none" pattern="*" />
<policy domain="filter" rights="none" pattern="*" />
<policy domain="coder" rights="none" pattern="*" />
<policy domain="coder" rights="read|write" pattern="{GIF,JPEG,PNG,WEBP}" />
-->
<policymap>
<!-- <policy domain="system" name="shred" value="2"/> -->
<!-- <policy domain="system" name="precision" value="6"/> -->
<!-- <policy domain="system" name="memory-map" value="anonymous"/> -->
<!-- <policy domain="system" name="max-memory-request" value="256MiB"/> -->
<!-- <policy domain="resource" name="temporary-path" value="/tmp"/> -->
<policy domain="resource" name="memory" value="256MiB"/>
<policy domain="resource" name="map" value="512MiB"/>
<policy domain="resource" name="width" value="16KP"/>
<policy domain="resource" name="height" value="16KP"/>
<!-- <policy domain="resource" name="list-length" value="128"/> -->
<policy domain="resource" name="area" value="128MB"/>
<policy domain="resource" name="disk" value="1GiB"/>
<!-- <policy domain="resource" name="file" value="768"/> -->
<!-- <policy domain="resource" name="thread" value="4"/> -->
<!-- <policy domain="resource" name="throttle" value="0"/> -->
<!-- <policy domain="resource" name="time" value="3600"/> -->
<!-- Paperless does not process SVG or ImageMagick scripting formats. -->
<policy domain="coder" rights="none" pattern="SVG" />
<policy domain="coder" rights="none" pattern="SVGZ" />
<policy domain="coder" rights="none" pattern="MSVG" />
<policy domain="coder" rights="none" pattern="RSVG" />
<policy domain="coder" rights="none" pattern="MSL" />
<policy domain="coder" rights="none" pattern="MVG" />
<policy domain="coder" rights="none" pattern="EPHEMERAL" />
<!-- <policy domain="module" rights="none" pattern="{PS,PDF,XPS}" /> -->
<!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> -->
<!-- <policy domain="path" rights="none" pattern="@*" /> -->
<!-- <policy domain="cache" name="memory-map" value="anonymous"/> -->
<!-- <policy domain="cache" name="synchronize" value="True"/> -->
<!-- <policy domain="cache" name="shared-secret" value="passphrase" stealth="true"/> -->
<!-- <policy domain="system" name="pixel-cache-memory" value="anonymous"/> -->
<!-- <policy domain="system" name="shred" value="2"/> -->
<!-- <policy domain="system" name="precision" value="6"/> -->
<!-- use curl -->
<policy domain="delegate" rights="none" pattern="URL" />
<policy domain="delegate" rights="none" pattern="HTTPS" />
<policy domain="delegate" rights="none" pattern="HTTP" />
<!-- in order to avoid to get image with password text -->
<policy domain="path" rights="none" pattern="@*"/>
<!-- disable ghostscript format types -->
<policy domain="coder" rights="none" pattern="PS" />
<policy domain="coder" rights="none" pattern="PS2" />
<policy domain="coder" rights="none" pattern="PS3" />
<policy domain="coder" rights="none" pattern="EPS" />
<policy domain="coder" rights="read|write" pattern="PDF" />
<policy domain="coder" rights="none" pattern="XPS" />
</policymap>
docs/configuration.md
+4 -1
View File
@@ -1873,7 +1873,10 @@ the literal path for that program.
#### [`PAPERLESS_CONVERT_BINARY=<path>`](#PAPERLESS_CONVERT_BINARY) {#PAPERLESS_CONVERT_BINARY}
: Defaults to "convert".
!!! warning
Deprecated and has no effect, since Paperless-ngx no longer uses
ImageMagick. It will be removed in a future release.
#### [`PAPERLESS_GS_BINARY=<path>`](#PAPERLESS_GS_BINARY) {#PAPERLESS_GS_BINARY}
docs/setup.md
+1 -9
View File
@@ -177,7 +177,6 @@ to a positive number to enable polling and disable native filesystem notificatio
- `pkg-config` for mysqlclient (python dependency)
- `fonts-liberation` for generating thumbnails for plain text
files
- `imagemagick` >= 6 for image alpha handling
- `gnupg` for decrypting GPG-encrypted email
- `libpq-dev` for PostgreSQL
- `libmagic-dev` for mime type detection
@@ -187,7 +186,7 @@ to a positive number to enable polling and disable native filesystem notificatio
Use this list for your preferred package management:
```
python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils
python3 python3-pip python3-dev fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils
```
These dependencies are required for OCRmyPDF, which is used for text
@@ -416,13 +415,6 @@ to a positive number to enable polling and disable native filesystem notificatio
You may need to change the path in the files. Example:
`ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO`
12. Harden ImageMagick by disabling formats that Paperless-ngx does not use.
PDF processing is not needed and should stay disabled.
Configure the active ImageMagick policy file (commonly
`/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and
adjust similar to [the docker policy file](https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/refs/heads/main/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml). You should also include restrictions as noted there.
**Optional: Install the [jbig2enc](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html) encoder.**
This will reduce the size of generated PDF documents. You'll most likely need to compile this yourself, because this
software has been patented until around 2017 and binary packages are not available for most distributions.
paperless.conf.example
-1
View File
@@ -88,5 +88,4 @@ PAPERLESS_SECRET_KEY=change-me
# Binaries
#PAPERLESS_CONVERT_BINARY=/usr/bin/convert
#PAPERLESS_GS_BINARY=/usr/bin/gs
src/documents/converters.py
+2 -11
View File
@@ -1,12 +1,11 @@
from pathlib import Path
import img2pdf
from django.conf import settings
from PIL import Image
from documents.utils import copy_basic_file_stats
from documents.utils import maybe_override_pixel_limit
from documents.utils import run_subprocess
from documents.utils import remove_alpha_channel
def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
@@ -27,15 +26,7 @@ def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
# Note the save into the temp folder, so as not to trigger a new
# consume
scratch_image = target_directory / tiff_path.name
run_subprocess(
[
settings.CONVERT_BINARY,
"-alpha",
"off",
tiff_path,
scratch_image,
],
)
remove_alpha_channel(tiff_path, scratch_image)
else:
# Not modifying the original, safe to use in place
scratch_image = tiff_path
src/documents/tests/test_utils.py
+101
View File
@@ -1,6 +1,11 @@
from pathlib import Path
import pytest
import pytest_mock
from PIL import Image
from documents.utils import QuerySetStream
from documents.utils import remove_alpha_channel
class TestQuerySetStream:
@@ -33,3 +38,99 @@ class TestQuerySetStream:
# the thing being verified here.
mock_queryset.count.assert_called_with()
mock_queryset.iterator.assert_called_once_with(chunk_size=1000)
class TestRemoveAlphaChannel:
@pytest.mark.parametrize(
("image_format", "mode", "expected_mode", "color", "expected_pixel"),
[
pytest.param(
"PNG",
"RGBA",
"RGB",
(10, 20, 30, 0),
(10, 20, 30),
id="png-rgba",
),
pytest.param(
"TIFF",
"RGBA",
"RGB",
(10, 20, 30, 128),
(10, 20, 30),
id="tiff-rgba",
),
pytest.param("PNG", "LA", "L", (40, 0), 40, id="png-la"),
],
)
def test_drops_alpha_and_keeps_format(
self,
tmp_path: Path,
image_format: str,
mode: str,
expected_mode: str,
color: tuple[int, ...],
expected_pixel: int | tuple[int, ...],
) -> None:
"""
GIVEN:
- An image with an alpha channel
WHEN:
- The alpha channel is removed to a destination without an extension
THEN:
- The destination has the matching opaque mode, the same size and
format, and the color values are kept rather than composited
"""
source = tmp_path / "source"
Image.new(mode, (8, 6), color).save(source, format=image_format)
dest = tmp_path / "no-alpha"
result = remove_alpha_channel(source, dest)
assert result == dest
with Image.open(dest) as out:
assert out.mode == expected_mode
assert out.size == (8, 6)
assert out.format == image_format
assert out.getpixel((0, 0)) == expected_pixel
def test_keeps_every_frame_of_a_multi_page_tiff(self, tmp_path: Path) -> None:
"""
GIVEN:
- A multi-page TIFF where every page has an alpha channel
WHEN:
- The alpha channel is removed
THEN:
- All pages are kept and none has an alpha channel
"""
source = tmp_path / "multi.tiff"
first = Image.new("RGBA", (8, 6), (255, 0, 0, 50))
rest = [Image.new("RGBA", (8, 6), (0, 255, 0, 50)) for _ in range(2)]
first.save(source, save_all=True, append_images=rest)
dest = tmp_path / "multi-no-alpha.tiff"
remove_alpha_channel(source, dest)
with Image.open(dest) as out:
assert out.n_frames == 3
for index in range(out.n_frames):
out.seek(index)
assert out.mode == "RGB"
def test_keeps_dpi(self, tmp_path: Path) -> None:
"""
GIVEN:
- An alpha image with a DPI set
WHEN:
- The alpha channel is removed
THEN:
- The DPI is unchanged, since OCR scaling depends on it
"""
source = tmp_path / "source.png"
Image.new("RGBA", (8, 6)).save(source, dpi=(300, 300))
dest = tmp_path / "dest.png"
remove_alpha_channel(source, dest)
with Image.open(dest) as out:
assert round(out.info["dpi"][0]) == 300
src/documents/utils.py
+26
View File
@@ -9,11 +9,13 @@ from pathlib import Path
from subprocess import CompletedProcess
from subprocess import run
from typing import TYPE_CHECKING
from typing import Any
from typing import Generic
from typing import TypeVar
from django.conf import settings
from PIL import Image
from PIL import ImageSequence
if TYPE_CHECKING:
from django.db.models import Model
@@ -128,6 +130,30 @@ def maybe_override_pixel_limit() -> None:
Image.MAX_IMAGE_PIXELS = pixel_count
def remove_alpha_channel(source: Path, dest: Path) -> Path:
"""
Writes a copy of the image with its alpha channel dropped, keeping the
format, every frame, the DPI and any ICC profile.
The channel is dropped, not composited, so transparent pixels keep their
underlying color. Returns dest.
"""
with Image.open(source) as im:
image_format = im.format
opaque_mode = "L" if im.mode == "LA" else "RGB"
frames = [frame.convert(opaque_mode) for frame in ImageSequence.Iterator(im)]
save_kwargs: dict[str, Any] = {}
if "dpi" in im.info:
save_kwargs["dpi"] = im.info["dpi"]
if "icc_profile" in im.info:
save_kwargs["icc_profile"] = im.info["icc_profile"]
if len(frames) > 1:
save_kwargs["save_all"] = True
save_kwargs["append_images"] = frames[1:]
frames[0].save(dest, format=image_format, **save_kwargs)
return dest
def run_subprocess(
arguments: list[str],
env: dict[str, str] | None = None,
src/paperless/checks.py
+1 -1
View File
@@ -82,7 +82,7 @@ def binaries_check(app_configs: Any, **kwargs: Any) -> list[Error]:
error = "Paperless can't find {}. Without it, consumption is impossible."
hint = "Either it's not in your ${PATH} or it's not installed."
binaries = (settings.CONVERT_BINARY, "tesseract", "gs")
binaries = ("tesseract", "gs")
check_messages = []
for binary in binaries:
src/paperless/parsers/tesseract.py
+2 -13
View File
@@ -19,7 +19,7 @@ from documents.parsers import ParseError
from documents.parsers import make_thumbnail_from_pdf
from documents.utils import copy_file_with_basic_stats
from documents.utils import maybe_override_pixel_limit
from documents.utils import run_subprocess
from documents.utils import remove_alpha_channel
from paperless.config import OcrConfig
from paperless.models import CleanChoices
from paperless.models import ModeChoices
@@ -197,18 +197,7 @@ class RasterisedDocumentParser:
return im.mode in ("RGBA", "LA")
def remove_alpha(self, image_path: Path) -> Path:
no_alpha_image = Path(self.tempdir) / "image-no-alpha"
run_subprocess(
[
settings.CONVERT_BINARY,
"-alpha",
"off",
str(image_path),
str(no_alpha_image),
],
logger=self.log,
)
return no_alpha_image
return remove_alpha_channel(image_path, Path(self.tempdir) / "image-no-alpha")
def get_dpi(self, image: Path) -> int | None:
try:
src/paperless/settings/__init__.py
-3
View File
@@ -984,9 +984,6 @@ MAX_IMAGE_PIXELS: Final[int | None] = get_int_from_env(
# GNUPG needs a home directory for some reason
GNUPG_HOME = os.getenv("HOME", "/tmp")
# Convert is part of the ImageMagick package
CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert")
GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs")
# Fallback layout for .eml consumption
src/paperless/tests/test_checks.py
+11 -3
View File
@@ -25,9 +25,17 @@ class TestChecks:
def test_binaries(self) -> None:
assert binaries_check(None) == []
def test_binaries_fail(self, settings: Settings) -> None:
settings.CONVERT_BINARY = "uuuhh"
assert len(binaries_check(None)) == 1
def test_binaries_fail(self, mocker: MockerFixture) -> None:
"""
GIVEN:
- Neither tesseract nor ghostscript can be found
WHEN:
- The binaries check runs
THEN:
- One warning is raised per missing binary
"""
mocker.patch("paperless.checks.shutil.which", return_value=None)
assert len(binaries_check(None)) == 2
@pytest.mark.usefixtures("paperless_dirs")
def test_paths_check(self) -> None: