mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-13 19:47:13 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea08908879 | ||
|
|
4c5927a45f | ||
|
|
884ddc401c | ||
|
|
e606dbffcd |
No files matched your search
@@ -39,7 +39,6 @@ ARG RUNTIME_PACKAGES="\
|
||||
ghostscript \
|
||||
gnupg \
|
||||
icc-profiles-free \
|
||||
imagemagick \
|
||||
# PostgreSQL
|
||||
postgresql-client \
|
||||
# MySQL / MariaDB
|
||||
@@ -93,21 +92,6 @@ RUN set -eux \
|
||||
https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
|
||||
&& dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb
|
||||
|
||||
# setup docker-specific things
|
||||
# These change sometimes, but rarely
|
||||
WORKDIR /usr/src/paperless/src/docker/
|
||||
|
||||
COPY [ \
|
||||
"docker/rootfs/etc/ImageMagick-6/paperless-policy.xml", \
|
||||
"./" \
|
||||
]
|
||||
|
||||
RUN set -eux \
|
||||
&& echo "Configuring ImageMagick" \
|
||||
&& mkdir -p /etc/ImageMagick-6 \
|
||||
&& mv paperless-policy.xml /etc/ImageMagick-6/policy.xml
|
||||
|
||||
|
||||
# Packages needed only for building a few quick Python
|
||||
# dependencies
|
||||
ARG BUILD_PACKAGES="\
|
||||
|
||||
@@ -111,10 +111,7 @@ jobs:
|
||||
timeout-minutes: 12
|
||||
uses: $/.github/actions/apt-install
|
||||
with:
|
||||
packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils
|
||||
- name: Configure ImageMagick
|
||||
run: |
|
||||
sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml
|
||||
packages: unpaper tesseract-ocr ghostscript poppler-utils qpdf
|
||||
- name: Install Python dependencies
|
||||
env:
|
||||
PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}
|
||||
|
||||
@@ -129,7 +129,6 @@ ARG RUNTIME_PACKAGES="\
|
||||
ghostscript \
|
||||
gnupg \
|
||||
icc-profiles-free \
|
||||
imagemagick \
|
||||
# PostgreSQL
|
||||
postgresql-client \
|
||||
# MySQL / MariaDB
|
||||
@@ -167,8 +166,6 @@ RUN set -eux \
|
||||
https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
|
||||
&& echo "Installing jbig2enc" \
|
||||
&& dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
|
||||
&& echo "Configuring imagemagick" \
|
||||
&& cp /etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml \
|
||||
&& echo "Cleaning up image layer" \
|
||||
&& rm --force --verbose *.deb \
|
||||
&& rm --recursive --force --verbose /var/lib/apt/lists/* \
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE policymap [
|
||||
<!ELEMENT policymap (policy)+>
|
||||
<!ATTLIST policymap xmlns CDATA #FIXED ''>
|
||||
<!ELEMENT policy EMPTY>
|
||||
<!ATTLIST policy xmlns CDATA #FIXED '' domain NMTOKEN #REQUIRED
|
||||
name NMTOKEN #IMPLIED pattern CDATA #IMPLIED rights NMTOKEN #IMPLIED
|
||||
stealth NMTOKEN #IMPLIED value CDATA #IMPLIED>
|
||||
]>
|
||||
<!--
|
||||
Configure ImageMagick policies.
|
||||
|
||||
Domains include system, delegate, coder, filter, path, or resource.
|
||||
|
||||
Rights include none, read, write, execute and all. Use | to combine them,
|
||||
for example: "read | write" to permit read from, or write to, a path.
|
||||
|
||||
Use a glob expression as a pattern.
|
||||
|
||||
Suppose we do not want users to process MPEG video images:
|
||||
|
||||
<policy domain="delegate" rights="none" pattern="mpeg:decode" />
|
||||
|
||||
Here we do not want users reading images from HTTP:
|
||||
|
||||
<policy domain="coder" rights="none" pattern="HTTP" />
|
||||
|
||||
The /repository file system is restricted to read only. We use a glob
|
||||
expression to match all paths that start with /repository:
|
||||
|
||||
<policy domain="path" rights="read" pattern="/repository/*" />
|
||||
|
||||
Lets prevent users from executing any image filters:
|
||||
|
||||
<policy domain="filter" rights="none" pattern="*" />
|
||||
|
||||
Any large image is cached to disk rather than memory:
|
||||
|
||||
<policy domain="resource" name="area" value="1GP"/>
|
||||
|
||||
Define arguments for the memory, map, area, width, height and disk resources
|
||||
with SI prefixes (.e.g 100MB). In addition, resource policies are maximums
|
||||
for each instance of ImageMagick (e.g. policy memory limit 1GB, -limit 2GB
|
||||
exceeds policy maximum so memory limit is 1GB).
|
||||
|
||||
Rules are processed in order. Here we want to restrict ImageMagick to only
|
||||
read or write a small subset of proven web-safe image types:
|
||||
|
||||
<policy domain="delegate" rights="none" pattern="*" />
|
||||
<policy domain="filter" rights="none" pattern="*" />
|
||||
<policy domain="coder" rights="none" pattern="*" />
|
||||
<policy domain="coder" rights="read|write" pattern="{GIF,JPEG,PNG,WEBP}" />
|
||||
-->
|
||||
<policymap>
|
||||
<!-- <policy domain="system" name="shred" value="2"/> -->
|
||||
<!-- <policy domain="system" name="precision" value="6"/> -->
|
||||
<!-- <policy domain="system" name="memory-map" value="anonymous"/> -->
|
||||
<!-- <policy domain="system" name="max-memory-request" value="256MiB"/> -->
|
||||
<!-- <policy domain="resource" name="temporary-path" value="/tmp"/> -->
|
||||
<policy domain="resource" name="memory" value="256MiB"/>
|
||||
<policy domain="resource" name="map" value="512MiB"/>
|
||||
<policy domain="resource" name="width" value="16KP"/>
|
||||
<policy domain="resource" name="height" value="16KP"/>
|
||||
<!-- <policy domain="resource" name="list-length" value="128"/> -->
|
||||
<policy domain="resource" name="area" value="128MB"/>
|
||||
<policy domain="resource" name="disk" value="1GiB"/>
|
||||
<!-- <policy domain="resource" name="file" value="768"/> -->
|
||||
<!-- <policy domain="resource" name="thread" value="4"/> -->
|
||||
<!-- <policy domain="resource" name="throttle" value="0"/> -->
|
||||
<!-- <policy domain="resource" name="time" value="3600"/> -->
|
||||
<!-- Paperless does not process SVG or ImageMagick scripting formats. -->
|
||||
<policy domain="coder" rights="none" pattern="SVG" />
|
||||
<policy domain="coder" rights="none" pattern="SVGZ" />
|
||||
<policy domain="coder" rights="none" pattern="MSVG" />
|
||||
<policy domain="coder" rights="none" pattern="RSVG" />
|
||||
<policy domain="coder" rights="none" pattern="MSL" />
|
||||
<policy domain="coder" rights="none" pattern="MVG" />
|
||||
<policy domain="coder" rights="none" pattern="EPHEMERAL" />
|
||||
<!-- <policy domain="module" rights="none" pattern="{PS,PDF,XPS}" /> -->
|
||||
<!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> -->
|
||||
<!-- <policy domain="path" rights="none" pattern="@*" /> -->
|
||||
<!-- <policy domain="cache" name="memory-map" value="anonymous"/> -->
|
||||
<!-- <policy domain="cache" name="synchronize" value="True"/> -->
|
||||
<!-- <policy domain="cache" name="shared-secret" value="passphrase" stealth="true"/> -->
|
||||
<!-- <policy domain="system" name="pixel-cache-memory" value="anonymous"/> -->
|
||||
<!-- <policy domain="system" name="shred" value="2"/> -->
|
||||
<!-- <policy domain="system" name="precision" value="6"/> -->
|
||||
<!-- use curl -->
|
||||
<policy domain="delegate" rights="none" pattern="URL" />
|
||||
<policy domain="delegate" rights="none" pattern="HTTPS" />
|
||||
<policy domain="delegate" rights="none" pattern="HTTP" />
|
||||
<!-- in order to avoid to get image with password text -->
|
||||
<policy domain="path" rights="none" pattern="@*"/>
|
||||
<!-- disable ghostscript format types -->
|
||||
<policy domain="coder" rights="none" pattern="PS" />
|
||||
<policy domain="coder" rights="none" pattern="PS2" />
|
||||
<policy domain="coder" rights="none" pattern="PS3" />
|
||||
<policy domain="coder" rights="none" pattern="EPS" />
|
||||
<policy domain="coder" rights="read|write" pattern="PDF" />
|
||||
<policy domain="coder" rights="none" pattern="XPS" />
|
||||
</policymap>
|
||||
+10
-30
@@ -794,17 +794,6 @@ system. See the corresponding
|
||||
|
||||
Defaults to None
|
||||
|
||||
#### [`PAPERLESS_SOCIAL_ACCOUNT_MFA_TRUSTED_AMR=<comma-separated-list>`](#PAPERLESS_SOCIAL_ACCOUNT_MFA_TRUSTED_AMR) {#PAPERLESS_SOCIAL_ACCOUNT_MFA_TRUSTED_AMR}
|
||||
|
||||
: A list of authentication method values which, if present in the `amr` claim of the ID token sent by an OpenID Connect provider, cause Paperless-ngx to skip its own two-factor authentication prompt for that login. This avoids users having to enter a second factor twice when the identity provider already performed multi-factor authentication. Logins with a username and password, or via a provider which does not send a matching `amr` value, still require the Paperless-ngx code.
|
||||
|
||||
Which values to trust depends entirely on your identity provider, e.g. Authelia sends `mfa` and Pocket ID sends `phr` for passkey logins (but `otp` for single-factor email codes, which should _not_ be trusted). Some providers, such as Keycloak, do not send an `amr` claim by default and need to be configured to do so. You can see what your provider sends from the Django admin as a superuser by checking `id_token` → `amr` in the "Extra data" of the social account.
|
||||
|
||||
!!! warning
|
||||
Only list values that mean a second factor was actually used for the login.
|
||||
|
||||
Defaults to `mfa`, the standard value for multi-factor authentication ([RFC 8176](https://www.rfc-editor.org/rfc/rfc8176.html)).
|
||||
|
||||
#### [`PAPERLESS_SOCIAL_ACCOUNT_DEFAULT_GROUPS=<comma-separated-list>`](#PAPERLESS_SOCIAL_ACCOUNT_DEFAULT_GROUPS) {#PAPERLESS_SOCIAL_ACCOUNT_DEFAULT_GROUPS}
|
||||
|
||||
: A list of group names that users who signup via social accounts will be added to upon signup. Groups listed here must already exist.
|
||||
@@ -1326,29 +1315,17 @@ valid crontab(5) expression describing when to run.
|
||||
|
||||
#### [`PAPERLESS_CONVERT_MEMORY_LIMIT=<num>`](#PAPERLESS_CONVERT_MEMORY_LIMIT) {#PAPERLESS_CONVERT_MEMORY_LIMIT}
|
||||
|
||||
: On smaller systems, or even in the case of Very Large Documents, the
|
||||
consumer may explode, complaining about how it's "unable to extend
|
||||
pixel cache". In such cases, try setting this to a reasonably low
|
||||
value, like 32. The default is to use whatever is necessary to do
|
||||
everything without writing to disk, and units are in megabytes.
|
||||
!!! warning
|
||||
|
||||
For more information on how to use this value, you should search the
|
||||
web for "MAGICK_MEMORY_LIMIT".
|
||||
|
||||
Defaults to 0, which disables the limit.
|
||||
Deprecated and has no effect, since PDF thumbnails no longer use
|
||||
ImageMagick. It will be removed in a future release.
|
||||
|
||||
#### [`PAPERLESS_CONVERT_TMPDIR=<path>`](#PAPERLESS_CONVERT_TMPDIR) {#PAPERLESS_CONVERT_TMPDIR}
|
||||
|
||||
: Similar to the memory limit, if you've got a small system and your
|
||||
OS mounts /tmp as tmpfs, you should set this to a path that's on a
|
||||
physical disk, like /home/your_user/tmp or something. ImageMagick
|
||||
will use this as scratch space when crunching through very large
|
||||
documents.
|
||||
!!! warning
|
||||
|
||||
For more information on how to use this value, you should search the
|
||||
web for "MAGICK_TMPDIR".
|
||||
|
||||
Default is none, which disables the temporary directory.
|
||||
Deprecated and has no effect, since PDF thumbnails no longer use
|
||||
ImageMagick. It will be removed in a future release.
|
||||
|
||||
#### [`PAPERLESS_APPS=<string>`](#PAPERLESS_APPS) {#PAPERLESS_APPS}
|
||||
|
||||
@@ -1896,7 +1873,10 @@ the literal path for that program.
|
||||
|
||||
#### [`PAPERLESS_CONVERT_BINARY=<path>`](#PAPERLESS_CONVERT_BINARY) {#PAPERLESS_CONVERT_BINARY}
|
||||
|
||||
: Defaults to "convert".
|
||||
!!! warning
|
||||
|
||||
Deprecated and has no effect, since Paperless-ngx no longer uses
|
||||
ImageMagick. It will be removed in a future release.
|
||||
|
||||
#### [`PAPERLESS_GS_BINARY=<path>`](#PAPERLESS_GS_BINARY) {#PAPERLESS_GS_BINARY}
|
||||
|
||||
|
||||
+2
-13
@@ -177,17 +177,16 @@ to a positive number to enable polling and disable native filesystem notificatio
|
||||
- `pkg-config` for mysqlclient (python dependency)
|
||||
- `fonts-liberation` for generating thumbnails for plain text
|
||||
files
|
||||
- `imagemagick` >= 6 for PDF conversion
|
||||
- `gnupg` for decrypting GPG-encrypted email
|
||||
- `libpq-dev` for PostgreSQL
|
||||
- `libmagic-dev` for mime type detection
|
||||
- `mariadb-client` for MariaDB compile time
|
||||
- `poppler-utils` for barcode detection
|
||||
- `poppler-utils` for thumbnail generation and barcode detection
|
||||
|
||||
Use this list for your preferred package management:
|
||||
|
||||
```
|
||||
python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils
|
||||
python3 python3-pip python3-dev fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils
|
||||
```
|
||||
|
||||
These dependencies are required for OCRmyPDF, which is used for text
|
||||
@@ -416,16 +415,6 @@ to a positive number to enable polling and disable native filesystem notificatio
|
||||
You may need to change the path in the files. Example:
|
||||
`ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO`
|
||||
|
||||
12. Configure ImageMagick to allow processing of PDF documents and disable
|
||||
formats that Paperless-ngx does not use. Most distributions disable PDF
|
||||
processing by default, since PDF documents can contain malware. If you
|
||||
don't enable it, Paperless-ngx will fall back to Ghostscript for certain
|
||||
steps such as thumbnail generation.
|
||||
|
||||
Configure the active ImageMagick policy file (commonly
|
||||
`/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and
|
||||
adjust similar to [the docker policy file](https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/refs/heads/main/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml). You should also include restrictions as noted there.
|
||||
|
||||
**Optional: Install the [jbig2enc](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html) encoder.**
|
||||
This will reduce the size of generated PDF documents. You'll most likely need to compile this yourself, because this
|
||||
software has been patented until around 2017 and binary packages are not available for most distributions.
|
||||
|
||||
@@ -473,8 +473,6 @@ Users can enable two-factor authentication (2FA) for their accounts from the 'My
|
||||
|
||||
Should a user lose access to their 2FA device and all recovery codes, a superuser can disable 2FA for the user from the 'Users & Groups' management screen.
|
||||
|
||||
If users log in via an OpenID Connect provider which already enforces multi-factor authentication, see [`PAPERLESS_SOCIAL_ACCOUNT_MFA_TRUSTED_AMR`](configuration.md#PAPERLESS_SOCIAL_ACCOUNT_MFA_TRUSTED_AMR) to skip the Paperless-ngx prompt for those logins.
|
||||
|
||||
## Workflows
|
||||
|
||||
!!! note
|
||||
|
||||
@@ -50,8 +50,6 @@ PAPERLESS_SECRET_KEY=change-me
|
||||
#PAPERLESS_OCR_ROTATE_PAGES=true
|
||||
#PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0
|
||||
#PAPERLESS_OCR_USER_ARGS={}
|
||||
#PAPERLESS_CONVERT_MEMORY_LIMIT=0
|
||||
#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
|
||||
|
||||
# Software tweaks
|
||||
|
||||
@@ -90,5 +88,4 @@ PAPERLESS_SECRET_KEY=change-me
|
||||
|
||||
# Binaries
|
||||
|
||||
#PAPERLESS_CONVERT_BINARY=/usr/bin/convert
|
||||
#PAPERLESS_GS_BINARY=/usr/bin/gs
|
||||
@@ -1,12 +1,11 @@
|
||||
from pathlib import Path
|
||||
|
||||
import img2pdf
|
||||
from django.conf import settings
|
||||
from PIL import Image
|
||||
|
||||
from documents.utils import copy_basic_file_stats
|
||||
from documents.utils import maybe_override_pixel_limit
|
||||
from documents.utils import run_subprocess
|
||||
from documents.utils import remove_alpha_channel
|
||||
|
||||
|
||||
def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
|
||||
@@ -27,15 +26,7 @@ def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
|
||||
# Note the save into the temp folder, so as not to trigger a new
|
||||
# consume
|
||||
scratch_image = target_directory / tiff_path.name
|
||||
run_subprocess(
|
||||
[
|
||||
settings.CONVERT_BINARY,
|
||||
"-alpha",
|
||||
"off",
|
||||
tiff_path,
|
||||
scratch_image,
|
||||
],
|
||||
)
|
||||
remove_alpha_channel(tiff_path, scratch_image)
|
||||
else:
|
||||
# Not modifying the original, safe to use in place
|
||||
scratch_image = tiff_path
|
||||
|
||||
+161
-98
@@ -1,8 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -68,58 +68,6 @@ def get_supported_file_extensions() -> set[str]:
|
||||
return extensions
|
||||
|
||||
|
||||
def run_convert(
|
||||
input_file,
|
||||
output_file,
|
||||
*,
|
||||
density=None,
|
||||
scale=None,
|
||||
alpha=None,
|
||||
strip=False,
|
||||
trim=False,
|
||||
type=None,
|
||||
depth=None,
|
||||
auto_orient=False,
|
||||
use_cropbox=False,
|
||||
extra=None,
|
||||
logging_group=None,
|
||||
) -> None:
|
||||
environment = os.environ.copy()
|
||||
if settings.CONVERT_MEMORY_LIMIT:
|
||||
# MAGICK_MEMORY_LIMIT sets the maximum amount of RAM the pixel cache can use.
|
||||
# MAGICK_MAP_LIMIT sets the maximum amount of memory-mapped I/O allowed.
|
||||
#
|
||||
# For large-format documents ImageMagick will hit the RAM limit and
|
||||
# immediately try to "map" the remaining data. If MAGICK_MAP_LIMIT isn't
|
||||
# also set, the process may trigger an OOM kill because the default
|
||||
# system/policy map limit is often too restrictive for these massive bitmaps.
|
||||
environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
|
||||
environment["MAGICK_MAP_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
|
||||
if settings.CONVERT_TMPDIR:
|
||||
environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR
|
||||
|
||||
args = [settings.CONVERT_BINARY]
|
||||
args += ["-density", str(density)] if density else []
|
||||
args += ["-scale", str(scale)] if scale else []
|
||||
args += ["-alpha", str(alpha)] if alpha else []
|
||||
args += ["-strip"] if strip else []
|
||||
args += ["-trim"] if trim else []
|
||||
args += ["-type", str(type)] if type else []
|
||||
args += ["-depth", str(depth)] if depth else []
|
||||
args += ["-auto-orient"] if auto_orient else []
|
||||
args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else []
|
||||
args += [str(input_file), str(output_file)]
|
||||
|
||||
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
|
||||
|
||||
try:
|
||||
run_subprocess(args, environment, logger)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise ParseError(f"Convert failed at {args}") from e
|
||||
except Exception as e: # pragma: no cover
|
||||
raise ParseError("Unknown error running convert") from e
|
||||
|
||||
|
||||
def get_default_thumbnail() -> Path:
|
||||
"""
|
||||
Returns the path to a generic thumbnail
|
||||
@@ -127,46 +75,168 @@ def get_default_thumbnail() -> Path:
|
||||
return (Path(__file__).parent / "resources" / "document.webp").resolve()
|
||||
|
||||
|
||||
def make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group=None) -> Path:
|
||||
out_path: Path = Path(temp_dir) / "convert_gs.webp"
|
||||
_THUMBNAIL_MAX_WIDTH = 500
|
||||
_THUMBNAIL_MAX_HEIGHT = 5000
|
||||
# Used only when the page geometry cannot be read
|
||||
_THUMBNAIL_FALLBACK_DPI = 150
|
||||
# Applied before supersampling, so tiny pages are not enlarged
|
||||
_THUMBNAIL_MAX_DPI = 300
|
||||
# Rendering at a multiple and downsampling keeps text crisper
|
||||
_THUMBNAIL_SUPERSAMPLE = 2
|
||||
|
||||
# if convert fails, fall back to extracting
|
||||
# the first PDF page as a PNG using Ghostscript
|
||||
logger.warning(
|
||||
"Thumbnail generation with ImageMagick failed, falling back "
|
||||
"to ghostscript. Check your /etc/ImageMagick-x/policy.xml!",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
# Ghostscript doesn't handle WebP outputs
|
||||
gs_out_path: Path = Path(temp_dir) / "gs_out.png"
|
||||
cmd = [settings.GS_BINARY, "-q", "-sDEVICE=pngalpha", "-o", gs_out_path, in_path]
|
||||
|
||||
def rasterize_pdf_page_to_png(
|
||||
in_path: Path,
|
||||
out_path: Path,
|
||||
*,
|
||||
dpi: int,
|
||||
logging_group=None,
|
||||
) -> None:
|
||||
"""
|
||||
Rasterizes the first page of a PDF to a PNG with pdftoppm.
|
||||
"""
|
||||
# -singlefile drops the page number and -png appends ".png", so pass the
|
||||
# path without its suffix
|
||||
args = [
|
||||
"pdftoppm",
|
||||
"-f",
|
||||
"1",
|
||||
"-l",
|
||||
"1",
|
||||
"-r",
|
||||
str(dpi),
|
||||
"-png",
|
||||
"-singlefile",
|
||||
"-cropbox",
|
||||
str(in_path),
|
||||
str(out_path.with_suffix("")),
|
||||
]
|
||||
|
||||
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
|
||||
|
||||
try:
|
||||
try:
|
||||
run_subprocess(cmd, logger=logger)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise ParseError(f"Thumbnail (gs) failed at {cmd}") from e
|
||||
# then run convert on the output from gs to make WebP
|
||||
run_convert(
|
||||
density=300,
|
||||
scale="500x5000>",
|
||||
alpha="remove",
|
||||
strip=True,
|
||||
trim=False,
|
||||
auto_orient=True,
|
||||
input_file=gs_out_path,
|
||||
output_file=out_path,
|
||||
logging_group=logging_group,
|
||||
)
|
||||
run_subprocess(args, logger=logger)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise ParseError(f"pdftoppm failed at {args}") from e
|
||||
except Exception as e: # pragma: no cover
|
||||
raise ParseError("Unknown error running pdftoppm") from e
|
||||
|
||||
|
||||
def encode_thumbnail_webp(
|
||||
png_path: Path,
|
||||
out_path: Path,
|
||||
*,
|
||||
supersample: int = 1,
|
||||
) -> None:
|
||||
"""
|
||||
Flattens alpha onto white, undoes supersampling, shrinks to fit and saves as WebP.
|
||||
"""
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
with Image.open(png_path) as im:
|
||||
if im.mode in ("RGBA", "LA"):
|
||||
flattened = Image.new("RGB", im.size, (255, 255, 255))
|
||||
flattened.paste(im, mask=im.split()[-1])
|
||||
else:
|
||||
flattened = im.convert("RGB")
|
||||
|
||||
if supersample > 1:
|
||||
flattened = flattened.resize(
|
||||
(
|
||||
max(1, round(flattened.width / supersample)),
|
||||
max(1, round(flattened.height / supersample)),
|
||||
),
|
||||
Image.Resampling.LANCZOS,
|
||||
)
|
||||
|
||||
flattened.thumbnail((_THUMBNAIL_MAX_WIDTH, _THUMBNAIL_MAX_HEIGHT))
|
||||
flattened.save(out_path, format="WEBP")
|
||||
except (OSError, Image.DecompressionBombError) as e:
|
||||
raise ParseError(f"Unable to encode thumbnail from {png_path}") from e
|
||||
|
||||
|
||||
def _compute_thumbnail_dpi(in_path: Path, logging_group=None) -> tuple[int, int]:
|
||||
"""
|
||||
Returns (dpi, supersample). Unknown geometry is not supersampled, since
|
||||
the render size cannot be bounded.
|
||||
"""
|
||||
from paperless.parsers.utils import get_pdf_first_page_size_points
|
||||
|
||||
size = get_pdf_first_page_size_points(in_path)
|
||||
if size is None:
|
||||
logger.debug(
|
||||
"Could not read PDF page size, using fallback DPI",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
return _THUMBNAIL_FALLBACK_DPI, 1
|
||||
|
||||
width_pts, height_pts = size
|
||||
dpi_for_width = _THUMBNAIL_MAX_WIDTH * 72 / width_pts
|
||||
dpi_for_height = _THUMBNAIL_MAX_HEIGHT * 72 / height_pts
|
||||
# Round up so the downsampled render is never a few pixels short of the
|
||||
# target; the shrink-only clamp in encode_thumbnail_webp trims the excess.
|
||||
dpi = max(
|
||||
1,
|
||||
math.ceil(min(_THUMBNAIL_MAX_DPI, dpi_for_width, dpi_for_height)),
|
||||
)
|
||||
# At the 1 DPI floor the page is already oversized, so do not supersample
|
||||
return dpi, 1 if dpi == 1 else _THUMBNAIL_SUPERSAMPLE
|
||||
|
||||
|
||||
def _render_pdf_thumbnail(
|
||||
in_path: Path,
|
||||
png_path: Path,
|
||||
out_path: Path,
|
||||
logging_group=None,
|
||||
) -> None:
|
||||
dpi, supersample = _compute_thumbnail_dpi(in_path, logging_group=logging_group)
|
||||
rasterize_pdf_page_to_png(
|
||||
in_path,
|
||||
png_path,
|
||||
dpi=dpi * supersample,
|
||||
logging_group=logging_group,
|
||||
)
|
||||
encode_thumbnail_webp(png_path, out_path, supersample=supersample)
|
||||
|
||||
|
||||
def _repair_pdf_with_qpdf(in_path: Path, out_path: Path) -> None:
|
||||
# qpdf exits 3 after a repair; --warning-exit-0 keeps that from failing
|
||||
try:
|
||||
shutil.copy(in_path, out_path)
|
||||
run_subprocess(
|
||||
["qpdf", "--warning-exit-0", "--replace-input", str(out_path)],
|
||||
logger=logger,
|
||||
)
|
||||
except (subprocess.CalledProcessError, OSError) as e:
|
||||
raise ParseError(f"qpdf repair failed for {in_path}") from e
|
||||
|
||||
|
||||
def make_thumbnail_from_pdf_qpdf_fallback(
|
||||
in_path: Path,
|
||||
temp_dir: Path,
|
||||
logging_group=None,
|
||||
) -> Path:
|
||||
png_path = temp_dir / "page1_repaired.png"
|
||||
out_path = temp_dir / "convert_qpdf.webp"
|
||||
repaired_path = temp_dir / "repaired.pdf"
|
||||
|
||||
logger.warning(
|
||||
"Thumbnail generation with pdftoppm failed, attempting qpdf repair and retry.",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
|
||||
try:
|
||||
_repair_pdf_with_qpdf(in_path, repaired_path)
|
||||
_render_pdf_thumbnail(repaired_path, png_path, out_path, logging_group)
|
||||
return out_path
|
||||
|
||||
except ParseError as e:
|
||||
logger.error(f"Unable to make thumbnail with Ghostscript: {e}")
|
||||
logger.error(f"Unable to make thumbnail after qpdf repair: {e}")
|
||||
# The caller might expect a generated thumbnail that can be moved,
|
||||
# so we need to copy it before it gets moved.
|
||||
# https://github.com/paperless-ngx/paperless-ngx/issues/3631
|
||||
default_thumbnail_path: Path = Path(temp_dir) / "document.webp"
|
||||
default_thumbnail_path = temp_dir / "document.webp"
|
||||
copy_file_with_basic_stats(get_default_thumbnail(), default_thumbnail_path)
|
||||
return default_thumbnail_path
|
||||
|
||||
@@ -175,25 +245,18 @@ def make_thumbnail_from_pdf(in_path: Path, temp_dir: Path, logging_group=None) -
|
||||
"""
|
||||
The thumbnail of a PDF is just a 500px wide image of the first page.
|
||||
"""
|
||||
png_path: Path = temp_dir / "page1.png"
|
||||
out_path: Path = temp_dir / "convert.webp"
|
||||
|
||||
# Run convert to get a decent thumbnail
|
||||
try:
|
||||
run_convert(
|
||||
density=300,
|
||||
scale="500x5000>",
|
||||
alpha="remove",
|
||||
strip=True,
|
||||
trim=False,
|
||||
auto_orient=True,
|
||||
use_cropbox=True,
|
||||
input_file=f"{in_path}[0]",
|
||||
output_file=str(out_path),
|
||||
logging_group=logging_group,
|
||||
)
|
||||
_render_pdf_thumbnail(in_path, png_path, out_path, logging_group)
|
||||
except ParseError as e:
|
||||
logger.error(f"Unable to make thumbnail with convert: {e}")
|
||||
out_path = make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group)
|
||||
logger.error(f"Unable to make thumbnail with pdftoppm: {e}")
|
||||
out_path = make_thumbnail_from_pdf_qpdf_fallback(
|
||||
in_path,
|
||||
temp_dir,
|
||||
logging_group,
|
||||
)
|
||||
|
||||
return out_path
|
||||
|
||||
|
||||
@@ -1,11 +1,22 @@
|
||||
import subprocess
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
import pikepdf
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from pytest_django.fixtures import Settings
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from documents.parsers import ParseError
|
||||
from documents.parsers import _compute_thumbnail_dpi
|
||||
from documents.parsers import encode_thumbnail_webp
|
||||
from documents.parsers import get_default_file_extension
|
||||
from documents.parsers import get_default_thumbnail
|
||||
from documents.parsers import get_supported_file_extensions
|
||||
from documents.parsers import is_file_ext_supported
|
||||
from documents.parsers import make_thumbnail_from_pdf
|
||||
from documents.parsers import rasterize_pdf_page_to_png
|
||||
from paperless.parsers.registry import get_parser_registry
|
||||
from paperless.parsers.registry import reset_parser_registry
|
||||
from paperless.parsers.tesseract import RasterisedDocumentParser
|
||||
@@ -125,3 +136,374 @@ class TestParserAvailability:
|
||||
assert is_file_ext_supported(".pdf")
|
||||
assert not is_file_ext_supported(".hsdfh")
|
||||
assert not is_file_ext_supported("")
|
||||
|
||||
|
||||
class TestComputeThumbnailDpi:
|
||||
@pytest.mark.parametrize(
|
||||
("size", "expected"),
|
||||
[
|
||||
pytest.param((612.0, 792.0), (59, 2), id="letter-width-bound"),
|
||||
pytest.param((792.0, 612.0), (46, 2), id="landscape-rounded-up"),
|
||||
pytest.param((612.0, 100000.0), (4, 2), id="tall-strip-height-bound"),
|
||||
pytest.param((200.0, 300.0), (180, 2), id="small-page-width-bound"),
|
||||
pytest.param((72.0, 72.0), (300, 2), id="tiny-page-capped-at-300"),
|
||||
pytest.param(
|
||||
(1000000.0, 1000000.0),
|
||||
(1, 1),
|
||||
id="huge-page-minimum-one-unsupersampled",
|
||||
),
|
||||
pytest.param(None, (150, 1), id="unreadable-geometry-fallback"),
|
||||
],
|
||||
)
|
||||
def test_dpi_from_page_size(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tmp_path: Path,
|
||||
size: tuple[float, float] | None,
|
||||
expected: tuple[int, int],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A first page of the given size, or unreadable geometry
|
||||
WHEN:
|
||||
- The thumbnail DPI is computed
|
||||
THEN:
|
||||
- The expected DPI and supersample factor are returned
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.parsers.utils.get_pdf_first_page_size_points",
|
||||
return_value=size,
|
||||
)
|
||||
assert _compute_thumbnail_dpi(tmp_path / "doc.pdf") == expected
|
||||
|
||||
|
||||
class TestMakeThumbnailFromPdf:
|
||||
@pytest.fixture
|
||||
def work_dir(self, tmp_path: Path) -> Path:
|
||||
path = tmp_path / "work"
|
||||
path.mkdir()
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _write_blank_pdf(path: Path, page_size: tuple[int, int] = (612, 792)) -> Path:
|
||||
pdf = pikepdf.new()
|
||||
pdf.add_blank_page(page_size=page_size)
|
||||
pdf.save(path, object_stream_mode=pikepdf.ObjectStreamMode.disable)
|
||||
return path
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("size", "expected_dpi", "expected_supersample"),
|
||||
[
|
||||
pytest.param((612.0, 792.0), 118, 2, id="known-geometry-2x"),
|
||||
pytest.param(None, 150, 1, id="unreadable-geometry-plain-fallback"),
|
||||
],
|
||||
)
|
||||
def test_render_dpi_requested(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tmp_path: Path,
|
||||
work_dir: Path,
|
||||
size: tuple[float, float] | None,
|
||||
expected_dpi: int,
|
||||
expected_supersample: int,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Readable or unreadable page geometry
|
||||
WHEN:
|
||||
- A thumbnail is made
|
||||
THEN:
|
||||
- Rasterize and encode get the matching DPI and supersample factor
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.parsers.utils.get_pdf_first_page_size_points",
|
||||
return_value=size,
|
||||
)
|
||||
rasterize = mocker.patch("documents.parsers.rasterize_pdf_page_to_png")
|
||||
encode = mocker.patch("documents.parsers.encode_thumbnail_webp")
|
||||
|
||||
make_thumbnail_from_pdf(tmp_path / "in.pdf", work_dir)
|
||||
|
||||
assert rasterize.call_args.kwargs["dpi"] == expected_dpi
|
||||
assert encode.call_args.kwargs["supersample"] == expected_supersample
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_size", "expected_width"),
|
||||
[
|
||||
pytest.param((612, 792), 500, id="letter"),
|
||||
pytest.param((792, 612), 500, id="landscape-letter"),
|
||||
pytest.param((595, 842), 500, id="a4"),
|
||||
pytest.param((200, 300), 500, id="small-page"),
|
||||
pytest.param((72, 72), 300, id="tiny-page-capped"),
|
||||
],
|
||||
)
|
||||
def test_thumbnail_width(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
work_dir: Path,
|
||||
page_size: tuple[int, int],
|
||||
expected_width: int,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF whose first page has the given size in points
|
||||
WHEN:
|
||||
- A thumbnail is made from it
|
||||
THEN:
|
||||
- The thumbnail has the expected width
|
||||
"""
|
||||
pdf_path = self._write_blank_pdf(tmp_path / "in.pdf", page_size)
|
||||
|
||||
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
|
||||
|
||||
assert thumb == work_dir / "convert.webp"
|
||||
with Image.open(thumb) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.width == expected_width
|
||||
|
||||
@classmethod
|
||||
def _write_pdf_without_xref(cls, path: Path) -> Path:
|
||||
"""
|
||||
Cuts off the xref and trailer, which pdftoppm cannot recover from but qpdf can.
|
||||
"""
|
||||
cls._write_blank_pdf(path)
|
||||
data = path.read_bytes()
|
||||
path.write_bytes(data[: data.rindex(b"\nxref")])
|
||||
return path
|
||||
|
||||
def test_qpdf_repair_produces_real_thumbnail(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
work_dir: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF with its xref table and trailer cut off
|
||||
WHEN:
|
||||
- A thumbnail is made from it
|
||||
THEN:
|
||||
- The thumbnail is rendered from a qpdf repaired copy
|
||||
- The original file is unchanged
|
||||
"""
|
||||
pdf_path = self._write_pdf_without_xref(tmp_path / "broken.pdf")
|
||||
original_bytes = pdf_path.read_bytes()
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
rasterize_pdf_page_to_png(pdf_path, work_dir / "probe.png", dpi=50)
|
||||
|
||||
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
|
||||
|
||||
assert thumb == work_dir / "convert_qpdf.webp"
|
||||
with Image.open(thumb) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.width == 500
|
||||
assert pdf_path.read_bytes() == original_bytes
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"qpdf_error",
|
||||
[
|
||||
pytest.param(subprocess.CalledProcessError(2, "qpdf"), id="qpdf-fails"),
|
||||
pytest.param(None, id="repaired-still-unrenderable"),
|
||||
],
|
||||
)
|
||||
def test_double_failure_uses_default_thumbnail(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tmp_path: Path,
|
||||
work_dir: Path,
|
||||
qpdf_error: subprocess.CalledProcessError | None,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF that cannot be rendered, even after qpdf repair
|
||||
WHEN:
|
||||
- A thumbnail is made from it
|
||||
THEN:
|
||||
- A copy of the default thumbnail is returned
|
||||
"""
|
||||
mocker.patch(
|
||||
"documents.parsers.rasterize_pdf_page_to_png",
|
||||
side_effect=ParseError("Does not compute."),
|
||||
)
|
||||
if qpdf_error is not None:
|
||||
mocker.patch("documents.parsers.run_subprocess", side_effect=qpdf_error)
|
||||
pdf_path = self._write_blank_pdf(tmp_path / "in.pdf")
|
||||
|
||||
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
|
||||
|
||||
assert thumb == work_dir / "document.webp"
|
||||
assert thumb.read_bytes() == get_default_thumbnail().read_bytes()
|
||||
|
||||
|
||||
class TestRasterizePdfPageToPng:
|
||||
@staticmethod
|
||||
def _write_pdf(
|
||||
path: Path,
|
||||
*,
|
||||
crop_box: tuple[float, float, float, float] | None = None,
|
||||
rotate: int | None = None,
|
||||
) -> Path:
|
||||
pdf = pikepdf.new()
|
||||
pdf.add_blank_page(page_size=(144, 72))
|
||||
pdf.add_blank_page(page_size=(300, 300))
|
||||
page = pdf.pages[0]
|
||||
if crop_box is not None:
|
||||
page.obj.CropBox = pikepdf.Array(crop_box)
|
||||
if rotate is not None:
|
||||
page.obj.Rotate = rotate
|
||||
pdf.save(path)
|
||||
return path
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("crop_box", "rotate", "expected_size"),
|
||||
[
|
||||
pytest.param(None, None, (144, 72), id="plain"),
|
||||
pytest.param(None, 90, (72, 144), id="rotated-90"),
|
||||
pytest.param((0, 0, 72, 36), None, (72, 36), id="crop-box"),
|
||||
],
|
||||
)
|
||||
def test_renders_first_page_to_exact_path(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
crop_box: tuple[float, float, float, float] | None,
|
||||
rotate: int | None,
|
||||
expected_size: tuple[int, int],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A two page PDF, first page optionally cropped or rotated
|
||||
WHEN:
|
||||
- The first page is rasterized at 72 DPI
|
||||
THEN:
|
||||
- Only out_path is written, sized to the first page's crop and rotation
|
||||
"""
|
||||
pdf_path = self._write_pdf(
|
||||
tmp_path / "in.pdf",
|
||||
crop_box=crop_box,
|
||||
rotate=rotate,
|
||||
)
|
||||
out_dir = tmp_path / "out"
|
||||
out_dir.mkdir()
|
||||
out_path = out_dir / "page1.png"
|
||||
|
||||
rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72)
|
||||
|
||||
assert list(out_dir.iterdir()) == [out_path]
|
||||
with Image.open(out_path) as im:
|
||||
assert im.format == "PNG"
|
||||
assert im.size == expected_size
|
||||
|
||||
def test_failure_raises_parse_error(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A file that is not a PDF
|
||||
WHEN:
|
||||
- Rasterization is attempted
|
||||
THEN:
|
||||
- A ParseError is raised
|
||||
"""
|
||||
bad = tmp_path / "bad.pdf"
|
||||
bad.write_bytes(b"not a pdf")
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72)
|
||||
|
||||
|
||||
class TestEncodeThumbnailWebp:
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "color"),
|
||||
[
|
||||
pytest.param("RGBA", (0, 0, 0, 0), id="rgba"),
|
||||
pytest.param("LA", (0, 0), id="la"),
|
||||
],
|
||||
)
|
||||
def test_alpha_flattened_onto_white(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
mode: str,
|
||||
color: tuple[int, ...],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A fully transparent PNG with an alpha channel
|
||||
WHEN:
|
||||
- It is encoded as a thumbnail
|
||||
THEN:
|
||||
- The WebP output is RGB with the transparency flattened to white
|
||||
"""
|
||||
png_path = tmp_path / "in.png"
|
||||
Image.new(mode, (20, 10), color).save(png_path)
|
||||
out_path = tmp_path / "out.webp"
|
||||
|
||||
encode_thumbnail_webp(png_path, out_path)
|
||||
|
||||
with Image.open(out_path) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.mode == "RGB"
|
||||
assert im.size == (20, 10)
|
||||
red, green, blue = im.getpixel((10, 5))
|
||||
assert min(red, green, blue) >= 250
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("in_size", "supersample", "expected_size"),
|
||||
[
|
||||
pytest.param((1000, 2000), 1, (500, 1000), id="too-wide-shrunk"),
|
||||
pytest.param((100, 10000), 1, (50, 5000), id="too-tall-shrunk"),
|
||||
pytest.param((100, 200), 1, (100, 200), id="small-not-enlarged"),
|
||||
pytest.param((1000, 1400), 2, (500, 700), id="2x-halved"),
|
||||
pytest.param((1001, 1401), 2, (500, 700), id="2x-odd-rounded"),
|
||||
pytest.param((600, 800), 2, (300, 400), id="2x-small-not-enlarged"),
|
||||
pytest.param((900, 1200), 2, (450, 600), id="2x-below-clamp"),
|
||||
],
|
||||
)
|
||||
def test_size_clamped_and_downsampled(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
in_size: tuple[int, int],
|
||||
supersample: int,
|
||||
expected_size: tuple[int, int],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A rendered image and its supersample factor
|
||||
WHEN:
|
||||
- It is encoded as a thumbnail with that factor
|
||||
THEN:
|
||||
- It is downsampled, fit within 500x5000 and never enlarged
|
||||
"""
|
||||
png_path = tmp_path / "in.png"
|
||||
Image.new("RGB", in_size, (255, 255, 255)).save(png_path)
|
||||
out_path = tmp_path / "out.webp"
|
||||
|
||||
encode_thumbnail_webp(png_path, out_path, supersample=supersample)
|
||||
|
||||
with Image.open(out_path) as im:
|
||||
assert im.size == expected_size
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
pytest.param(OSError("broken image"), id="os-error"),
|
||||
pytest.param(Image.DecompressionBombError("too large"), id="bomb"),
|
||||
],
|
||||
)
|
||||
def test_decode_failure_raises_parse_error(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
mocker: MockerFixture,
|
||||
error: Exception,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Opening the rendered image fails
|
||||
WHEN:
|
||||
- It is encoded as a thumbnail
|
||||
THEN:
|
||||
- A ParseError is raised
|
||||
"""
|
||||
png_path = tmp_path / "in.png"
|
||||
Image.new("RGB", (10, 10)).save(png_path)
|
||||
mocker.patch("PIL.Image.open", side_effect=error)
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
encode_thumbnail_webp(png_path, tmp_path / "out.webp")
|
||||
@@ -1,6 +1,11 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import pytest_mock
|
||||
from PIL import Image
|
||||
|
||||
from documents.utils import QuerySetStream
|
||||
from documents.utils import remove_alpha_channel
|
||||
|
||||
|
||||
class TestQuerySetStream:
|
||||
@@ -33,3 +38,99 @@ class TestQuerySetStream:
|
||||
# the thing being verified here.
|
||||
mock_queryset.count.assert_called_with()
|
||||
mock_queryset.iterator.assert_called_once_with(chunk_size=1000)
|
||||
|
||||
|
||||
class TestRemoveAlphaChannel:
|
||||
@pytest.mark.parametrize(
|
||||
("image_format", "mode", "expected_mode", "color", "expected_pixel"),
|
||||
[
|
||||
pytest.param(
|
||||
"PNG",
|
||||
"RGBA",
|
||||
"RGB",
|
||||
(10, 20, 30, 0),
|
||||
(10, 20, 30),
|
||||
id="png-rgba",
|
||||
),
|
||||
pytest.param(
|
||||
"TIFF",
|
||||
"RGBA",
|
||||
"RGB",
|
||||
(10, 20, 30, 128),
|
||||
(10, 20, 30),
|
||||
id="tiff-rgba",
|
||||
),
|
||||
pytest.param("PNG", "LA", "L", (40, 0), 40, id="png-la"),
|
||||
],
|
||||
)
|
||||
def test_drops_alpha_and_keeps_format(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
image_format: str,
|
||||
mode: str,
|
||||
expected_mode: str,
|
||||
color: tuple[int, ...],
|
||||
expected_pixel: int | tuple[int, ...],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An image with an alpha channel
|
||||
WHEN:
|
||||
- The alpha channel is removed to a destination without an extension
|
||||
THEN:
|
||||
- The destination has the matching opaque mode, the same size and
|
||||
format, and the color values are kept rather than composited
|
||||
"""
|
||||
source = tmp_path / "source"
|
||||
Image.new(mode, (8, 6), color).save(source, format=image_format)
|
||||
dest = tmp_path / "no-alpha"
|
||||
|
||||
result = remove_alpha_channel(source, dest)
|
||||
|
||||
assert result == dest
|
||||
with Image.open(dest) as out:
|
||||
assert out.mode == expected_mode
|
||||
assert out.size == (8, 6)
|
||||
assert out.format == image_format
|
||||
assert out.getpixel((0, 0)) == expected_pixel
|
||||
|
||||
def test_keeps_every_frame_of_a_multi_page_tiff(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A multi-page TIFF where every page has an alpha channel
|
||||
WHEN:
|
||||
- The alpha channel is removed
|
||||
THEN:
|
||||
- All pages are kept and none has an alpha channel
|
||||
"""
|
||||
source = tmp_path / "multi.tiff"
|
||||
first = Image.new("RGBA", (8, 6), (255, 0, 0, 50))
|
||||
rest = [Image.new("RGBA", (8, 6), (0, 255, 0, 50)) for _ in range(2)]
|
||||
first.save(source, save_all=True, append_images=rest)
|
||||
dest = tmp_path / "multi-no-alpha.tiff"
|
||||
|
||||
remove_alpha_channel(source, dest)
|
||||
|
||||
with Image.open(dest) as out:
|
||||
assert out.n_frames == 3
|
||||
for index in range(out.n_frames):
|
||||
out.seek(index)
|
||||
assert out.mode == "RGB"
|
||||
|
||||
def test_keeps_dpi(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An alpha image with a DPI set
|
||||
WHEN:
|
||||
- The alpha channel is removed
|
||||
THEN:
|
||||
- The DPI is unchanged, since OCR scaling depends on it
|
||||
"""
|
||||
source = tmp_path / "source.png"
|
||||
Image.new("RGBA", (8, 6)).save(source, dpi=(300, 300))
|
||||
dest = tmp_path / "dest.png"
|
||||
|
||||
remove_alpha_channel(source, dest)
|
||||
|
||||
with Image.open(dest) as out:
|
||||
assert round(out.info["dpi"][0]) == 300
|
||||
@@ -9,11 +9,13 @@ from pathlib import Path
|
||||
from subprocess import CompletedProcess
|
||||
from subprocess import run
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
from typing import Generic
|
||||
from typing import TypeVar
|
||||
|
||||
from django.conf import settings
|
||||
from PIL import Image
|
||||
from PIL import ImageSequence
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.db.models import Model
|
||||
@@ -128,6 +130,30 @@ def maybe_override_pixel_limit() -> None:
|
||||
Image.MAX_IMAGE_PIXELS = pixel_count
|
||||
|
||||
|
||||
def remove_alpha_channel(source: Path, dest: Path) -> Path:
|
||||
"""
|
||||
Writes a copy of the image with its alpha channel dropped, keeping the
|
||||
format, every frame, the DPI and any ICC profile.
|
||||
|
||||
The channel is dropped, not composited, so transparent pixels keep their
|
||||
underlying color. Returns dest.
|
||||
"""
|
||||
with Image.open(source) as im:
|
||||
image_format = im.format
|
||||
opaque_mode = "L" if im.mode == "LA" else "RGB"
|
||||
frames = [frame.convert(opaque_mode) for frame in ImageSequence.Iterator(im)]
|
||||
save_kwargs: dict[str, Any] = {}
|
||||
if "dpi" in im.info:
|
||||
save_kwargs["dpi"] = im.info["dpi"]
|
||||
if "icc_profile" in im.info:
|
||||
save_kwargs["icc_profile"] = im.info["icc_profile"]
|
||||
if len(frames) > 1:
|
||||
save_kwargs["save_all"] = True
|
||||
save_kwargs["append_images"] = frames[1:]
|
||||
frames[0].save(dest, format=image_format, **save_kwargs)
|
||||
return dest
|
||||
|
||||
|
||||
def run_subprocess(
|
||||
arguments: list[str],
|
||||
env: dict[str, str] | None = None,
|
||||
|
||||
@@ -4,7 +4,6 @@ from urllib.parse import quote
|
||||
from allauth.account.adapter import DefaultAccountAdapter
|
||||
from allauth.core import context
|
||||
from allauth.headless.tokens.strategies.sessions import SessionTokenStrategy
|
||||
from allauth.mfa.stages import AuthenticateStage
|
||||
from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import Group
|
||||
@@ -20,34 +19,7 @@ from paperless.signals import handle_social_account_updated
|
||||
logger = logging.getLogger("paperless.auth")
|
||||
|
||||
|
||||
class SocialAccountAuthenticateStage(AuthenticateStage):
|
||||
"""
|
||||
Skips the 2FA prompt for a social login if the ID token's `amr` claim shows the
|
||||
identity provider already used one of the SOCIAL_ACCOUNT_MFA_TRUSTED_AMR methods.
|
||||
"""
|
||||
|
||||
def _should_handle(self, request: HttpRequest) -> bool:
|
||||
sociallogin = (self.login.signal_kwargs or {}).get("sociallogin")
|
||||
if sociallogin is not None and settings.SOCIAL_ACCOUNT_MFA_TRUSTED_AMR:
|
||||
id_token = (sociallogin.account.extra_data or {}).get("id_token") or {}
|
||||
amr = id_token.get("amr") or []
|
||||
if set(amr) & set(settings.SOCIAL_ACCOUNT_MFA_TRUSTED_AMR):
|
||||
logger.debug(
|
||||
f"Skipping 2FA for `{self.login.user}`, provider `{sociallogin.account.provider}` reported amr {amr}",
|
||||
)
|
||||
return False
|
||||
return super()._should_handle(request)
|
||||
|
||||
|
||||
class CustomAccountAdapter(DefaultAccountAdapter):
|
||||
def get_login_stages(self) -> list[str]:
|
||||
return [
|
||||
"paperless.adapter.SocialAccountAuthenticateStage"
|
||||
if stage == "allauth.mfa.stages.AuthenticateStage"
|
||||
else stage
|
||||
for stage in super().get_login_stages()
|
||||
]
|
||||
|
||||
def is_open_for_signup(self, request):
|
||||
"""
|
||||
Check whether the site is open for signups, which can be
|
||||
|
||||
+11
-12
@@ -82,7 +82,7 @@ def binaries_check(app_configs: Any, **kwargs: Any) -> list[Error]:
|
||||
error = "Paperless can't find {}. Without it, consumption is impossible."
|
||||
hint = "Either it's not in your ${PATH} or it's not installed."
|
||||
|
||||
binaries = (settings.CONVERT_BINARY, "tesseract", "gs")
|
||||
binaries = ("tesseract", "gs")
|
||||
|
||||
check_messages = []
|
||||
for binary in binaries:
|
||||
@@ -272,27 +272,26 @@ def check_deprecated_db_settings(
|
||||
Detects legacy advanced options that should be migrated to
|
||||
PAPERLESS_DB_OPTIONS. Returns one Warning per deprecated variable found.
|
||||
"""
|
||||
deprecated_vars: dict[str, str] = {
|
||||
"PAPERLESS_DB_TIMEOUT": "timeout",
|
||||
"PAPERLESS_DB_POOLSIZE": "pool.min_size / pool.max_size",
|
||||
"PAPERLESS_DBSSLMODE": "sslmode",
|
||||
"PAPERLESS_DBSSLROOTCERT": "sslrootcert",
|
||||
"PAPERLESS_DBSSLCERT": "sslcert",
|
||||
"PAPERLESS_DBSSLKEY": "sslkey",
|
||||
}
|
||||
deprecated_vars = (
|
||||
"PAPERLESS_DB_TIMEOUT",
|
||||
"PAPERLESS_DB_POOLSIZE",
|
||||
"PAPERLESS_DBSSLMODE",
|
||||
"PAPERLESS_DBSSLROOTCERT",
|
||||
"PAPERLESS_DBSSLCERT",
|
||||
"PAPERLESS_DBSSLKEY",
|
||||
)
|
||||
|
||||
warnings: list[Warning] = []
|
||||
|
||||
for var_name, db_option_key in deprecated_vars.items():
|
||||
for var_name in deprecated_vars:
|
||||
if not os.getenv(var_name):
|
||||
continue
|
||||
warnings.append(
|
||||
Warning(
|
||||
f"Deprecated environment variable: {var_name}",
|
||||
hint=(
|
||||
f"{var_name} is no longer supported and will be removed in v3.2. "
|
||||
f"{var_name} is deprecated. "
|
||||
f"Set the equivalent option via PAPERLESS_DB_OPTIONS instead. "
|
||||
f'Example: PAPERLESS_DB_OPTIONS=\'{{"{db_option_key}": "<value>"}}\'. '
|
||||
"See https://docs.paperless-ngx.com/migration-v3/ for the full reference."
|
||||
),
|
||||
id="paperless.W001",
|
||||
|
||||
@@ -19,7 +19,7 @@ from documents.parsers import ParseError
|
||||
from documents.parsers import make_thumbnail_from_pdf
|
||||
from documents.utils import copy_file_with_basic_stats
|
||||
from documents.utils import maybe_override_pixel_limit
|
||||
from documents.utils import run_subprocess
|
||||
from documents.utils import remove_alpha_channel
|
||||
from paperless.config import OcrConfig
|
||||
from paperless.models import CleanChoices
|
||||
from paperless.models import ModeChoices
|
||||
@@ -197,18 +197,7 @@ class RasterisedDocumentParser:
|
||||
return im.mode in ("RGBA", "LA")
|
||||
|
||||
def remove_alpha(self, image_path: Path) -> Path:
|
||||
no_alpha_image = Path(self.tempdir) / "image-no-alpha"
|
||||
run_subprocess(
|
||||
[
|
||||
settings.CONVERT_BINARY,
|
||||
"-alpha",
|
||||
"off",
|
||||
str(image_path),
|
||||
str(no_alpha_image),
|
||||
],
|
||||
logger=self.log,
|
||||
)
|
||||
return no_alpha_image
|
||||
return remove_alpha_channel(image_path, Path(self.tempdir) / "image-no-alpha")
|
||||
|
||||
def get_dpi(self, image: Path) -> int | None:
|
||||
try:
|
||||
|
||||
@@ -265,6 +265,52 @@ def get_page_count_for_pdf(
|
||||
return None
|
||||
|
||||
|
||||
def get_pdf_first_page_size_points(
|
||||
path: Path,
|
||||
log: logging.Logger | None = None,
|
||||
) -> tuple[float, float] | None:
|
||||
"""Return the first page's (width, height) in PDF points, post-rotation.
|
||||
|
||||
Uses the CropBox (MediaBox if absent), which must match pdftoppm's
|
||||
``-cropbox`` or the computed DPI targets the wrong box.
|
||||
|
||||
Swaps width and height for 90/270 rotation. ``page.rotation`` resolves
|
||||
inherited and negative ``/Rotate`` values, a raw lookup does not.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
Absolute path to the PDF file.
|
||||
log:
|
||||
Logger for warnings. Falls back to the module-level logger when omitted.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[float, float] | None
|
||||
``(width_points, height_points)``, or ``None`` if the file cannot be
|
||||
opened, has no pages, or the page box is degenerate.
|
||||
"""
|
||||
import pikepdf
|
||||
|
||||
_log = log or logger
|
||||
try:
|
||||
with pikepdf.Pdf.open(path) as pdf:
|
||||
if len(pdf.pages) == 0:
|
||||
return None
|
||||
page = pdf.pages[0]
|
||||
llx, lly, urx, ury = (float(v) for v in page.cropbox)
|
||||
width = abs(urx - llx)
|
||||
height = abs(ury - lly)
|
||||
if width <= 0 or height <= 0:
|
||||
return None
|
||||
if page.rotation in (90, 270):
|
||||
width, height = height, width
|
||||
return width, height
|
||||
except Exception as e:
|
||||
_log.warning("Could not determine PDF page size for %s: %s", path, e)
|
||||
return None
|
||||
|
||||
|
||||
def extract_pdf_metadata(
|
||||
document_path: Path,
|
||||
log: logging.Logger | None = None,
|
||||
|
||||
@@ -348,11 +348,6 @@ SOCIAL_ACCOUNT_SYNC_SUPERUSER_GROUP: Final[str | None] = os.getenv(
|
||||
SOCIAL_ACCOUNT_SYNC_STAFF_GROUP: Final[str | None] = os.getenv(
|
||||
"PAPERLESS_SOCIAL_ACCOUNT_SYNC_STAFF_GROUP",
|
||||
)
|
||||
SOCIAL_ACCOUNT_MFA_TRUSTED_AMR = [
|
||||
value.strip()
|
||||
for value in os.getenv("PAPERLESS_SOCIAL_ACCOUNT_MFA_TRUSTED_AMR", "mfa").split(",")
|
||||
if value.strip()
|
||||
]
|
||||
|
||||
HEADLESS_TOKEN_STRATEGY = "paperless.adapter.DrfTokenStrategy"
|
||||
|
||||
@@ -989,11 +984,6 @@ MAX_IMAGE_PIXELS: Final[int | None] = get_int_from_env(
|
||||
# GNUPG needs a home directory for some reason
|
||||
GNUPG_HOME = os.getenv("HOME", "/tmp")
|
||||
|
||||
# Convert is part of the ImageMagick package
|
||||
CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert")
|
||||
CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
|
||||
CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
|
||||
|
||||
GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs")
|
||||
|
||||
# Fallback layout for .eml consumption
|
||||
|
||||
@@ -15,9 +15,10 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from ocrmypdf import SubprocessOutputError
|
||||
from PIL import Image
|
||||
|
||||
from documents.parsers import ParseError
|
||||
from documents.parsers import run_convert
|
||||
from documents.parsers import rasterize_pdf_page_to_png
|
||||
from paperless.models import ModeChoices
|
||||
from paperless.parsers import ParserProtocol
|
||||
from paperless.parsers.tesseract import RasterisedDocumentParser
|
||||
@@ -280,24 +281,71 @@ class TestGetThumbnail:
|
||||
)
|
||||
assert thumb.is_file()
|
||||
|
||||
def test_thumbnail_fallback_on_convert_error(
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "expected_height"),
|
||||
[
|
||||
pytest.param("simple-digital.pdf", 647, id="portrait-letter"),
|
||||
pytest.param("rotated.pdf", 386, id="landscape"),
|
||||
],
|
||||
)
|
||||
def test_thumbnail_is_correct_format_and_size(
|
||||
self,
|
||||
tesseract_parser: RasterisedDocumentParser,
|
||||
tesseract_samples_dir: Path,
|
||||
filename: str,
|
||||
expected_height: int,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A portrait or landscape PDF
|
||||
WHEN:
|
||||
- A thumbnail is generated
|
||||
THEN:
|
||||
- A 500px wide WebP keeping the page's aspect ratio
|
||||
"""
|
||||
thumb = tesseract_parser.get_thumbnail(
|
||||
tesseract_samples_dir / filename,
|
||||
"application/pdf",
|
||||
)
|
||||
with Image.open(thumb) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.width == 500
|
||||
assert im.height == pytest.approx(expected_height, abs=2)
|
||||
|
||||
def test_thumbnail_fallback_on_pdftoppm_error(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tesseract_parser: RasterisedDocumentParser,
|
||||
tesseract_samples_dir: Path,
|
||||
) -> None:
|
||||
def _raise_on_pdf(input_file, output_file, **kwargs) -> None:
|
||||
if ".pdf" in str(input_file):
|
||||
"""
|
||||
GIVEN:
|
||||
- Rasterizing the original PDF fails
|
||||
WHEN:
|
||||
- A thumbnail is generated
|
||||
THEN:
|
||||
- The PDF is repaired with qpdf and a real thumbnail is rendered
|
||||
"""
|
||||
original = tesseract_samples_dir / "simple-digital.pdf"
|
||||
|
||||
def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None:
|
||||
if in_path == original:
|
||||
raise ParseError("Does not compute.")
|
||||
run_convert(input_file=input_file, output_file=output_file, **kwargs)
|
||||
rasterize_pdf_page_to_png(in_path, out_path, **kwargs)
|
||||
|
||||
mocker.patch("documents.parsers.run_convert", side_effect=_raise_on_pdf)
|
||||
|
||||
thumb = tesseract_parser.get_thumbnail(
|
||||
tesseract_samples_dir / "simple-digital.pdf",
|
||||
"application/pdf",
|
||||
rasterize = mocker.patch(
|
||||
"documents.parsers.rasterize_pdf_page_to_png",
|
||||
side_effect=_fail_on_original,
|
||||
)
|
||||
|
||||
thumb = tesseract_parser.get_thumbnail(original, "application/pdf")
|
||||
|
||||
assert rasterize.call_count == 2
|
||||
assert thumb.is_file()
|
||||
assert thumb.name == "convert_qpdf.webp"
|
||||
with Image.open(thumb) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.width == 500
|
||||
|
||||
def test_thumbnail_encrypted_pdf(
|
||||
self,
|
||||
|
||||
@@ -2,26 +2,19 @@ import logging
|
||||
|
||||
import pytest
|
||||
from allauth.account.adapter import get_adapter
|
||||
from allauth.account.models import Login
|
||||
from allauth.core import context
|
||||
from allauth.mfa.totp.internal.auth import TOTP
|
||||
from allauth.mfa.totp.internal.auth import generate_totp_secret
|
||||
from allauth.socialaccount.adapter import get_adapter as get_social_adapter
|
||||
from allauth.socialaccount.models import SocialAccount
|
||||
from allauth.socialaccount.models import SocialLogin
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import User
|
||||
from django.forms import ValidationError
|
||||
from django.http import HttpRequest
|
||||
from django.test import RequestFactory
|
||||
from django.urls import reverse
|
||||
from pytest_django.fixtures import Settings
|
||||
from pytest_mock import MockerFixture
|
||||
from rest_framework.authtoken.models import Token
|
||||
|
||||
from paperless.adapter import DrfTokenStrategy
|
||||
from paperless.adapter import SocialAccountAuthenticateStage
|
||||
from paperless_testing.factories import UserFactory
|
||||
|
||||
|
||||
@@ -136,89 +129,6 @@ class TestCustomAccountAdapter:
|
||||
assert not user2.is_superuser
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestSocialAccountAuthenticateStage:
|
||||
@staticmethod
|
||||
def _should_handle(user: User, extra_data: dict | None) -> bool:
|
||||
signal_kwargs = None
|
||||
if extra_data is not None:
|
||||
account = SocialAccount(
|
||||
provider="openid_connect",
|
||||
uid="uid",
|
||||
extra_data=extra_data,
|
||||
)
|
||||
signal_kwargs = {"sociallogin": SocialLogin(user=user, account=account)}
|
||||
request = RequestFactory().get("/")
|
||||
request.session = {}
|
||||
stage = SocialAccountAuthenticateStage(
|
||||
None,
|
||||
request,
|
||||
Login(user=user, signal_kwargs=signal_kwargs),
|
||||
)
|
||||
return stage._should_handle(request)
|
||||
|
||||
def test_stage_replaces_allauth_stage(self) -> None:
|
||||
stages = get_adapter().get_login_stages()
|
||||
|
||||
assert "paperless.adapter.SocialAccountAuthenticateStage" in stages
|
||||
assert "allauth.mfa.stages.AuthenticateStage" not in stages
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("trusted_amr", "extra_data", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
["mfa"],
|
||||
{"id_token": {"amr": ["pwd", "mfa"]}},
|
||||
False,
|
||||
id="trusted-amr",
|
||||
),
|
||||
pytest.param(
|
||||
["phr"],
|
||||
{"id_token": {"amr": ["otp"]}},
|
||||
True,
|
||||
id="untrusted-amr",
|
||||
),
|
||||
pytest.param([], {"id_token": {"amr": ["mfa"]}}, True, id="setting-unset"),
|
||||
pytest.param(["mfa"], {"id_token": {}}, True, id="no-amr-claim"),
|
||||
pytest.param(
|
||||
["mfa"],
|
||||
{"userinfo": {"amr": ["mfa"]}},
|
||||
True,
|
||||
id="userinfo-only",
|
||||
),
|
||||
pytest.param(["mfa"], None, True, id="regular-login"),
|
||||
],
|
||||
)
|
||||
def test_mfa_skipped_only_for_trusted_amr(
|
||||
self,
|
||||
settings: Settings,
|
||||
trusted_amr: list[str],
|
||||
extra_data: dict | None,
|
||||
*,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A user with TOTP enabled
|
||||
WHEN:
|
||||
- The user logs in, via a social account or not
|
||||
THEN:
|
||||
- The 2FA prompt is only skipped if the ID token's amr claim contains
|
||||
one of SOCIAL_ACCOUNT_MFA_TRUSTED_AMR
|
||||
"""
|
||||
settings.SOCIAL_ACCOUNT_MFA_TRUSTED_AMR = trusted_amr
|
||||
user = UserFactory(username="testuser")
|
||||
TOTP.activate(user, generate_totp_secret())
|
||||
|
||||
assert self._should_handle(user, extra_data) is expected
|
||||
|
||||
def test_no_mfa_for_user_without_authenticator(self, settings: Settings) -> None:
|
||||
settings.SOCIAL_ACCOUNT_MFA_TRUSTED_AMR = ["mfa"]
|
||||
user = UserFactory(username="testuser")
|
||||
|
||||
assert not self._should_handle(user, {"id_token": {"amr": ["pwd"]}})
|
||||
|
||||
|
||||
class TestCustomSocialAccountAdapter:
|
||||
@pytest.mark.django_db
|
||||
def test_is_open_for_signup(self, settings: Settings) -> None:
|
||||
|
||||
@@ -25,9 +25,17 @@ class TestChecks:
|
||||
def test_binaries(self) -> None:
|
||||
assert binaries_check(None) == []
|
||||
|
||||
def test_binaries_fail(self, settings: Settings) -> None:
|
||||
settings.CONVERT_BINARY = "uuuhh"
|
||||
assert len(binaries_check(None)) == 1
|
||||
def test_binaries_fail(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Neither tesseract nor ghostscript can be found
|
||||
WHEN:
|
||||
- The binaries check runs
|
||||
THEN:
|
||||
- One warning is raised per missing binary
|
||||
"""
|
||||
mocker.patch("paperless.checks.shutil.which", return_value=None)
|
||||
assert len(binaries_check(None)) == 2
|
||||
|
||||
@pytest.mark.usefixtures("paperless_dirs")
|
||||
def test_paths_check(self) -> None:
|
||||
@@ -211,14 +219,14 @@ class TestAuditLogChecks:
|
||||
assert "auditlog table was found but audit log is disabled." in msgs[0].msg
|
||||
|
||||
|
||||
DEPRECATED_VARS: dict[str, str] = {
|
||||
"PAPERLESS_DB_TIMEOUT": "timeout",
|
||||
"PAPERLESS_DB_POOLSIZE": "pool.min_size / pool.max_size",
|
||||
"PAPERLESS_DBSSLMODE": "sslmode",
|
||||
"PAPERLESS_DBSSLROOTCERT": "sslrootcert",
|
||||
"PAPERLESS_DBSSLCERT": "sslcert",
|
||||
"PAPERLESS_DBSSLKEY": "sslkey",
|
||||
}
|
||||
DEPRECATED_VARS = (
|
||||
"PAPERLESS_DB_TIMEOUT",
|
||||
"PAPERLESS_DB_POOLSIZE",
|
||||
"PAPERLESS_DBSSLMODE",
|
||||
"PAPERLESS_DBSSLROOTCERT",
|
||||
"PAPERLESS_DBSSLCERT",
|
||||
"PAPERLESS_DBSSLKEY",
|
||||
)
|
||||
|
||||
|
||||
class TestDeprecatedDbSettings:
|
||||
@@ -234,26 +242,11 @@ class TestDeprecatedDbSettings:
|
||||
result = check_deprecated_db_settings(None)
|
||||
assert result == []
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env_var", "db_option_key"),
|
||||
[
|
||||
pytest.param("PAPERLESS_DB_TIMEOUT", "timeout", id="db-timeout"),
|
||||
pytest.param(
|
||||
"PAPERLESS_DB_POOLSIZE",
|
||||
"pool.min_size / pool.max_size",
|
||||
id="db-poolsize",
|
||||
),
|
||||
pytest.param("PAPERLESS_DBSSLMODE", "sslmode", id="ssl-mode"),
|
||||
pytest.param("PAPERLESS_DBSSLROOTCERT", "sslrootcert", id="ssl-rootcert"),
|
||||
pytest.param("PAPERLESS_DBSSLCERT", "sslcert", id="ssl-cert"),
|
||||
pytest.param("PAPERLESS_DBSSLKEY", "sslkey", id="ssl-key"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("env_var", DEPRECATED_VARS)
|
||||
def test_single_deprecated_var_produces_one_warning(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
env_var: str,
|
||||
db_option_key: str,
|
||||
) -> None:
|
||||
"""Each deprecated var in isolation produces exactly one warning."""
|
||||
mocker.patch.dict(os.environ, {env_var: "some_value"}, clear=True)
|
||||
@@ -264,7 +257,8 @@ class TestDeprecatedDbSettings:
|
||||
assert isinstance(warning, Warning)
|
||||
assert warning.id == "paperless.W001"
|
||||
assert env_var in warning.hint
|
||||
assert db_option_key in warning.hint
|
||||
assert "PAPERLESS_DB_OPTIONS" in warning.hint
|
||||
assert "https://docs.paperless-ngx.com/migration-v3/" in warning.hint
|
||||
|
||||
def test_multiple_deprecated_vars_produce_one_warning_each(
|
||||
self,
|
||||
|
||||
@@ -6,8 +6,10 @@ import codecs
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pikepdf
|
||||
import pytest
|
||||
|
||||
from paperless.parsers.utils import get_pdf_first_page_size_points
|
||||
from paperless.parsers.utils import is_tagged_pdf
|
||||
from paperless.parsers.utils import pdf_born_digital_text
|
||||
from paperless.parsers.utils import post_process_text
|
||||
@@ -70,6 +72,149 @@ class TestIsTaggedPdf:
|
||||
assert is_tagged_pdf(bad) is False
|
||||
|
||||
|
||||
class TestGetPdfFirstPageSizePoints:
|
||||
@staticmethod
|
||||
def _write_pdf(
|
||||
path: Path,
|
||||
*,
|
||||
media_box: tuple[float, float, float, float] = (0, 0, 600, 800),
|
||||
crop_box: tuple[float, float, float, float] | None = None,
|
||||
page_rotate: int | None = None,
|
||||
inherited_rotate: int | None = None,
|
||||
) -> Path:
|
||||
pdf = pikepdf.new()
|
||||
pdf.add_blank_page(page_size=(media_box[2], media_box[3]))
|
||||
page = pdf.pages[0]
|
||||
page.obj.MediaBox = pikepdf.Array(media_box)
|
||||
if crop_box is not None:
|
||||
page.obj.CropBox = pikepdf.Array(crop_box)
|
||||
if page_rotate is not None:
|
||||
page.obj.Rotate = page_rotate
|
||||
if inherited_rotate is not None:
|
||||
pdf.Root.Pages.Rotate = inherited_rotate
|
||||
pdf.save(path)
|
||||
return path
|
||||
|
||||
def test_letter_sample(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A US Letter sample PDF with no CropBox and no rotation
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- The MediaBox size in points is returned
|
||||
"""
|
||||
assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") == (
|
||||
612.0,
|
||||
792.0,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rotate", "expected"),
|
||||
[
|
||||
pytest.param(0, (600.0, 800.0), id="rotate-0"),
|
||||
pytest.param(90, (800.0, 600.0), id="rotate-90"),
|
||||
pytest.param(180, (600.0, 800.0), id="rotate-180"),
|
||||
pytest.param(270, (800.0, 600.0), id="rotate-270"),
|
||||
pytest.param(-90, (800.0, 600.0), id="rotate-negative-90"),
|
||||
],
|
||||
)
|
||||
def test_page_rotation_swaps_dimensions(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
rotate: int,
|
||||
expected: tuple[float, float],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A portrait PDF page with /Rotate set directly on the page
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- Width and height are swapped for quarter-turn rotations only
|
||||
"""
|
||||
pdf_path = self._write_pdf(tmp_path / "rotated.pdf", page_rotate=rotate)
|
||||
assert get_pdf_first_page_size_points(pdf_path) == expected
|
||||
|
||||
def test_inherited_rotation_swaps_dimensions(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A page inheriting /Rotate 90 from the /Pages node
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- Width and height are swapped
|
||||
"""
|
||||
pdf_path = self._write_pdf(tmp_path / "inherited.pdf", inherited_rotate=90)
|
||||
assert get_pdf_first_page_size_points(pdf_path) == (800.0, 600.0)
|
||||
|
||||
def test_crop_box_preferred_over_media_box(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF page with a CropBox smaller than its MediaBox
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- The CropBox size is returned
|
||||
"""
|
||||
pdf_path = self._write_pdf(
|
||||
tmp_path / "cropped.pdf",
|
||||
crop_box=(50, 100, 350, 500),
|
||||
)
|
||||
assert get_pdf_first_page_size_points(pdf_path) == (300.0, 400.0)
|
||||
|
||||
def test_degenerate_box_returns_none(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF page whose box has zero width
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- None is returned
|
||||
"""
|
||||
pdf_path = self._write_pdf(
|
||||
tmp_path / "degenerate.pdf",
|
||||
media_box=(0, 0, 600, 800),
|
||||
crop_box=(100, 0, 100, 800),
|
||||
)
|
||||
assert get_pdf_first_page_size_points(pdf_path) is None
|
||||
|
||||
def test_nonexistent_path_returns_none(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A path that does not exist
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- None is returned and nothing is raised
|
||||
"""
|
||||
assert get_pdf_first_page_size_points(Path("/nonexistent/file.pdf")) is None
|
||||
|
||||
def test_corrupt_pdf_returns_none(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A file that is not a PDF
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- None is returned and nothing is raised
|
||||
"""
|
||||
bad = tmp_path / "bad.pdf"
|
||||
bad.write_bytes(b"not a pdf")
|
||||
assert get_pdf_first_page_size_points(bad) is None
|
||||
|
||||
def test_encrypted_pdf_returns_none(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A password protected PDF
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- None is returned and nothing is raised
|
||||
"""
|
||||
assert get_pdf_first_page_size_points(SAMPLES / "encrypted.pdf") is None
|
||||
|
||||
|
||||
class TestPostProcessText:
|
||||
@pytest.mark.parametrize(
|
||||
("source", "expected"),
|
||||
|
||||
Reference in new issue
Block a user