Compare commits

..
Author SHA1 Message Date
stumpylog ea08908879 Feature/Chore: Replaces all usages of ImageMagik's convert binary with a simple Pillow based helper
This removes ImageMagik as a dependency entirly and all documentation is cleaned up from it.
We no longer need to install it, have a policy for it to allow PDFs and no longer shell
out to it either.
2026-10-13 10:24:44 -07:00
stumpylogandClaude Sonnet 5.5 4c5927a45f Chore: Supersample the PDF thumbnail render 2x
Render the PDF page at twice the target size and downsample, with edge case
handling for small and odd sized pages, simplified helpers and tests, and
trimmed comments.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-10 15:09:59 -07:00
stumpylogandClaude Sonnet 5.5 884ddc401c Chore: Generate PDF thumbnails with pdftoppm and Pillow instead of ImageMagick
Replace the ImageMagick convert based PDF thumbnail path with pdftoppm and
Pillow. Keeps thumbnails at 500px wide for small and odd sized pages, covers
the qpdf repair and double failure paths, removes the unused run_convert
helper, hardens thumbnail encoding, deprecates PAPERLESS_CONVERT_MEMORY_LIMIT
and PAPERLESS_CONVERT_TMPDIR, and corrects the related docs and CI.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-10 15:09:59 -07:00
shamoon e606dbffcd Chore: update the db options warning (#14398) 2026-10-10 07:17:54 -07:00
shamoon 138160cbda Fix: bulk reprocess latest version for root documents (#14385) 2026-10-08 18:50:44 +00:00
GitHub Actions 6d960c11d5 Auto translate strings 2026-10-08 18:40:22 +00:00
shamoon a35bd6e238 Fix: consistently use root doc for version action permissions (#14384) 2026-10-08 18:39:05 +00:00
Trenton H 474c630aa4 Fix: Don't attempt to index created dates which are not representable inside the FTS index (#14390) 2026-10-08 08:36:35 -07:00
Trenton H d7a9894400 Fix: retry a search index rebuild that was interrupted (#14379)
An interrupted rebuild left an empty index stamped as current, so the next
start reported it as up to date. Mark the rebuild as in progress and only
clear the marker once it completes.
2026-10-07 08:54:36 -07:00
GitHub Actions ee34a6598e Auto translate strings 2026-10-06 15:13:08 +00:00
Trenton H 3a3b3ef66a Chore: Upgrade runners to Ubuntu 26.04 (#14347)
* Moves runners to 26.04 and a few jobs to -slim variant

* Probably fixing the imagemagik 7 problems and maybe the frontend playwright thing?

* Compare thumbnails, but allow a little difference in the perceptual hash
2026-10-06 08:11:44 -07:00
54 changed files with 1810 additions and 561 deletions

No files matched your search

-16
View File
@@ -39,7 +39,6 @@ ARG RUNTIME_PACKAGES="\
ghostscript \
gnupg \
icc-profiles-free \
imagemagick \
# PostgreSQL
postgresql-client \
# MySQL / MariaDB
@@ -93,21 +92,6 @@ RUN set -eux \
https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb
# setup docker-specific things
# These change sometimes, but rarely
WORKDIR /usr/src/paperless/src/docker/
COPY [ \
"docker/rootfs/etc/ImageMagick-6/paperless-policy.xml", \
"./" \
]
RUN set -eux \
&& echo "Configuring ImageMagick" \
&& mkdir -p /etc/ImageMagick-6 \
&& mv paperless-policy.xml /etc/ImageMagick-6/policy.xml
# Packages needed only for building a few quick Python
# dependencies
ARG BUILD_PACKAGES="\
+3 -6
View File
@@ -80,7 +80,7 @@ jobs:
needs: changes
if: needs.changes.outputs.backend_changed == 'true'
name: "Python ${{ matrix.python-version }}"
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
strategy:
@@ -111,10 +111,7 @@ jobs:
timeout-minutes: 12
uses: $/.github/actions/apt-install
with:
packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils
- name: Configure ImageMagick
run: |
sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml
packages: unpaper tesseract-ocr ghostscript poppler-utils qpdf
- name: Install Python dependencies
env:
PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}
@@ -158,7 +155,7 @@ jobs:
needs: changes
if: needs.changes.outputs.backend_changed == 'true'
name: Check project typing
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
env:
+3 -3
View File
@@ -24,10 +24,10 @@ jobs:
fail-fast: false
matrix:
include:
- runner: ubuntu-24.04
- runner: ubuntu-26.04
arch: amd64
platform: linux/amd64
- runner: ubuntu-24.04-arm
- runner: ubuntu-26.04-arm
arch: arm64
platform: linux/arm64
runs-on: ${{ matrix.runner }}
@@ -163,7 +163,7 @@ jobs:
archive: false
merge-and-push:
name: Merge and Push Manifest
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
needs: build-arch
if: needs.build-arch.outputs.should-push == 'true'
environment: image-publishing
+2 -2
View File
@@ -65,7 +65,7 @@ jobs:
needs: changes
if: needs.changes.outputs.docs_changed == 'true'
name: Build Documentation
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
steps:
- uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
- name: Checkout
@@ -102,7 +102,7 @@ jobs:
name: Deploy Documentation
needs: [changes, build]
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.docs_changed == 'true'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
pages: write
id-token: write
+6 -6
View File
@@ -72,7 +72,7 @@ jobs:
needs: changes
if: needs.changes.outputs.frontend_changed == 'true'
name: Install Dependencies
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
steps:
@@ -104,7 +104,7 @@ jobs:
name: Lint
needs: [changes, install-dependencies]
if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
steps:
@@ -137,7 +137,7 @@ jobs:
name: "Unit Tests (${{ matrix.shard-index }}/${{ matrix.shard-count }})"
needs: [changes, install-dependencies]
if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
strategy:
@@ -188,10 +188,10 @@ jobs:
name: E2E Tests
needs: [changes, install-dependencies]
if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
container: mcr.microsoft.com/playwright:v1.62.1-noble
container: mcr.microsoft.com/playwright:v1.62.1-resolute
env:
PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
@@ -246,7 +246,7 @@ jobs:
name: Frontend Build
needs: [changes, unit-tests, e2e-tests]
if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
steps:
+5 -5
View File
@@ -14,7 +14,7 @@ permissions: {}
jobs:
wait-for-docker:
name: Wait for Docker Build
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
checks: read
statuses: read
@@ -30,7 +30,7 @@ jobs:
build-release:
name: Build Release
needs: wait-for-docker
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
steps:
@@ -73,7 +73,7 @@ jobs:
timeout-minutes: 12
uses: $/.github/actions/apt-install
with:
packages: gettext liblept5
packages: gettext libleptonica6
# ---- Build Documentation ----
- name: Build documentation
env:
@@ -145,7 +145,7 @@ jobs:
publish-release:
name: Publish Release
needs: build-release
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: write
pull-requests: write
@@ -197,7 +197,7 @@ jobs:
name: Append Changelog
needs: publish-release
if: needs.publish-release.outputs.prerelease == 'false'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: write
pull-requests: write
+2 -2
View File
@@ -15,7 +15,7 @@ permissions:
jobs:
zizmor:
name: Run zizmor
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
contents: read
actions: read
@@ -29,7 +29,7 @@ jobs:
uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4
semgrep:
name: Semgrep CE
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
container:
image: semgrep/semgrep:1.155.0@sha256:cc869c685dcc0fe497c86258da9f205397d8108e56d21a86082ea4886e52784d
if: github.actor != 'dependabot[bot]'
+2 -2
View File
@@ -17,7 +17,7 @@ jobs:
cleanup-images:
name: Cleanup Image Tags for ${{ matrix.primary-name }}
if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
environment: registry-maintenance
strategy:
fail-fast: false
@@ -42,7 +42,7 @@ jobs:
cleanup-untagged-images:
name: Cleanup Untagged Images Tags for ${{ matrix.primary-name }}
if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
needs:
- cleanup-images
environment: registry-maintenance
+1 -1
View File
@@ -21,7 +21,7 @@ on:
jobs:
analyze:
name: Analyze
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
actions: read
contents: read
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
synchronize-with-crowdin:
name: Crowdin Sync
if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
environment: translation-sync
steps:
- name: Checkout
+1 -1
View File
@@ -7,7 +7,7 @@ jobs:
# Note: peakoss/anti-slop does not support the `issues` event yet (all of its
# issue inputs are still commented out upstream), so the checks that the PR Bot
# workflow gets from the action are implemented manually here.
runs-on: ubuntu-latest
runs-on: ubuntu-slim
permissions:
issues: write
steps:
+2 -2
View File
@@ -4,7 +4,7 @@ on:
types: [opened]
jobs:
Anti-slop:
runs-on: ubuntu-latest
runs-on: ubuntu-slim
permissions:
contents: read
issues: read
@@ -24,7 +24,7 @@ jobs:
ASLOP-PR-VERIFY
pr-bot:
name: Automated PR Bot
runs-on: ubuntu-latest
runs-on: ubuntu-slim
# Runs after Anti-slop so the welcome comment can see whether the PR was closed
# instead of racing it. Still runs if that job fails, so labeling is not lost.
needs: Anti-slop
+1 -1
View File
@@ -12,7 +12,7 @@ permissions:
jobs:
pr_opened_or_reopened:
name: pr_opened_or_reopened
runs-on: ubuntu-24.04
runs-on: ubuntu-slim
permissions:
# write permission is required for autolabeler
pull-requests: write
+5 -5
View File
@@ -9,7 +9,7 @@ jobs:
stale:
name: 'Stale'
if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
issues: write
pull-requests: write
@@ -34,7 +34,7 @@ jobs:
lock-threads:
name: 'Lock Old Threads'
if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04
runs-on: ubuntu-26.04
permissions:
issues: write
pull-requests: write
@@ -58,7 +58,7 @@ jobs:
close-answered-discussions:
name: 'Close Answered Discussions'
if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04
runs-on: ubuntu-slim
permissions:
discussions: write
steps:
@@ -117,7 +117,7 @@ jobs:
close-outdated-discussions:
name: 'Close Outdated Discussions'
if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04
runs-on: ubuntu-slim
permissions:
discussions: write
steps:
@@ -211,7 +211,7 @@ jobs:
close-unsupported-feature-requests:
name: 'Close Unsupported Feature Requests'
if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04
runs-on: ubuntu-slim
permissions:
discussions: write
steps:
+1 -1
View File
@@ -8,7 +8,7 @@ env:
jobs:
generate-translate-strings:
name: Generate Translation Strings
runs-on: ubuntu-latest
runs-on: ubuntu-26.04
environment: translation-sync
permissions:
contents: write
-3
View File
@@ -129,7 +129,6 @@ ARG RUNTIME_PACKAGES="\
ghostscript \
gnupg \
icc-profiles-free \
imagemagick \
# PostgreSQL
postgresql-client \
# MySQL / MariaDB
@@ -167,8 +166,6 @@ RUN set -eux \
https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& echo "Installing jbig2enc" \
&& dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& echo "Configuring imagemagick" \
&& cp /etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml \
&& echo "Cleaning up image layer" \
&& rm --force --verbose *.deb \
&& rm --recursive --force --verbose /var/lib/apt/lists/* \
@@ -1,101 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE policymap [
<!ELEMENT policymap (policy)+>
<!ATTLIST policymap xmlns CDATA #FIXED ''>
<!ELEMENT policy EMPTY>
<!ATTLIST policy xmlns CDATA #FIXED '' domain NMTOKEN #REQUIRED
name NMTOKEN #IMPLIED pattern CDATA #IMPLIED rights NMTOKEN #IMPLIED
stealth NMTOKEN #IMPLIED value CDATA #IMPLIED>
]>
<!--
Configure ImageMagick policies.
Domains include system, delegate, coder, filter, path, or resource.
Rights include none, read, write, execute and all. Use | to combine them,
for example: "read | write" to permit read from, or write to, a path.
Use a glob expression as a pattern.
Suppose we do not want users to process MPEG video images:
<policy domain="delegate" rights="none" pattern="mpeg:decode" />
Here we do not want users reading images from HTTP:
<policy domain="coder" rights="none" pattern="HTTP" />
The /repository file system is restricted to read only. We use a glob
expression to match all paths that start with /repository:
<policy domain="path" rights="read" pattern="/repository/*" />
Lets prevent users from executing any image filters:
<policy domain="filter" rights="none" pattern="*" />
Any large image is cached to disk rather than memory:
<policy domain="resource" name="area" value="1GP"/>
Define arguments for the memory, map, area, width, height and disk resources
with SI prefixes (.e.g 100MB). In addition, resource policies are maximums
for each instance of ImageMagick (e.g. policy memory limit 1GB, -limit 2GB
exceeds policy maximum so memory limit is 1GB).
Rules are processed in order. Here we want to restrict ImageMagick to only
read or write a small subset of proven web-safe image types:
<policy domain="delegate" rights="none" pattern="*" />
<policy domain="filter" rights="none" pattern="*" />
<policy domain="coder" rights="none" pattern="*" />
<policy domain="coder" rights="read|write" pattern="{GIF,JPEG,PNG,WEBP}" />
-->
<policymap>
<!-- <policy domain="system" name="shred" value="2"/> -->
<!-- <policy domain="system" name="precision" value="6"/> -->
<!-- <policy domain="system" name="memory-map" value="anonymous"/> -->
<!-- <policy domain="system" name="max-memory-request" value="256MiB"/> -->
<!-- <policy domain="resource" name="temporary-path" value="/tmp"/> -->
<policy domain="resource" name="memory" value="256MiB"/>
<policy domain="resource" name="map" value="512MiB"/>
<policy domain="resource" name="width" value="16KP"/>
<policy domain="resource" name="height" value="16KP"/>
<!-- <policy domain="resource" name="list-length" value="128"/> -->
<policy domain="resource" name="area" value="128MB"/>
<policy domain="resource" name="disk" value="1GiB"/>
<!-- <policy domain="resource" name="file" value="768"/> -->
<!-- <policy domain="resource" name="thread" value="4"/> -->
<!-- <policy domain="resource" name="throttle" value="0"/> -->
<!-- <policy domain="resource" name="time" value="3600"/> -->
<!-- Paperless does not process SVG or ImageMagick scripting formats. -->
<policy domain="coder" rights="none" pattern="SVG" />
<policy domain="coder" rights="none" pattern="SVGZ" />
<policy domain="coder" rights="none" pattern="MSVG" />
<policy domain="coder" rights="none" pattern="RSVG" />
<policy domain="coder" rights="none" pattern="MSL" />
<policy domain="coder" rights="none" pattern="MVG" />
<policy domain="coder" rights="none" pattern="EPHEMERAL" />
<!-- <policy domain="module" rights="none" pattern="{PS,PDF,XPS}" /> -->
<!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> -->
<!-- <policy domain="path" rights="none" pattern="@*" /> -->
<!-- <policy domain="cache" name="memory-map" value="anonymous"/> -->
<!-- <policy domain="cache" name="synchronize" value="True"/> -->
<!-- <policy domain="cache" name="shared-secret" value="passphrase" stealth="true"/> -->
<!-- <policy domain="system" name="pixel-cache-memory" value="anonymous"/> -->
<!-- <policy domain="system" name="shred" value="2"/> -->
<!-- <policy domain="system" name="precision" value="6"/> -->
<!-- use curl -->
<policy domain="delegate" rights="none" pattern="URL" />
<policy domain="delegate" rights="none" pattern="HTTPS" />
<policy domain="delegate" rights="none" pattern="HTTP" />
<!-- in order to avoid to get image with password text -->
<policy domain="path" rights="none" pattern="@*"/>
<!-- disable ghostscript format types -->
<policy domain="coder" rights="none" pattern="PS" />
<policy domain="coder" rights="none" pattern="PS2" />
<policy domain="coder" rights="none" pattern="PS3" />
<policy domain="coder" rights="none" pattern="EPS" />
<policy domain="coder" rights="read|write" pattern="PDF" />
<policy domain="coder" rights="none" pattern="XPS" />
</policymap>
+10 -19
View File
@@ -1315,29 +1315,17 @@ valid crontab(5) expression describing when to run.
#### [`PAPERLESS_CONVERT_MEMORY_LIMIT=<num>`](#PAPERLESS_CONVERT_MEMORY_LIMIT) {#PAPERLESS_CONVERT_MEMORY_LIMIT}
: On smaller systems, or even in the case of Very Large Documents, the
consumer may explode, complaining about how it's "unable to extend
pixel cache". In such cases, try setting this to a reasonably low
value, like 32. The default is to use whatever is necessary to do
everything without writing to disk, and units are in megabytes.
!!! warning
For more information on how to use this value, you should search the
web for "MAGICK_MEMORY_LIMIT".
Defaults to 0, which disables the limit.
Deprecated and has no effect, since PDF thumbnails no longer use
ImageMagick. It will be removed in a future release.
#### [`PAPERLESS_CONVERT_TMPDIR=<path>`](#PAPERLESS_CONVERT_TMPDIR) {#PAPERLESS_CONVERT_TMPDIR}
: Similar to the memory limit, if you've got a small system and your
OS mounts /tmp as tmpfs, you should set this to a path that's on a
physical disk, like /home/your_user/tmp or something. ImageMagick
will use this as scratch space when crunching through very large
documents.
!!! warning
For more information on how to use this value, you should search the
web for "MAGICK_TMPDIR".
Default is none, which disables the temporary directory.
Deprecated and has no effect, since PDF thumbnails no longer use
ImageMagick. It will be removed in a future release.
#### [`PAPERLESS_APPS=<string>`](#PAPERLESS_APPS) {#PAPERLESS_APPS}
@@ -1885,7 +1873,10 @@ the literal path for that program.
#### [`PAPERLESS_CONVERT_BINARY=<path>`](#PAPERLESS_CONVERT_BINARY) {#PAPERLESS_CONVERT_BINARY}
: Defaults to "convert".
!!! warning
Deprecated and has no effect, since Paperless-ngx no longer uses
ImageMagick. It will be removed in a future release.
#### [`PAPERLESS_GS_BINARY=<path>`](#PAPERLESS_GS_BINARY) {#PAPERLESS_GS_BINARY}
+1 -3
View File
@@ -76,9 +76,7 @@ is not supported by any of the available parsers.
**A:** Not by default. As of v3, a file whose contents match an existing document is still
consumed, and the duplicate is flagged in the UI — open the document and check the
**Duplicates** tab to review documents that share the same content, or filter the document
list by **Duplicates** to find all of them (see
[Duplicate documents](usage.md#duplicate-documents)). If you prefer the old
**Duplicates** tab to review documents that share the same content. If you prefer the old
behavior of rejecting duplicates during consumption, set
[`PAPERLESS_CONSUMER_DELETE_DUPLICATES`](configuration.md#PAPERLESS_CONSUMER_DELETE_DUPLICATES)
to `true`.
+2 -13
View File
@@ -177,17 +177,16 @@ to a positive number to enable polling and disable native filesystem notificatio
- `pkg-config` for mysqlclient (python dependency)
- `fonts-liberation` for generating thumbnails for plain text
files
- `imagemagick` >= 6 for PDF conversion
- `gnupg` for decrypting GPG-encrypted email
- `libpq-dev` for PostgreSQL
- `libmagic-dev` for mime type detection
- `mariadb-client` for MariaDB compile time
- `poppler-utils` for barcode detection
- `poppler-utils` for thumbnail generation and barcode detection
Use this list for your preferred package management:
```
python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils
python3 python3-pip python3-dev fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils
```
These dependencies are required for OCRmyPDF, which is used for text
@@ -416,16 +415,6 @@ to a positive number to enable polling and disable native filesystem notificatio
You may need to change the path in the files. Example:
`ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO`
12. Configure ImageMagick to allow processing of PDF documents and disable
formats that Paperless-ngx does not use. Most distributions disable PDF
processing by default, since PDF documents can contain malware. If you
don't enable it, Paperless-ngx will fall back to Ghostscript for certain
steps such as thumbnail generation.
Configure the active ImageMagick policy file (commonly
`/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and
adjust similar to [the docker policy file](https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/refs/heads/main/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml). You should also include restrictions as noted there.
**Optional: Install the [jbig2enc](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html) encoder.**
This will reduce the size of generated PDF documents. You'll most likely need to compile this yourself, because this
software has been patented until around 2017 and binary packages are not available for most distributions.
+8 -7
View File
@@ -299,18 +299,19 @@ for details.
### Duplicate documents
By default, Paperless-ngx **does not reject duplicates**. If you consume a file whose
contents match an existing document (same original or archive checksum), the new copy is
still consumed and a warning is logged.
contents exactly match an existing document (same checksum), the new copy is still
consumed and a warning is logged. The task entry for the upload also flags that a
duplicate was detected and links to the existing document(s).
When a document has duplicates, a **Duplicates** tab appears on its detail page, listing
the other documents you can view that share the same content (including any in the trash).
To find all documents with duplicates, choose **Duplicates** in the document list's text
filter dropdown, or use `has_duplicates=true` in the REST API.
To review duplicates, open a document and switch to the **Duplicates** tab on the
document detail page. It lists other documents that share the same content, including any
that are in the trash (shown with a badge), and links to each so you can decide which to
keep.
If you would rather reject duplicates at consumption time (the pre-v3 behavior), set
[`PAPERLESS_CONSUMER_DELETE_DUPLICATES`](configuration.md#PAPERLESS_CONSUMER_DELETE_DUPLICATES)
to `true`. The duplicate file is then deleted instead of consumed, and the task fails with
a "Document already exists" message linking to the existing document.
a "document already exists" message.
## Document Suggestions
-3
View File
@@ -50,8 +50,6 @@ PAPERLESS_SECRET_KEY=change-me
#PAPERLESS_OCR_ROTATE_PAGES=true
#PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0
#PAPERLESS_OCR_USER_ARGS={}
#PAPERLESS_CONVERT_MEMORY_LIMIT=0
#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
# Software tweaks
@@ -90,5 +88,4 @@ PAPERLESS_SECRET_KEY=change-me
# Binaries
#PAPERLESS_CONVERT_BINARY=/usr/bin/convert
#PAPERLESS_GS_BINARY=/usr/bin/gs
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "paperless-ngx"
version = "3.3.0"
version = "3.2.1"
description = """\
A community-supported supercharged document management system: scan, index and archive all your physical documents\
"""
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "paperless-ngx-ui",
"version": "3.3.0",
"version": "3.2.1",
"scripts": {
"preinstall": "npx only-allow pnpm",
"ng": "ng",
+1 -1
View File
@@ -8,7 +8,7 @@ export const environment = {
apiVersion: '10', // match src/paperless/settings.py
appTitle: DEFAULT_APP_TITLE,
tag: 'prod',
version: '3.3.0',
version: '3.2.1',
webSocketHost: window.location.host,
webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
webSocketBaseUrl: base_url.pathname + 'ws/',
+27 -2
View File
@@ -13,8 +13,14 @@ from celery import group
from celery import shared_task
from django.conf import settings
from django.db import transaction
from django.db.models import Case
from django.db.models import F
from django.db.models import Max
from django.db.models import OuterRef
from django.db.models import Q
from django.db.models import Subquery
from django.db.models import When
from django.db.models.functions import Coalesce
from django.utils import timezone
from documents.data_models import ConsumableDocument
@@ -36,6 +42,7 @@ from documents.tasks import remove_document_from_index
from documents.tasks import update_document_content_maybe_archive_file
from documents.versioning import get_latest_version_for_root
from documents.versioning import get_root_document
from documents.versioning import versions_newest_first
if TYPE_CHECKING:
from collections.abc import Mapping
@@ -408,10 +415,28 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
Consumption workflows do not run here, so ``remote_ocr`` is how the user
asks for the remote engine when it is not configured to handle everything.
A root document with versions reprocesses its latest version, which is the
file whose content, archive and thumbnail are shown for it.
"""
for document_id in doc_ids:
latest_version = versions_newest_first(
Document.objects.filter(root_document=OuterRef("pk")),
).values("id")[:1]
source_ids = (
Document.objects.filter(id__in=doc_ids)
.annotate(
source_id=Case(
When(root_document__isnull=False, then=F("id")),
default=Coalesce(Subquery(latest_version), F("id")),
),
)
.order_by()
.values_list("source_id", flat=True)
.distinct()
)
for source_id in source_ids:
update_document_content_maybe_archive_file.apply_async(
kwargs={"document_id": document_id, "remote_ocr": remote_ocr},
kwargs={"document_id": source_id, "remote_ocr": remote_ocr},
headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
)
+2 -11
View File
@@ -1,12 +1,11 @@
from pathlib import Path
import img2pdf
from django.conf import settings
from PIL import Image
from documents.utils import copy_basic_file_stats
from documents.utils import maybe_override_pixel_limit
from documents.utils import run_subprocess
from documents.utils import remove_alpha_channel
def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
@@ -27,15 +26,7 @@ def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
# Note the save into the temp folder, so as not to trigger a new
# consume
scratch_image = target_directory / tiff_path.name
run_subprocess(
[
settings.CONVERT_BINARY,
"-alpha",
"off",
tiff_path,
scratch_image,
],
)
remove_alpha_channel(tiff_path, scratch_image)
else:
# Not modifying the original, safe to use in place
scratch_image = tiff_path
+161 -98
View File
@@ -1,8 +1,8 @@
from __future__ import annotations
import logging
import math
import mimetypes
import os
import shutil
import subprocess
import tempfile
@@ -68,58 +68,6 @@ def get_supported_file_extensions() -> set[str]:
return extensions
def run_convert(
input_file,
output_file,
*,
density=None,
scale=None,
alpha=None,
strip=False,
trim=False,
type=None,
depth=None,
auto_orient=False,
use_cropbox=False,
extra=None,
logging_group=None,
) -> None:
environment = os.environ.copy()
if settings.CONVERT_MEMORY_LIMIT:
# MAGICK_MEMORY_LIMIT sets the maximum amount of RAM the pixel cache can use.
# MAGICK_MAP_LIMIT sets the maximum amount of memory-mapped I/O allowed.
#
# For large-format documents ImageMagick will hit the RAM limit and
# immediately try to "map" the remaining data. If MAGICK_MAP_LIMIT isn't
# also set, the process may trigger an OOM kill because the default
# system/policy map limit is often too restrictive for these massive bitmaps.
environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
environment["MAGICK_MAP_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
if settings.CONVERT_TMPDIR:
environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR
args = [settings.CONVERT_BINARY]
args += ["-density", str(density)] if density else []
args += ["-scale", str(scale)] if scale else []
args += ["-alpha", str(alpha)] if alpha else []
args += ["-strip"] if strip else []
args += ["-trim"] if trim else []
args += ["-type", str(type)] if type else []
args += ["-depth", str(depth)] if depth else []
args += ["-auto-orient"] if auto_orient else []
args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else []
args += [str(input_file), str(output_file)]
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
try:
run_subprocess(args, environment, logger)
except subprocess.CalledProcessError as e:
raise ParseError(f"Convert failed at {args}") from e
except Exception as e: # pragma: no cover
raise ParseError("Unknown error running convert") from e
def get_default_thumbnail() -> Path:
"""
Returns the path to a generic thumbnail
@@ -127,46 +75,168 @@ def get_default_thumbnail() -> Path:
return (Path(__file__).parent / "resources" / "document.webp").resolve()
def make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group=None) -> Path:
out_path: Path = Path(temp_dir) / "convert_gs.webp"
_THUMBNAIL_MAX_WIDTH = 500
_THUMBNAIL_MAX_HEIGHT = 5000
# Used only when the page geometry cannot be read
_THUMBNAIL_FALLBACK_DPI = 150
# Applied before supersampling, so tiny pages are not enlarged
_THUMBNAIL_MAX_DPI = 300
# Rendering at a multiple and downsampling keeps text crisper
_THUMBNAIL_SUPERSAMPLE = 2
# if convert fails, fall back to extracting
# the first PDF page as a PNG using Ghostscript
logger.warning(
"Thumbnail generation with ImageMagick failed, falling back "
"to ghostscript. Check your /etc/ImageMagick-x/policy.xml!",
extra={"group": logging_group},
)
# Ghostscript doesn't handle WebP outputs
gs_out_path: Path = Path(temp_dir) / "gs_out.png"
cmd = [settings.GS_BINARY, "-q", "-sDEVICE=pngalpha", "-o", gs_out_path, in_path]
def rasterize_pdf_page_to_png(
in_path: Path,
out_path: Path,
*,
dpi: int,
logging_group=None,
) -> None:
"""
Rasterizes the first page of a PDF to a PNG with pdftoppm.
"""
# -singlefile drops the page number and -png appends ".png", so pass the
# path without its suffix
args = [
"pdftoppm",
"-f",
"1",
"-l",
"1",
"-r",
str(dpi),
"-png",
"-singlefile",
"-cropbox",
str(in_path),
str(out_path.with_suffix("")),
]
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
try:
try:
run_subprocess(cmd, logger=logger)
except subprocess.CalledProcessError as e:
raise ParseError(f"Thumbnail (gs) failed at {cmd}") from e
# then run convert on the output from gs to make WebP
run_convert(
density=300,
scale="500x5000>",
alpha="remove",
strip=True,
trim=False,
auto_orient=True,
input_file=gs_out_path,
output_file=out_path,
logging_group=logging_group,
)
run_subprocess(args, logger=logger)
except subprocess.CalledProcessError as e:
raise ParseError(f"pdftoppm failed at {args}") from e
except Exception as e: # pragma: no cover
raise ParseError("Unknown error running pdftoppm") from e
def encode_thumbnail_webp(
png_path: Path,
out_path: Path,
*,
supersample: int = 1,
) -> None:
"""
Flattens alpha onto white, undoes supersampling, shrinks to fit and saves as WebP.
"""
from PIL import Image
try:
with Image.open(png_path) as im:
if im.mode in ("RGBA", "LA"):
flattened = Image.new("RGB", im.size, (255, 255, 255))
flattened.paste(im, mask=im.split()[-1])
else:
flattened = im.convert("RGB")
if supersample > 1:
flattened = flattened.resize(
(
max(1, round(flattened.width / supersample)),
max(1, round(flattened.height / supersample)),
),
Image.Resampling.LANCZOS,
)
flattened.thumbnail((_THUMBNAIL_MAX_WIDTH, _THUMBNAIL_MAX_HEIGHT))
flattened.save(out_path, format="WEBP")
except (OSError, Image.DecompressionBombError) as e:
raise ParseError(f"Unable to encode thumbnail from {png_path}") from e
def _compute_thumbnail_dpi(in_path: Path, logging_group=None) -> tuple[int, int]:
"""
Returns (dpi, supersample). Unknown geometry is not supersampled, since
the render size cannot be bounded.
"""
from paperless.parsers.utils import get_pdf_first_page_size_points
size = get_pdf_first_page_size_points(in_path)
if size is None:
logger.debug(
"Could not read PDF page size, using fallback DPI",
extra={"group": logging_group},
)
return _THUMBNAIL_FALLBACK_DPI, 1
width_pts, height_pts = size
dpi_for_width = _THUMBNAIL_MAX_WIDTH * 72 / width_pts
dpi_for_height = _THUMBNAIL_MAX_HEIGHT * 72 / height_pts
# Round up so the downsampled render is never a few pixels short of the
# target; the shrink-only clamp in encode_thumbnail_webp trims the excess.
dpi = max(
1,
math.ceil(min(_THUMBNAIL_MAX_DPI, dpi_for_width, dpi_for_height)),
)
# At the 1 DPI floor the page is already oversized, so do not supersample
return dpi, 1 if dpi == 1 else _THUMBNAIL_SUPERSAMPLE
def _render_pdf_thumbnail(
in_path: Path,
png_path: Path,
out_path: Path,
logging_group=None,
) -> None:
dpi, supersample = _compute_thumbnail_dpi(in_path, logging_group=logging_group)
rasterize_pdf_page_to_png(
in_path,
png_path,
dpi=dpi * supersample,
logging_group=logging_group,
)
encode_thumbnail_webp(png_path, out_path, supersample=supersample)
def _repair_pdf_with_qpdf(in_path: Path, out_path: Path) -> None:
# qpdf exits 3 after a repair; --warning-exit-0 keeps that from failing
try:
shutil.copy(in_path, out_path)
run_subprocess(
["qpdf", "--warning-exit-0", "--replace-input", str(out_path)],
logger=logger,
)
except (subprocess.CalledProcessError, OSError) as e:
raise ParseError(f"qpdf repair failed for {in_path}") from e
def make_thumbnail_from_pdf_qpdf_fallback(
in_path: Path,
temp_dir: Path,
logging_group=None,
) -> Path:
png_path = temp_dir / "page1_repaired.png"
out_path = temp_dir / "convert_qpdf.webp"
repaired_path = temp_dir / "repaired.pdf"
logger.warning(
"Thumbnail generation with pdftoppm failed, attempting qpdf repair and retry.",
extra={"group": logging_group},
)
try:
_repair_pdf_with_qpdf(in_path, repaired_path)
_render_pdf_thumbnail(repaired_path, png_path, out_path, logging_group)
return out_path
except ParseError as e:
logger.error(f"Unable to make thumbnail with Ghostscript: {e}")
logger.error(f"Unable to make thumbnail after qpdf repair: {e}")
# The caller might expect a generated thumbnail that can be moved,
# so we need to copy it before it gets moved.
# https://github.com/paperless-ngx/paperless-ngx/issues/3631
default_thumbnail_path: Path = Path(temp_dir) / "document.webp"
default_thumbnail_path = temp_dir / "document.webp"
copy_file_with_basic_stats(get_default_thumbnail(), default_thumbnail_path)
return default_thumbnail_path
@@ -175,25 +245,18 @@ def make_thumbnail_from_pdf(in_path: Path, temp_dir: Path, logging_group=None) -
"""
The thumbnail of a PDF is just a 500px wide image of the first page.
"""
png_path: Path = temp_dir / "page1.png"
out_path: Path = temp_dir / "convert.webp"
# Run convert to get a decent thumbnail
try:
run_convert(
density=300,
scale="500x5000>",
alpha="remove",
strip=True,
trim=False,
auto_orient=True,
use_cropbox=True,
input_file=f"{in_path}[0]",
output_file=str(out_path),
logging_group=logging_group,
)
_render_pdf_thumbnail(in_path, png_path, out_path, logging_group)
except ParseError as e:
logger.error(f"Unable to make thumbnail with convert: {e}")
out_path = make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group)
logger.error(f"Unable to make thumbnail with pdftoppm: {e}")
out_path = make_thumbnail_from_pdf_qpdf_fallback(
in_path,
temp_dir,
logging_group,
)
return out_path
+24
View File
@@ -14,6 +14,7 @@ from django.db.models import QuerySet
from django.db.models import Value
from django.db.models import When
from django.db.models.functions import Cast
from django.db.models.functions import Coalesce
from guardian.core import ObjectPermissionChecker
from guardian.models import GroupObjectPermission
from guardian.models import UserObjectPermission
@@ -474,6 +475,29 @@ def permitted_document_ids(
return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
def documents_without_permitted_root(
documents: QuerySet[Document],
user: User | None,
*,
perm: str = "view_document",
include_deleted: bool = False,
) -> QuerySet[Document]:
"""
The documents the user lacks ``perm`` on. Versions are authorized by their
root document, so a version's own owner is ignored. A single query, without
loading the documents or joining the root.
"""
return documents.annotate(
root_id=Coalesce("root_document_id", "id"),
).exclude(
root_id__in=permitted_document_ids(
user,
perm=perm,
include_deleted=include_deleted,
),
)
def get_document_count_filter_for_user(user, related_name: str = "documents"):
"""
Return the Q object used to filter document counts for the given user.
+61 -33
View File
@@ -7,6 +7,7 @@ import threading
import time
from datetime import UTC
from datetime import datetime
from datetime import timedelta
from enum import StrEnum
from itertools import islice
from typing import TYPE_CHECKING
@@ -31,6 +32,7 @@ from documents.search._query import parse_user_query
from documents.search._schema import _write_sentinels
from documents.search._schema import build_schema
from documents.search._schema import open_or_rebuild_index
from documents.search._schema import rebuild_in_progress
from documents.search._schema import wipe_index
from documents.search._tokenizer import ascii_fold
from documents.search._tokenizer import autocomplete_tokens
@@ -54,6 +56,19 @@ if TYPE_CHECKING:
logger = logging.getLogger("paperless.search")
# tantivy stores dates as signed 64-bit nanoseconds since the Unix epoch, which
# covers 1677-09-21T00:12:43 to 2262-04-11T23:47:16 UTC
_INDEX_DATE_NANOS_MIN: Final[int] = -(2**63)
_INDEX_DATE_NANOS_MAX: Final[int] = 2**63 - 1
_UNIX_EPOCH: Final[datetime] = datetime(1970, 1, 1, tzinfo=UTC)
def _is_indexable_date(value: datetime) -> bool:
"""Whether value, at whole-second precision, fits tantivy's date range."""
nanos = ((value - _UNIX_EPOCH) // timedelta(seconds=1)) * 1_000_000_000
return _INDEX_DATE_NANOS_MIN <= nanos <= _INDEX_DATE_NANOS_MAX
_LOCK_TIMEOUT_SECONDS: Final[float] = 10.0 # per-attempt acquire timeout
_LOCK_RETRY_ATTEMPTS: Final[int] = 4 # total attempts (1 initial + 3 retries)
_LOCK_BACKOFF_BASE: Final[float] = 1.0 # seconds
@@ -627,7 +642,15 @@ class TantivyBackend:
document.created.day,
tzinfo=UTC,
)
doc.add_date("created", created_date)
if _is_indexable_date(created_date):
doc.add_date("created", created_date)
else:
logger.warning(
"Document %s has a created date (%s) outside the range the search "
"index can store; it will be indexed without a created date",
document.pk,
document.created,
)
doc.add_date("modified", document.modified)
doc.add_date("added", document.added)
@@ -1110,39 +1133,44 @@ class TantivyBackend:
flushing a segment, deferring merge work; they do not avoid it.
"""
wipe_index(self._path)
new_index = tantivy.Index(build_schema(), path=str(self._path))
_write_sentinels(self._path)
register_tokenizers(new_index, settings.SEARCH_LANGUAGE)
# The marker covers the window where the empty index is already stamped
# as current but not yet populated, so an interrupted rebuild is retried.
with rebuild_in_progress(self._path):
new_index = tantivy.Index(build_schema(), path=str(self._path))
_write_sentinels(self._path)
register_tokenizers(new_index, settings.SEARCH_LANGUAGE)
# Point instance at the new index so _build_tantivy_doc uses it
old_index, old_schema = self._raw_index, self._raw_schema
self._raw_index = new_index
self._raw_schema = new_index.schema
# Stream documents one-by-one (so the progress bar advances per
# document) while fetching viewer permissions one SQL query per chunk.
# The stream is Sized, so iter_wrapper can still discover the total.
documents_stream = _DocumentViewerStream(documents, chunk_size=1000)
try:
writer = new_index.writer(heap_size=writer_heap_bytes)
for document, (viewer_ids, viewer_group_ids) in iter_wrapper(
documents_stream,
):
doc = self._build_tantivy_doc(
document,
viewer_ids=viewer_ids,
viewer_group_ids=viewer_group_ids,
)
writer.add_document(doc)
writer.commit()
# Wait for background merge threads to finish so all segments are
# fully merged and persisted before the index is considered rebuilt.
writer.wait_merging_threads()
new_index.reload()
except BaseException: # pragma: no cover
# Restore old index on failure so the backend remains usable
self._raw_index = old_index
self._raw_schema = old_schema
raise
# Point instance at the new index so _build_tantivy_doc uses it
old_index, old_schema = self._raw_index, self._raw_schema
self._raw_index = new_index
self._raw_schema = new_index.schema
# Stream documents one-by-one (so the progress bar advances per
# document) while fetching viewer permissions one SQL query per
# chunk. The stream is Sized, so iter_wrapper can still discover
# the total.
documents_stream = _DocumentViewerStream(documents, chunk_size=1000)
try:
writer = new_index.writer(heap_size=writer_heap_bytes)
for document, (viewer_ids, viewer_group_ids) in iter_wrapper(
documents_stream,
):
doc = self._build_tantivy_doc(
document,
viewer_ids=viewer_ids,
viewer_group_ids=viewer_group_ids,
)
writer.add_document(doc)
writer.commit()
# Wait for background merge threads to finish so all segments
# are fully merged and persisted before the index is considered
# rebuilt.
writer.wait_merging_threads()
new_index.reload()
except BaseException: # pragma: no cover
# Restore old index on failure so the backend remains usable
self._raw_index = old_index
self._raw_schema = old_schema
raise
def chunked(iterable, size):
+45 -4
View File
@@ -4,6 +4,7 @@ import hashlib
import json
import logging
import shutil
from contextlib import contextmanager
from typing import TYPE_CHECKING
from typing import Final
from typing import NamedTuple
@@ -16,6 +17,7 @@ from whoosh_compat import FieldKind
from documents.search._fields import PUBLIC_FIELDS
if TYPE_CHECKING:
from collections.abc import Iterator
from pathlib import Path
logger = logging.getLogger("paperless.search")
@@ -28,6 +30,11 @@ logger = logging.getLogger("paperless.search")
# v3 - barcodes JSON field for stored barcode contents
SCHEMA_VERSION: Final[int] = 3
# Present in the index directory from the moment a full rebuild starts until it
# finishes. If a rebuild is interrupted it is left behind, so the half-built
# index is not mistaken for a complete one.
REBUILD_MARKER: Final[str] = ".rebuilding"
class FieldDescriptor(NamedTuple):
"""One tantivy field, in declaration order.
@@ -255,9 +262,9 @@ def needs_rebuild(index_dir: Path) -> bool:
"""
Check if the search index needs rebuilding.
Reads .index_settings.json to compare the stored schema version, search
language and schema fingerprint against the current configuration. Returns
True if the file is missing, unparsable, or any value mismatches.
True if a previous full rebuild never finished (the rebuild marker is still
present), or if the index's stamped settings no longer match the current
configuration. See _settings_mismatch().
Args:
index_dir: Path to the search index directory
@@ -265,6 +272,40 @@ def needs_rebuild(index_dir: Path) -> bool:
Returns:
True if the index needs rebuilding, False if it's up to date
"""
if (index_dir / REBUILD_MARKER).exists():
logger.warning("Previous search index rebuild did not finish - rebuilding.")
return True
return _settings_mismatch(index_dir)
@contextmanager
def rebuild_in_progress(index_dir: Path) -> Iterator[None]:
"""
Flag the index as incomplete for the duration of a full rebuild.
The marker is cleared only if the block exits cleanly. There is deliberately
no try/finally: an exception must leave the marker behind so the next
needs_rebuild() check retries the rebuild.
"""
marker = index_dir / REBUILD_MARKER
marker.touch()
yield
marker.unlink(missing_ok=True)
def _settings_mismatch(index_dir: Path) -> bool:
"""
Check the stamped settings against the current configuration.
Reads .index_settings.json to compare the stored schema version, search
language and schema fingerprint. Returns True if the file is missing,
unparsable, or any value mismatches.
This deliberately ignores the rebuild marker: open_or_rebuild_index() uses it
so that a process opening the index while another process is mid-rebuild
(or after one died) does not wipe the partial index out from under it.
Repopulating is the job of ``document_index reindex``.
"""
settings_file = index_dir / ".index_settings.json"
if not settings_file.exists():
return True
@@ -333,7 +374,7 @@ def open_or_rebuild_index(index_dir: Path | None = None) -> tantivy.Index:
index_dir = cast("Path", settings.INDEX_DIR)
if not index_dir.exists():
return tantivy.Index(build_schema())
if needs_rebuild(index_dir):
if _settings_mismatch(index_dir):
wipe_index(index_dir)
idx = tantivy.Index(build_schema(), path=str(index_dir))
_write_sentinels(index_dir)
+2 -1
View File
@@ -90,6 +90,7 @@ from documents.templating.utils import convert_format_str_to_template_format
from documents.templating.workflows import validate_workflow_template
from documents.validators import uri_validator
from documents.validators import url_validator
from documents.versioning import get_root_document
from documents.versioning import has_prefetched_effective_content
from documents.versioning import sort_versions_newest_first
@@ -2894,7 +2895,7 @@ class ShareLinkSerializer(OwnedObjectSerializer):
and has_perms_owner_aware(
self.user,
"view_document",
document,
get_root_document(document),
)
):
return document
+8 -3
View File
@@ -490,18 +490,23 @@ def update_document_content_maybe_archive_file(
shutil.move(thumbnail, document.thumbnail_path)
document.refresh_from_db()
root_document = (
document.root_document if document.root_document_id else document
)
logger.info(
f"Updating index for document {document_id} ({document.archive_checksum})",
f"Updating index for document {root_document.pk} ({document.archive_checksum})",
)
from documents.search import get_backend
get_backend().add_or_update(document)
get_backend().add_or_update(root_document)
ai_config = AIConfig()
if ai_config.llm_index_enabled:
llm_index_add_or_update_document(document)
llm_index_add_or_update_document(root_document)
clear_document_caches(document.pk)
if root_document.pk != document.pk:
clear_document_caches(root_document.pk)
except Exception:
logger.exception(
+125
View File
@@ -1,4 +1,6 @@
import json
import logging
from datetime import date
from pathlib import Path
import pytest
@@ -16,7 +18,10 @@ from documents.search._backend import TantivyBackend
from documents.search._backend import WriteBatch
from documents.search._backend import get_backend
from documents.search._backend import reset_backend
from documents.search._schema import REBUILD_MARKER
from documents.search._schema import needs_rebuild
from documents.signals.handlers import add_to_index
from paperless_testing.dirs import PaperlessDirs
from paperless_testing.factories import CorrespondentFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import DocumentTypeFactory
@@ -823,6 +828,53 @@ class TestRebuild:
backend.rebuild(Document.objects.all(), iter_wrapper=wrapper)
assert 30 in seen
def test_successful_rebuild_leaves_index_up_to_date(
self,
backend: TantivyBackend,
paperless_dirs: PaperlessDirs,
) -> None:
"""
GIVEN:
- A backend and one document
WHEN:
- rebuild() completes
THEN:
- needs_rebuild() is False and no rebuild marker remains
"""
DocumentFactory.create()
backend.rebuild(Document.objects.all())
assert needs_rebuild(paperless_dirs.index_dir) is False
assert not (paperless_dirs.index_dir / REBUILD_MARKER).exists()
def test_interrupted_rebuild_is_retried(
self,
backend: TantivyBackend,
paperless_dirs: PaperlessDirs,
) -> None:
"""
GIVEN:
- A rebuild that dies while indexing documents (e.g. the database
connection is lost)
WHEN:
- needs_rebuild() is checked afterwards
THEN:
- It is True, even though the empty index was already stamped with
current settings, so the next start rebuilds instead of reporting
the index as up to date
"""
DocumentFactory.create()
def die(pairs):
raise RuntimeError("terminating connection due to administrator command")
yield # pragma: no cover
with pytest.raises(RuntimeError):
backend.rebuild(Document.objects.all(), iter_wrapper=die)
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_includes_group_granted_viewers(self, backend: TantivyBackend) -> None:
"""Rebuild must index viewer ids for group-only grants, not just direct ones.
@@ -854,6 +906,79 @@ class TestRebuild:
assert ids == [doc.pk]
class TestCreatedDateOutOfRange:
"""The index stores dates as nanosecond i64 values (1677-09-22 to 2262-04-11).
A document whose created date falls outside that window must not abort
indexing: it is indexed without a created value and a warning names it.
"""
@pytest.mark.parametrize(
("created", "expected_warnings"),
[
pytest.param(date(1677, 9, 22), 0, id="first-representable-day"),
pytest.param(date(2262, 4, 11), 0, id="last-representable-day"),
pytest.param(date(1677, 9, 21), 1, id="day-before-first"),
pytest.param(date(2262, 4, 12), 1, id="day-after-last"),
pytest.param(date(16, 8, 30), 1, id="two-digit-year-read-as-year-16"),
pytest.param(date(9999, 12, 31), 1, id="max-python-date"),
],
)
def test_add_or_update_indexes_document_and_warns_when_out_of_range(
self,
backend: TantivyBackend,
caplog: pytest.LogCaptureFixture,
created: date,
expected_warnings: int,
) -> None:
"""
GIVEN:
- A document with a created date at or beyond the index date limits
WHEN:
- The document is added to the index
THEN:
- The document is indexed and searchable either way
- A warning naming the document is logged only for out-of-range dates
"""
doc = DocumentFactory(created=created, content="boundarycontent")
with caplog.at_level(logging.WARNING, logger="paperless.search"):
backend.add_or_update(doc)
assert backend.search_ids("boundarycontent", user=None) == [doc.pk]
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
assert len(warnings) == expected_warnings
if expected_warnings:
assert f"Document {doc.pk}" in warnings[0].getMessage()
def test_rebuild_continues_past_out_of_range_document(
self,
backend: TantivyBackend,
caplog: pytest.LogCaptureFixture,
) -> None:
"""
GIVEN:
- A document with an unrepresentable created date among valid ones
WHEN:
- The index is rebuilt
THEN:
- Rebuild completes and every document is searchable
- A warning names the offending document
"""
good = DocumentFactory(created=date(2016, 8, 30), content="rebuildcontent")
bad = DocumentFactory(created=date(16, 8, 30), content="rebuildcontent")
with caplog.at_level(logging.WARNING, logger="paperless.search"):
backend.rebuild(Document.objects.all())
assert sorted(backend.search_ids("rebuildcontent", user=None)) == sorted(
[good.pk, bad.pk],
)
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
assert len(warnings) == 1
assert f"Document {bad.pk}" in warnings[0].getMessage()
class TestAutocomplete:
"""Test autocomplete functionality."""
@@ -8,6 +8,7 @@ from auditlog.models import LogEntry # type: ignore[import-untyped]
from django.contrib.contenttypes.models import ContentType
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase as DjangoTestCase
from django.test import override_settings
from django.utils import timezone
from rest_framework import status
from rest_framework.test import APITestCase
@@ -16,13 +17,17 @@ from documents.data_models import DocumentSource
from documents.filters import EffectiveContentFilter
from documents.filters import TitleContentFilter
from documents.models import Document
from documents.models import Note
from documents.models import ShareLink
from documents.versioning import annotate_effective_content
from documents.views import DocumentSelectionMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_all_global
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
if TYPE_CHECKING:
from pathlib import Path
@@ -1043,3 +1048,152 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
)
self.assertEqual(selected, [root.id])
class TestVersionActionPermissions(DirectoriesMixin, APITestCase):
def setUp(self):
super().setUp()
self.user = UserFactory()
grant_all_global(self.user)
self.client.force_authenticate(self.user)
self.root = DocumentFactory(owner=UserFactory())
self.version = DocumentFactory(root_document=self.root, owner=None)
@override_settings(AUDIT_LOG_ENABLED=True)
def test_actions_reject_stale_version_ownership(self):
note = Note.objects.create(document=self.version, note="Version note")
for owner in (None, self.user):
self.version.owner = owner
self.version.save(update_fields=["owner"])
for action in (
"notes",
"suggestions",
"ai_suggestions",
"history",
"share_links",
):
with self.subTest(owner=owner, action=action):
response = self.client.get(
f"/api/documents/{self.version.pk}/{action}/",
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
f"/api/documents/{self.version.pk}/notes/",
{"note": "New note"},
)
self.assertEqual(response.status_code, 403)
response = self.client.delete(
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
"/api/share_links/",
{"document": self.version.pk, "file_version": "original"},
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
"/api/share_link_bundles/",
{"document_ids": [self.version.pk], "file_version": "original"},
format="json",
)
self.assertEqual(response.status_code, 400)
response = self.client.post(
"/api/documents/email/",
{
"documents": [self.version.pk],
"addresses": "recipient@example.com",
"subject": "Version",
"message": "Version",
},
format="json",
)
self.assertEqual(response.status_code, 403)
with (
mock.patch("documents.views.AIConfig") as ai_config,
mock.patch("documents.views.stream_chat_with_documents") as chat,
):
ai_config.return_value.ai_enabled = True
response = self.client.post(
"/api/documents/chat/",
{"q": "Version?", "document_id": self.version.pk},
format="json",
)
self.assertEqual(response.status_code, 403)
chat.assert_not_called()
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
self.assertFalse(ShareLink.objects.exists())
@mock.patch("documents.views.build_share_link_bundle.apply_async")
def test_root_permissions_allow_sharing_a_private_version(self, build_mock):
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
grant_object(self.user, self.root, "view_document", "change_document")
note = Note.objects.create(document=self.version, note="Version note")
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data[0]["id"], note.pk)
response = self.client.post(
"/api/share_links/",
{"document": self.version.pk, "file_version": "original"},
)
self.assertEqual(response.status_code, 201)
self.assertEqual(ShareLink.objects.get().document_id, self.version.pk)
response = self.client.get(f"/api/documents/{self.version.pk}/share_links/")
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
response = self.client.post(
"/api/share_link_bundles/",
{"document_ids": [self.version.pk], "file_version": "original"},
format="json",
)
self.assertEqual(response.status_code, 201)
build_mock.assert_called_once()
def test_root_view_permission_does_not_allow_note_changes(self):
grant_object(self.user, self.root, "view_document")
note = Note.objects.create(document=self.version, note="Version note")
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
self.assertEqual(response.status_code, 200)
response = self.client.post(
f"/api/documents/{self.version.pk}/notes/",
{"note": "New note"},
)
self.assertEqual(response.status_code, 403)
response = self.client.delete(
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
)
self.assertEqual(response.status_code, 403)
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
@override_settings(AUDIT_LOG_ENABLED=True)
def test_history_uses_root_ownership(self):
self.root.owner = self.user
self.root.save(update_fields=["owner"])
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
response = self.client.get(f"/api/documents/{self.version.pk}/history/")
self.assertEqual(response.status_code, 200)
def test_selection_data_rejects_stale_version_ownership(self):
for owner in (None, self.user):
self.version.owner = owner
self.version.save(update_fields=["owner"])
with self.subTest(owner=owner):
response = self.client.post(
"/api/documents/selection_data/",
{"documents": [self.version.pk]},
format="json",
)
self.assertEqual(response.status_code, 403)
def test_selection_data_allows_private_version_of_permitted_root(self):
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
grant_object(self.user, self.root, "view_document")
other = DocumentFactory(owner=self.user)
response = self.client.post(
"/api/documents/selection_data/",
{"documents": [self.version.pk, other.pk]},
format="json",
)
self.assertEqual(response.status_code, 200)
+51
View File
@@ -6,6 +6,7 @@ from rest_framework.test import APITestCase
from documents.models import Document
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_all_global
@@ -279,3 +280,53 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
Document.objects.filter(root_document=root).values_list("id", flat=True),
[version.pk for version in versions],
)
def test_api_trash_version_follows_root_owner(self) -> None:
"""
GIVEN:
- A deleted version of user2's document, owned by nobody
- A deleted version of the user's document, owned by user2
WHEN:
- The user lists the trash and tries to restore or empty the versions
THEN:
- Only the version of the user's own document is listed
- The other version can't be restored or emptied
- The version of the user's own document can be restored
"""
user2 = UserFactory(username="user2")
other_version = DocumentFactory(
root_document=DocumentFactory(owner=user2),
version_index=1,
)
other_version.delete()
own_version = DocumentFactory(
owner=user2,
root_document=DocumentFactory(owner=self.user),
version_index=1,
)
own_version.delete()
resp = self.client.get("/api/trash/")
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(
[doc["id"] for doc in resp.data["results"]],
[own_version.pk],
)
for action in ("restore", "empty"):
with self.subTest(action=action):
resp = self.client.post(
"/api/trash/",
{"action": action, "documents": [other_version.pk]},
)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(
Document.deleted_objects.filter(pk=other_version.pk).exists(),
)
resp = self.client.post(
"/api/trash/",
{"action": "restore", "documents": [own_version.pk]},
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertTrue(Document.objects.filter(pk=own_version.pk).exists())
+91 -27
View File
@@ -4,6 +4,7 @@ from pathlib import Path
from unittest import mock
import pikepdf
import pytest
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
@@ -12,6 +13,7 @@ from django.test import TestCase
from django.test.utils import CaptureQueriesContext
from guardian.shortcuts import get_groups_with_perms
from guardian.shortcuts import get_users_with_perms
from pytest_mock import MockerFixture
from documents import bulk_edit
from documents.models import Correspondent
@@ -23,6 +25,7 @@ from documents.models import StoragePath
from documents.models import Tag
from documents.permissions import set_permissions_for_objects
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.permissions import grant_object
@@ -1970,18 +1973,22 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertIn("Error removing password from document", cm.output[0])
class TestBulkEditReprocess(DirectoriesMixin, TestCase):
def setUp(self) -> None:
super().setUp()
self.doc = Document.objects.create(
title="test",
checksum="A",
mime_type="application/pdf",
@pytest.mark.django_db
class TestBulkEditReprocess:
@pytest.fixture
def mock_task(self, mocker: MockerFixture) -> mock.MagicMock:
return mocker.patch(
"documents.bulk_edit.update_document_content_maybe_archive_file",
)
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
def test_reprocess_defaults_to_local(self, mock_task: mock.Mock) -> None:
@staticmethod
def _queued_ids(mock_task: mock.MagicMock) -> list[int]:
return [
call.kwargs["kwargs"]["document_id"]
for call in mock_task.apply_async.call_args_list
]
def test_reprocess_defaults_to_local(self, mock_task: mock.MagicMock) -> None:
"""
GIVEN:
- A reprocess request that says nothing about remote OCR
@@ -1990,18 +1997,17 @@ class TestBulkEditReprocess(DirectoriesMixin, TestCase):
THEN:
- The task is queued without asking for the remote engine
"""
result = bulk_edit.reprocess([self.doc.id])
doc = DocumentFactory()
assert bulk_edit.reprocess([doc.id]) == "OK"
self.assertEqual(result, "OK")
mock_task.apply_async.assert_called_once()
_, kwargs = mock_task.apply_async.call_args
self.assertEqual(
kwargs["kwargs"],
{"document_id": self.doc.id, "remote_ocr": False},
)
assert mock_task.apply_async.call_args.kwargs["kwargs"] == {
"document_id": doc.id,
"remote_ocr": False,
}
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
def test_reprocess_passes_remote_ocr(self, mock_task: mock.Mock) -> None:
def test_reprocess_passes_remote_ocr(self, mock_task: mock.MagicMock) -> None:
"""
GIVEN:
- A reprocess request that explicitly asks for remote OCR
@@ -2010,14 +2016,72 @@ class TestBulkEditReprocess(DirectoriesMixin, TestCase):
THEN:
- The request is forwarded to the task for every document
"""
other = Document.objects.create(
title="test2",
checksum="B",
mime_type="application/pdf",
docs = DocumentFactory.create_batch(2)
bulk_edit.reprocess([doc.id for doc in docs], remote_ocr=True)
assert mock_task.apply_async.call_count == 2
for call in mock_task.apply_async.call_args_list:
assert call.kwargs["kwargs"]["remote_ocr"]
def test_reprocess_root_uses_latest_version(
self,
mock_task: mock.MagicMock,
) -> None:
"""
GIVEN:
- A root document with two versions
WHEN:
- reprocess is called with the root document
THEN:
- The latest version is reprocessed, not the root's original file
"""
root = DocumentFactory()
DocumentFactory(root_document=root, version_index=1)
latest = DocumentFactory(root_document=root, version_index=2)
bulk_edit.reprocess([root.id])
assert self._queued_ids(mock_task) == [latest.id]
def test_reprocess_explicit_version(self, mock_task: mock.MagicMock) -> None:
"""
GIVEN:
- A root document with two versions
WHEN:
- reprocess is called with the older version
THEN:
- That version is reprocessed
"""
root = DocumentFactory()
older = DocumentFactory(root_document=root, version_index=1)
DocumentFactory(root_document=root, version_index=2)
bulk_edit.reprocess([older.id])
assert self._queued_ids(mock_task) == [older.id]
def test_reprocess_root_and_latest_version_dispatches_once(
self,
mock_task: mock.MagicMock,
) -> None:
"""
GIVEN:
- A root document with two versions, the latest created on a
different date than the root
WHEN:
- reprocess is called with both the root and its latest version
THEN:
- The latest version is reprocessed only once
"""
root = DocumentFactory(created=date(2024, 1, 1))
DocumentFactory(root_document=root, version_index=1)
latest = DocumentFactory(
root_document=root,
version_index=2,
created=date(2025, 1, 1),
)
bulk_edit.reprocess([self.doc.id, other.id], remote_ocr=True)
bulk_edit.reprocess([root.id, latest.id])
self.assertEqual(mock_task.apply_async.call_count, 2)
for call in mock_task.apply_async.call_args_list:
self.assertTrue(call.kwargs["kwargs"]["remote_ocr"])
assert self._queued_ids(mock_task) == [latest.id]
+382
View File
@@ -1,11 +1,22 @@
import subprocess
from collections.abc import Generator
from pathlib import Path
import pikepdf
import pytest
from PIL import Image
from pytest_django.fixtures import Settings
from pytest_mock import MockerFixture
from documents.parsers import ParseError
from documents.parsers import _compute_thumbnail_dpi
from documents.parsers import encode_thumbnail_webp
from documents.parsers import get_default_file_extension
from documents.parsers import get_default_thumbnail
from documents.parsers import get_supported_file_extensions
from documents.parsers import is_file_ext_supported
from documents.parsers import make_thumbnail_from_pdf
from documents.parsers import rasterize_pdf_page_to_png
from paperless.parsers.registry import get_parser_registry
from paperless.parsers.registry import reset_parser_registry
from paperless.parsers.tesseract import RasterisedDocumentParser
@@ -125,3 +136,374 @@ class TestParserAvailability:
assert is_file_ext_supported(".pdf")
assert not is_file_ext_supported(".hsdfh")
assert not is_file_ext_supported("")
class TestComputeThumbnailDpi:
@pytest.mark.parametrize(
("size", "expected"),
[
pytest.param((612.0, 792.0), (59, 2), id="letter-width-bound"),
pytest.param((792.0, 612.0), (46, 2), id="landscape-rounded-up"),
pytest.param((612.0, 100000.0), (4, 2), id="tall-strip-height-bound"),
pytest.param((200.0, 300.0), (180, 2), id="small-page-width-bound"),
pytest.param((72.0, 72.0), (300, 2), id="tiny-page-capped-at-300"),
pytest.param(
(1000000.0, 1000000.0),
(1, 1),
id="huge-page-minimum-one-unsupersampled",
),
pytest.param(None, (150, 1), id="unreadable-geometry-fallback"),
],
)
def test_dpi_from_page_size(
self,
mocker: MockerFixture,
tmp_path: Path,
size: tuple[float, float] | None,
expected: tuple[int, int],
) -> None:
"""
GIVEN:
- A first page of the given size, or unreadable geometry
WHEN:
- The thumbnail DPI is computed
THEN:
- The expected DPI and supersample factor are returned
"""
mocker.patch(
"paperless.parsers.utils.get_pdf_first_page_size_points",
return_value=size,
)
assert _compute_thumbnail_dpi(tmp_path / "doc.pdf") == expected
class TestMakeThumbnailFromPdf:
@pytest.fixture
def work_dir(self, tmp_path: Path) -> Path:
path = tmp_path / "work"
path.mkdir()
return path
@staticmethod
def _write_blank_pdf(path: Path, page_size: tuple[int, int] = (612, 792)) -> Path:
pdf = pikepdf.new()
pdf.add_blank_page(page_size=page_size)
pdf.save(path, object_stream_mode=pikepdf.ObjectStreamMode.disable)
return path
@pytest.mark.parametrize(
("size", "expected_dpi", "expected_supersample"),
[
pytest.param((612.0, 792.0), 118, 2, id="known-geometry-2x"),
pytest.param(None, 150, 1, id="unreadable-geometry-plain-fallback"),
],
)
def test_render_dpi_requested(
self,
mocker: MockerFixture,
tmp_path: Path,
work_dir: Path,
size: tuple[float, float] | None,
expected_dpi: int,
expected_supersample: int,
) -> None:
"""
GIVEN:
- Readable or unreadable page geometry
WHEN:
- A thumbnail is made
THEN:
- Rasterize and encode get the matching DPI and supersample factor
"""
mocker.patch(
"paperless.parsers.utils.get_pdf_first_page_size_points",
return_value=size,
)
rasterize = mocker.patch("documents.parsers.rasterize_pdf_page_to_png")
encode = mocker.patch("documents.parsers.encode_thumbnail_webp")
make_thumbnail_from_pdf(tmp_path / "in.pdf", work_dir)
assert rasterize.call_args.kwargs["dpi"] == expected_dpi
assert encode.call_args.kwargs["supersample"] == expected_supersample
@pytest.mark.parametrize(
("page_size", "expected_width"),
[
pytest.param((612, 792), 500, id="letter"),
pytest.param((792, 612), 500, id="landscape-letter"),
pytest.param((595, 842), 500, id="a4"),
pytest.param((200, 300), 500, id="small-page"),
pytest.param((72, 72), 300, id="tiny-page-capped"),
],
)
def test_thumbnail_width(
self,
tmp_path: Path,
work_dir: Path,
page_size: tuple[int, int],
expected_width: int,
) -> None:
"""
GIVEN:
- A PDF whose first page has the given size in points
WHEN:
- A thumbnail is made from it
THEN:
- The thumbnail has the expected width
"""
pdf_path = self._write_blank_pdf(tmp_path / "in.pdf", page_size)
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
assert thumb == work_dir / "convert.webp"
with Image.open(thumb) as im:
assert im.format == "WEBP"
assert im.width == expected_width
@classmethod
def _write_pdf_without_xref(cls, path: Path) -> Path:
"""
Cuts off the xref and trailer, which pdftoppm cannot recover from but qpdf can.
"""
cls._write_blank_pdf(path)
data = path.read_bytes()
path.write_bytes(data[: data.rindex(b"\nxref")])
return path
def test_qpdf_repair_produces_real_thumbnail(
self,
tmp_path: Path,
work_dir: Path,
) -> None:
"""
GIVEN:
- A PDF with its xref table and trailer cut off
WHEN:
- A thumbnail is made from it
THEN:
- The thumbnail is rendered from a qpdf repaired copy
- The original file is unchanged
"""
pdf_path = self._write_pdf_without_xref(tmp_path / "broken.pdf")
original_bytes = pdf_path.read_bytes()
with pytest.raises(ParseError):
rasterize_pdf_page_to_png(pdf_path, work_dir / "probe.png", dpi=50)
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
assert thumb == work_dir / "convert_qpdf.webp"
with Image.open(thumb) as im:
assert im.format == "WEBP"
assert im.width == 500
assert pdf_path.read_bytes() == original_bytes
@pytest.mark.parametrize(
"qpdf_error",
[
pytest.param(subprocess.CalledProcessError(2, "qpdf"), id="qpdf-fails"),
pytest.param(None, id="repaired-still-unrenderable"),
],
)
def test_double_failure_uses_default_thumbnail(
self,
mocker: MockerFixture,
tmp_path: Path,
work_dir: Path,
qpdf_error: subprocess.CalledProcessError | None,
) -> None:
"""
GIVEN:
- A PDF that cannot be rendered, even after qpdf repair
WHEN:
- A thumbnail is made from it
THEN:
- A copy of the default thumbnail is returned
"""
mocker.patch(
"documents.parsers.rasterize_pdf_page_to_png",
side_effect=ParseError("Does not compute."),
)
if qpdf_error is not None:
mocker.patch("documents.parsers.run_subprocess", side_effect=qpdf_error)
pdf_path = self._write_blank_pdf(tmp_path / "in.pdf")
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
assert thumb == work_dir / "document.webp"
assert thumb.read_bytes() == get_default_thumbnail().read_bytes()
class TestRasterizePdfPageToPng:
@staticmethod
def _write_pdf(
path: Path,
*,
crop_box: tuple[float, float, float, float] | None = None,
rotate: int | None = None,
) -> Path:
pdf = pikepdf.new()
pdf.add_blank_page(page_size=(144, 72))
pdf.add_blank_page(page_size=(300, 300))
page = pdf.pages[0]
if crop_box is not None:
page.obj.CropBox = pikepdf.Array(crop_box)
if rotate is not None:
page.obj.Rotate = rotate
pdf.save(path)
return path
@pytest.mark.parametrize(
("crop_box", "rotate", "expected_size"),
[
pytest.param(None, None, (144, 72), id="plain"),
pytest.param(None, 90, (72, 144), id="rotated-90"),
pytest.param((0, 0, 72, 36), None, (72, 36), id="crop-box"),
],
)
def test_renders_first_page_to_exact_path(
self,
tmp_path: Path,
crop_box: tuple[float, float, float, float] | None,
rotate: int | None,
expected_size: tuple[int, int],
) -> None:
"""
GIVEN:
- A two page PDF, first page optionally cropped or rotated
WHEN:
- The first page is rasterized at 72 DPI
THEN:
- Only out_path is written, sized to the first page's crop and rotation
"""
pdf_path = self._write_pdf(
tmp_path / "in.pdf",
crop_box=crop_box,
rotate=rotate,
)
out_dir = tmp_path / "out"
out_dir.mkdir()
out_path = out_dir / "page1.png"
rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72)
assert list(out_dir.iterdir()) == [out_path]
with Image.open(out_path) as im:
assert im.format == "PNG"
assert im.size == expected_size
def test_failure_raises_parse_error(self, tmp_path: Path) -> None:
"""
GIVEN:
- A file that is not a PDF
WHEN:
- Rasterization is attempted
THEN:
- A ParseError is raised
"""
bad = tmp_path / "bad.pdf"
bad.write_bytes(b"not a pdf")
with pytest.raises(ParseError):
rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72)
class TestEncodeThumbnailWebp:
@pytest.mark.parametrize(
("mode", "color"),
[
pytest.param("RGBA", (0, 0, 0, 0), id="rgba"),
pytest.param("LA", (0, 0), id="la"),
],
)
def test_alpha_flattened_onto_white(
self,
tmp_path: Path,
mode: str,
color: tuple[int, ...],
) -> None:
"""
GIVEN:
- A fully transparent PNG with an alpha channel
WHEN:
- It is encoded as a thumbnail
THEN:
- The WebP output is RGB with the transparency flattened to white
"""
png_path = tmp_path / "in.png"
Image.new(mode, (20, 10), color).save(png_path)
out_path = tmp_path / "out.webp"
encode_thumbnail_webp(png_path, out_path)
with Image.open(out_path) as im:
assert im.format == "WEBP"
assert im.mode == "RGB"
assert im.size == (20, 10)
red, green, blue = im.getpixel((10, 5))
assert min(red, green, blue) >= 250
@pytest.mark.parametrize(
("in_size", "supersample", "expected_size"),
[
pytest.param((1000, 2000), 1, (500, 1000), id="too-wide-shrunk"),
pytest.param((100, 10000), 1, (50, 5000), id="too-tall-shrunk"),
pytest.param((100, 200), 1, (100, 200), id="small-not-enlarged"),
pytest.param((1000, 1400), 2, (500, 700), id="2x-halved"),
pytest.param((1001, 1401), 2, (500, 700), id="2x-odd-rounded"),
pytest.param((600, 800), 2, (300, 400), id="2x-small-not-enlarged"),
pytest.param((900, 1200), 2, (450, 600), id="2x-below-clamp"),
],
)
def test_size_clamped_and_downsampled(
self,
tmp_path: Path,
in_size: tuple[int, int],
supersample: int,
expected_size: tuple[int, int],
) -> None:
"""
GIVEN:
- A rendered image and its supersample factor
WHEN:
- It is encoded as a thumbnail with that factor
THEN:
- It is downsampled, fit within 500x5000 and never enlarged
"""
png_path = tmp_path / "in.png"
Image.new("RGB", in_size, (255, 255, 255)).save(png_path)
out_path = tmp_path / "out.webp"
encode_thumbnail_webp(png_path, out_path, supersample=supersample)
with Image.open(out_path) as im:
assert im.size == expected_size
@pytest.mark.parametrize(
"error",
[
pytest.param(OSError("broken image"), id="os-error"),
pytest.param(Image.DecompressionBombError("too large"), id="bomb"),
],
)
def test_decode_failure_raises_parse_error(
self,
tmp_path: Path,
mocker: MockerFixture,
error: Exception,
) -> None:
"""
GIVEN:
- Opening the rendered image fails
WHEN:
- It is encoded as a thumbnail
THEN:
- A ParseError is raised
"""
png_path = tmp_path / "in.png"
Image.new("RGB", (10, 10)).save(png_path)
mocker.patch("PIL.Image.open", side_effect=error)
with pytest.raises(ParseError):
encode_thumbnail_webp(png_path, tmp_path / "out.webp")
@@ -90,10 +90,13 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("document_ids", response.data)
@mock.patch("documents.views.permitted_document_ids", return_value=set())
def test_create_bundle_rejects_insufficient_permissions(self, perms_mock) -> None:
def test_create_bundle_rejects_insufficient_permissions(self) -> None:
requester = UserFactory(username="bundle_creator")
grant_global(requester, "add_sharelinkbundle", "view_document")
self.client.force_authenticate(requester)
document = DocumentFactory(owner=UserFactory(username="document_owner"))
payload = {
"document_ids": [self.document.pk],
"document_ids": [self.document.pk, document.pk],
"file_version": ShareLink.FileVersion.ARCHIVE,
"expiration_days": 7,
}
@@ -101,8 +104,8 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
response = self.client.post(self.ENDPOINT, payload, format="json")
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("document_ids", response.data)
perms_mock.assert_called()
self.assertIn(str(document.pk), str(response.data["document_ids"]))
self.assertFalse(ShareLinkBundle.objects.exists())
@mock.patch("documents.views.build_share_link_bundle.apply_async")
def test_rebuild_bundle_resets_state(self, delay_mock) -> None:
+77
View File
@@ -20,6 +20,7 @@ from documents.sanity_checker import SanityCheckMessages
from documents.tests.helpers import dummy_preprocess
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
@pytest.mark.django_db
@@ -287,6 +288,82 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
tasks.update_document_content_maybe_archive_file(doc.pk)
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
def _create_root_with_version(self) -> tuple[Document, Document]:
sample1 = self.dirs.scratch_dir / "sample.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf",
sample1,
)
root = DocumentFactory(content="root content", mime_type="application/pdf")
version = DocumentFactory(
content="my document",
filename=sample1,
mime_type="application/pdf",
root_document=root,
version_index=1,
)
return root, version
@mock.patch("documents.tasks.clear_document_caches")
@mock.patch("documents.search.get_backend")
def test_update_content_version_indexes_root(
self,
mock_get_backend: mock.Mock,
mock_clear_caches: mock.Mock,
) -> None:
"""
GIVEN:
- A root document with a version
WHEN:
- Update content task is called for the version
THEN:
- The version's content is updated
- The root document is indexed rather than the version
- Caches are cleared for both
"""
root, version = self._create_root_with_version()
tasks.update_document_content_maybe_archive_file(version.pk)
self.assertNotEqual(
Document.objects.get(pk=version.pk).content,
"my document",
)
self.assertEqual(Document.objects.get(pk=root.pk).content, "root content")
indexed = mock_get_backend.return_value.add_or_update.call_args.args[0]
self.assertEqual(indexed.pk, root.pk)
mock_clear_caches.assert_has_calls(
[mock.call(version.pk), mock.call(root.pk)],
)
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
@mock.patch("documents.tasks.llm_index_add_or_update_document")
@mock.patch("documents.search.get_backend")
def test_update_content_version_updates_llm_index_for_root(
self,
mock_get_backend: mock.Mock,
mock_llm_index: mock.Mock,
) -> None:
"""
GIVEN:
- A root document with a version
- The LLM index is enabled
WHEN:
- Update content task is called for the version
THEN:
- The LLM index is updated for the root document, not the version
"""
root, version = self._create_root_with_version()
tasks.update_document_content_maybe_archive_file(version.pk)
mock_llm_index.assert_called_once()
self.assertEqual(mock_llm_index.call_args.args[0].pk, root.pk)
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
"""
+101
View File
@@ -1,6 +1,11 @@
from pathlib import Path
import pytest
import pytest_mock
from PIL import Image
from documents.utils import QuerySetStream
from documents.utils import remove_alpha_channel
class TestQuerySetStream:
@@ -33,3 +38,99 @@ class TestQuerySetStream:
# the thing being verified here.
mock_queryset.count.assert_called_with()
mock_queryset.iterator.assert_called_once_with(chunk_size=1000)
class TestRemoveAlphaChannel:
@pytest.mark.parametrize(
("image_format", "mode", "expected_mode", "color", "expected_pixel"),
[
pytest.param(
"PNG",
"RGBA",
"RGB",
(10, 20, 30, 0),
(10, 20, 30),
id="png-rgba",
),
pytest.param(
"TIFF",
"RGBA",
"RGB",
(10, 20, 30, 128),
(10, 20, 30),
id="tiff-rgba",
),
pytest.param("PNG", "LA", "L", (40, 0), 40, id="png-la"),
],
)
def test_drops_alpha_and_keeps_format(
self,
tmp_path: Path,
image_format: str,
mode: str,
expected_mode: str,
color: tuple[int, ...],
expected_pixel: int | tuple[int, ...],
) -> None:
"""
GIVEN:
- An image with an alpha channel
WHEN:
- The alpha channel is removed to a destination without an extension
THEN:
- The destination has the matching opaque mode, the same size and
format, and the color values are kept rather than composited
"""
source = tmp_path / "source"
Image.new(mode, (8, 6), color).save(source, format=image_format)
dest = tmp_path / "no-alpha"
result = remove_alpha_channel(source, dest)
assert result == dest
with Image.open(dest) as out:
assert out.mode == expected_mode
assert out.size == (8, 6)
assert out.format == image_format
assert out.getpixel((0, 0)) == expected_pixel
def test_keeps_every_frame_of_a_multi_page_tiff(self, tmp_path: Path) -> None:
"""
GIVEN:
- A multi-page TIFF where every page has an alpha channel
WHEN:
- The alpha channel is removed
THEN:
- All pages are kept and none has an alpha channel
"""
source = tmp_path / "multi.tiff"
first = Image.new("RGBA", (8, 6), (255, 0, 0, 50))
rest = [Image.new("RGBA", (8, 6), (0, 255, 0, 50)) for _ in range(2)]
first.save(source, save_all=True, append_images=rest)
dest = tmp_path / "multi-no-alpha.tiff"
remove_alpha_channel(source, dest)
with Image.open(dest) as out:
assert out.n_frames == 3
for index in range(out.n_frames):
out.seek(index)
assert out.mode == "RGB"
def test_keeps_dpi(self, tmp_path: Path) -> None:
"""
GIVEN:
- An alpha image with a DPI set
WHEN:
- The alpha channel is removed
THEN:
- The DPI is unchanged, since OCR scaling depends on it
"""
source = tmp_path / "source.png"
Image.new("RGBA", (8, 6)).save(source, dpi=(300, 300))
dest = tmp_path / "dest.png"
remove_alpha_channel(source, dest)
with Image.open(dest) as out:
assert round(out.info["dpi"][0]) == 300
+26
View File
@@ -9,11 +9,13 @@ from pathlib import Path
from subprocess import CompletedProcess
from subprocess import run
from typing import TYPE_CHECKING
from typing import Any
from typing import Generic
from typing import TypeVar
from django.conf import settings
from PIL import Image
from PIL import ImageSequence
if TYPE_CHECKING:
from django.db.models import Model
@@ -128,6 +130,30 @@ def maybe_override_pixel_limit() -> None:
Image.MAX_IMAGE_PIXELS = pixel_count
def remove_alpha_channel(source: Path, dest: Path) -> Path:
"""
Writes a copy of the image with its alpha channel dropped, keeping the
format, every frame, the DPI and any ICC profile.
The channel is dropped, not composited, so transparent pixels keep their
underlying color. Returns dest.
"""
with Image.open(source) as im:
image_format = im.format
opaque_mode = "L" if im.mode == "LA" else "RGB"
frames = [frame.convert(opaque_mode) for frame in ImageSequence.Iterator(im)]
save_kwargs: dict[str, Any] = {}
if "dpi" in im.info:
save_kwargs["dpi"] = im.info["dpi"]
if "icc_profile" in im.info:
save_kwargs["icc_profile"] = im.info["icc_profile"]
if len(frames) > 1:
save_kwargs["save_all"] = True
save_kwargs["append_images"] = frames[1:]
frames[0].save(dest, format=image_format, **save_kwargs)
return dest
def run_subprocess(
arguments: list[str],
env: dict[str, str] | None = None,
+81 -41
View File
@@ -174,6 +174,7 @@ from documents.permissions import TrashPermissions
from documents.permissions import ViewDocumentsPermissions
from documents.permissions import annotate_document_count_by_ids
from documents.permissions import annotate_document_count_for_related_queryset
from documents.permissions import documents_without_permitted_root
from documents.permissions import get_document_count_filter_for_user
from documents.permissions import get_objects_for_user_owner_aware
from documents.permissions import has_global_statistics_permission
@@ -1550,13 +1551,16 @@ class DocumentViewSet(
)
def suggestions(self, request, pk=None):
doc = get_object_or_404(
Document.objects.select_related("owner").prefetch_related("versions"),
Document.objects.select_related(
"owner",
"root_document__owner",
).prefetch_related("versions"),
pk=pk,
)
if request.user is not None and not has_perms_owner_aware(
request.user,
"change_document",
doc,
get_root_document(doc),
):
return HttpResponseForbidden("Insufficient permissions")
@@ -1610,13 +1614,16 @@ class DocumentViewSet(
@method_decorator(cache_control(no_cache=True))
def ai_suggestions(self, request, pk=None):
doc = get_object_or_404(
Document.objects.select_related("owner").prefetch_related("versions"),
Document.objects.select_related(
"owner",
"root_document__owner",
).prefetch_related("versions"),
pk=pk,
)
if request.user is not None and not has_perms_owner_aware(
request.user,
"change_document",
doc,
get_root_document(doc),
):
return HttpResponseForbidden("Insufficient permissions")
@@ -1856,15 +1863,20 @@ class DocumentViewSet(
currentUser = request.user
try:
doc = (
Document.objects.select_related("owner")
Document.objects.select_related("owner", "root_document__owner")
.prefetch_related("notes")
.only("pk", "owner__id")
.only(
"pk",
"owner__id",
"root_document__id",
"root_document__owner__id",
)
.get(pk=pk)
)
if currentUser is not None and not has_perms_owner_aware(
currentUser,
"view_document",
doc,
get_root_document(doc),
):
return HttpResponseForbidden("Insufficient permissions to view notes")
except Document.DoesNotExist:
@@ -1886,7 +1898,7 @@ class DocumentViewSet(
if currentUser is not None and not has_perms_owner_aware(
currentUser,
"change_document",
doc,
get_root_document(doc),
):
return HttpResponseForbidden(
"Insufficient permissions to create notes",
@@ -1929,7 +1941,7 @@ class DocumentViewSet(
if currentUser is not None and not has_perms_owner_aware(
currentUser,
"change_document",
doc,
get_root_document(doc),
):
return HttpResponseForbidden("Insufficient permissions to delete notes")
@@ -1973,11 +1985,13 @@ class DocumentViewSet(
def share_links(self, request, pk=None):
currentUser = request.user
try:
doc = Document.objects.select_related("owner").get(pk=pk)
doc = Document.objects.select_related("owner", "root_document__owner").get(
pk=pk,
)
if currentUser is not None and not has_perms_owner_aware(
currentUser,
"change_document",
doc,
get_root_document(doc),
):
return HttpResponseForbidden(
"Insufficient permissions to add share link",
@@ -2008,10 +2022,11 @@ class DocumentViewSet(
if not settings.AUDIT_LOG_ENABLED:
return HttpResponseBadRequest("Audit log is disabled")
try:
doc = Document.objects.get(pk=pk)
doc = Document.objects.select_related("root_document__owner").get(pk=pk)
root_doc = get_root_document(doc)
if not request.user.has_perm("auditlog.view_logentry") or (
doc.owner is not None
and doc.owner != request.user
root_doc.owner is not None
and root_doc.owner != request.user
and not request.user.is_superuser
):
return HttpResponseForbidden(
@@ -2102,9 +2117,7 @@ class DocumentViewSet(
documents = Document.objects.filter(pk__in=document_ids)
if (
request.user is not None
and documents.exclude(
pk__in=permitted_document_ids(request.user),
).exists()
and documents_without_permitted_root(documents, request.user).exists()
):
return HttpResponseForbidden("Insufficient permissions")
@@ -2430,11 +2443,17 @@ class ChatStreamingView(GenericAPIView[Any]):
if doc_id:
try:
document = Document.objects.get(id=doc_id)
document = Document.objects.select_related(
"root_document__owner",
).get(id=doc_id)
except Document.DoesNotExist:
return HttpResponseBadRequest("Document not found")
if not has_perms_owner_aware(request.user, "view_document", document):
if not has_perms_owner_aware(
request.user,
"view_document",
get_root_document(document),
):
return HttpResponseForbidden("Insufficient permissions")
documents = Document.objects.filter(pk=document.pk)
@@ -3605,10 +3624,11 @@ class SelectionDataView(DocumentSelectionMixin, GenericAPIView[Any]):
user=request.user,
validated_data=serializer.validated_data,
)
permitted_documents = Document.objects.filter(
id__in=permitted_document_ids(request.user),
)
if permitted_documents.filter(pk__in=ids).count() != len(ids):
documents = Document.objects.filter(pk__in=ids)
if (
documents.count() != len(ids)
or documents_without_permitted_root(documents, request.user).exists()
):
return HttpResponseForbidden("Insufficient permissions")
correspondents = Correspondent.objects.annotate(
@@ -4790,19 +4810,23 @@ class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
},
)
documents = list(documents_qs)
permitted_ids = set(permitted_document_ids(request.user))
for document in documents:
if document.pk not in permitted_ids:
raise ValidationError(
{
"document_ids": _(
"Insufficient permissions to share document %(id)s.",
)
% {"id": document.pk},
},
)
denied_id = (
documents_without_permitted_root(documents_qs, request.user)
.order_by("pk")
.values_list("pk", flat=True)
.first()
)
if denied_id is not None:
raise ValidationError(
{
"document_ids": _(
"Insufficient permissions to share document %(id)s.",
)
% {"id": denied_id},
},
)
documents = list(documents_qs)
document_map = {document.pk: document for document in documents}
ordered_documents = [document_map[doc_id] for doc_id in document_ids]
@@ -5587,6 +5611,23 @@ class TrashView(ListModelMixin, PassUserMixin):
class _TrashPermittedObjectsFilter(PermittedObjectsFilter):
include_granted = False
def filter_queryset(self, request, queryset, view):
if request.user.is_superuser or not request.user.is_active:
return super().filter_queryset(request, queryset, view)
# A version belongs to whoever owns its root
def owned_or_unowned(prefix: str) -> Q:
return Q(**{f"{prefix}owner": request.user}) | Q(
**{f"{prefix}owner__isnull": True},
)
return queryset.filter(
(Q(root_document__isnull=True) & owned_or_unowned(""))
| (
Q(root_document__isnull=False) & owned_or_unowned("root_document__")
),
)
filter_backends = (_TrashPermittedObjectsFilter,)
pagination_class = StandardPagination
@@ -5616,12 +5657,11 @@ class TrashView(ListModelMixin, PassUserMixin):
if doc_ids is not None
else self.filter_queryset(self.get_queryset()).all()
)
if docs.exclude(
pk__in=permitted_document_ids(
request.user,
perm="delete_document",
include_deleted=True,
),
if documents_without_permitted_root(
docs,
request.user,
perm="delete_document",
include_deleted=True,
).exists():
return HttpResponseForbidden("Insufficient permissions")
action = serializer.validated_data.get("action")
+22 -39
View File
@@ -2,7 +2,7 @@ msgid ""
msgstr ""
"Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2026-10-05 16:26+0000\n"
"POT-Creation-Date: 2026-10-08 18:39+0000\n"
"PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n"
"Language-Team: English\n"
@@ -1652,49 +1652,49 @@ msgstr ""
msgid "workflow runs"
msgstr ""
#: documents/serialisers.py:515 documents/serialisers.py:872
#: documents/serialisers.py:2902 documents/views.py:343 documents/views.py:2732
#: documents/serialisers.py:516 documents/serialisers.py:873
#: documents/serialisers.py:2903 documents/views.py:344 documents/views.py:2751
#: paperless_mail/serialisers.py:156
msgid "Insufficient permissions."
msgstr ""
#: documents/serialisers.py:708
#: documents/serialisers.py:709
msgid "Invalid color."
msgstr ""
#: documents/serialisers.py:2369
#: documents/serialisers.py:2370
#, python-format
msgid "File type %(type)s not supported"
msgstr ""
#: documents/serialisers.py:2413
#: documents/serialisers.py:2414
#, python-format
msgid "Custom field id must be an integer: %(id)s"
msgstr ""
#: documents/serialisers.py:2420
#: documents/serialisers.py:2421
#, python-format
msgid "Custom field with id %(id)s does not exist"
msgstr ""
#: documents/serialisers.py:2437 documents/serialisers.py:2447
#: documents/serialisers.py:2438 documents/serialisers.py:2448
msgid ""
"Custom fields must be a list of integers or an object mapping ids to values."
msgstr ""
#: documents/serialisers.py:2442
#: documents/serialisers.py:2443
msgid "Some custom fields don't exist or were specified twice."
msgstr ""
#: documents/serialisers.py:2589
#: documents/serialisers.py:2590
msgid "Invalid variable detected."
msgstr ""
#: documents/serialisers.py:2958
#: documents/serialisers.py:2959
msgid "Duplicate document identifiers are not allowed."
msgstr ""
#: documents/serialisers.py:2988 documents/views.py:4787
#: documents/serialisers.py:2989 documents/views.py:4807
#, python-format
msgid "Documents not found: %(ids)s"
msgstr ""
@@ -1941,61 +1941,44 @@ msgstr ""
msgid "As a final step, please complete the following form:"
msgstr ""
#: documents/validators.py:24
#, python-brace-format
msgid "Unable to parse URI {value}, missing scheme"
msgstr ""
#: documents/validators.py:29
#, python-brace-format
msgid "Unable to parse URI {value}, missing net location or path"
msgstr ""
#: documents/validators.py:36
msgid ""
"URI scheme '{parts.scheme}' is not allowed. Allowed schemes: {', '."
"join(allowed_schemes)}"
msgid ", "
msgstr ""
#: documents/validators.py:45
#, python-brace-format
msgid "Unable to parse URI {value}"
msgstr ""
#: documents/views.py:336 documents/views.py:2729
#: documents/views.py:337 documents/views.py:2748
msgid "Invalid more_like_id"
msgstr ""
#: documents/views.py:1676
#: documents/views.py:1683
msgid "Invalid AI configuration."
msgstr ""
#: documents/views.py:1687
#: documents/views.py:1694
msgid "AI backend request timed out."
msgstr ""
#: documents/views.py:1699
#: documents/views.py:1706
msgid "AI backend rejected the request. Check logs for details."
msgstr ""
#: documents/views.py:2554 documents/views.py:2870
#: documents/views.py:2573 documents/views.py:2889
msgid "Specify only one of text, title_search, query, or more_like_id."
msgstr ""
#: documents/views.py:4800
#: documents/views.py:4823
#, python-format
msgid "Insufficient permissions to share document %(id)s."
msgstr ""
#: documents/views.py:4846
#: documents/views.py:4870
msgid "Bundle is already being processed."
msgstr ""
#: documents/views.py:4910
#: documents/views.py:4934
msgid "The share link bundle is still being prepared. Please try again later."
msgstr ""
#: documents/views.py:4924
#: documents/views.py:4948
msgid "The share link bundle is unavailable."
msgstr ""
+11 -12
View File
@@ -82,7 +82,7 @@ def binaries_check(app_configs: Any, **kwargs: Any) -> list[Error]:
error = "Paperless can't find {}. Without it, consumption is impossible."
hint = "Either it's not in your ${PATH} or it's not installed."
binaries = (settings.CONVERT_BINARY, "tesseract", "gs")
binaries = ("tesseract", "gs")
check_messages = []
for binary in binaries:
@@ -272,27 +272,26 @@ def check_deprecated_db_settings(
Detects legacy advanced options that should be migrated to
PAPERLESS_DB_OPTIONS. Returns one Warning per deprecated variable found.
"""
deprecated_vars: dict[str, str] = {
"PAPERLESS_DB_TIMEOUT": "timeout",
"PAPERLESS_DB_POOLSIZE": "pool.min_size / pool.max_size",
"PAPERLESS_DBSSLMODE": "sslmode",
"PAPERLESS_DBSSLROOTCERT": "sslrootcert",
"PAPERLESS_DBSSLCERT": "sslcert",
"PAPERLESS_DBSSLKEY": "sslkey",
}
deprecated_vars = (
"PAPERLESS_DB_TIMEOUT",
"PAPERLESS_DB_POOLSIZE",
"PAPERLESS_DBSSLMODE",
"PAPERLESS_DBSSLROOTCERT",
"PAPERLESS_DBSSLCERT",
"PAPERLESS_DBSSLKEY",
)
warnings: list[Warning] = []
for var_name, db_option_key in deprecated_vars.items():
for var_name in deprecated_vars:
if not os.getenv(var_name):
continue
warnings.append(
Warning(
f"Deprecated environment variable: {var_name}",
hint=(
f"{var_name} is no longer supported and will be removed in v3.2. "
f"{var_name} is deprecated. "
f"Set the equivalent option via PAPERLESS_DB_OPTIONS instead. "
f'Example: PAPERLESS_DB_OPTIONS=\'{{"{db_option_key}": "<value>"}}\'. '
"See https://docs.paperless-ngx.com/migration-v3/ for the full reference."
),
id="paperless.W001",
+2 -13
View File
@@ -19,7 +19,7 @@ from documents.parsers import ParseError
from documents.parsers import make_thumbnail_from_pdf
from documents.utils import copy_file_with_basic_stats
from documents.utils import maybe_override_pixel_limit
from documents.utils import run_subprocess
from documents.utils import remove_alpha_channel
from paperless.config import OcrConfig
from paperless.models import CleanChoices
from paperless.models import ModeChoices
@@ -197,18 +197,7 @@ class RasterisedDocumentParser:
return im.mode in ("RGBA", "LA")
def remove_alpha(self, image_path: Path) -> Path:
no_alpha_image = Path(self.tempdir) / "image-no-alpha"
run_subprocess(
[
settings.CONVERT_BINARY,
"-alpha",
"off",
str(image_path),
str(no_alpha_image),
],
logger=self.log,
)
return no_alpha_image
return remove_alpha_channel(image_path, Path(self.tempdir) / "image-no-alpha")
def get_dpi(self, image: Path) -> int | None:
try:
+46
View File
@@ -265,6 +265,52 @@ def get_page_count_for_pdf(
return None
def get_pdf_first_page_size_points(
path: Path,
log: logging.Logger | None = None,
) -> tuple[float, float] | None:
"""Return the first page's (width, height) in PDF points, post-rotation.
Uses the CropBox (MediaBox if absent), which must match pdftoppm's
``-cropbox`` or the computed DPI targets the wrong box.
Swaps width and height for 90/270 rotation. ``page.rotation`` resolves
inherited and negative ``/Rotate`` values, a raw lookup does not.
Parameters
----------
path:
Absolute path to the PDF file.
log:
Logger for warnings. Falls back to the module-level logger when omitted.
Returns
-------
tuple[float, float] | None
``(width_points, height_points)``, or ``None`` if the file cannot be
opened, has no pages, or the page box is degenerate.
"""
import pikepdf
_log = log or logger
try:
with pikepdf.Pdf.open(path) as pdf:
if len(pdf.pages) == 0:
return None
page = pdf.pages[0]
llx, lly, urx, ury = (float(v) for v in page.cropbox)
width = abs(urx - llx)
height = abs(ury - lly)
if width <= 0 or height <= 0:
return None
if page.rotation in (90, 270):
width, height = height, width
return width, height
except Exception as e:
_log.warning("Could not determine PDF page size for %s: %s", path, e)
return None
def extract_pdf_metadata(
document_path: Path,
log: logging.Logger | None = None,
-5
View File
@@ -984,11 +984,6 @@ MAX_IMAGE_PIXELS: Final[int | None] = get_int_from_env(
# GNUPG needs a home directory for some reason
GNUPG_HOME = os.getenv("HOME", "/tmp")
# Convert is part of the ImageMagick package
CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert")
CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs")
# Fallback layout for .eml consumption
@@ -137,9 +137,20 @@ class TestNginxService:
reason="No Gotenberg/Tika servers to test with",
)
class TestParserLive:
@staticmethod
def imagehash(file: Path, hash_size: int = 18) -> str:
return f"{average_hash(Image.open(file), hash_size)}"
# Rasterizer versions shift a few pixels, so compare perceptual hashes by
# Hamming distance (out of 18 * 18 = 324 bits) rather than for equality
MAX_HASH_DISTANCE = 8
@classmethod
def assert_thumbnails_similar(cls, generated: Path, expected: Path) -> None:
distance = average_hash(Image.open(generated), 18) - average_hash(
Image.open(expected),
18,
)
assert distance <= cls.MAX_HASH_DISTANCE, (
f"Thumbnail {generated} differs from {expected} by {distance} bits "
f"(max {cls.MAX_HASH_DISTANCE})"
)
def test_get_thumbnail(
self,
@@ -168,12 +179,7 @@ class TestParserLive:
assert thumb.exists()
assert thumb.is_file()
assert self.imagehash(thumb) == self.imagehash(
simple_txt_email_thumbnail_file,
), (
f"Created thumbnail {thumb} differs from expected file "
f"{simple_txt_email_thumbnail_file}"
)
self.assert_thumbnails_similar(thumb, simple_txt_email_thumbnail_file)
def test_tika_parse_successful(self, mail_parser: MailDocumentParser) -> None:
"""
@@ -255,7 +261,7 @@ class TestParserLive:
THEN:
- Gotenberg shall be called to generate the PDF
- The archive PDF shall contain the expected content
- The generated thumbnail shall match the expected image hash
- The generated thumbnail shall be perceptually close to the expected image
"""
util_call_with_backoff(mail_parser.parse, [html_email_file, "message/rfc822"])
@@ -272,14 +278,4 @@ class TestParserLive:
html_email_file,
"message/rfc822",
)
generated_thumbnail_hash = self.imagehash(generated_thumbnail)
# The created PDF is not reproducible, but the converted image
# should always look the same
expected_hash = self.imagehash(html_email_thumbnail_file)
assert generated_thumbnail_hash == expected_hash, (
f"PDF thumbnail differs from expected. "
f"Generated: {generated_thumbnail}, "
f"Hash: {generated_thumbnail_hash} vs {expected_hash}"
)
self.assert_thumbnails_similar(generated_thumbnail, html_email_thumbnail_file)
@@ -15,9 +15,10 @@ from typing import TYPE_CHECKING
import pytest
from ocrmypdf import SubprocessOutputError
from PIL import Image
from documents.parsers import ParseError
from documents.parsers import run_convert
from documents.parsers import rasterize_pdf_page_to_png
from paperless.models import ModeChoices
from paperless.parsers import ParserProtocol
from paperless.parsers.tesseract import RasterisedDocumentParser
@@ -280,24 +281,71 @@ class TestGetThumbnail:
)
assert thumb.is_file()
def test_thumbnail_fallback_on_convert_error(
@pytest.mark.parametrize(
("filename", "expected_height"),
[
pytest.param("simple-digital.pdf", 647, id="portrait-letter"),
pytest.param("rotated.pdf", 386, id="landscape"),
],
)
def test_thumbnail_is_correct_format_and_size(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
filename: str,
expected_height: int,
) -> None:
"""
GIVEN:
- A portrait or landscape PDF
WHEN:
- A thumbnail is generated
THEN:
- A 500px wide WebP keeping the page's aspect ratio
"""
thumb = tesseract_parser.get_thumbnail(
tesseract_samples_dir / filename,
"application/pdf",
)
with Image.open(thumb) as im:
assert im.format == "WEBP"
assert im.width == 500
assert im.height == pytest.approx(expected_height, abs=2)
def test_thumbnail_fallback_on_pdftoppm_error(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
def _raise_on_pdf(input_file, output_file, **kwargs) -> None:
if ".pdf" in str(input_file):
"""
GIVEN:
- Rasterizing the original PDF fails
WHEN:
- A thumbnail is generated
THEN:
- The PDF is repaired with qpdf and a real thumbnail is rendered
"""
original = tesseract_samples_dir / "simple-digital.pdf"
def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None:
if in_path == original:
raise ParseError("Does not compute.")
run_convert(input_file=input_file, output_file=output_file, **kwargs)
rasterize_pdf_page_to_png(in_path, out_path, **kwargs)
mocker.patch("documents.parsers.run_convert", side_effect=_raise_on_pdf)
thumb = tesseract_parser.get_thumbnail(
tesseract_samples_dir / "simple-digital.pdf",
"application/pdf",
rasterize = mocker.patch(
"documents.parsers.rasterize_pdf_page_to_png",
side_effect=_fail_on_original,
)
thumb = tesseract_parser.get_thumbnail(original, "application/pdf")
assert rasterize.call_count == 2
assert thumb.is_file()
assert thumb.name == "convert_qpdf.webp"
with Image.open(thumb) as im:
assert im.format == "WEBP"
assert im.width == 500
def test_thumbnail_encrypted_pdf(
self,
+22 -28
View File
@@ -25,9 +25,17 @@ class TestChecks:
def test_binaries(self) -> None:
assert binaries_check(None) == []
def test_binaries_fail(self, settings: Settings) -> None:
settings.CONVERT_BINARY = "uuuhh"
assert len(binaries_check(None)) == 1
def test_binaries_fail(self, mocker: MockerFixture) -> None:
"""
GIVEN:
- Neither tesseract nor ghostscript can be found
WHEN:
- The binaries check runs
THEN:
- One warning is raised per missing binary
"""
mocker.patch("paperless.checks.shutil.which", return_value=None)
assert len(binaries_check(None)) == 2
@pytest.mark.usefixtures("paperless_dirs")
def test_paths_check(self) -> None:
@@ -211,14 +219,14 @@ class TestAuditLogChecks:
assert "auditlog table was found but audit log is disabled." in msgs[0].msg
DEPRECATED_VARS: dict[str, str] = {
"PAPERLESS_DB_TIMEOUT": "timeout",
"PAPERLESS_DB_POOLSIZE": "pool.min_size / pool.max_size",
"PAPERLESS_DBSSLMODE": "sslmode",
"PAPERLESS_DBSSLROOTCERT": "sslrootcert",
"PAPERLESS_DBSSLCERT": "sslcert",
"PAPERLESS_DBSSLKEY": "sslkey",
}
DEPRECATED_VARS = (
"PAPERLESS_DB_TIMEOUT",
"PAPERLESS_DB_POOLSIZE",
"PAPERLESS_DBSSLMODE",
"PAPERLESS_DBSSLROOTCERT",
"PAPERLESS_DBSSLCERT",
"PAPERLESS_DBSSLKEY",
)
class TestDeprecatedDbSettings:
@@ -234,26 +242,11 @@ class TestDeprecatedDbSettings:
result = check_deprecated_db_settings(None)
assert result == []
@pytest.mark.parametrize(
("env_var", "db_option_key"),
[
pytest.param("PAPERLESS_DB_TIMEOUT", "timeout", id="db-timeout"),
pytest.param(
"PAPERLESS_DB_POOLSIZE",
"pool.min_size / pool.max_size",
id="db-poolsize",
),
pytest.param("PAPERLESS_DBSSLMODE", "sslmode", id="ssl-mode"),
pytest.param("PAPERLESS_DBSSLROOTCERT", "sslrootcert", id="ssl-rootcert"),
pytest.param("PAPERLESS_DBSSLCERT", "sslcert", id="ssl-cert"),
pytest.param("PAPERLESS_DBSSLKEY", "sslkey", id="ssl-key"),
],
)
@pytest.mark.parametrize("env_var", DEPRECATED_VARS)
def test_single_deprecated_var_produces_one_warning(
self,
mocker: MockerFixture,
env_var: str,
db_option_key: str,
) -> None:
"""Each deprecated var in isolation produces exactly one warning."""
mocker.patch.dict(os.environ, {env_var: "some_value"}, clear=True)
@@ -264,7 +257,8 @@ class TestDeprecatedDbSettings:
assert isinstance(warning, Warning)
assert warning.id == "paperless.W001"
assert env_var in warning.hint
assert db_option_key in warning.hint
assert "PAPERLESS_DB_OPTIONS" in warning.hint
assert "https://docs.paperless-ngx.com/migration-v3/" in warning.hint
def test_multiple_deprecated_vars_produce_one_warning_each(
self,
+145
View File
@@ -6,8 +6,10 @@ import codecs
from pathlib import Path
from typing import TYPE_CHECKING
import pikepdf
import pytest
from paperless.parsers.utils import get_pdf_first_page_size_points
from paperless.parsers.utils import is_tagged_pdf
from paperless.parsers.utils import pdf_born_digital_text
from paperless.parsers.utils import post_process_text
@@ -70,6 +72,149 @@ class TestIsTaggedPdf:
assert is_tagged_pdf(bad) is False
class TestGetPdfFirstPageSizePoints:
@staticmethod
def _write_pdf(
path: Path,
*,
media_box: tuple[float, float, float, float] = (0, 0, 600, 800),
crop_box: tuple[float, float, float, float] | None = None,
page_rotate: int | None = None,
inherited_rotate: int | None = None,
) -> Path:
pdf = pikepdf.new()
pdf.add_blank_page(page_size=(media_box[2], media_box[3]))
page = pdf.pages[0]
page.obj.MediaBox = pikepdf.Array(media_box)
if crop_box is not None:
page.obj.CropBox = pikepdf.Array(crop_box)
if page_rotate is not None:
page.obj.Rotate = page_rotate
if inherited_rotate is not None:
pdf.Root.Pages.Rotate = inherited_rotate
pdf.save(path)
return path
def test_letter_sample(self) -> None:
"""
GIVEN:
- A US Letter sample PDF with no CropBox and no rotation
WHEN:
- The first page size is requested
THEN:
- The MediaBox size in points is returned
"""
assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") == (
612.0,
792.0,
)
@pytest.mark.parametrize(
("rotate", "expected"),
[
pytest.param(0, (600.0, 800.0), id="rotate-0"),
pytest.param(90, (800.0, 600.0), id="rotate-90"),
pytest.param(180, (600.0, 800.0), id="rotate-180"),
pytest.param(270, (800.0, 600.0), id="rotate-270"),
pytest.param(-90, (800.0, 600.0), id="rotate-negative-90"),
],
)
def test_page_rotation_swaps_dimensions(
self,
tmp_path: Path,
rotate: int,
expected: tuple[float, float],
) -> None:
"""
GIVEN:
- A portrait PDF page with /Rotate set directly on the page
WHEN:
- The first page size is requested
THEN:
- Width and height are swapped for quarter-turn rotations only
"""
pdf_path = self._write_pdf(tmp_path / "rotated.pdf", page_rotate=rotate)
assert get_pdf_first_page_size_points(pdf_path) == expected
def test_inherited_rotation_swaps_dimensions(self, tmp_path: Path) -> None:
"""
GIVEN:
- A page inheriting /Rotate 90 from the /Pages node
WHEN:
- The first page size is requested
THEN:
- Width and height are swapped
"""
pdf_path = self._write_pdf(tmp_path / "inherited.pdf", inherited_rotate=90)
assert get_pdf_first_page_size_points(pdf_path) == (800.0, 600.0)
def test_crop_box_preferred_over_media_box(self, tmp_path: Path) -> None:
"""
GIVEN:
- A PDF page with a CropBox smaller than its MediaBox
WHEN:
- The first page size is requested
THEN:
- The CropBox size is returned
"""
pdf_path = self._write_pdf(
tmp_path / "cropped.pdf",
crop_box=(50, 100, 350, 500),
)
assert get_pdf_first_page_size_points(pdf_path) == (300.0, 400.0)
def test_degenerate_box_returns_none(self, tmp_path: Path) -> None:
"""
GIVEN:
- A PDF page whose box has zero width
WHEN:
- The first page size is requested
THEN:
- None is returned
"""
pdf_path = self._write_pdf(
tmp_path / "degenerate.pdf",
media_box=(0, 0, 600, 800),
crop_box=(100, 0, 100, 800),
)
assert get_pdf_first_page_size_points(pdf_path) is None
def test_nonexistent_path_returns_none(self) -> None:
"""
GIVEN:
- A path that does not exist
WHEN:
- The first page size is requested
THEN:
- None is returned and nothing is raised
"""
assert get_pdf_first_page_size_points(Path("/nonexistent/file.pdf")) is None
def test_corrupt_pdf_returns_none(self, tmp_path: Path) -> None:
"""
GIVEN:
- A file that is not a PDF
WHEN:
- The first page size is requested
THEN:
- None is returned and nothing is raised
"""
bad = tmp_path / "bad.pdf"
bad.write_bytes(b"not a pdf")
assert get_pdf_first_page_size_points(bad) is None
def test_encrypted_pdf_returns_none(self) -> None:
"""
GIVEN:
- A password protected PDF
WHEN:
- The first page size is requested
THEN:
- None is returned and nothing is raised
"""
assert get_pdf_first_page_size_points(SAMPLES / "encrypted.pdf") is None
class TestPostProcessText:
@pytest.mark.parametrize(
("source", "expected"),
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import Final
__version__: Final[tuple[int, int, int]] = (3, 3, 0)
__version__: Final[tuple[int, int, int]] = (3, 2, 1)
# Version string like X.Y.Z
__full_version_str__: Final[str] = ".".join(map(str, __version__))
# Version string like X.Y
Generated
+1 -1
View File
@@ -2971,7 +2971,7 @@ wheels = [
[[package]]
name = "paperless-ngx"
version = "3.3.0"
version = "3.2.1"
source = { virtual = "." }
dependencies = [
{ name = "azure-ai-documentintelligence" },