Feature/Chore: Replaces all usages of ImageMagik's convert binary with a simple Pillow based helper

This removes ImageMagik as a dependency entirly and all documentation is cleaned up from it. We no longer need to install it, have a policy for it to allow PDFs and no longer shell out to it either.
Chore: Supersample the PDF thumbnail render 2x
2026-10-14 20:17:13 +00:00 · 2026-10-13 10:24:44 -07:00 · 2026-10-10 15:09:59 -07:00 · 2026-10-10 15:09:59 -07:00
35 changed files with 1049 additions and 983 deletions
@@ -39,7 +39,6 @@ ARG RUNTIME_PACKAGES="\
  ghostscript \
  gnupg \
  icc-profiles-free \
  imagemagick \
  # PostgreSQL
  postgresql-client \
  # MySQL / MariaDB
@@ -93,21 +92,6 @@ RUN set -eux \
        https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
      && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb
 # setup docker-specific things
 # These change sometimes, but rarely
 WORKDIR /usr/src/paperless/src/docker/
 COPY [ \
  "docker/rootfs/etc/ImageMagick-6/paperless-policy.xml", \
  "./" \
 ]
 RUN set -eux \
  && echo "Configuring ImageMagick" \
    && mkdir -p /etc/ImageMagick-6 \
    && mv paperless-policy.xml /etc/ImageMagick-6/policy.xml
 # Packages needed only for building a few quick Python
 # dependencies
 ARG BUILD_PACKAGES="\
@@ -111,10 +111,7 @@ jobs:
        timeout-minutes: 12
        uses: $/.github/actions/apt-install
        with:
-          packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils
+          packages: unpaper tesseract-ocr ghostscript poppler-utils qpdf
      - name: Configure ImageMagick
        run: |
          sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml
      - name: Install Python dependencies
        env:
          PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}
@@ -129,7 +129,6 @@ ARG RUNTIME_PACKAGES="\
  ghostscript \
  gnupg \
  icc-profiles-free \
  imagemagick \
  # PostgreSQL
  postgresql-client \
  # MySQL / MariaDB
@@ -167,8 +166,6 @@ RUN set -eux \
        https://github.com/paperless-ngx/builder/releases/download/jbig2enc-trixie-v${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
      && echo "Installing jbig2enc" \
        && dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
      && echo "Configuring imagemagick" \
        && cp /etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml \
      && echo "Cleaning up image layer" \
        && rm --force --verbose *.deb \
    && rm --recursive --force --verbose /var/lib/apt/lists/* \
@@ -1,101 +0,0 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE policymap [
  <!ELEMENT policymap (policy)+>
  <!ATTLIST policymap xmlns CDATA #FIXED ''>
  <!ELEMENT policy EMPTY>
  <!ATTLIST policy xmlns CDATA #FIXED '' domain NMTOKEN #REQUIRED
    name NMTOKEN #IMPLIED pattern CDATA #IMPLIED rights NMTOKEN #IMPLIED
    stealth NMTOKEN #IMPLIED value CDATA #IMPLIED>
 ]>
 <!--
  Configure ImageMagick policies.
  Domains include system, delegate, coder, filter, path, or resource.
  Rights include none, read, write, execute and all.  Use | to combine them,
  for example: "read | write" to permit read from, or write to, a path.
  Use a glob expression as a pattern.
  Suppose we do not want users to process MPEG video images:
    <policy domain="delegate" rights="none" pattern="mpeg:decode" />
  Here we do not want users reading images from HTTP:
    <policy domain="coder" rights="none" pattern="HTTP" />
  The /repository file system is restricted to read only.  We use a glob
  expression to match all paths that start with /repository:
    <policy domain="path" rights="read" pattern="/repository/*" />
  Lets prevent users from executing any image filters:
    <policy domain="filter" rights="none" pattern="*" />
  Any large image is cached to disk rather than memory:
    <policy domain="resource" name="area" value="1GP"/>
  Define arguments for the memory, map, area, width, height and disk resources
  with SI prefixes (.e.g 100MB).  In addition, resource policies are maximums
  for each instance of ImageMagick (e.g. policy memory limit 1GB, -limit 2GB
  exceeds policy maximum so memory limit is 1GB).
  Rules are processed in order.  Here we want to restrict ImageMagick to only
  read or write a small subset of proven web-safe image types:
    <policy domain="delegate" rights="none" pattern="*" />
    <policy domain="filter" rights="none" pattern="*" />
    <policy domain="coder" rights="none" pattern="*" />
    <policy domain="coder" rights="read|write" pattern="{GIF,JPEG,PNG,WEBP}" />
 -->
 <policymap>
  <!-- <policy domain="system" name="shred" value="2"/> -->
  <!-- <policy domain="system" name="precision" value="6"/> -->
  <!-- <policy domain="system" name="memory-map" value="anonymous"/> -->
  <!-- <policy domain="system" name="max-memory-request" value="256MiB"/> -->
  <!-- <policy domain="resource" name="temporary-path" value="/tmp"/> -->
  <policy domain="resource" name="memory" value="256MiB"/>
  <policy domain="resource" name="map" value="512MiB"/>
  <policy domain="resource" name="width" value="16KP"/>
  <policy domain="resource" name="height" value="16KP"/>
  <!-- <policy domain="resource" name="list-length" value="128"/> -->
  <policy domain="resource" name="area" value="128MB"/>
  <policy domain="resource" name="disk" value="1GiB"/>
  <!-- <policy domain="resource" name="file" value="768"/> -->
  <!-- <policy domain="resource" name="thread" value="4"/> -->
  <!-- <policy domain="resource" name="throttle" value="0"/> -->
  <!-- <policy domain="resource" name="time" value="3600"/> -->
  <!-- Paperless does not process SVG or ImageMagick scripting formats. -->
  <policy domain="coder" rights="none" pattern="SVG" />
  <policy domain="coder" rights="none" pattern="SVGZ" />
  <policy domain="coder" rights="none" pattern="MSVG" />
  <policy domain="coder" rights="none" pattern="RSVG" />
  <policy domain="coder" rights="none" pattern="MSL" />
  <policy domain="coder" rights="none" pattern="MVG" />
  <policy domain="coder" rights="none" pattern="EPHEMERAL" />
  <!-- <policy domain="module" rights="none" pattern="{PS,PDF,XPS}" /> -->
  <!-- <policy domain="delegate" rights="none" pattern="HTTPS" /> -->
  <!-- <policy domain="path" rights="none" pattern="@*" /> -->
  <!-- <policy domain="cache" name="memory-map" value="anonymous"/> -->
  <!-- <policy domain="cache" name="synchronize" value="True"/> -->
  <!-- <policy domain="cache" name="shared-secret" value="passphrase" stealth="true"/> -->
  <!-- <policy domain="system" name="pixel-cache-memory" value="anonymous"/> -->
  <!-- <policy domain="system" name="shred" value="2"/> -->
  <!-- <policy domain="system" name="precision" value="6"/> -->
  <!-- use curl -->
  <policy domain="delegate" rights="none" pattern="URL" />
  <policy domain="delegate" rights="none" pattern="HTTPS" />
  <policy domain="delegate" rights="none" pattern="HTTP" />
  <!-- in order to avoid to get image with password text -->
  <policy domain="path" rights="none" pattern="@*"/>
  <!-- disable ghostscript format types -->
  <policy domain="coder" rights="none" pattern="PS" />
  <policy domain="coder" rights="none" pattern="PS2" />
  <policy domain="coder" rights="none" pattern="PS3" />
  <policy domain="coder" rights="none" pattern="EPS" />
  <policy domain="coder" rights="read|write" pattern="PDF" />
  <policy domain="coder" rights="none" pattern="XPS" />
 </policymap>
@@ -1315,29 +1315,17 @@ valid crontab(5) expression describing when to run.
 #### [`PAPERLESS_CONVERT_MEMORY_LIMIT=<num>`](#PAPERLESS_CONVERT_MEMORY_LIMIT) {#PAPERLESS_CONVERT_MEMORY_LIMIT}
-: On smaller systems, or even in the case of Very Large Documents, the
+!!! warning
 consumer may explode, complaining about how it's "unable to extend
 pixel cache". In such cases, try setting this to a reasonably low
 value, like 32. The default is to use whatever is necessary to do
 everything without writing to disk, and units are in megabytes.
-    For more information on how to use this value, you should search the
+    Deprecated and has no effect, since PDF thumbnails no longer use
-    web for "MAGICK_MEMORY_LIMIT".
+    ImageMagick. It will be removed in a future release.
    Defaults to 0, which disables the limit.
 #### [`PAPERLESS_CONVERT_TMPDIR=<path>`](#PAPERLESS_CONVERT_TMPDIR) {#PAPERLESS_CONVERT_TMPDIR}
-: Similar to the memory limit, if you've got a small system and your
+!!! warning
 OS mounts /tmp as tmpfs, you should set this to a path that's on a
 physical disk, like /home/your_user/tmp or something. ImageMagick
 will use this as scratch space when crunching through very large
 documents.
-    For more information on how to use this value, you should search the
+    Deprecated and has no effect, since PDF thumbnails no longer use
-    web for "MAGICK_TMPDIR".
+    ImageMagick. It will be removed in a future release.
    Default is none, which disables the temporary directory.
 #### [`PAPERLESS_APPS=<string>`](#PAPERLESS_APPS) {#PAPERLESS_APPS}
@@ -1885,7 +1873,10 @@ the literal path for that program.
 #### [`PAPERLESS_CONVERT_BINARY=<path>`](#PAPERLESS_CONVERT_BINARY) {#PAPERLESS_CONVERT_BINARY}
-: Defaults to "convert".
+!!! warning
    Deprecated and has no effect, since Paperless-ngx no longer uses
    ImageMagick. It will be removed in a future release.
 #### [`PAPERLESS_GS_BINARY=<path>`](#PAPERLESS_GS_BINARY) {#PAPERLESS_GS_BINARY}
@@ -177,17 +177,16 @@ to a positive number to enable polling and disable native filesystem notificatio
    - `pkg-config` for mysqlclient (python dependency)
    - `fonts-liberation` for generating thumbnails for plain text
      files
    - `imagemagick` >= 6 for PDF conversion
    - `gnupg` for decrypting GPG-encrypted email
    - `libpq-dev` for PostgreSQL
    - `libmagic-dev` for mime type detection
    - `mariadb-client` for MariaDB compile time
-    - `poppler-utils` for barcode detection
+    - `poppler-utils` for thumbnail generation and barcode detection
    Use this list for your preferred package management:
    ```
-    python3 python3-pip python3-dev imagemagick fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils
+    python3 python3-pip python3-dev fonts-liberation gnupg libpq-dev default-libmysqlclient-dev pkg-config libmagic-dev poppler-utils
    ```
    These dependencies are required for OCRmyPDF, which is used for text
@@ -416,16 +415,6 @@ to a positive number to enable polling and disable native filesystem notificatio
        You may need to change the path in the files. Example:
        `ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO`
 12. Configure ImageMagick to allow processing of PDF documents and disable
    formats that Paperless-ngx does not use. Most distributions disable PDF
    processing by default, since PDF documents can contain malware. If you
    don't enable it, Paperless-ngx will fall back to Ghostscript for certain
    steps such as thumbnail generation.
    Configure the active ImageMagick policy file (commonly
    `/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and
    adjust similar to [the docker policy file](https://raw.githubusercontent.com/paperless-ngx/paperless-ngx/refs/heads/main/docker/rootfs/etc/ImageMagick-6/paperless-policy.xml). You should also include restrictions as noted there.
 **Optional: Install the [jbig2enc](https://ocrmypdf.readthedocs.io/en/latest/jbig2.html) encoder.**
 This will reduce the size of generated PDF documents. You'll most likely need to compile this yourself, because this
 software has been patented until around 2017 and binary packages are not available for most distributions.
@@ -50,8 +50,6 @@ PAPERLESS_SECRET_KEY=change-me
 #PAPERLESS_OCR_ROTATE_PAGES=true
 #PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0
 #PAPERLESS_OCR_USER_ARGS={}
 #PAPERLESS_CONVERT_MEMORY_LIMIT=0
 #PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
 # Software tweaks
@@ -90,5 +88,4 @@ PAPERLESS_SECRET_KEY=change-me
 # Binaries
 #PAPERLESS_CONVERT_BINARY=/usr/bin/convert
 #PAPERLESS_GS_BINARY=/usr/bin/gs
@@ -26,7 +26,7 @@ module.exports = {
    'abstract-paperless-service',
  ],
  transformIgnorePatterns: [
-    'node_modules/(?!.*(\\.mjs$|tslib|lodash-es|normalize-diacritics|marked|@angular/common/locales/.*\\.js$))',
+    'node_modules/(?!.*(\\.mjs$|tslib|lodash-es|normalize-diacritics|@angular/common/locales/.*\\.js$))',
  ],
  moduleNameMapper: {
    ...esmPreset.moduleNameMapper,
@@ -30,7 +30,6 @@
    "bootstrap": "^5.3.8",
    "file-saver": "^2.0.5",
    "lodash-es": "^4.18.1",
    "marked": "~18.0.14",
    "mime-names": "^1.0.0",
    "ngx-bootstrap-icons": "^1.9.3",
    "ngx-color": "^10.1.0",
@@ -53,9 +53,6 @@ importers:
      lodash-es:
        specifier: ^4.18.1
        version: 4.18.1
      marked:
        specifier: ~18.0.14
        version: 18.0.14
      mime-names:
        specifier: ^1.0.0
        version: 1.0.0
@@ -3229,11 +3226,6 @@ packages:
  make-error@1.3.6:
    resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
  marked@18.0.14:
    resolution: {integrity: sha512-mBHK6FBHuBAlhgRe88w9F0O1AbwwXJUcQibUbC/QcdTbVGAD7aWza+xt3N6oT/jCZx3/OMeS+8rnuiHZcQ9s7A==}
    engines: {node: '>= 20'}
    hasBin: true
  material-colors@1.2.6:
    resolution: {integrity: sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg==}
@@ -7417,8 +7409,6 @@ snapshots:
  make-error@1.3.6: {}
  marked@18.0.14: {}
  material-colors@1.2.6: {}
  merge-stream@2.0.0: {}
@@ -5,16 +5,14 @@
  </button>
  <div ngbDropdownMenu class="dropdown-menu-end shadow p-3" aria-labelledby="chatDropdown">
    <div class="chat-container bg-light p-2">
-      <div class="chat-messages small">
+      <div class="chat-messages font-monospace small">
        @for (message of messages(); track message) {
          <div class="message d-flex flex-row small" [class.justify-content-end]="message.role === 'user'">
            <div class="p-2 m-2" [class.bg-body]="message.role === 'user'">
-              @if (message.role === 'assistant') {
+              <span class="text-break">
-                <div class="chat-markdown text-break text-wrap" [innerHTML]="message.content | markdown"></div>
+                {{ message.content }}
                @if (message.isStreaming) { <span class="blinking-cursor">|</span> }
-              } @else {
+              </span>
                <span class="text-break">{{ message.content }}</span>
              }
              @if (message.role === 'assistant' && message.references?.length) {
                <div class="chat-references list-group mt-3">
                  @for (reference of message.references; track reference.id) {
@@ -8,6 +8,10 @@
  white-space: pre-wrap;
 }
 .chat-references {
  font-family: var(--bs-font-sans-serif);
 }
 .dropdown-toggle::after {
  display: none;
 }
@@ -36,19 +40,3 @@
    opacity: 1;
  }
 }
 .chat-markdown ::ng-deep {
  h1, h2, h3, h4, h5, h6 {
    font-size: 1em;
    font-weight: bold;
  }
  th, td {
    padding: 0.15rem 0.4rem;
    border: 1px solid var(--bs-border-color);
  }
  > :last-child {
    margin-bottom: 0;
  }
 }
@@ -207,33 +207,4 @@ describe('ChatComponent', () => {
    component.searchInputKeyDown(event)
    expect(component.sendMessage).not.toHaveBeenCalled()
  })
  it('should render markdown for assistant messages only', () => {
    component.messages.set([
      { role: 'user', content: '**user**' },
      { role: 'assistant', content: '**assistant** <script>x</script>' },
    ])
    fixture.detectChanges()
    const el: HTMLElement = fixture.nativeElement
    expect(el.querySelector('.chat-markdown strong')?.textContent).toBe(
      'assistant'
    )
    expect(el.querySelector('.chat-markdown script')).toBeNull()
    expect(el.textContent).toContain('**user**')
  })
  it('should keep the destination hostname visible after sanitizing chat links', () => {
    component.messages.set([
      {
        role: 'assistant',
        content:
          '[View original document](https://otherdomain.example/?data=secret)',
      },
    ])
    fixture.detectChanges()
    const el: HTMLElement = fixture.nativeElement
    const link = el.querySelector('.chat-markdown a')
    expect(link?.textContent).toBe('View original document')
    expect(link?.nextElementSibling?.textContent).toBe('(otherdomain.example)')
  })
 })
@@ -10,7 +10,6 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms'
 import { NavigationEnd, Router, RouterModule } from '@angular/router'
 import { NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
 import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
 import { MarkdownPipe } from 'src/app/pipes/markdown.pipe'
 import { filter, map } from 'rxjs'
 import {
  ChatMessage,
@@ -26,7 +25,6 @@ import {
    RouterModule,
    NgxBootstrapIconsModule,
    NgbDropdownModule,
    MarkdownPipe,
  ],
  templateUrl: './chat.component.html',
  styleUrl: './chat.component.scss',
@@ -1,90 +0,0 @@
 import { MarkdownPipe } from './markdown.pipe'
 describe('MarkdownPipe', () => {
  const pipe = new MarkdownPipe()
  it('should return empty string for empty input', () => {
    expect(pipe.transform(null)).toEqual('')
    expect(pipe.transform(undefined)).toEqual('')
    expect(pipe.transform('')).toEqual('')
  })
  it('should render basic markdown', () => {
    const html = pipe.transform(
      '**bold** _em_ `code`\n\n- one\n- two\n\n| a | b |\n|---|---|\n| 1 | 2 |'
    )
    expect(html).toContain('<strong>bold</strong>')
    expect(html).toContain('<em>em</em>')
    expect(html).toContain('<code>code</code>')
    expect(html).toContain('<li>one</li>')
    expect(html).toContain('<table>')
  })
  it('should escape raw html', () => {
    const html = pipe.transform(
      'hi <img src=x onerror="alert(1)"> <b>x</b>\n\n<script>alert(1)</script>'
    )
    expect(html).not.toContain('<img')
    expect(html).not.toContain('<script')
    expect(html).not.toContain('<b>')
    expect(html).toContain('&lt;script&gt;')
  })
  it('should not render images', () => {
    const html = pipe.transform(
      '![secret](https://otherdomain.example/x.png?d=1)'
    )
    expect(html).not.toContain('<img')
    expect(html).not.toContain('otherdomain.example')
    expect(html).toContain('secret')
  })
  it('should render safe links with target and rel', () => {
    const html = pipe.transform('[docs](https://docs.paperless-ngx.com "Docs")')
    expect(html).toContain(
      '<a target="_blank" rel="noopener noreferrer nofollow" href="https://docs.paperless-ngx.com" title="Docs">docs</a>'
    )
    expect(pipe.transform('[mail](mailto:a@b.c)')).toContain(
      'href="mailto:a@b.c"'
    )
  })
  it('should drop unsafe or relative links but keep their text', () => {
    for (const href of [
      'javascript:alert(1)',
      'JaVaScRiPt:alert(1)',
      'data:text/html,<script>alert(1)</script>',
      'vbscript:msgbox',
      '/api/documents/',
    ]) {
      const html = pipe.transform(`[click](${href})`)
      expect(html).not.toContain('<a')
      expect(html).toContain('click')
    }
  })
  it('should show the actual hostname beside web links with misleading labels', () => {
    for (const protocol of ['http', 'https']) {
      const html = pipe.transform(
        `[docs.paperless-ngx.com](${protocol}://docs.paperless-ngx.com@otherdomain.example/path?secret=1)`
      )
      expect(html).toContain(
        '</a> <span class="text-muted small">(otherdomain.example)</span>'
      )
      expect(html).not.toContain('(docs.paperless-ngx.com)')
    }
  })
  it('should omit the hostname label for mailto links', () => {
    expect(pipe.transform('[mail](mailto:a@b.c)')).toBe(
      '<p><a target="_blank" rel="noopener noreferrer nofollow" href="mailto:a@b.c">mail</a></p>\n'
    )
  })
  it('should escape link titles', () => {
    const html = pipe.transform(
      '[x](https://a.example "a\\" onmouseover=\\"1")'
    )
    expect(html).not.toMatch(/"\s*onmouseover=/)
  })
 })
@@ -1,52 +0,0 @@
 import { Pipe, PipeTransform } from '@angular/core'
 import { Marked, Renderer, Tokens } from 'marked'
 const ALLOWED_LINK_PROTOCOLS = new Set(['http:', 'https:', 'mailto:'])
 function allowedUrl(href: string): URL | null {
  try {
    const url = new URL(href)
    return ALLOWED_LINK_PROTOCOLS.has(url.protocol) ? url : null
  } catch {
    return null
  }
 }
 // Treat chat content as untrusted: no raw HTML, no images, and only
 // absolute links. All escaping is done by marked's default renderer, and
 // Angular's sanitizer runs on top of this as well.
 const renderer: Partial<Renderer> = {
  html(this: Renderer, { raw, text }: Tokens.HTML | Tokens.Tag): string {
    return this.text({ type: 'text', raw, text })
  },
  image(this: Renderer, { raw, text }: Tokens.Image): string {
    return this.text({ type: 'text', raw, text })
  },
  link(this: Renderer, token: Tokens.Link): string {
    const url = allowedUrl(token.href)
    if (!url) return this.parser.parseInline(token.tokens)
    const anchor = Renderer.prototype.link
      .call(this, token)
      .replace('<a ', '<a target="_blank" rel="noopener noreferrer nofollow" ')
    if (!url.hostname) return anchor
    const hostname = this.text({ type: 'text', raw: '', text: url.hostname })
    return `${anchor} <span class="text-muted small">(${hostname})</span>`
  },
 }
 const markdown = new Marked({
  async: false,
  gfm: true,
  breaks: true,
  renderer,
 })
@Pipe({
  name: 'markdown',
 })
 export class MarkdownPipe implements PipeTransform {
  transform(value: string | null | undefined): string {
    if (!value) return ''
    return markdown.parse(value) as string
  }
 }
@@ -15,7 +15,7 @@
    "target": "ES2022",
    "module": "es2020",
    "lib": [
-      "es2022",
+      "es2020",
      "dom"
    ],
    "useDefineForClassFields": false
@@ -1,12 +1,11 @@
 from pathlib import Path
 import img2pdf
 from django.conf import settings
 from PIL import Image
 from documents.utils import copy_basic_file_stats
 from documents.utils import maybe_override_pixel_limit
-from documents.utils import run_subprocess
+from documents.utils import remove_alpha_channel
 def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
@@ -27,15 +26,7 @@ def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
        # Note the save into the temp folder, so as not to trigger a new
        # consume
        scratch_image = target_directory / tiff_path.name
-        run_subprocess(
+        remove_alpha_channel(tiff_path, scratch_image)
            [
                settings.CONVERT_BINARY,
                "-alpha",
                "off",
                tiff_path,
                scratch_image,
            ],
        )
    else:
        # Not modifying the original, safe to use in place
        scratch_image = tiff_path
@@ -1,8 +1,8 @@
 from __future__ import annotations
 import logging
 import math
 import mimetypes
 import os
 import shutil
 import subprocess
 import tempfile
@@ -68,58 +68,6 @@ def get_supported_file_extensions() -> set[str]:
    return extensions
 def run_convert(
    input_file,
    output_file,
    *,
    density=None,
    scale=None,
    alpha=None,
    strip=False,
    trim=False,
    type=None,
    depth=None,
    auto_orient=False,
    use_cropbox=False,
    extra=None,
    logging_group=None,
 ) -> None:
    environment = os.environ.copy()
    if settings.CONVERT_MEMORY_LIMIT:
        # MAGICK_MEMORY_LIMIT sets the maximum amount of RAM the pixel cache can use.
        # MAGICK_MAP_LIMIT sets the maximum amount of memory-mapped I/O allowed.
        #
        # For large-format documents  ImageMagick will hit the RAM limit and
        # immediately try to "map" the remaining data. If MAGICK_MAP_LIMIT isn't
        # also set, the process may trigger an OOM kill because the default
        # system/policy map limit is often too restrictive for these massive bitmaps.
        environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
        environment["MAGICK_MAP_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
    if settings.CONVERT_TMPDIR:
        environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR
    args = [settings.CONVERT_BINARY]
    args += ["-density", str(density)] if density else []
    args += ["-scale", str(scale)] if scale else []
    args += ["-alpha", str(alpha)] if alpha else []
    args += ["-strip"] if strip else []
    args += ["-trim"] if trim else []
    args += ["-type", str(type)] if type else []
    args += ["-depth", str(depth)] if depth else []
    args += ["-auto-orient"] if auto_orient else []
    args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else []
    args += [str(input_file), str(output_file)]
    logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
    try:
        run_subprocess(args, environment, logger)
    except subprocess.CalledProcessError as e:
        raise ParseError(f"Convert failed at {args}") from e
    except Exception as e:  # pragma: no cover
        raise ParseError("Unknown error running convert") from e
 def get_default_thumbnail() -> Path:
    """
    Returns the path to a generic thumbnail
@@ -127,46 +75,168 @@ def get_default_thumbnail() -> Path:
    return (Path(__file__).parent / "resources" / "document.webp").resolve()
-def make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group=None) -> Path:
+_THUMBNAIL_MAX_WIDTH = 500
-    out_path: Path = Path(temp_dir) / "convert_gs.webp"
+_THUMBNAIL_MAX_HEIGHT = 5000
 # Used only when the page geometry cannot be read
 _THUMBNAIL_FALLBACK_DPI = 150
 # Applied before supersampling, so tiny pages are not enlarged
 _THUMBNAIL_MAX_DPI = 300
 # Rendering at a multiple and downsampling keeps text crisper
 _THUMBNAIL_SUPERSAMPLE = 2
-    # if convert fails, fall back to extracting
+
-    # the first PDF page as a PNG using Ghostscript
+def rasterize_pdf_page_to_png(
-    logger.warning(
+    in_path: Path,
-        "Thumbnail generation with ImageMagick failed, falling back "
+    out_path: Path,
-        "to ghostscript. Check your /etc/ImageMagick-x/policy.xml!",
+    *,
-        extra={"group": logging_group},
+    dpi: int,
-    )
+    logging_group=None,
-    # Ghostscript doesn't handle WebP outputs
+) -> None:
-    gs_out_path: Path = Path(temp_dir) / "gs_out.png"
+    """
-    cmd = [settings.GS_BINARY, "-q", "-sDEVICE=pngalpha", "-o", gs_out_path, in_path]
+    Rasterizes the first page of a PDF to a PNG with pdftoppm.
    """
    # -singlefile drops the page number and -png appends ".png", so pass the
    # path without its suffix
    args = [
        "pdftoppm",
        "-f",
        "1",
        "-l",
        "1",
        "-r",
        str(dpi),
        "-png",
        "-singlefile",
        "-cropbox",
        str(in_path),
        str(out_path.with_suffix("")),
    ]
    logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
    try:
-        try:
+        run_subprocess(args, logger=logger)
-            run_subprocess(cmd, logger=logger)
+    except subprocess.CalledProcessError as e:
-        except subprocess.CalledProcessError as e:
+        raise ParseError(f"pdftoppm failed at {args}") from e
-            raise ParseError(f"Thumbnail (gs) failed at {cmd}") from e
+    except Exception as e:  # pragma: no cover
-        # then run convert on the output from gs to make WebP
+        raise ParseError("Unknown error running pdftoppm") from e
        run_convert(
            density=300,
            scale="500x5000>",
            alpha="remove",
            strip=True,
            trim=False,
            auto_orient=True,
            input_file=gs_out_path,
            output_file=out_path,
            logging_group=logging_group,
        )
 def encode_thumbnail_webp(
    png_path: Path,
    out_path: Path,
    *,
    supersample: int = 1,
 ) -> None:
    """
    Flattens alpha onto white, undoes supersampling, shrinks to fit and saves as WebP.
    """
    from PIL import Image
    try:
        with Image.open(png_path) as im:
            if im.mode in ("RGBA", "LA"):
                flattened = Image.new("RGB", im.size, (255, 255, 255))
                flattened.paste(im, mask=im.split()[-1])
            else:
                flattened = im.convert("RGB")
        if supersample > 1:
            flattened = flattened.resize(
                (
                    max(1, round(flattened.width / supersample)),
                    max(1, round(flattened.height / supersample)),
                ),
                Image.Resampling.LANCZOS,
            )
        flattened.thumbnail((_THUMBNAIL_MAX_WIDTH, _THUMBNAIL_MAX_HEIGHT))
        flattened.save(out_path, format="WEBP")
    except (OSError, Image.DecompressionBombError) as e:
        raise ParseError(f"Unable to encode thumbnail from {png_path}") from e
 def _compute_thumbnail_dpi(in_path: Path, logging_group=None) -> tuple[int, int]:
    """
    Returns (dpi, supersample). Unknown geometry is not supersampled, since
    the render size cannot be bounded.
    """
    from paperless.parsers.utils import get_pdf_first_page_size_points
    size = get_pdf_first_page_size_points(in_path)
    if size is None:
        logger.debug(
            "Could not read PDF page size, using fallback DPI",
            extra={"group": logging_group},
        )
        return _THUMBNAIL_FALLBACK_DPI, 1
    width_pts, height_pts = size
    dpi_for_width = _THUMBNAIL_MAX_WIDTH * 72 / width_pts
    dpi_for_height = _THUMBNAIL_MAX_HEIGHT * 72 / height_pts
    # Round up so the downsampled render is never a few pixels short of the
    # target; the shrink-only clamp in encode_thumbnail_webp trims the excess.
    dpi = max(
        1,
        math.ceil(min(_THUMBNAIL_MAX_DPI, dpi_for_width, dpi_for_height)),
    )
    # At the 1 DPI floor the page is already oversized, so do not supersample
    return dpi, 1 if dpi == 1 else _THUMBNAIL_SUPERSAMPLE
 def _render_pdf_thumbnail(
    in_path: Path,
    png_path: Path,
    out_path: Path,
    logging_group=None,
 ) -> None:
    dpi, supersample = _compute_thumbnail_dpi(in_path, logging_group=logging_group)
    rasterize_pdf_page_to_png(
        in_path,
        png_path,
        dpi=dpi * supersample,
        logging_group=logging_group,
    )
    encode_thumbnail_webp(png_path, out_path, supersample=supersample)
 def _repair_pdf_with_qpdf(in_path: Path, out_path: Path) -> None:
    # qpdf exits 3 after a repair; --warning-exit-0 keeps that from failing
    try:
        shutil.copy(in_path, out_path)
        run_subprocess(
            ["qpdf", "--warning-exit-0", "--replace-input", str(out_path)],
            logger=logger,
        )
    except (subprocess.CalledProcessError, OSError) as e:
        raise ParseError(f"qpdf repair failed for {in_path}") from e
 def make_thumbnail_from_pdf_qpdf_fallback(
    in_path: Path,
    temp_dir: Path,
    logging_group=None,
 ) -> Path:
    png_path = temp_dir / "page1_repaired.png"
    out_path = temp_dir / "convert_qpdf.webp"
    repaired_path = temp_dir / "repaired.pdf"
    logger.warning(
        "Thumbnail generation with pdftoppm failed, attempting qpdf repair and retry.",
        extra={"group": logging_group},
    )
    try:
        _repair_pdf_with_qpdf(in_path, repaired_path)
        _render_pdf_thumbnail(repaired_path, png_path, out_path, logging_group)
        return out_path
    except ParseError as e:
-        logger.error(f"Unable to make thumbnail with Ghostscript: {e}")
+        logger.error(f"Unable to make thumbnail after qpdf repair: {e}")
        # The caller might expect a generated thumbnail that can be moved,
        # so we need to copy it before it gets moved.
        # https://github.com/paperless-ngx/paperless-ngx/issues/3631
-        default_thumbnail_path: Path = Path(temp_dir) / "document.webp"
+        default_thumbnail_path = temp_dir / "document.webp"
        copy_file_with_basic_stats(get_default_thumbnail(), default_thumbnail_path)
        return default_thumbnail_path
@@ -175,25 +245,18 @@ def make_thumbnail_from_pdf(in_path: Path, temp_dir: Path, logging_group=None) -
    """
    The thumbnail of a PDF is just a 500px wide image of the first page.
    """
    png_path: Path = temp_dir / "page1.png"
    out_path: Path = temp_dir / "convert.webp"
    # Run convert to get a decent thumbnail
    try:
-        run_convert(
+        _render_pdf_thumbnail(in_path, png_path, out_path, logging_group)
            density=300,
            scale="500x5000>",
            alpha="remove",
            strip=True,
            trim=False,
            auto_orient=True,
            use_cropbox=True,
            input_file=f"{in_path}[0]",
            output_file=str(out_path),
            logging_group=logging_group,
        )
    except ParseError as e:
-        logger.error(f"Unable to make thumbnail with convert: {e}")
+        logger.error(f"Unable to make thumbnail with pdftoppm: {e}")
-        out_path = make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group)
+        out_path = make_thumbnail_from_pdf_qpdf_fallback(
            in_path,
            temp_dir,
            logging_group,
        )
    return out_path
@@ -6,9 +6,7 @@ from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.db.models import Case
 from django.db.models import CharField
 from django.db.models import Count
 from django.db.models import F
 from django.db.models import IntegerField
 from django.db.models import Model
 from django.db.models import Q
@@ -351,7 +349,6 @@ def permitted_object_ids(
    perm: str,
    *,
    include_deleted: bool = False,
    parent_field: str | None = None,
 ) -> QuerySet[int]:
    """
    Generic version of ``permitted_document_ids`` for any model with an
@@ -360,24 +357,6 @@ def permitted_object_ids(
    soft-delete pattern (currently only ``Document``); for every other model
    it is accepted but has no effect, since those models have no soft-delete
    concept.
    ``parent_field`` names a self-referencing foreign key whose target
    authorizes the row (``Document.root_document``). A row with a parent is
    visible exactly when its parent is, judged by the parent's owner and
    grants, so the row's own owner and grants are ignored.
    Guardian stores ``object_pk`` as a string, so the row key is cast to a
    string and tested against the user's and groups' grants with a single
    uncorrelated ``IN``. Postgres and SQLite build that set once. MariaDB
    evaluates it as an index probe per row, which is cheap because the
    lookups use guardian's unique indexes. Casting every ``object_pk`` to an
    integer instead cannot use an index, and MariaDB cannot materialize it
    inside the owner ``OR``, so it re-scans the user's grants for every row.
    A correlated ``EXISTS`` per grant fixes MariaDB too, but Postgres and
    SQLite re-run it for every row and end up slower than the original. The
    user's groups are matched with an ``IN`` subquery rather than a join
    through the membership table, which SQLite plans badly once the grant
    tables grow.
    """
    has_soft_delete = hasattr(model, "global_objects")
    manager = (
@@ -385,21 +364,8 @@ def permitted_object_ids(
    )
    base_qs = manager.all().only("id", "owner")
    owner_field, key_field = "owner", "pk"
    if parent_field is not None:
        owner_field, key_field = "authorizing_owner", "authorizing_id"
        base_qs = base_qs.annotate(
            authorizing_id=Coalesce(f"{parent_field}_id", "id"),
            authorizing_owner=Case(
                When(**{f"{parent_field}_id__isnull": True}, then=F("owner_id")),
                default=F(f"{parent_field}__owner_id"),
                output_field=IntegerField(),
            ),
        )
    unowned = Q(**{f"{owner_field}__isnull": True})
    if user is None or not getattr(user, "is_authenticated", False):
-        return base_qs.filter(unowned).values_list("id", flat=True)
+        return base_qs.filter(owner__isnull=True).values_list("id", flat=True)
    # Deactivated users get nothing, deactivated superusers included, so this
    # has to come before the superuser shortcut. guardian's
@@ -423,26 +389,21 @@ def permitted_object_ids(
        "permission__content_type": content_type,
    }
-    # Both grant sets are compared to the row key as strings, exactly as
+    user_perm_ids = (
-    # guardian stores them, and are uncorrelated, so each engine can build the
+        UserObjectPermission.objects.filter(user=user, **perm_filter)
-    # set once instead of probing per row.
+        .annotate(object_pk_int=Cast("object_pk", IntegerField()))
-    user_keys = UserObjectPermission.objects.filter(
+        .values_list("object_pk_int", flat=True)
        user=user,
        **perm_filter,
    ).values_list("object_pk", flat=True)
    group_keys = GroupObjectPermission.objects.filter(
        group_id__in=user.groups.values("id"),
        **perm_filter,
    ).values_list("object_pk", flat=True)
    permitted_keys = user_keys.union(group_keys, all=True)
    return (
        base_qs.annotate(permitted_key=Cast(key_field, CharField(max_length=64)))
        .filter(
            Q(**{owner_field: user.pk}) | unowned | Q(permitted_key__in=permitted_keys),
        )
        .values_list("id", flat=True)
    )
    group_perm_ids = (
        GroupObjectPermission.objects.filter(group__user=user, **perm_filter)
        .annotate(object_pk_int=Cast("object_pk", IntegerField()))
        .values_list("object_pk_int", flat=True)
    )
    permitted_ids = user_perm_ids.union(group_perm_ids)
    return base_qs.filter(
        Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_ids),
    ).values_list("id", flat=True)
 ModelT = TypeVar("ModelT", bound=Model)
@@ -510,16 +471,30 @@ def permitted_document_ids(
    ``include_deleted=True`` for callers that need to check permission on
    soft-deleted documents (e.g. trash restore). This intentionally avoids
    ``get_objects_for_user`` to keep the subquery small and index-friendly.
    A version is authorized by its root document, so a version's own owner and
    grants never matter.
    """
-    return permitted_object_ids(
+    return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
-        user,
+
-        Document,
+
-        perm,
+def documents_without_permitted_root(
-        include_deleted=include_deleted,
+    documents: QuerySet[Document],
-        parent_field="root_document",
+    user: User | None,
    *,
    perm: str = "view_document",
    include_deleted: bool = False,
 ) -> QuerySet[Document]:
    """
    The documents the user lacks ``perm`` on. Versions are authorized by their
    root document, so a version's own owner is ignored. A single query, without
    loading the documents or joining the root.
    """
    return documents.annotate(
        root_id=Coalesce("root_document_id", "id"),
    ).exclude(
        root_id__in=permitted_document_ids(
            user,
            perm=perm,
            include_deleted=include_deleted,
        ),
    )
@@ -1,11 +1,22 @@
 import subprocess
 from collections.abc import Generator
 from pathlib import Path
 import pikepdf
 import pytest
 from PIL import Image
 from pytest_django.fixtures import Settings
 from pytest_mock import MockerFixture
 from documents.parsers import ParseError
 from documents.parsers import _compute_thumbnail_dpi
 from documents.parsers import encode_thumbnail_webp
 from documents.parsers import get_default_file_extension
 from documents.parsers import get_default_thumbnail
 from documents.parsers import get_supported_file_extensions
 from documents.parsers import is_file_ext_supported
 from documents.parsers import make_thumbnail_from_pdf
 from documents.parsers import rasterize_pdf_page_to_png
 from paperless.parsers.registry import get_parser_registry
 from paperless.parsers.registry import reset_parser_registry
 from paperless.parsers.tesseract import RasterisedDocumentParser
@@ -125,3 +136,374 @@ class TestParserAvailability:
        assert is_file_ext_supported(".pdf")
        assert not is_file_ext_supported(".hsdfh")
        assert not is_file_ext_supported("")
 class TestComputeThumbnailDpi:
    @pytest.mark.parametrize(
        ("size", "expected"),
        [
            pytest.param((612.0, 792.0), (59, 2), id="letter-width-bound"),
            pytest.param((792.0, 612.0), (46, 2), id="landscape-rounded-up"),
            pytest.param((612.0, 100000.0), (4, 2), id="tall-strip-height-bound"),
            pytest.param((200.0, 300.0), (180, 2), id="small-page-width-bound"),
            pytest.param((72.0, 72.0), (300, 2), id="tiny-page-capped-at-300"),
            pytest.param(
                (1000000.0, 1000000.0),
                (1, 1),
                id="huge-page-minimum-one-unsupersampled",
            ),
            pytest.param(None, (150, 1), id="unreadable-geometry-fallback"),
        ],
    )
    def test_dpi_from_page_size(
        self,
        mocker: MockerFixture,
        tmp_path: Path,
        size: tuple[float, float] | None,
        expected: tuple[int, int],
    ) -> None:
        """
        GIVEN:
            - A first page of the given size, or unreadable geometry
        WHEN:
            - The thumbnail DPI is computed
        THEN:
            - The expected DPI and supersample factor are returned
        """
        mocker.patch(
            "paperless.parsers.utils.get_pdf_first_page_size_points",
            return_value=size,
        )
        assert _compute_thumbnail_dpi(tmp_path / "doc.pdf") == expected
 class TestMakeThumbnailFromPdf:
    @pytest.fixture
    def work_dir(self, tmp_path: Path) -> Path:
        path = tmp_path / "work"
        path.mkdir()
        return path
    @staticmethod
    def _write_blank_pdf(path: Path, page_size: tuple[int, int] = (612, 792)) -> Path:
        pdf = pikepdf.new()
        pdf.add_blank_page(page_size=page_size)
        pdf.save(path, object_stream_mode=pikepdf.ObjectStreamMode.disable)
        return path
    @pytest.mark.parametrize(
        ("size", "expected_dpi", "expected_supersample"),
        [
            pytest.param((612.0, 792.0), 118, 2, id="known-geometry-2x"),
            pytest.param(None, 150, 1, id="unreadable-geometry-plain-fallback"),
        ],
    )
    def test_render_dpi_requested(
        self,
        mocker: MockerFixture,
        tmp_path: Path,
        work_dir: Path,
        size: tuple[float, float] | None,
        expected_dpi: int,
        expected_supersample: int,
    ) -> None:
        """
        GIVEN:
            - Readable or unreadable page geometry
        WHEN:
            - A thumbnail is made
        THEN:
            - Rasterize and encode get the matching DPI and supersample factor
        """
        mocker.patch(
            "paperless.parsers.utils.get_pdf_first_page_size_points",
            return_value=size,
        )
        rasterize = mocker.patch("documents.parsers.rasterize_pdf_page_to_png")
        encode = mocker.patch("documents.parsers.encode_thumbnail_webp")
        make_thumbnail_from_pdf(tmp_path / "in.pdf", work_dir)
        assert rasterize.call_args.kwargs["dpi"] == expected_dpi
        assert encode.call_args.kwargs["supersample"] == expected_supersample
    @pytest.mark.parametrize(
        ("page_size", "expected_width"),
        [
            pytest.param((612, 792), 500, id="letter"),
            pytest.param((792, 612), 500, id="landscape-letter"),
            pytest.param((595, 842), 500, id="a4"),
            pytest.param((200, 300), 500, id="small-page"),
            pytest.param((72, 72), 300, id="tiny-page-capped"),
        ],
    )
    def test_thumbnail_width(
        self,
        tmp_path: Path,
        work_dir: Path,
        page_size: tuple[int, int],
        expected_width: int,
    ) -> None:
        """
        GIVEN:
            - A PDF whose first page has the given size in points
        WHEN:
            - A thumbnail is made from it
        THEN:
            - The thumbnail has the expected width
        """
        pdf_path = self._write_blank_pdf(tmp_path / "in.pdf", page_size)
        thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
        assert thumb == work_dir / "convert.webp"
        with Image.open(thumb) as im:
            assert im.format == "WEBP"
            assert im.width == expected_width
    @classmethod
    def _write_pdf_without_xref(cls, path: Path) -> Path:
        """
        Cuts off the xref and trailer, which pdftoppm cannot recover from but qpdf can.
        """
        cls._write_blank_pdf(path)
        data = path.read_bytes()
        path.write_bytes(data[: data.rindex(b"\nxref")])
        return path
    def test_qpdf_repair_produces_real_thumbnail(
        self,
        tmp_path: Path,
        work_dir: Path,
    ) -> None:
        """
        GIVEN:
            - A PDF with its xref table and trailer cut off
        WHEN:
            - A thumbnail is made from it
        THEN:
            - The thumbnail is rendered from a qpdf repaired copy
            - The original file is unchanged
        """
        pdf_path = self._write_pdf_without_xref(tmp_path / "broken.pdf")
        original_bytes = pdf_path.read_bytes()
        with pytest.raises(ParseError):
            rasterize_pdf_page_to_png(pdf_path, work_dir / "probe.png", dpi=50)
        thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
        assert thumb == work_dir / "convert_qpdf.webp"
        with Image.open(thumb) as im:
            assert im.format == "WEBP"
            assert im.width == 500
        assert pdf_path.read_bytes() == original_bytes
    @pytest.mark.parametrize(
        "qpdf_error",
        [
            pytest.param(subprocess.CalledProcessError(2, "qpdf"), id="qpdf-fails"),
            pytest.param(None, id="repaired-still-unrenderable"),
        ],
    )
    def test_double_failure_uses_default_thumbnail(
        self,
        mocker: MockerFixture,
        tmp_path: Path,
        work_dir: Path,
        qpdf_error: subprocess.CalledProcessError | None,
    ) -> None:
        """
        GIVEN:
            - A PDF that cannot be rendered, even after qpdf repair
        WHEN:
            - A thumbnail is made from it
        THEN:
            - A copy of the default thumbnail is returned
        """
        mocker.patch(
            "documents.parsers.rasterize_pdf_page_to_png",
            side_effect=ParseError("Does not compute."),
        )
        if qpdf_error is not None:
            mocker.patch("documents.parsers.run_subprocess", side_effect=qpdf_error)
        pdf_path = self._write_blank_pdf(tmp_path / "in.pdf")
        thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
        assert thumb == work_dir / "document.webp"
        assert thumb.read_bytes() == get_default_thumbnail().read_bytes()
 class TestRasterizePdfPageToPng:
    @staticmethod
    def _write_pdf(
        path: Path,
        *,
        crop_box: tuple[float, float, float, float] | None = None,
        rotate: int | None = None,
    ) -> Path:
        pdf = pikepdf.new()
        pdf.add_blank_page(page_size=(144, 72))
        pdf.add_blank_page(page_size=(300, 300))
        page = pdf.pages[0]
        if crop_box is not None:
            page.obj.CropBox = pikepdf.Array(crop_box)
        if rotate is not None:
            page.obj.Rotate = rotate
        pdf.save(path)
        return path
    @pytest.mark.parametrize(
        ("crop_box", "rotate", "expected_size"),
        [
            pytest.param(None, None, (144, 72), id="plain"),
            pytest.param(None, 90, (72, 144), id="rotated-90"),
            pytest.param((0, 0, 72, 36), None, (72, 36), id="crop-box"),
        ],
    )
    def test_renders_first_page_to_exact_path(
        self,
        tmp_path: Path,
        crop_box: tuple[float, float, float, float] | None,
        rotate: int | None,
        expected_size: tuple[int, int],
    ) -> None:
        """
        GIVEN:
            - A two page PDF, first page optionally cropped or rotated
        WHEN:
            - The first page is rasterized at 72 DPI
        THEN:
            - Only out_path is written, sized to the first page's crop and rotation
        """
        pdf_path = self._write_pdf(
            tmp_path / "in.pdf",
            crop_box=crop_box,
            rotate=rotate,
        )
        out_dir = tmp_path / "out"
        out_dir.mkdir()
        out_path = out_dir / "page1.png"
        rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72)
        assert list(out_dir.iterdir()) == [out_path]
        with Image.open(out_path) as im:
            assert im.format == "PNG"
            assert im.size == expected_size
    def test_failure_raises_parse_error(self, tmp_path: Path) -> None:
        """
        GIVEN:
            - A file that is not a PDF
        WHEN:
            - Rasterization is attempted
        THEN:
            - A ParseError is raised
        """
        bad = tmp_path / "bad.pdf"
        bad.write_bytes(b"not a pdf")
        with pytest.raises(ParseError):
            rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72)
 class TestEncodeThumbnailWebp:
    @pytest.mark.parametrize(
        ("mode", "color"),
        [
            pytest.param("RGBA", (0, 0, 0, 0), id="rgba"),
            pytest.param("LA", (0, 0), id="la"),
        ],
    )
    def test_alpha_flattened_onto_white(
        self,
        tmp_path: Path,
        mode: str,
        color: tuple[int, ...],
    ) -> None:
        """
        GIVEN:
            - A fully transparent PNG with an alpha channel
        WHEN:
            - It is encoded as a thumbnail
        THEN:
            - The WebP output is RGB with the transparency flattened to white
        """
        png_path = tmp_path / "in.png"
        Image.new(mode, (20, 10), color).save(png_path)
        out_path = tmp_path / "out.webp"
        encode_thumbnail_webp(png_path, out_path)
        with Image.open(out_path) as im:
            assert im.format == "WEBP"
            assert im.mode == "RGB"
            assert im.size == (20, 10)
            red, green, blue = im.getpixel((10, 5))
            assert min(red, green, blue) >= 250
    @pytest.mark.parametrize(
        ("in_size", "supersample", "expected_size"),
        [
            pytest.param((1000, 2000), 1, (500, 1000), id="too-wide-shrunk"),
            pytest.param((100, 10000), 1, (50, 5000), id="too-tall-shrunk"),
            pytest.param((100, 200), 1, (100, 200), id="small-not-enlarged"),
            pytest.param((1000, 1400), 2, (500, 700), id="2x-halved"),
            pytest.param((1001, 1401), 2, (500, 700), id="2x-odd-rounded"),
            pytest.param((600, 800), 2, (300, 400), id="2x-small-not-enlarged"),
            pytest.param((900, 1200), 2, (450, 600), id="2x-below-clamp"),
        ],
    )
    def test_size_clamped_and_downsampled(
        self,
        tmp_path: Path,
        in_size: tuple[int, int],
        supersample: int,
        expected_size: tuple[int, int],
    ) -> None:
        """
        GIVEN:
            - A rendered image and its supersample factor
        WHEN:
            - It is encoded as a thumbnail with that factor
        THEN:
            - It is downsampled, fit within 500x5000 and never enlarged
        """
        png_path = tmp_path / "in.png"
        Image.new("RGB", in_size, (255, 255, 255)).save(png_path)
        out_path = tmp_path / "out.webp"
        encode_thumbnail_webp(png_path, out_path, supersample=supersample)
        with Image.open(out_path) as im:
            assert im.size == expected_size
    @pytest.mark.parametrize(
        "error",
        [
            pytest.param(OSError("broken image"), id="os-error"),
            pytest.param(Image.DecompressionBombError("too large"), id="bomb"),
        ],
    )
    def test_decode_failure_raises_parse_error(
        self,
        tmp_path: Path,
        mocker: MockerFixture,
        error: Exception,
    ) -> None:
        """
        GIVEN:
            - Opening the rendered image fails
        WHEN:
            - It is encoded as a thumbnail
        THEN:
            - A ParseError is raised
        """
        png_path = tmp_path / "in.png"
        Image.new("RGB", (10, 10)).save(png_path)
        mocker.patch("PIL.Image.open", side_effect=error)
        with pytest.raises(ParseError):
            encode_thumbnail_webp(png_path, tmp_path / "out.webp")
@@ -32,8 +32,6 @@ from paperless_testing.permissions import grant_global
 from paperless_testing.permissions import grant_object
 if TYPE_CHECKING:
    from django.contrib.auth.models import User
    from paperless_testing.dirs import PaperlessDirs
@@ -180,296 +178,6 @@ class TestPermittedDocumentIdsIncludeDeleted:
        )
@pytest.mark.django_db
 class TestPermittedDocumentIdsVersions:
    """
    A version is authorized by its root document: the version's own owner and
    grants never matter.
    """
    @pytest.mark.parametrize(
        ("root_owner", "version_owner", "expected_visible"),
        [
            pytest.param(
                "other",
                "nobody",
                False,
                id="unowned-version-of-private-root",
            ),
            pytest.param("other", "user", False, id="own-version-of-private-root"),
            pytest.param("user", "other", True, id="foreign-version-of-own-root"),
            pytest.param("user", "nobody", True, id="unowned-version-of-own-root"),
            pytest.param("nobody", "other", True, id="private-version-of-unowned-root"),
        ],
    )
    def test_version_follows_root_owner(
        self,
        root_owner: str,
        version_owner: str,
        *,
        expected_visible: bool,
    ) -> None:
        """
        GIVEN:
            - A root document and a version with differing owners
        WHEN:
            - The permitted document ids are resolved for the user
        THEN:
            - The version is visible exactly when its root is
        """
        user = UserFactory()
        owners = {"user": user, "other": UserFactory(), "nobody": None}
        root = DocumentFactory(owner=owners[root_owner])
        version = DocumentFactory(root_document=root, owner=owners[version_owner])
        visible = set(permitted_document_ids(user))
        assert (version.pk in visible) is expected_visible
        assert (root.pk in visible) is expected_visible
    @staticmethod
    def grantee(user: User, kind: str) -> User | Group:
        """The user itself, or a new group the user belongs to."""
        if kind == "user":
            return user
        group = Group.objects.create(name="shared")
        user.groups.add(group)
        return group
    @pytest.mark.parametrize(
        "grantee_kind",
        [pytest.param("user", id="user"), pytest.param("group", id="group")],
    )
    def test_grant_on_root_applies_to_version(self, grantee_kind: str) -> None:
        """
        GIVEN:
            - A private root document shared with a user or one of their groups
            - A version of it owned by someone else
        WHEN:
            - The permitted document ids are resolved for the user
        THEN:
            - Both the root and the version are visible
            - A user without the grant sees neither
        """
        user = UserFactory()
        stranger = UserFactory()
        root = DocumentFactory(owner=UserFactory())
        version = DocumentFactory(root_document=root, owner=UserFactory())
        grant_object(self.grantee(user, grantee_kind), root, "view_document")
        assert_visible_document_ids(
            permitted_document_ids(user),
            expected_visible=[root.pk, version.pk],
            expected_hidden=[],
        )
        assert_visible_document_ids(
            permitted_document_ids(stranger),
            expected_visible=[],
            expected_hidden=[root.pk, version.pk],
        )
    @pytest.mark.parametrize(
        "grantee_kind",
        [pytest.param("user", id="user"), pytest.param("group", id="group")],
    )
    def test_grant_on_version_is_ignored(self, grantee_kind: str) -> None:
        """
        GIVEN:
            - A private root document
            - A version with an explicit grant for the user or one of their groups
        WHEN:
            - The permitted document ids are resolved for the user
        THEN:
            - Neither the root nor the version is visible
        """
        user = UserFactory()
        root = DocumentFactory(owner=UserFactory())
        version = DocumentFactory(root_document=root, owner=UserFactory())
        grant_object(self.grantee(user, grantee_kind), version, "view_document")
        assert_visible_document_ids(
            permitted_document_ids(user),
            expected_visible=[],
            expected_hidden=[root.pk, version.pk],
        )
    def test_grant_on_one_root_does_not_reach_another_roots_version(self) -> None:
        """
        GIVEN:
            - Two private roots, each with a version
            - The user may view only the first root
        WHEN:
            - The permitted document ids are resolved for the user
        THEN:
            - Only the first root and its version are visible
        """
        user = UserFactory()
        first = DocumentFactory(owner=UserFactory())
        first_version = DocumentFactory(root_document=first, owner=UserFactory())
        second = DocumentFactory(owner=UserFactory())
        second_version = DocumentFactory(root_document=second, owner=user)
        grant_object(user, first, "view_document")
        assert_visible_document_ids(
            permitted_document_ids(user),
            expected_visible=[first.pk, first_version.pk],
            expected_hidden=[second.pk, second_version.pk],
        )
    def test_user_in_several_groups(self) -> None:
        """
        GIVEN:
            - A user in two groups
            - Two private roots shared with one group each, and a third shared with nobody
            - A version of each root
        WHEN:
            - The permitted document ids are resolved for the user
        THEN:
            - The two shared roots and their versions are visible
            - The third root and its version are not
        """
        user = UserFactory()
        groups = [Group.objects.create(name=f"group{i}") for i in range(2)]
        user.groups.add(*groups)
        shared = [DocumentFactory(owner=UserFactory()) for _ in groups]
        for root, group in zip(shared, groups, strict=True):
            grant_object(group, root, "view_document")
        unshared = DocumentFactory(owner=UserFactory())
        shared_versions = [
            DocumentFactory(root_document=root, owner=UserFactory()) for root in shared
        ]
        unshared_version = DocumentFactory(root_document=unshared, owner=None)
        assert_visible_document_ids(
            permitted_document_ids(user),
            expected_visible=[
                *(root.pk for root in shared),
                *(version.pk for version in shared_versions),
            ],
            expected_hidden=[unshared.pk, unshared_version.pk],
        )
    def test_permission_is_resolved_through_the_root(self) -> None:
        """
        GIVEN:
            - A private root document where the user may view and change
        WHEN:
            - The permitted ids are resolved for view, change and delete
        THEN:
            - The version is visible for view and change only
        """
        user = UserFactory()
        root = DocumentFactory(owner=UserFactory())
        version = DocumentFactory(root_document=root, owner=UserFactory())
        grant_object(user, root, "view_document", "change_document")
        assert version.pk in set(permitted_document_ids(user))
        assert version.pk in set(permitted_document_ids(user, perm="change_document"))
        assert version.pk in set(
            permitted_document_ids(user, perm="documents.change_document"),
        )
        assert version.pk not in set(
            permitted_document_ids(user, perm="delete_document"),
        )
    def test_anonymous_sees_versions_of_unowned_roots_only(self) -> None:
        """
        GIVEN:
            - A version owned by nobody under a private root
            - A version owned by someone under an unowned root
        WHEN:
            - The permitted document ids are resolved for an anonymous user
        THEN:
            - Only the version of the unowned root is visible
        """
        private_root = DocumentFactory(owner=UserFactory())
        private_version = DocumentFactory(root_document=private_root, owner=None)
        open_root = DocumentFactory(owner=None)
        open_version = DocumentFactory(root_document=open_root, owner=UserFactory())
        assert_visible_document_ids(
            permitted_document_ids(AnonymousUser()),
            expected_visible=[open_root.pk, open_version.pk],
            expected_hidden=[private_root.pk, private_version.pk],
        )
    def test_deleted_versions_follow_their_deleted_root(self) -> None:
        """
        GIVEN:
            - A soft-deleted root document and its version, which deleting the
              root soft-deletes too; the version is owned by someone else
        WHEN:
            - The permitted document ids are resolved with and without deleted
              documents
        THEN:
            - Nothing is visible by default
            - With deleted documents included, the version is visible to the
              root's owner and not to the version's own owner
        """
        owner = UserFactory()
        version_owner = UserFactory()
        root = DocumentFactory(owner=owner)
        version = DocumentFactory(root_document=root, owner=version_owner)
        root.delete()
        assert not {root.pk, version.pk} & set(permitted_document_ids(owner))
        assert_visible_document_ids(
            permitted_document_ids(owner, include_deleted=True),
            expected_visible=[root.pk, version.pk],
            expected_hidden=[],
        )
        assert_visible_document_ids(
            permitted_document_ids(version_owner, include_deleted=True),
            expected_visible=[],
            expected_hidden=[root.pk, version.pk],
        )
    @pytest.mark.parametrize(
        "is_superuser",
        [
            pytest.param(False, id="regular-user"),
            pytest.param(True, id="superuser"),
        ],
    )
    def test_inactive_user_sees_no_versions(self, *, is_superuser: bool) -> None:
        """
        GIVEN:
            - An inactive user, possibly a superuser, who owns a root and its version
        WHEN:
            - The permitted document ids are resolved for them
        THEN:
            - Nothing is visible
        """
        user = UserFactory(is_active=False, is_superuser=is_superuser)
        root = DocumentFactory(owner=user)
        version = DocumentFactory(root_document=root, owner=user)
        assert_visible_document_ids(
            permitted_document_ids(user),
            expected_visible=[],
            expected_hidden=[root.pk, version.pk],
        )
    def test_superuser_sees_all_versions(self) -> None:
        """
        GIVEN:
            - A private root owned by someone else, with a version
        WHEN:
            - The permitted document ids are resolved for a superuser
        THEN:
            - Both the root and the version are visible
        """
        superuser = UserFactory(superuser=True)
        root = DocumentFactory(owner=UserFactory())
        version = DocumentFactory(root_document=root, owner=UserFactory())
        assert_visible_document_ids(
            permitted_document_ids(superuser),
            expected_visible=[root.pk, version.pk],
            expected_hidden=[],
        )
@pytest.mark.django_db
 class TestAiChatAllDocumentsPermissionBoundary:
    """
@@ -1,6 +1,11 @@
 from pathlib import Path
 import pytest
 import pytest_mock
 from PIL import Image
 from documents.utils import QuerySetStream
 from documents.utils import remove_alpha_channel
 class TestQuerySetStream:
@@ -33,3 +38,99 @@ class TestQuerySetStream:
        # the thing being verified here.
        mock_queryset.count.assert_called_with()
        mock_queryset.iterator.assert_called_once_with(chunk_size=1000)
 class TestRemoveAlphaChannel:
    @pytest.mark.parametrize(
        ("image_format", "mode", "expected_mode", "color", "expected_pixel"),
        [
            pytest.param(
                "PNG",
                "RGBA",
                "RGB",
                (10, 20, 30, 0),
                (10, 20, 30),
                id="png-rgba",
            ),
            pytest.param(
                "TIFF",
                "RGBA",
                "RGB",
                (10, 20, 30, 128),
                (10, 20, 30),
                id="tiff-rgba",
            ),
            pytest.param("PNG", "LA", "L", (40, 0), 40, id="png-la"),
        ],
    )
    def test_drops_alpha_and_keeps_format(
        self,
        tmp_path: Path,
        image_format: str,
        mode: str,
        expected_mode: str,
        color: tuple[int, ...],
        expected_pixel: int | tuple[int, ...],
    ) -> None:
        """
        GIVEN:
            - An image with an alpha channel
        WHEN:
            - The alpha channel is removed to a destination without an extension
        THEN:
            - The destination has the matching opaque mode, the same size and
              format, and the color values are kept rather than composited
        """
        source = tmp_path / "source"
        Image.new(mode, (8, 6), color).save(source, format=image_format)
        dest = tmp_path / "no-alpha"
        result = remove_alpha_channel(source, dest)
        assert result == dest
        with Image.open(dest) as out:
            assert out.mode == expected_mode
            assert out.size == (8, 6)
            assert out.format == image_format
            assert out.getpixel((0, 0)) == expected_pixel
    def test_keeps_every_frame_of_a_multi_page_tiff(self, tmp_path: Path) -> None:
        """
        GIVEN:
            - A multi-page TIFF where every page has an alpha channel
        WHEN:
            - The alpha channel is removed
        THEN:
            - All pages are kept and none has an alpha channel
        """
        source = tmp_path / "multi.tiff"
        first = Image.new("RGBA", (8, 6), (255, 0, 0, 50))
        rest = [Image.new("RGBA", (8, 6), (0, 255, 0, 50)) for _ in range(2)]
        first.save(source, save_all=True, append_images=rest)
        dest = tmp_path / "multi-no-alpha.tiff"
        remove_alpha_channel(source, dest)
        with Image.open(dest) as out:
            assert out.n_frames == 3
            for index in range(out.n_frames):
                out.seek(index)
                assert out.mode == "RGB"
    def test_keeps_dpi(self, tmp_path: Path) -> None:
        """
        GIVEN:
            - An alpha image with a DPI set
        WHEN:
            - The alpha channel is removed
        THEN:
            - The DPI is unchanged, since OCR scaling depends on it
        """
        source = tmp_path / "source.png"
        Image.new("RGBA", (8, 6)).save(source, dpi=(300, 300))
        dest = tmp_path / "dest.png"
        remove_alpha_channel(source, dest)
        with Image.open(dest) as out:
            assert round(out.info["dpi"][0]) == 300
@@ -9,11 +9,13 @@ from pathlib import Path
 from subprocess import CompletedProcess
 from subprocess import run
 from typing import TYPE_CHECKING
 from typing import Any
 from typing import Generic
 from typing import TypeVar
 from django.conf import settings
 from PIL import Image
 from PIL import ImageSequence
 if TYPE_CHECKING:
    from django.db.models import Model
@@ -128,6 +130,30 @@ def maybe_override_pixel_limit() -> None:
        Image.MAX_IMAGE_PIXELS = pixel_count
 def remove_alpha_channel(source: Path, dest: Path) -> Path:
    """
    Writes a copy of the image with its alpha channel dropped, keeping the
    format, every frame, the DPI and any ICC profile.
    The channel is dropped, not composited, so transparent pixels keep their
    underlying color. Returns dest.
    """
    with Image.open(source) as im:
        image_format = im.format
        opaque_mode = "L" if im.mode == "LA" else "RGB"
        frames = [frame.convert(opaque_mode) for frame in ImageSequence.Iterator(im)]
        save_kwargs: dict[str, Any] = {}
        if "dpi" in im.info:
            save_kwargs["dpi"] = im.info["dpi"]
        if "icc_profile" in im.info:
            save_kwargs["icc_profile"] = im.info["icc_profile"]
        if len(frames) > 1:
            save_kwargs["save_all"] = True
            save_kwargs["append_images"] = frames[1:]
    frames[0].save(dest, format=image_format, **save_kwargs)
    return dest
 def run_subprocess(
    arguments: list[str],
    env: dict[str, str] | None = None,
@@ -174,6 +174,7 @@ from documents.permissions import TrashPermissions
 from documents.permissions import ViewDocumentsPermissions
 from documents.permissions import annotate_document_count_by_ids
 from documents.permissions import annotate_document_count_for_related_queryset
 from documents.permissions import documents_without_permitted_root
 from documents.permissions import get_document_count_filter_for_user
 from documents.permissions import get_objects_for_user_owner_aware
 from documents.permissions import has_global_statistics_permission
@@ -2116,7 +2117,7 @@ class DocumentViewSet(
        documents = Document.objects.filter(pk__in=document_ids)
        if (
            request.user is not None
-            and documents.exclude(id__in=permitted_document_ids(request.user)).exists()
+            and documents_without_permitted_root(documents, request.user).exists()
        ):
            return HttpResponseForbidden("Insufficient permissions")
@@ -3008,8 +3009,12 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
            user.has_perm(
                "documents.change_document",
            )
-            and not Document.global_objects.filter(pk__in=documents)
+            and not Document.global_objects.filter(
-            .exclude(pk__in=permitted_document_ids(user, perm="change_document"))
+                pk__in=[doc.pk for doc in root_docs],
            )
            .exclude(
                pk__in=permitted_document_ids(user, perm="change_document"),
            )
            .exists()
        )
@@ -3622,7 +3627,7 @@ class SelectionDataView(DocumentSelectionMixin, GenericAPIView[Any]):
        documents = Document.objects.filter(pk__in=ids)
        if (
            documents.count() != len(ids)
-            or documents.exclude(id__in=permitted_document_ids(request.user)).exists()
+            or documents_without_permitted_root(documents, request.user).exists()
        ):
            return HttpResponseForbidden("Insufficient permissions")
@@ -4119,16 +4124,21 @@ class BulkDownloadView(DocumentSelectionMixin, GenericAPIView[Any]):
            validated_data=serializer.validated_data,
        )
        documents = Document.objects.filter(pk__in=ids)
        versioned_documents = []
        compression = serializer.validated_data.get("compression")
        content = serializer.validated_data.get("content")
        follow_filename_format = serializer.validated_data.get("follow_formatting")
-        if documents.exclude(id__in=permitted_document_ids(request.user)).exists():
+        permitted_ids = set(permitted_document_ids(request.user))
-            return HttpResponseForbidden("Insufficient permissions")
+        for document in documents:
-        versioned_documents = [
+            root_doc = get_root_document(document)
-            get_latest_version_for_root(get_root_document(document))
+            if root_doc.pk not in permitted_ids:
-            for document in documents
+                return HttpResponseForbidden("Insufficient permissions")
-        ]
+            versioned_documents.append(
                get_latest_version_for_root(
                    root_doc,
                ),
            )
        if content == "both":
            strategy_class = OriginalAndArchiveStrategy
@@ -4801,7 +4811,7 @@ class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
            )
        denied_id = (
-            documents_qs.exclude(id__in=permitted_document_ids(request.user))
+            documents_without_permitted_root(documents_qs, request.user)
            .order_by("pk")
            .values_list("pk", flat=True)
            .first()
@@ -5647,12 +5657,11 @@ class TrashView(ListModelMixin, PassUserMixin):
            if doc_ids is not None
            else self.filter_queryset(self.get_queryset()).all()
        )
-        if docs.exclude(
+        if documents_without_permitted_root(
-            id__in=permitted_document_ids(
+            docs,
-                request.user,
+            request.user,
-                perm="delete_document",
+            perm="delete_document",
-                include_deleted=True,
+            include_deleted=True,
            ),
        ).exists():
            return HttpResponseForbidden("Insufficient permissions")
        action = serializer.validated_data.get("action")
@@ -2,7 +2,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: paperless-ngx\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2026-10-10 22:04+0000\n"
+"POT-Creation-Date: 2026-10-08 18:39+0000\n"
 "PO-Revision-Date: 2022-02-17 04:17\n"
 "Last-Translator: \n"
 "Language-Team: English\n"
@@ -1653,7 +1653,7 @@ msgid "workflow runs"
 msgstr ""
 #: documents/serialisers.py:516 documents/serialisers.py:873
-#: documents/serialisers.py:2903 documents/views.py:343 documents/views.py:2750
+#: documents/serialisers.py:2903 documents/views.py:344 documents/views.py:2751
 #: paperless_mail/serialisers.py:156
 msgid "Insufficient permissions."
 msgstr ""
@@ -1694,7 +1694,7 @@ msgstr ""
 msgid "Duplicate document identifiers are not allowed."
 msgstr ""
-#: documents/serialisers.py:2989 documents/views.py:4797
+#: documents/serialisers.py:2989 documents/views.py:4807
 #, python-format
 msgid "Documents not found: %(ids)s"
 msgstr ""
@@ -1945,40 +1945,40 @@ msgstr ""
 msgid ", "
 msgstr ""
-#: documents/views.py:336 documents/views.py:2747
+#: documents/views.py:337 documents/views.py:2748
 msgid "Invalid more_like_id"
 msgstr ""
-#: documents/views.py:1682
+#: documents/views.py:1683
 msgid "Invalid AI configuration."
 msgstr ""
-#: documents/views.py:1693
+#: documents/views.py:1694
 msgid "AI backend request timed out."
 msgstr ""
-#: documents/views.py:1705
+#: documents/views.py:1706
 msgid "AI backend rejected the request. Check logs for details."
 msgstr ""
-#: documents/views.py:2572 documents/views.py:2888
+#: documents/views.py:2573 documents/views.py:2889
 msgid "Specify only one of text, title_search, query, or more_like_id."
 msgstr ""
-#: documents/views.py:4813
+#: documents/views.py:4823
 #, python-format
 msgid "Insufficient permissions to share document %(id)s."
 msgstr ""
-#: documents/views.py:4860
+#: documents/views.py:4870
 msgid "Bundle is already being processed."
 msgstr ""
-#: documents/views.py:4924
+#: documents/views.py:4934
 msgid "The share link bundle is still being prepared. Please try again later."
 msgstr ""
-#: documents/views.py:4938
+#: documents/views.py:4948
 msgid "The share link bundle is unavailable."
 msgstr ""
@@ -82,7 +82,7 @@ def binaries_check(app_configs: Any, **kwargs: Any) -> list[Error]:
    error = "Paperless can't find {}. Without it, consumption is impossible."
    hint = "Either it's not in your ${PATH} or it's not installed."
-    binaries = (settings.CONVERT_BINARY, "tesseract", "gs")
+    binaries = ("tesseract", "gs")
    check_messages = []
    for binary in binaries:
@@ -19,7 +19,7 @@ from documents.parsers import ParseError
 from documents.parsers import make_thumbnail_from_pdf
 from documents.utils import copy_file_with_basic_stats
 from documents.utils import maybe_override_pixel_limit
-from documents.utils import run_subprocess
+from documents.utils import remove_alpha_channel
 from paperless.config import OcrConfig
 from paperless.models import CleanChoices
 from paperless.models import ModeChoices
@@ -197,18 +197,7 @@ class RasterisedDocumentParser:
            return im.mode in ("RGBA", "LA")
    def remove_alpha(self, image_path: Path) -> Path:
-        no_alpha_image = Path(self.tempdir) / "image-no-alpha"
+        return remove_alpha_channel(image_path, Path(self.tempdir) / "image-no-alpha")
        run_subprocess(
            [
                settings.CONVERT_BINARY,
                "-alpha",
                "off",
                str(image_path),
                str(no_alpha_image),
            ],
            logger=self.log,
        )
        return no_alpha_image
    def get_dpi(self, image: Path) -> int | None:
        try:
@@ -265,6 +265,52 @@ def get_page_count_for_pdf(
        return None
 def get_pdf_first_page_size_points(
    path: Path,
    log: logging.Logger | None = None,
 ) -> tuple[float, float] | None:
    """Return the first page's (width, height) in PDF points, post-rotation.
    Uses the CropBox (MediaBox if absent), which must match pdftoppm's
    ``-cropbox`` or the computed DPI targets the wrong box.
    Swaps width and height for 90/270 rotation. ``page.rotation`` resolves
    inherited and negative ``/Rotate`` values, a raw lookup does not.
    Parameters
    ----------
    path:
        Absolute path to the PDF file.
    log:
        Logger for warnings.  Falls back to the module-level logger when omitted.
    Returns
    -------
    tuple[float, float] | None
        ``(width_points, height_points)``, or ``None`` if the file cannot be
        opened, has no pages, or the page box is degenerate.
    """
    import pikepdf
    _log = log or logger
    try:
        with pikepdf.Pdf.open(path) as pdf:
            if len(pdf.pages) == 0:
                return None
            page = pdf.pages[0]
            llx, lly, urx, ury = (float(v) for v in page.cropbox)
            width = abs(urx - llx)
            height = abs(ury - lly)
            if width <= 0 or height <= 0:
                return None
            if page.rotation in (90, 270):
                width, height = height, width
            return width, height
    except Exception as e:
        _log.warning("Could not determine PDF page size for %s: %s", path, e)
        return None
 def extract_pdf_metadata(
    document_path: Path,
    log: logging.Logger | None = None,
@@ -984,11 +984,6 @@ MAX_IMAGE_PIXELS: Final[int | None] = get_int_from_env(
 # GNUPG needs a home directory for some reason
 GNUPG_HOME = os.getenv("HOME", "/tmp")
 # Convert is part of the ImageMagick package
 CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert")
 CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
 CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
 GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs")
 # Fallback layout for .eml consumption
@@ -15,9 +15,10 @@ from typing import TYPE_CHECKING
 import pytest
 from ocrmypdf import SubprocessOutputError
 from PIL import Image
 from documents.parsers import ParseError
-from documents.parsers import run_convert
+from documents.parsers import rasterize_pdf_page_to_png
 from paperless.models import ModeChoices
 from paperless.parsers import ParserProtocol
 from paperless.parsers.tesseract import RasterisedDocumentParser
@@ -280,24 +281,71 @@ class TestGetThumbnail:
        )
        assert thumb.is_file()
-    def test_thumbnail_fallback_on_convert_error(
+    @pytest.mark.parametrize(
        ("filename", "expected_height"),
        [
            pytest.param("simple-digital.pdf", 647, id="portrait-letter"),
            pytest.param("rotated.pdf", 386, id="landscape"),
        ],
    )
    def test_thumbnail_is_correct_format_and_size(
        self,
        tesseract_parser: RasterisedDocumentParser,
        tesseract_samples_dir: Path,
        filename: str,
        expected_height: int,
    ) -> None:
        """
        GIVEN:
            - A portrait or landscape PDF
        WHEN:
            - A thumbnail is generated
        THEN:
            - A 500px wide WebP keeping the page's aspect ratio
        """
        thumb = tesseract_parser.get_thumbnail(
            tesseract_samples_dir / filename,
            "application/pdf",
        )
        with Image.open(thumb) as im:
            assert im.format == "WEBP"
            assert im.width == 500
            assert im.height == pytest.approx(expected_height, abs=2)
    def test_thumbnail_fallback_on_pdftoppm_error(
        self,
        mocker: MockerFixture,
        tesseract_parser: RasterisedDocumentParser,
        tesseract_samples_dir: Path,
    ) -> None:
-        def _raise_on_pdf(input_file, output_file, **kwargs) -> None:
+        """
-            if ".pdf" in str(input_file):
+        GIVEN:
            - Rasterizing the original PDF fails
        WHEN:
            - A thumbnail is generated
        THEN:
            - The PDF is repaired with qpdf and a real thumbnail is rendered
        """
        original = tesseract_samples_dir / "simple-digital.pdf"
        def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None:
            if in_path == original:
                raise ParseError("Does not compute.")
-            run_convert(input_file=input_file, output_file=output_file, **kwargs)
+            rasterize_pdf_page_to_png(in_path, out_path, **kwargs)
-        mocker.patch("documents.parsers.run_convert", side_effect=_raise_on_pdf)
+        rasterize = mocker.patch(
-
+            "documents.parsers.rasterize_pdf_page_to_png",
-        thumb = tesseract_parser.get_thumbnail(
+            side_effect=_fail_on_original,
            tesseract_samples_dir / "simple-digital.pdf",
            "application/pdf",
        )
        thumb = tesseract_parser.get_thumbnail(original, "application/pdf")
        assert rasterize.call_count == 2
        assert thumb.is_file()
        assert thumb.name == "convert_qpdf.webp"
        with Image.open(thumb) as im:
            assert im.format == "WEBP"
            assert im.width == 500
    def test_thumbnail_encrypted_pdf(
        self,
@@ -25,9 +25,17 @@ class TestChecks:
    def test_binaries(self) -> None:
        assert binaries_check(None) == []
-    def test_binaries_fail(self, settings: Settings) -> None:
+    def test_binaries_fail(self, mocker: MockerFixture) -> None:
-        settings.CONVERT_BINARY = "uuuhh"
+        """
-        assert len(binaries_check(None)) == 1
+        GIVEN:
            - Neither tesseract nor ghostscript can be found
        WHEN:
            - The binaries check runs
        THEN:
            - One warning is raised per missing binary
        """
        mocker.patch("paperless.checks.shutil.which", return_value=None)
        assert len(binaries_check(None)) == 2
    @pytest.mark.usefixtures("paperless_dirs")
    def test_paths_check(self) -> None:
@@ -6,8 +6,10 @@ import codecs
 from pathlib import Path
 from typing import TYPE_CHECKING
 import pikepdf
 import pytest
 from paperless.parsers.utils import get_pdf_first_page_size_points
 from paperless.parsers.utils import is_tagged_pdf
 from paperless.parsers.utils import pdf_born_digital_text
 from paperless.parsers.utils import post_process_text
@@ -70,6 +72,149 @@ class TestIsTaggedPdf:
        assert is_tagged_pdf(bad) is False
 class TestGetPdfFirstPageSizePoints:
    @staticmethod
    def _write_pdf(
        path: Path,
        *,
        media_box: tuple[float, float, float, float] = (0, 0, 600, 800),
        crop_box: tuple[float, float, float, float] | None = None,
        page_rotate: int | None = None,
        inherited_rotate: int | None = None,
    ) -> Path:
        pdf = pikepdf.new()
        pdf.add_blank_page(page_size=(media_box[2], media_box[3]))
        page = pdf.pages[0]
        page.obj.MediaBox = pikepdf.Array(media_box)
        if crop_box is not None:
            page.obj.CropBox = pikepdf.Array(crop_box)
        if page_rotate is not None:
            page.obj.Rotate = page_rotate
        if inherited_rotate is not None:
            pdf.Root.Pages.Rotate = inherited_rotate
        pdf.save(path)
        return path
    def test_letter_sample(self) -> None:
        """
        GIVEN:
            - A US Letter sample PDF with no CropBox and no rotation
        WHEN:
            - The first page size is requested
        THEN:
            - The MediaBox size in points is returned
        """
        assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") == (
            612.0,
            792.0,
        )
    @pytest.mark.parametrize(
        ("rotate", "expected"),
        [
            pytest.param(0, (600.0, 800.0), id="rotate-0"),
            pytest.param(90, (800.0, 600.0), id="rotate-90"),
            pytest.param(180, (600.0, 800.0), id="rotate-180"),
            pytest.param(270, (800.0, 600.0), id="rotate-270"),
            pytest.param(-90, (800.0, 600.0), id="rotate-negative-90"),
        ],
    )
    def test_page_rotation_swaps_dimensions(
        self,
        tmp_path: Path,
        rotate: int,
        expected: tuple[float, float],
    ) -> None:
        """
        GIVEN:
            - A portrait PDF page with /Rotate set directly on the page
        WHEN:
            - The first page size is requested
        THEN:
            - Width and height are swapped for quarter-turn rotations only
        """
        pdf_path = self._write_pdf(tmp_path / "rotated.pdf", page_rotate=rotate)
        assert get_pdf_first_page_size_points(pdf_path) == expected
    def test_inherited_rotation_swaps_dimensions(self, tmp_path: Path) -> None:
        """
        GIVEN:
            - A page inheriting /Rotate 90 from the /Pages node
        WHEN:
            - The first page size is requested
        THEN:
            - Width and height are swapped
        """
        pdf_path = self._write_pdf(tmp_path / "inherited.pdf", inherited_rotate=90)
        assert get_pdf_first_page_size_points(pdf_path) == (800.0, 600.0)
    def test_crop_box_preferred_over_media_box(self, tmp_path: Path) -> None:
        """
        GIVEN:
            - A PDF page with a CropBox smaller than its MediaBox
        WHEN:
            - The first page size is requested
        THEN:
            - The CropBox size is returned
        """
        pdf_path = self._write_pdf(
            tmp_path / "cropped.pdf",
            crop_box=(50, 100, 350, 500),
        )
        assert get_pdf_first_page_size_points(pdf_path) == (300.0, 400.0)
    def test_degenerate_box_returns_none(self, tmp_path: Path) -> None:
        """
        GIVEN:
            - A PDF page whose box has zero width
        WHEN:
            - The first page size is requested
        THEN:
            - None is returned
        """
        pdf_path = self._write_pdf(
            tmp_path / "degenerate.pdf",
            media_box=(0, 0, 600, 800),
            crop_box=(100, 0, 100, 800),
        )
        assert get_pdf_first_page_size_points(pdf_path) is None
    def test_nonexistent_path_returns_none(self) -> None:
        """
        GIVEN:
            - A path that does not exist
        WHEN:
            - The first page size is requested
        THEN:
            - None is returned and nothing is raised
        """
        assert get_pdf_first_page_size_points(Path("/nonexistent/file.pdf")) is None
    def test_corrupt_pdf_returns_none(self, tmp_path: Path) -> None:
        """
        GIVEN:
            - A file that is not a PDF
        WHEN:
            - The first page size is requested
        THEN:
            - None is returned and nothing is raised
        """
        bad = tmp_path / "bad.pdf"
        bad.write_bytes(b"not a pdf")
        assert get_pdf_first_page_size_points(bad) is None
    def test_encrypted_pdf_returns_none(self) -> None:
        """
        GIVEN:
            - A password protected PDF
        WHEN:
            - The first page size is requested
        THEN:
            - None is returned and nothing is raised
        """
        assert get_pdf_first_page_size_points(SAMPLES / "encrypted.pdf") is None
 class TestPostProcessText:
    @pytest.mark.parametrize(
        ("source", "expected"),
@@ -4,7 +4,8 @@ from django.conf import settings
 from django.contrib.auth.models import User
 from documents.models import Document
-from documents.permissions import permitted_document_ids
+from documents.permissions import permitted_object_ids
 from documents.permissions import restrict_queryset_to_visible
 from documents.permissions import user_is_unrestricted
 from paperless.config import AIConfig
 from paperless_ai.base_model import ClassificationSuggestions
@@ -53,8 +54,8 @@ def _fulltext_similar_documents(
    active superuser - see user_is_unrestricted) is normalized to ``None``
    before calling, since the backend's permission filter has no superuser
    short-circuit of its own. Results are re-checked with
-    permitted_document_ids() since Tantivy's indexed permission fields lag the
+    restrict_queryset_to_visible() since Tantivy's indexed permission fields
-    DB via async reindexing and judge a version by its own owner.
+    lag the DB via async reindexing.
    """
    from documents.search import get_backend
@@ -68,9 +69,10 @@ def _fulltext_similar_documents(
    )
    if not unrestricted:
        allowed_ids = set(
-            Document.objects.filter(
+            restrict_queryset_to_visible(
-                pk__in=similar_ids,
+                Document.objects.filter(pk__in=similar_ids),
-                id__in=permitted_document_ids(user),
+                user,
                "view_document",
            ).values_list("pk", flat=True),
        )
        similar_ids = [doc_id for doc_id in similar_ids if doc_id in allowed_ids]
@@ -198,13 +200,13 @@ def get_taxonomy_context(
            # quadratic scan in the vector store at best, and past ~32,763
            # documents a hard sqlite3.OperationalError (SQLite's
            # bound-parameter limit) at worst.
-            # permitted_document_ids() has its own superuser shortcut that would
+            # permitted_object_ids() has its own superuser shortcut that would
            # return every Document's id anyway, so this changes nothing about
            # which documents are considered -- only how we get there.
            visible_document_ids = (
                None
                if user_is_unrestricted(user)
-                else list(permitted_document_ids(user))
+                else list(permitted_object_ids(user, Document, "view_document"))
            )
            nodes = retrieve_similar_nodes(
                document,
@@ -552,7 +552,7 @@ class TestGetTaxonomyContextVisibility:
            return_value=[],
        )
        mock_permitted = mocker.patch(
-            "paperless_ai.ai_classifier.permitted_document_ids",
+            "paperless_ai.ai_classifier.permitted_object_ids",
        )
        user = UserFactory.create(is_superuser=True)
@@ -582,7 +582,7 @@ class TestGetTaxonomyContextVisibility:
            return_value=[],
        )
        mock_permitted = mocker.patch(
-            "paperless_ai.ai_classifier.permitted_document_ids",
+            "paperless_ai.ai_classifier.permitted_object_ids",
        )
        get_taxonomy_context(document, None)
@@ -611,55 +611,16 @@ class TestGetTaxonomyContextVisibility:
            return_value=[],
        )
        mock_permitted = mocker.patch(
-            "paperless_ai.ai_classifier.permitted_document_ids",
+            "paperless_ai.ai_classifier.permitted_object_ids",
            return_value=[1, 2, 3],
        )
        user = UserFactory.create(is_superuser=False)
        get_taxonomy_context(document, user)
-        mock_permitted.assert_called_once_with(user)
+        mock_permitted.assert_called_once_with(user, Document, "view_document")
        assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
    @pytest.mark.django_db
    @override_settings(LLM_EMBEDDING_BACKEND="huggingface")
    def test_version_of_private_root_is_not_visible(
        self,
        mocker: pytest_mock.MockerFixture,
    ) -> None:
        """
        GIVEN:
            - A private root document owned by someone else
            - A version of it whose own owner is unset, as when the root
              changed hands after the version was created
        WHEN:
            - get_taxonomy_context() is called for a non-superuser
        THEN:
            - Neither the root nor the version is in the visible ids passed to
              retrieve_similar_nodes(), since a version follows its root
        """
        owner = UserFactory.create()
        viewer = UserFactory.create(is_superuser=False)
        root = DocumentFactory.create(content="private", owner=owner)
        version = DocumentFactory.create(
            content="private",
            owner=None,
            root_document=root,
            version_index=1,
        )
        source = DocumentFactory.create(content="Some content", owner=viewer)
        mock_retrieve = mocker.patch(
            "paperless_ai.ai_classifier.retrieve_similar_nodes",
            return_value=[],
        )
        get_taxonomy_context(source, viewer)
        visible = mock_retrieve.call_args.kwargs["document_ids"]
        assert source.pk in visible
        assert root.pk not in visible
        assert version.pk not in visible
@pytest.mark.django_db
 class TestFulltextSimilarDocuments:
@@ -842,7 +803,7 @@ class TestFulltextSimilarDocuments:
            - _fulltext_similar_documents() is called with that user
        THEN:
            - Only the still-permitted document is returned - the DB
-              re-check via permitted_document_ids() must catch the
+              re-check via restrict_queryset_to_visible() must catch the
              document Tantivy's stale index still thinks is visible
        """
        owner = UserFactory.create()
@@ -873,45 +834,6 @@ class TestFulltextSimilarDocuments:
        assert [s["document_id"] for s in result] == [permitted.pk]
    def test_excludes_version_of_private_root_for_regular_user(
        self,
        fulltext_backend: TantivyBackend,
    ) -> None:
        """
        GIVEN:
            - A regular user and a private root owned by someone else
            - A version of that root with no owner of its own, which the
              Tantivy index therefore treats as visible to everyone
        WHEN:
            - _fulltext_similar_documents() is called with that user
        THEN:
            - The version is not returned, since the DB re-check judges it by
              its root
        """
        owner = UserFactory.create()
        viewer = UserFactory.create(is_superuser=False)
        source = DocumentFactory.create(
            content="shared content phrase",
            owner=viewer,
        )
        root = DocumentFactory.create(
            content="shared content phrase",
            owner=owner,
        )
        version = DocumentFactory.create(
            content="shared content phrase",
            owner=None,
            root_document=root,
            version_index=1,
        )
        fulltext_backend.add_or_update(source)
        fulltext_backend.add_or_update(root)
        fulltext_backend.add_or_update(version)
        result = _fulltext_similar_documents(source, user=viewer, top_k=5)
        assert version.pk not in [s["document_id"] for s in result]
@pytest.mark.django_db
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")