Refactor: split views.py into documents/views/ module

documents/views.py suffers the same problem as serialisers.py: a
single 5,395-line file covering every REST resource in the app.
Restructures it into a package, one module per domain area, matching
the same layout used for documents/serialisers/.

paperless/urls.py and paperless_mail/views.py are updated to import
from the new submodules. Test files that imported or mock.patch'd
documents.views or documents.serialisers symbols directly are updated
to point at the correct submodule for both splits. The mypy and
pyrefly baselines are updated to reference the new file paths, and a
stale comment in chat_qa.j2 pointing at the old location of
_get_llm_output_language is corrected.

.dockerignore

@@ -15,6 +15,8 @@
 # Test related
 **/.pytest_cache
 **/tests
+src/paperless_testing
+src/conftest.py
 **/*.spec.ts
 **/htmlcov
 # Local folders

.github/actions/apt-install/action.yml

@@ -0,0 +1,53 @@
+name: Install apt packages
+description: >
+  Installs apt packages with a per-attempt timeout and retries. The hosted runner's Azure apt mirror intermittently serves archives at a crawl, which apt's own timeout does not catch. A stuck attempt is killed and retried, which usually lands on a healthy mirror node.
+
+inputs:
+  packages:
+    description: Space separated list of packages to install
+    required: true
+  attempts:
+    description: Maximum number of attempts
+    default: "3"
+  attempt-timeout:
+    description: Seconds before a single attempt is killed
+    default: "180"
+runs:
+  using: composite
+  steps:
+    - name: Install ${{ inputs.packages }}
+      shell: bash
+      env:
+        PACKAGES: ${{ inputs.packages }}
+        ATTEMPTS: ${{ inputs.attempts }}
+        ATTEMPT_TIMEOUT: ${{ inputs.attempt-timeout }}
+      run: |
+        SUDO=""
+        if [ "$(id -u)" -ne 0 ]; then
+          SUDO="sudo"
+        fi
+
+        attempt_install() {
+          $SUDO apt-get update -q
+          # Intentionally unquoted, this is a list of package names
+          $SUDO apt-get install -q --yes --no-install-recommends $PACKAGES
+        }
+        export -f attempt_install
+        export SUDO PACKAGES
+
+        for attempt in $(seq 1 "${ATTEMPTS}"); do
+          echo "::group::apt install attempt ${attempt}/${ATTEMPTS}"
+          start=$(date +%s)
+          if timeout --kill-after=10 "${ATTEMPT_TIMEOUT}" bash -c attempt_install; then
+            echo "::endgroup::"
+            echo "Installed in $(( $(date +%s) - start ))s"
+            exit 0
+          fi
+          echo "::endgroup::"
+          echo "::warning::apt install attempt ${attempt}/${ATTEMPTS} failed or exceeded ${ATTEMPT_TIMEOUT}s"
+          # A killed attempt can leave dpkg half configured
+          $SUDO dpkg --configure -a || true
+        done
+
+        echo "::error::apt install failed after ${ATTEMPTS} attempts"
+        exit 1

.github/dependabot.yml

@@ -59,7 +59,6 @@ updates:
           - "drf-*"
           - "djangorestframework"
           - "whitenoise"
-          - "bleach"
           - "jinja2"
       # Async, Task Queuing & Caching
       async-tasks:

.github/workflows/ci-backend.yml

@@ -102,17 +102,16 @@ jobs:
         with:
           python-version: "${{ matrix.python-version }}"
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: ${{ env.DEFAULT_UV_VERSION }}
           enable-cache: true
           python-version: ${{ steps.setup-python.outputs.python-version }}
       - name: Install system dependencies
-        timeout-minutes: 10
+        timeout-minutes: 12
-        run: |
+        uses: $/.github/actions/apt-install
-          sudo apt-get update -qq
+        with:
-          sudo apt-get install -qq --no-install-recommends \
+          packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils
-            unpaper tesseract-ocr imagemagick ghostscript poppler-utils
       - name: Configure ImageMagick
         run: |
           sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml
@@ -139,13 +138,13 @@ jobs:
             pytest
       - name: Upload test results to Codecov
         if: always()
-        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           flags: backend-python-${{ matrix.python-version }}
           files: junit.xml
           report_type: test_results
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           flags: backend-python-${{ matrix.python-version }}
           files: coverage.xml
@@ -176,7 +175,7 @@ jobs:
         with:
           python-version: "${{ env.DEFAULT_PYTHON }}"
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: ${{ env.DEFAULT_UV_VERSION }}
           enable-cache: true

.github/workflows/ci-docker.yml

@@ -106,7 +106,7 @@ jobs:
           echo "repository=${repo_name}"
           echo "name=${repo_name}" >> $GITHUB_OUTPUT
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+        uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
       - name: Login to GitHub Container Registry
         uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:
@@ -132,7 +132,7 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
       - name: Build and push by digest
         id: build
-        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
           context: .
           file: ./Dockerfile
@@ -182,7 +182,7 @@ jobs:
           echo "Downloaded digests:"
           ls -la /tmp/digests/
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
+        uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
       - name: Login to GitHub Container Registry
         uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
         with:

.github/workflows/ci-docs.yml

@@ -78,7 +78,7 @@ jobs:
         with:
           python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: ${{ env.DEFAULT_UV_VERSION }}
           enable-cache: true

.github/workflows/ci-frontend.yml

@@ -174,13 +174,13 @@ jobs:
         run: cd src-ui && pnpm run test --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }}
       - name: Upload test results to Codecov
         if: always()
-        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           flags: frontend-node-${{ matrix.node-version }}
           directory: src-ui/
           report_type: test_results
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
         with:
           flags: frontend-node-${{ matrix.node-version }}
           directory: src-ui/coverage/
@@ -216,7 +216,7 @@ jobs:
         with:
           python-version: '3.12'
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: '0.12.x'
           enable-cache: false
@@ -231,9 +231,10 @@ jobs:
       - name: Install dependencies
         run: cd src-ui && pnpm install --frozen-lockfile
       - name: Install backend system dependencies
-        run: |
+        timeout-minutes: 12
-          apt-get update
+        uses: $/.github/actions/apt-install
-          apt-get install --yes --quiet --no-install-recommends libmagic1
+        with:
+          packages: libmagic1
       - name: Install backend dependencies
         env:
           PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}

.github/workflows/ci-release.yml

@@ -59,7 +59,7 @@ jobs:
         with:
           python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: ${{ env.DEFAULT_UV_VERSION }}
           enable-cache: false
@@ -70,9 +70,10 @@ jobs:
         run: |
           uv sync --python "${PYTHON_VERSION}" --dev --frozen
       - name: Install system dependencies
-        run: |
+        timeout-minutes: 12
-          sudo apt-get update -qq
+        uses: $/.github/actions/apt-install
-          sudo apt-get install -qq --no-install-recommends gettext liblept5
+        with:
+          packages: gettext liblept5
       # ---- Build Documentation ----
       - name: Build documentation
         env:
@@ -212,7 +213,7 @@ jobs:
         with:
           python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: ${{ env.DEFAULT_UV_VERSION }}
           enable-cache: false

.github/workflows/ci-static-analysis.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Run Semgrep
         run: semgrep scan --config auto --sarif-output results.sarif
       - name: Upload results to GitHub code scanning
-        uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
+        uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         if: always()
         with:
           sarif_file: results.sarif

.github/workflows/cleanup-tags.yml

@@ -29,7 +29,7 @@ jobs:
     steps:
       - name: Clean temporary images
         if: "${{ env.TOKEN != '' }}"
-        uses: stumpylog/image-cleaner-action/ephemeral@4fe057d991d63b8f6d5d22c40f17c1bca2226537 # v0.12.0
+        uses: stumpylog/image-cleaner-action/ephemeral@21f875bab2376314e0525c614e923c669434f0e4 # v0.13.0
         with:
           token: "${{ env.TOKEN }}"
           owner: "${{ github.repository_owner }}"
@@ -56,7 +56,7 @@ jobs:
     steps:
       - name: Clean untagged images
         if: "${{ env.TOKEN != '' }}"
-        uses: stumpylog/image-cleaner-action/untagged@4fe057d991d63b8f6d5d22c40f17c1bca2226537 # v0.12.0
+        uses: stumpylog/image-cleaner-action/untagged@21f875bab2376314e0525c614e923c669434f0e4 # v0.13.0
         with:
           token: "${{ env.TOKEN }}"
           owner: "${{ github.repository_owner }}"

.github/workflows/codeql-analysis.yml

@@ -39,7 +39,7 @@ jobs:
           persist-credentials: false
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
+        uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -47,4 +47,4 @@ jobs:
           # Prefix the list here with "+" to use these queries and those in the config file.
           # queries: ./path/to/local/query, your-org/your-repo/queries@main
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
+        uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2

.github/workflows/crowdin.yml

@@ -22,7 +22,7 @@ jobs:
           token: ${{ secrets.PNGX_BOT_PAT }}
           persist-credentials: false
       - name: crowdin action
-        uses: crowdin/github-action@0d5670f539973aea2f01abce61a8989934df0025 # v3.0.2
+        uses: crowdin/github-action@9c23991700c0ec5256fd41089b9d9d7d540e424e # v3.3.0
         with:
           upload_translations: false
           download_translations: true

.github/workflows/translate-strings.yml

@@ -25,11 +25,12 @@ jobs:
         id: setup-python
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
       - name: Install system dependencies
-        run: |
+        timeout-minutes: 12
-          sudo apt-get update -qq
+        uses: $/.github/actions/apt-install
-          sudo apt-get install -qq --no-install-recommends gettext
+        with:
+          packages: gettext
       - name: Install uv
-        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
         with:
           version: ${{ env.DEFAULT_UV_VERSION }}
           enable-cache: true

.pre-commit-config.yaml

@@ -38,7 +38,7 @@ repos:
           - json
   # See https://github.com/prettier/prettier/issues/15742 for the fork reason
   - repo: https://github.com/rbubley/mirrors-prettier
-    rev: 'v3.9.6'
+    rev: 'v3.9.9'
     hooks:
       - id: prettier
         types_or:
@@ -46,16 +46,16 @@ repos:
           - ts
           - markdown
         additional_dependencies:
-          - prettier@3.9.6
+          - prettier@3.9.9
           - 'prettier-plugin-organize-imports@4.3.0'
   # Python hooks
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.16.5
+    rev: v0.16.9
     hooks:
       - id: ruff-check
       - id: ruff-format
   - repo: https://github.com/tox-dev/pyproject-fmt
-    rev: "v2.28.1"
+    rev: "v2.29.4"
     hooks:
       - id: pyproject-fmt
         additional_dependencies: [tomli]

Dockerfile

@@ -30,7 +30,7 @@ RUN set -eux \
 # Purpose: Installs s6-overlay and rootfs
 # Comments:
 #  - Don't leave anything extra in here either
-FROM ghcr.io/astral-sh/uv:0.12.16-python3.14-trixie-slim AS s6-overlay-base
+FROM ghcr.io/astral-sh/uv:0.12.23-python3.14-trixie-slim AS s6-overlay-base
 
 WORKDIR /usr/src/s6
 
@@ -171,7 +171,9 @@ RUN set -eux \
         && cp /etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml \
       && echo "Cleaning up image layer" \
         && rm --force --verbose *.deb \
-    && rm --recursive --force --verbose /var/lib/apt/lists/*
+    && rm --recursive --force --verbose /var/lib/apt/lists/* \
+    && echo "Configuring interactive shells to source the s6 container environment" \
+      && echo '. /etc/profile.d/contenv.sh' >> /etc/bash.bashrc
 
 WORKDIR /usr/src/paperless/src/
 

docker/compose/docker-compose.ci-test.yml

@@ -24,7 +24,7 @@ services:
     network_mode: host
     restart: unless-stopped
   greenmail:
-    image: docker.io/greenmail/standalone:2.1.13
+    image: docker.io/greenmail/standalone:2.1.14
     hostname: greenmail
     container_name: greenmail
     environment:

docker/rootfs/etc/profile.d/contenv.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+# Source s6 container environment for interactive shells.
+# Ensures variables resolved from *_FILE secret injection are visible
+# when using 'docker exec bash'. Does not affect s6 services (those
+# use with-contenv directly). Has no effect in non-container contexts
+# because the directory will not exist.
+# Note: sh/dash shells opened via 'docker exec sh' are not covered;
+# only bash-based sessions benefit from this file.
+_pngx_contenv="/run/s6/container_environment"
+if [ -d "${_pngx_contenv}" ]; then
+	for _pngx_f in "${_pngx_contenv}"/*; do
+		[ -f "${_pngx_f}" ] || continue
+		_pngx_name=$(basename "${_pngx_f}")
+		_pngx_val=$(cat "${_pngx_f}")
+		export "${_pngx_name}=${_pngx_val}"
+	done
+fi
+unset _pngx_contenv _pngx_f _pngx_name _pngx_val

docs/advanced_usage.md

@@ -136,13 +136,15 @@ for suggested generation and embedding models.
 ### AI-assisted suggestions
 
 With AI enabled, Paperless-ngx can suggest a title, tags, correspondent, document type,
-storage path and dates by sending the document to the LLM. This is **opt-in per request**
+storage path and dates by sending the document to the LLM using "Suggest" button on the document
-and surfaces through the "Suggest" control on the document detail page, alongside the
+detail page. You can choose which type of suggestions are requested by default under Settings >
-classic classifier-based suggestions — it does not disable them. Suggestions are requested
+Documents, either ML (classifier-based) suggestions, AI suggestions, or both. When both are requested
-automatically when you open a document that carries an inbox tag unless "Automatically request
+the results are combined.
-suggestions for inbox documents" under Settings > Documents is disabled. Suggestion output
+
-language can be steered with
+Suggestions are requested automatically when you open a document that carries an inbox tag
-[`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE`](configuration.md#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE)
+unless "Automatically request suggestions for inbox documents" under Settings > Documents is disabled.
+
+Suggestion output language can be steered with [`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE`](configuration.md#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE)
 (otherwise it follows the user's UI language).
 
 ### The LLM index (RAG) and similar documents
@@ -153,8 +155,11 @@ in similar existing documents, and the document chat can retrieve relevant conte
 
 Enable it by setting
 [`PAPERLESS_AI_LLM_EMBEDDING_BACKEND`](configuration.md#PAPERLESS_AI_LLM_EMBEDDING_BACKEND)
-(`huggingface` for fully-local embeddings, or `ollama` / `openai-like`). The index is only
+(`huggingface` for fully-local embeddings, or `ollama` / `openai-like`). By default, the main
-built when AI is enabled **and** an embedding backend is set.
+LLM API key and endpoint are used, but an optional embedding-specific[API key](configuration.md#PAPERLESS_AI_LLM_EMBEDDING_API_KEY)
+and [endpoint](configuration.md#PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT) can be configured.
+
+The index is only built when AI is enabled **and** an embedding backend is set.
 
 The index is updated automatically on a schedule controlled by
 [`PAPERLESS_LLM_INDEX_TASK_CRON`](configuration.md#PAPERLESS_LLM_INDEX_TASK_CRON) (daily by

docs/changelog.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## paperless-ngx 3.2.1
+
+### Bug Fixes
+
+- Fix: only pass --conf to flower when flowerconfig.py exists [@bitfoo1](https://github.com/bitfoo1) ([#14182](https://github.com/paperless-ngx/paperless-ngx/pull/14182))
+- Fix: replace stale mail-fetch overlap check with a self-expiring lock [@stumpylog](https://github.com/stumpylog) ([#14189](https://github.com/paperless-ngx/paperless-ngx/pull/14189))
+- Fix: bump ocrmypdf to 17.12 to pick up the ligature text-layer fix [@stumpylog](https://github.com/stumpylog) ([#14190](https://github.com/paperless-ngx/paperless-ngx/pull/14190))
+- Fix: rebuild the search index automatically when it is missing Tantivy files [@stumpylog](https://github.com/stumpylog) ([#14180](https://github.com/paperless-ngx/paperless-ngx/pull/14180))
+
+### Dependencies
+
+- Chore(deps): Bump anyio from 4.12.1 to 4.14.2 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#14175](https://github.com/paperless-ngx/paperless-ngx/pull/14175))
+
+### All App Changes
+
+<details>
+<summary>4 changes</summary>
+
+- Chore(deps): Bump anyio from 4.12.1 to 4.14.2 in the uv group across 1 directory @[dependabot[bot]](https://github.com/apps/dependabot) ([#14175](https://github.com/paperless-ngx/paperless-ngx/pull/14175))
+- Fix: replace stale mail-fetch overlap check with a self-expiring lock [@stumpylog](https://github.com/stumpylog) ([#14189](https://github.com/paperless-ngx/paperless-ngx/pull/14189))
+- Fix: bump ocrmypdf to 17.12 to pick up the ligature text-layer fix [@stumpylog](https://github.com/stumpylog) ([#14190](https://github.com/paperless-ngx/paperless-ngx/pull/14190))
+- Fix: rebuild the search index automatically when it is missing Tantivy files [@stumpylog](https://github.com/stumpylog) ([#14180](https://github.com/paperless-ngx/paperless-ngx/pull/14180))
+
+</details>
+
 ## paperless-ngx 3.2.0
 
 ### Features / Enhancements

docs/configuration.md

@@ -2133,6 +2133,13 @@ for language and resource considerations.
 
     Defaults to None.
 
+#### [`PAPERLESS_AI_LLM_EMBEDDING_API_KEY=<str>`](#PAPERLESS_AI_LLM_EMBEDDING_API_KEY) {#PAPERLESS_AI_LLM_EMBEDDING_API_KEY}
+
+: The API key to use for the embedding backend. If not supplied, embeddings use
+`PAPERLESS_AI_LLM_API_KEY`.
+
+    Defaults to None.
+
 #### [`PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT=<str>`](#PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT) {#PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT}
 
 : The endpoint / url to use for the embedding backend. If not supplied, embeddings use
@@ -2217,6 +2224,19 @@ used with the OpenAI-compatible backend to target a custom provider or local gat
 
     Defaults to true, which allows internal endpoints.
 
+#### [`PAPERLESS_AI_LLM_EXTRA_PARAMS=<json>`](#PAPERLESS_AI_LLM_EXTRA_PARAMS) {#PAPERLESS_AI_LLM_EXTRA_PARAMS}
+
+: A JSON object of extra parameters sent with every LLM request, for providers that require a parameter Paperless does not
+set itself. Values here override Paperless' own, and no validation is performed. Whatever you put here is passed to the
+backend as-is, so an invalid parameter will simply be rejected by your provider. For example, current OpenAI reasoning
+models refuse tool calls on the chat completions API unless reasoning is off:
+
+    ```
+    PAPERLESS_AI_LLM_EXTRA_PARAMS={"reasoning_effort": "none"}
+    ```
+
+    Defaults to empty, which adds nothing to requests.
+
 #### [`PAPERLESS_LLM_INDEX_TASK_CRON=<cron expression>`](#PAPERLESS_LLM_INDEX_TASK_CRON) {#PAPERLESS_LLM_INDEX_TASK_CRON}
 
 : Configures the schedule to update the AI embeddings of text content and metadata for all documents. Only performed if

docs/development.md

@@ -150,6 +150,7 @@ pnpm ng build --configuration production
   is loaded as well. However, the tests rely on the default
   configuration. This is not ideal. But for now, make sure no settings
   except for DEBUG are overridden when testing.
+- Tests run in a random order each session, so that one test cannot quietly depend on another having run first. The seed is printed at the top of the run; pass `--randomly-seed=<seed>` to replay that exact order, or `--randomly-seed=last` to repeat the previous run.
 
 !!! note
 

pyproject.toml

@@ -1,7 +1,9 @@
 [project]
 name = "paperless-ngx"
 version = "3.2.1"
-description = "A community-supported supercharged document management system: scan, index and archive all your physical documents"
+description = """\
+  A community-supported supercharged document management system: scan, index and archive all your physical documents\
+  """
 readme = "README.md"
 requires-python = ">=3.11"
 classifiers = [
@@ -10,13 +12,13 @@ classifiers = [
   "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: 3.13",
   "Programming Language :: Python :: 3.14",
+  "Programming Language :: Python :: 3.15",
 ]
 # TODO: Move certain things to groups and then utilize that further
 # This will allow testing to not install a webserver, mysql, etc
 dependencies = [
   "azure-ai-documentintelligence>=1.0.2",
   "babel>=2.17",
-  "bleach~=6.4.0",
   "celery[redis]~=5.6.2",
   "channels~=4.2",
   "channels-redis~=4.2",
@@ -32,7 +34,7 @@ dependencies = [
   "django-cors-headers~=4.9.0",
   "django-extensions~=4.1",
   "django-filter~=25.1",
-  "django-guardian>=3.3.3,<3.5",
+  "django-guardian>=3.3.3,<3.6",
   "django-multiselectfield~=1.0.1",
   "django-rich~=2.2.0",
   "django-soft-delete~=1.0.18",
@@ -42,7 +44,7 @@ dependencies = [
   "drf-spectacular-sidecar>=2026.7.1,<2026.10",
   "drf-writable-nested~=0.7.1",
   "filelock~=3.32.0",
-  "flower>=2.0.1,<2.2",
+  "flower>=2.0.1,<2.3",
   "gotenberg-client[httpx]~=1.0",
   "httpx-oauth~=0.17",
   "ijson>=3.5.1",
@@ -74,6 +76,7 @@ dependencies = [
   "tantivy~=0.26.0",
   "tika-client[httpx]~=1.0",
   "torch>=2.13,<2.15",
+  "turbohtml~=1.10.0",
   "watchfiles>=1.2",
   "whitenoise~=6.11",
   "whoosh-compat[tantivy]==0.3",
@@ -81,13 +84,13 @@ dependencies = [
 ]
 [project.optional-dependencies]
 mariadb = [
-  "mysqlclient~=2.2.7",
+  "mysqlclient>=2.2.7,<2.4",
 ]
 postgres = [
   "psycopg[c,pool]==3.3.4",
   # Direct dependency for proper resolution of the pre-built wheels
   "psycopg-c==3.3.4",
-  "psycopg-pool==3.3.1",
+  "psycopg-pool==3.3.2",
 ]
 webserver = [
   "granian[uvloop]>=2.7,<2.9",
@@ -109,7 +112,7 @@ lint = [
 testing = [
   "daphne",
   "factory-boy~=3.3.1",
-  "faker>=40.36,<40.39",
+  "faker>=40.36,<40.40",
   "imagehash",
   "pytest~=9.1.1",
   "pytest-cov~=7.1.0",
@@ -117,7 +120,7 @@ testing = [
   "pytest-env~=1.7.0",
   "pytest-httpx",
   "pytest-mock~=3.15.1",
-  # "pytest-randomly~=4.0.1",
+  "pytest-randomly~=5.0.0",
   "pytest-rerunfailures~=16.4",
   "pytest-sugar",
   "pytest-xdist~=3.8.0",
@@ -133,7 +136,6 @@ typing = [
   "mypy",
   "mypy-baseline",
   "pyrefly",
-  "types-bleach",
   "types-channels",
   "types-colorama",
   "types-dateparser",
@@ -148,6 +150,7 @@ typing = [
 
 [tool.uv]
 required-version = ">=0.9.0"
+prerelease = "disallow"
 environments = [
   "sys_platform == 'darwin'",
   "sys_platform == 'linux'",
@@ -244,6 +247,10 @@ per-file-ignores."docker/wait-for-redis.py" = [
 per-file-ignores."src/documents/models.py" = [
   "SIM115",
 ]
+per-file-ignores."src/documents/tests/*.py" = [
+  "TID251",
+]
+flake8-tidy-imports.banned-api."documents.tests".msg = "Shared test infrastructure lives in src/paperless_testing/."
 isort.force-single-line = true
 
 [tool.codespell]
@@ -271,9 +278,9 @@ plugins = [
 ]
 
 [tool.pyrefly]
-baseline = ".pyrefly-baseline.json"
 python-platform = "linux"
 search-path = [ "src" ]
+baseline = ".pyrefly-baseline.json"
 
 [tool.django-stubs]
 django_settings_module = "paperless.settings"
@@ -326,6 +333,8 @@ PAPERLESS_CACHE_BACKEND = "django.core.cache.backends.locmem.LocMemCache"
 PAPERLESS_CHANNELS_BACKEND = "channels.layers.InMemoryChannelLayer"
 # I don't think anything hits this, but just in case, basically infinite
 PAPERLESS_TOKEN_THROTTLE_RATE = "1000/min"
+# The 0.1s production default trips on a stalled CI runner, the date parsing tests then find no dates
+PAPERLESS_MATCH_REGEX_TIMEOUT_SECONDS = "5"
 
 [tool.coverage.run]
 source = [
@@ -334,13 +343,15 @@ source = [
 omit = [
   "*/tests/*",
   "manage.py",
-  "paperless/wsgi.py",
   "paperless/auth.py",
+  "paperless/wsgi.py",
+  "src/conftest.py",
+  "src/paperless_testing/*",
 ]
 [tool.coverage.report]
 exclude_also = [
-  "if settings.AUDIT_LOG_ENABLED:",
   "if AUDIT_LOG_ENABLED:",
+  "if settings.AUDIT_LOG_ENABLED:",
   "if TYPE_CHECKING:",
 ]
 

src-ui/angular.json

@@ -91,6 +91,11 @@
                 "glob": "**/*",
                 "input": "node_modules/pdfjs-dist/iccs/",
                 "output": "/assets/iccs/"
+              },
+              {
+                "glob": "**/*",
+                "input": "node_modules/pdfjs-dist/cmaps/",
+                "output": "/assets/cmaps/"
               }
             ],
             "styles": [

src-ui/package.json

@@ -15,16 +15,16 @@
   },
   "private": true,
   "dependencies": {
-    "@angular/cdk": "^22.1.6",
+    "@angular/cdk": "^22.2.0",
-    "@angular/common": "~22.1.6",
+    "@angular/common": "~22.2.0",
-    "@angular/compiler": "~22.1.6",
+    "@angular/compiler": "~22.2.0",
-    "@angular/core": "~22.1.6",
+    "@angular/core": "~22.2.0",
-    "@angular/forms": "~22.1.6",
+    "@angular/forms": "~22.2.0",
-    "@angular/localize": "~22.1.6",
+    "@angular/localize": "~22.2.0",
-    "@angular/platform-browser": "~22.1.6",
+    "@angular/platform-browser": "~22.2.0",
-    "@angular/router": "~22.1.6",
+    "@angular/router": "~22.2.0",
     "@ng-bootstrap/ng-bootstrap": "^21.0.0",
-    "@ng-select/ng-select": "~24.1.1",
+    "@ng-select/ng-select": "~24.1.2",
     "@ngneat/dirty-check-forms": "^3.0.3",
     "@popperjs/core": "^2.11.8",
     "bootstrap": "^5.3.8",
@@ -45,29 +45,29 @@
   },
   "devDependencies": {
     "@angular-builders/jest": "^22.0.1",
-    "@angular-devkit/core": "^22.1.8",
+    "@angular-devkit/core": "^22.2.0",
-    "@angular-devkit/schematics": "^22.1.8",
+    "@angular-devkit/schematics": "^22.2.0",
     "@angular-eslint/builder": "22.5.0",
     "@angular-eslint/eslint-plugin": "22.5.0",
     "@angular-eslint/eslint-plugin-template": "22.5.0",
     "@angular-eslint/schematics": "22.5.0",
     "@angular-eslint/template-parser": "22.5.0",
-    "@angular/build": "22.1.8",
+    "@angular/build": "22.2.0",
-    "@angular/cli": "22.1.8",
+    "@angular/cli": "22.2.0",
-    "@angular/compiler-cli": "~22.1.6",
+    "@angular/compiler-cli": "~22.2.0",
     "@playwright/test": "^1.62.1",
     "@types/jest": "^30.0.0",
-    "@types/node": "^26.5.0",
+    "@types/node": "^26.6.2",
-    "@typescript-eslint/eslint-plugin": "^8.70.0",
+    "@typescript-eslint/eslint-plugin": "^8.70.1",
-    "@typescript-eslint/parser": "^8.70.0",
+    "@typescript-eslint/parser": "^8.70.1",
-    "@typescript-eslint/utils": "^8.70.0",
+    "@typescript-eslint/utils": "^8.70.1",
-    "eslint": "^10.10.0",
+    "eslint": "^10.11.0",
-    "jest": "30.5.1",
+    "jest": "30.5.2",
-    "jest-environment-jsdom": "^30.5.1",
+    "jest-environment-jsdom": "^30.5.2",
     "jest-junit": "^17.0.0",
     "jest-preset-angular": "^17.0.0",
     "jest-websocket-mock": "^2.5.0",
-    "prettier": "^3.9.6",
+    "prettier": "^3.9.9",
     "prettier-plugin-organize-imports": "^4.3.0",
     "ts-node": "~10.9.2",
     "typescript": "^6.0.3"

src-ui/src/app/components/admin/settings/settings.component.html

@@ -253,6 +253,24 @@
               </div>
             </div>
 
+            @if (aiEnabled) {
+              <div class="row mb-3">
+                <div class="col-md-3 col-form-label pt-0">
+                  <span i18n>Suggestions default to</span>
+                </div>
+                <div class="col">
+                  <fieldset class="btn-group btn-group-sm">
+                    <input type="radio" class="btn-check" id="suggestionSourceBoth" [value]="SuggestionSource.Both" formControlName="documentEditingSuggestionSource">
+                    <label class="btn btn-outline-primary" for="suggestionSourceBoth"><ng-container i18n>Both</ng-container></label>
+                    <input type="radio" class="btn-check" id="suggestionSourceML" [value]="SuggestionSource.ML" formControlName="documentEditingSuggestionSource">
+                    <label class="btn btn-outline-primary" for="suggestionSourceML"><i-bs class="me-1" name="cpu"></i-bs><ng-container i18n>ML only</ng-container></label>
+                    <input type="radio" class="btn-check" id="suggestionSourceAI" [value]="SuggestionSource.AI" formControlName="documentEditingSuggestionSource">
+                    <label class="btn btn-outline-primary" for="suggestionSourceAI"><i-bs class="me-1" name="stars"></i-bs><ng-container i18n>AI only</ng-container></label>
+                  </fieldset>
+                </div>
+              </div>
+            }
+
             <div class="row">
               <div class="col">
                 <pngx-input-check i18n-title title="Automatically request suggestions for inbox documents" i18n-hint hint="If un-checked, suggestions must be requested via the Suggest button." formControlName="documentEditingAutoSuggest"></pngx-input-check>

src-ui/src/app/components/admin/settings/settings.component.spec.ts

@@ -307,7 +307,7 @@ describe('SettingsComponent', () => {
     expect(toastErrorSpy).toHaveBeenCalled()
     expect(storeSpy).toHaveBeenCalled()
     expect(appearanceSettingsSpy).not.toHaveBeenCalled()
-    expect(setSpy).toHaveBeenCalledTimes(34)
+    expect(setSpy).toHaveBeenCalledTimes(35)
     expect(setSpy).toHaveBeenCalledWith(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [
       HideableSidebarItemID.Workflows,
     ])

src-ui/src/app/components/admin/settings/settings.component.ts

@@ -44,6 +44,7 @@ import {
   HIDEABLE_SIDEBAR_ITEM_IDS,
   HideableSidebarItemID,
   SETTINGS_KEYS,
+  SuggestionSource,
 } from 'src/app/data/ui-settings'
 import { User } from 'src/app/data/user'
 import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
@@ -184,6 +185,7 @@ export class SettingsComponent
     documentEditingRemoveInboxTags: new FormControl(null),
     documentEditingOverlayThumbnail: new FormControl(null),
     documentEditingAutoSuggest: new FormControl(null),
+    documentEditingSuggestionSource: new FormControl(null),
     documentDetailsHiddenFields: new FormControl([]),
     searchDbOnly: new FormControl(null),
     searchLink: new FormControl(null),
@@ -217,6 +219,11 @@ export class SettingsComponent
   public readonly PdfZoomScale = PdfZoomScale
 
   public readonly PdfEditorEditMode = PdfEditorEditMode
+  public readonly SuggestionSource = SuggestionSource
+
+  get aiEnabled(): boolean {
+    return this.settings.get(SETTINGS_KEYS.AI_ENABLED)
+  }
 
   public readonly documentDetailFieldOptions = documentDetailFieldOptions
   public readonly sidebarItemOptions = HIDEABLE_SIDEBAR_ITEM_IDS.map((id) => ({
@@ -404,6 +411,9 @@ export class SettingsComponent
       documentEditingAutoSuggest: this.settings.get(
         SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST
       ),
+      documentEditingSuggestionSource: this.settings.get(
+        SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE
+      ),
       documentDetailsHiddenFields: this.settings.get(
         SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS
       ),
@@ -625,6 +635,10 @@ export class SettingsComponent
       SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST,
       this.settingsForm.value.documentEditingAutoSuggest
     )
+    this.settings.set(
+      SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
+      this.settingsForm.value.documentEditingSuggestionSource
+    )
     this.settings.set(
       SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS,
       this.settingsForm.value.documentDetailsHiddenFields

src-ui/src/app/components/common/pdf-viewer/pdf-viewer.component.scss

@@ -154,11 +154,29 @@
   & section {
     position: absolute;
     text-align: initial;
+    pointer-events: auto;
     box-sizing: border-box;
     transform-origin: 0 0;
   }
 
+  & :is(.linkAnnotation, .buttonWidgetAnnotation.pushButton) > a {
+    position: absolute;
+    inset: 0;
+    font-size: 1em;
+    transition: none;
+  }
+
+  & :is(.linkAnnotation, .buttonWidgetAnnotation.pushButton):not(.hasBorder)
+    > a:hover {
+    opacity: 0.2;
+    background-color: rgb(255 255 0);
+  }
+
   & .annotationTextContent {
     opacity: 0;
   }
 }
+
+:host ::ng-deep .textLayer.selecting ~ .annotationLayer section {
+  pointer-events: none;
+}

src-ui/src/app/components/common/pdf-viewer/pdf-viewer.component.spec.ts

@@ -1,7 +1,11 @@
 import { SimpleChange } from '@angular/core'
 import { ComponentFixture, TestBed } from '@angular/core/testing'
 import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs'
-import { PDFSinglePageViewer, PDFViewer } from 'pdfjs-dist/web/pdf_viewer.mjs'
+import {
+  LinkTarget,
+  PDFSinglePageViewer,
+  PDFViewer,
+} from 'pdfjs-dist/web/pdf_viewer.mjs'
 import { PngxPdfViewerComponent } from './pdf-viewer.component'
 import { PdfRenderMode, PdfZoomLevel, PdfZoomScale } from './pdf-viewer.types'
 
@@ -58,6 +62,16 @@ describe('PngxPdfViewerComponent', () => {
     expect((component as any).pdfViewer).toBeInstanceOf(PDFViewer)
   })
 
+  it('opens external links in a new tab', () => {
+    const linkService = (component as any).linkService
+    expect(linkService.options).toEqual(
+      expect.objectContaining({
+        externalLinkTarget: LinkTarget.BLANK,
+        externalLinkRel: 'noopener noreferrer nofollow',
+      })
+    )
+  })
+
   it('resolves the worker source relative to the document base URI', async () => {
     setBaseHref('/paperless/')
     const getDocumentSpy = jest.spyOn(pdfjs, 'getDocument')
@@ -76,6 +90,7 @@ describe('PngxPdfViewerComponent', () => {
       withCredentials: true,
       wasmUrl: expect.stringContaining('/paperless/assets/wasm/'),
       iccUrl: expect.stringContaining('/paperless/assets/iccs/'),
+      cMapUrl: expect.stringContaining('/paperless/assets/cmaps/'),
     })
   })
 

src-ui/src/app/components/common/pdf-viewer/pdf-viewer.component.ts

@@ -21,6 +21,7 @@ import {
 } from 'pdfjs-dist/legacy/build/pdf.mjs'
 import {
   EventBus,
+  LinkTarget,
   PDFFindController,
   PDFLinkService,
   PDFSinglePageViewer,
@@ -75,7 +76,11 @@ export class PngxPdfViewerComponent
   private lastViewerPage?: number
 
   private readonly eventBus = new EventBus()
-  private readonly linkService = new PDFLinkService({ eventBus: this.eventBus })
+  private readonly linkService = new PDFLinkService({
+    eventBus: this.eventBus,
+    externalLinkTarget: LinkTarget.BLANK,
+    externalLinkRel: 'noopener noreferrer nofollow',
+  })
   private readonly findController = new PDFFindController({
     eventBus: this.eventBus,
     linkService: this.linkService,
@@ -185,6 +190,7 @@ export class PngxPdfViewerComponent
       withCredentials: true,
       wasmUrl: new URL('assets/wasm/', this.document.baseURI).toString(),
       iccUrl: new URL('assets/iccs/', this.document.baseURI).toString(),
+      cMapUrl: new URL('assets/cmaps/', this.document.baseURI).toString(),
     }
     this.loadingTask = getDocument(initOptions)
     try {

src-ui/src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html

@@ -1,58 +1,84 @@
-<div class="btn-group">
+<div class="d-flex align-items-center">
-  <button type="button" class="btn btn-sm btn-outline-primary" (click)="clickSuggest()" [disabled]="disabled() || loading() || (suggestions() && !aiEnabled())" [aria-label]="noSuggestions ? 'No suggestions' : 'Suggest'" i18n-aria-label>
+  <div class="btn-group">
-    @if (loading()) {
+    <button type="button" class="btn btn-sm btn-outline-primary" (click)="clickSuggest()" [disabled]="disabled() || loading() || (suggestions() && !aiEnabled())" [aria-label]="noSuggestions ? 'No suggestions' : 'Suggest'" i18n-aria-label>
-      <div class="spinner-border spinner-border-sm" role="status"></div>
+      @if (loading()) {
-    } @else if (noSuggestions) {
+        <div class="spinner-border spinner-border-sm" role="status"></div>
-      <i-bs width="1.2em" height="1.2em" name="check-circle"></i-bs>
+      } @else if (noSuggestions) {
-    } @else {
+        <i-bs width="1.2em" height="1.2em" name="check-circle"></i-bs>
-      <i-bs width="1.2em" height="1.2em" name="stars"></i-bs>
+      } @else {
+        <i-bs width="1.2em" height="1.2em" name="lightbulb"></i-bs>
+      }
+      @if (noSuggestions) {
+        <span class="d-none d-lg-inline ps-1" i18n>No suggestions</span>
+      } @else {
+        <span class="d-none d-lg-inline ps-1" i18n>Suggest</span>
+      }
+      @if (totalSuggestions > 0) {
+        <span class="badge bg-primary ms-2">{{ totalSuggestions }}</span>
+      }
+    </button>
+
+    @if (aiEnabled()) {
+      <div class="btn-group" ngbDropdown #dropdown="ngbDropdown" [popperOptions]="popperOptions">
+        <button type="button" class="btn btn-sm btn-outline-primary" ngbDropdownToggle [disabled]="disabled() || loading() || !suggestions()" aria-expanded="false" aria-controls="suggestionsDropdown" aria-label="Suggestions dropdown">
+          <span class="visually-hidden" i18n>Show suggestions</span>
+        </button>
+
+        <div ngbDropdownMenu aria-labelledby="suggestionsDropdown" class="shadow suggestions-dropdown">
+          <div class="list-group list-group-flush small pb-0">
+            @if (novelSuggestions === 0 && reusableSuggestions === 0) {
+              <div class="list-group-item text-muted fst-italic">
+                <small class="text-muted small fst-italic" i18n>No novel suggestions</small>
+              </div>
+            }
+            @if (suggestions()?.suggested_tags?.length > 0) {
+              <small class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="tags"></i-bs><ng-container i18n>Tags</ng-container></small>
+              @for (tag of suggestions().suggested_tags; track tag) {
+                <button type="button" class="list-group-item list-group-item-action bg-light" (click)="addTag.emit(tag)">{{ tag }}</button>
+              }
+            }
+            @if (suggestions()?.suggested_document_types?.length > 0) {
+              <div class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="hash"></i-bs><ng-container i18n>Document Types</ng-container></div>
+              @for (type of suggestions().suggested_document_types; track type) {
+                <button type="button" class="list-group-item list-group-item-action bg-light" (click)="addDocumentType.emit(type)">{{ type }}</button>
+              }
+            }
+            @if (suggestions()?.suggested_correspondents?.length > 0) {
+              <div class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="person"></i-bs><ng-container i18n>Correspondents</ng-container></div>
+              @for (correspondent of suggestions().suggested_correspondents; track correspondent) {
+                <button type="button" class="list-group-item list-group-item-action bg-light" (click)="addCorrespondent.emit(correspondent)">{{ correspondent }}</button>
+              }
+            }
+            @if (reusableSuggestions > 0) {
+              <div class="list-group-item text-muted fst-italic">
+                <small class="text-muted small fst-italic" i18n>{reusableSuggestions, plural, =1 {1 existing value suggested below} other {{{reusableSuggestions}} existing values suggested below}}</small>
+              </div>
+            }
+          </div>
+        </div>
+      </div>
     }
-    @if (noSuggestions) {
+  </div>
-      <span class="d-none d-lg-inline ps-1" i18n>No suggestions</span>
-    } @else {
-      <span class="d-none d-lg-inline ps-1" i18n>Suggest</span>
-    }
-    @if (totalSuggestions > 0) {
-      <span class="badge bg-primary ms-2">{{ totalSuggestions }}</span>
-    }
-  </button>
 
   @if (aiEnabled()) {
-    <div class="btn-group" ngbDropdown #dropdown="ngbDropdown" [popperOptions]="popperOptions">
+    <div ngbDropdown autoClose="outside" placement="bottom-end" [popperOptions]="popperOptions">
-      <button type="button" class="btn btn-sm btn-outline-primary" ngbDropdownToggle [disabled]="disabled() || loading() || !suggestions()" aria-expanded="false" aria-controls="suggestionsDropdown" aria-label="Suggestions dropdown">
+      <button type="button" class="btn btn-sm btn-link position-relative" ngbDropdownToggle [disabled]="disabled() || loading()" i18n-title title="Suggestion options">
-        <span class="visually-hidden" i18n>Show suggestions</span>
+        <i-bs name="three-dots"></i-bs>
+        @if (source() !== defaultSource()) {
+          <span class="position-absolute top-0 start-100 translate-middle p-1 bg-primary border border-light rounded-circle">
+            <span class="visually-hidden" i18n>Not using default</span>
+          </span>
+        }
       </button>
-
+      <div ngbDropdownMenu class="shadow p-3">
-      <div ngbDropdownMenu aria-labelledby="suggestionsDropdown" class="shadow suggestions-dropdown">
+        <div class="small text-muted mb-2" i18n>Suggest using:</div>
-        <div class="list-group list-group-flush small pb-0">
+        <div class="form-check small">
-          @if (novelSuggestions === 0 && reusableSuggestions === 0) {
+          <input class="form-check-input" type="checkbox" id="suggestionSourceML" [checked]="useML" [disabled]="useML && !useAI" (change)="setSources($event.target.checked, useAI)">
-            <div class="list-group-item text-muted fst-italic">
+          <label class="form-check-label d-inline-flex align-items-center gap-1" for="suggestionSourceML"><i-bs name="cpu"></i-bs><ng-container i18n>ML</ng-container></label>
-              <small class="text-muted small fst-italic" i18n>No novel suggestions</small>
+        </div>
-            </div>
+        <div class="form-check small">
-          }
+          <input class="form-check-input" type="checkbox" id="suggestionSourceAI" [checked]="useAI" [disabled]="useAI && !useML" (change)="setSources(useML, $event.target.checked)">
-          @if (suggestions()?.suggested_tags?.length > 0) {
+          <label class="form-check-label d-inline-flex align-items-center gap-1" for="suggestionSourceAI"><i-bs name="stars"></i-bs><ng-container i18n>AI</ng-container></label>
-            <small class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="tags"></i-bs><ng-container i18n>Tags</ng-container></small>
-            @for (tag of suggestions().suggested_tags; track tag) {
-              <button type="button" class="list-group-item list-group-item-action bg-light" (click)="addTag.emit(tag)">{{ tag }}</button>
-            }
-          }
-          @if (suggestions()?.suggested_document_types?.length > 0) {
-            <div class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="hash"></i-bs><ng-container i18n>Document Types</ng-container></div>
-            @for (type of suggestions().suggested_document_types; track type) {
-              <button type="button" class="list-group-item list-group-item-action bg-light" (click)="addDocumentType.emit(type)">{{ type }}</button>
-            }
-          }
-          @if (suggestions()?.suggested_correspondents?.length > 0) {
-            <div class="list-group-item text-uppercase text-muted small"><i-bs class="me-2" name="person"></i-bs><ng-container i18n>Correspondents</ng-container></div>
-            @for (correspondent of suggestions().suggested_correspondents; track correspondent) {
-              <button type="button" class="list-group-item list-group-item-action bg-light" (click)="addCorrespondent.emit(correspondent)">{{ correspondent }}</button>
-            }
-          }
-          @if (reusableSuggestions > 0) {
-            <div class="list-group-item text-muted fst-italic">
-              <small class="text-muted small fst-italic" i18n>{reusableSuggestions, plural, =1 {1 existing value suggested below} other {{{reusableSuggestions}} existing values suggested below}}</small>
-            </div>
-          }
         </div>
       </div>
     </div>

src-ui/src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.scss

@@ -1,3 +1,7 @@
 .suggestions-dropdown {
   min-width: 250px;
 }
+
+.btn-link.dropdown-toggle::after {
+  display: none;
+}

src-ui/src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.spec.ts

@@ -1,6 +1,7 @@
 import { ComponentFixture, TestBed } from '@angular/core/testing'
 import { NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
 import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
+import { SuggestionSource } from 'src/app/data/ui-settings'
 import { SuggestionsDropdownComponent } from './suggestions-dropdown.component'
 
 describe('SuggestionsDropdownComponent', () => {
@@ -179,14 +180,71 @@ describe('SuggestionsDropdownComponent', () => {
 
   it('should toggle dropdown when clickSuggest is called and suggestions are not null', () => {
     fixture.componentRef.setInput('aiEnabled', true)
+    fixture.componentRef.setInput('fetchedSources', [SuggestionSource.ML])
     fixture.detectChanges()
     fixture.componentRef.setInput('suggestions', {
       suggested_correspondents: [],
       suggested_tags: [],
       suggested_document_types: [],
     })
+    fixture.detectChanges()
     component.clickSuggest()
-    expect(component.dropdown.open).toBeTruthy()
+    expect(component.dropdown.isOpen()).toBeTruthy()
     expect(fixture.nativeElement.textContent).toContain('No novel suggestions')
   })
+
+  it('should fetch unfetched sources and show existing suggestions', () => {
+    jest.spyOn(component.getSuggestions, 'emit')
+    fixture.componentRef.setInput('aiEnabled', true)
+    fixture.componentRef.setInput('source', SuggestionSource.Both)
+    fixture.componentRef.setInput('fetchedSources', [SuggestionSource.ML])
+    fixture.componentRef.setInput('suggestions', { tags: [1] })
+    fixture.detectChanges()
+    component.clickSuggest()
+    expect(component.getSuggestions.emit).toHaveBeenCalledWith(
+      SuggestionSource.Both
+    )
+    expect(component.dropdown.isOpen()).toBeTruthy()
+  })
+
+  it('should only show source options when AI is enabled', () => {
+    expect(
+      fixture.nativeElement.querySelector('#suggestionSourceML')
+    ).toBeNull()
+    fixture.componentRef.setInput('aiEnabled', true)
+    fixture.detectChanges()
+    fixture.nativeElement
+      .querySelector('button[title="Suggestion options"]')
+      .click()
+    fixture.detectChanges()
+    expect(
+      fixture.nativeElement.querySelector('#suggestionSourceML')
+    ).not.toBeNull()
+  })
+
+  it('should emit source changes and never allow no source', () => {
+    const emitSpy = jest.spyOn(component.sourceChange, 'emit')
+    component.setSources(true, true)
+    expect(emitSpy).toHaveBeenCalledWith(SuggestionSource.Both)
+    component.setSources(true, false)
+    expect(emitSpy).toHaveBeenCalledWith(SuggestionSource.ML)
+    component.setSources(false, true)
+    expect(emitSpy).toHaveBeenCalledWith(SuggestionSource.AI)
+
+    emitSpy.mockClear()
+    component.setSources(false, false)
+    expect(emitSpy).not.toHaveBeenCalled()
+  })
+
+  it('should indicate a non-default source', () => {
+    fixture.componentRef.setInput('aiEnabled', true)
+    fixture.componentRef.setInput('source', SuggestionSource.AI)
+    fixture.componentRef.setInput('defaultSource', SuggestionSource.AI)
+    fixture.detectChanges()
+    expect(fixture.nativeElement.textContent).not.toContain('Not using default')
+
+    fixture.componentRef.setInput('source', SuggestionSource.Both)
+    fixture.detectChanges()
+    expect(fixture.nativeElement.textContent).toContain('Not using default')
+  })
 })

src-ui/src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.ts

@@ -8,6 +8,7 @@ import {
 import { NgbDropdown, NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
 import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
 import { DocumentSuggestions } from 'src/app/data/document-suggestions'
+import { SuggestionSource } from 'src/app/data/ui-settings'
 import { pngxPopperOptions } from 'src/app/utils/popper-options'
 
 @Component({
@@ -18,12 +19,16 @@ import { pngxPopperOptions } from 'src/app/utils/popper-options'
 })
 export class SuggestionsDropdownComponent {
   public popperOptions = pngxPopperOptions
+  public readonly SuggestionSource = SuggestionSource
 
   @ViewChild('dropdown') dropdown: NgbDropdown
   readonly suggestions = input<DocumentSuggestions>(null)
   readonly aiEnabled = input(false)
   readonly loading = input(false)
   readonly disabled = input(false)
+  readonly source = input<SuggestionSource>(SuggestionSource.ML)
+  readonly defaultSource = input<SuggestionSource>(SuggestionSource.ML)
+  readonly fetchedSources = input<SuggestionSource[]>([])
 
   readonly appliedTags = input<number[]>([])
   readonly appliedCorrespondent = input<number>(null)
@@ -31,8 +36,10 @@ export class SuggestionsDropdownComponent {
   readonly appliedStoragePath = input<number>(null)
 
   @Output()
-  getSuggestions: EventEmitter<SuggestionsDropdownComponent> =
+  getSuggestions: EventEmitter<SuggestionSource> = new EventEmitter()
-    new EventEmitter()
+
+  @Output()
+  sourceChange: EventEmitter<SuggestionSource> = new EventEmitter()
 
   @Output()
   addTag: EventEmitter<string> = new EventEmitter()
@@ -53,12 +60,42 @@ export class SuggestionsDropdownComponent {
     }
 
     if (!this.suggestions()) {
-      this.getSuggestions.emit(this)
+      this.getSuggestions.emit(this.source())
+    } else if (this.hasUnfetchedSources) {
+      // sources changed, fetch the rest and show what we have meanwhile
+      this.getSuggestions.emit(this.source())
+      this.dropdown?.open()
     } else {
       this.dropdown?.toggle()
     }
   }
 
+  get useML(): boolean {
+    return this.source() !== SuggestionSource.AI
+  }
+
+  get useAI(): boolean {
+    return this.source() !== SuggestionSource.ML
+  }
+
+  get hasUnfetchedSources(): boolean {
+    const fetched = this.fetchedSources()
+    return (
+      (this.useML && !fetched.includes(SuggestionSource.ML)) ||
+      (this.useAI && !fetched.includes(SuggestionSource.AI))
+    )
+  }
+
+  public setSources(ml: boolean, ai: boolean) {
+    if (ml && ai) {
+      this.sourceChange.emit(SuggestionSource.Both)
+    } else if (ml) {
+      this.sourceChange.emit(SuggestionSource.ML)
+    } else if (ai) {
+      this.sourceChange.emit(SuggestionSource.AI)
+    }
+  }
+
   get novelSuggestions(): number {
     return (
       (this.suggestions()?.suggested_correspondents?.length ?? 0) +

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -134,11 +134,15 @@
                 [loading]="suggestionsLoading()"
                 [suggestions]="suggestions()"
                 [aiEnabled]="aiEnabled"
+                [source]="suggestionSource"
+                [defaultSource]="defaultSuggestionSource"
+                [fetchedSources]="fetchedSuggestionSources()"
                 [appliedTags]="documentForm.value.tags"
                 [appliedCorrespondent]="documentForm.value.correspondent"
                 [appliedDocumentType]="documentForm.value.document_type"
                 [appliedStoragePath]="documentForm.value.storage_path"
-                (getSuggestions)="getSuggestions()"
+                (getSuggestions)="getSuggestions($event)"
+                (sourceChange)="suggestionSourceOverride.set($event)"
                 (addTag)="createTag($event)"
                 (addDocumentType)="createDocumentType($event)"
                 (addCorrespondent)="createCorrespondent($event)">

src-ui/src/app/components/document-detail/document-detail.component.spec.ts

@@ -43,7 +43,7 @@ import {
 } from 'src/app/data/filter-rule-type'
 import { StoragePath } from 'src/app/data/storage-path'
 import { Tag } from 'src/app/data/tag'
-import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
+import { SETTINGS_KEYS, SuggestionSource } from 'src/app/data/ui-settings'
 import { PermissionsGuard } from 'src/app/guards/permissions.guard'
 import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
 import { DocumentTitlePipe } from 'src/app/pipes/document-title.pipe'
@@ -1528,6 +1528,113 @@ describe('DocumentDetailComponent', () => {
     expect(component.suggestionsLoading()).toBeFalsy()
   })
 
+  it('should get and merge ML and AI suggestions when source is both', () => {
+    settingsService.set(
+      SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
+      SuggestionSource.Both
+    )
+    const getSetting = settingsService.get.bind(settingsService)
+    jest
+      .spyOn(settingsService, 'get')
+      .mockImplementation((key) =>
+        key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
+      )
+    const suggestionsSpy = jest
+      .spyOn(documentService, 'getSuggestions')
+      .mockReturnValue(of({ tags: [42], dates: ['2024-01-01'] }))
+    const aiSuggestionsSpy = jest
+      .spyOn(documentService, 'getAiSuggestions')
+      .mockReturnValue(
+        of({ title: 'AI title', tags: [42, 43], suggested_tags: ['New'] })
+      )
+    initNormally()
+    expect(suggestionsSpy).toHaveBeenCalled()
+    expect(aiSuggestionsSpy).toHaveBeenCalled()
+    expect(component.suggestions().title).toEqual('AI title')
+    expect(component.suggestions().tags).toEqual([42, 43])
+    expect(component.suggestions().suggested_tags).toEqual(['New'])
+    expect(component.suggestions().dates).toEqual(['2024-01-01'])
+  })
+
+  it('should only fetch sources not yet fetched for the document', () => {
+    settingsService.set(SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST, false)
+    settingsService.set(
+      SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
+      SuggestionSource.ML
+    )
+    const getSetting = settingsService.get.bind(settingsService)
+    jest
+      .spyOn(settingsService, 'get')
+      .mockImplementation((key) =>
+        key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
+      )
+    const suggestionsSpy = jest
+      .spyOn(documentService, 'getSuggestions')
+      .mockReturnValue(of({ tags: [42] }))
+    const aiSuggestionsSpy = jest
+      .spyOn(documentService, 'getAiSuggestions')
+      .mockReturnValue(of({ tags: [43] }))
+    initNormally()
+
+    component.getSuggestions()
+    expect(suggestionsSpy).toHaveBeenCalledTimes(1)
+    expect(aiSuggestionsSpy).not.toHaveBeenCalled()
+
+    component.getSuggestions(SuggestionSource.Both)
+    expect(suggestionsSpy).toHaveBeenCalledTimes(1)
+    expect(aiSuggestionsSpy).toHaveBeenCalledTimes(1)
+    expect(component.suggestions().tags).toEqual([42, 43])
+
+    component.getSuggestions(SuggestionSource.Both)
+    expect(suggestionsSpy).toHaveBeenCalledTimes(1)
+    expect(aiSuggestionsSpy).toHaveBeenCalledTimes(1)
+  })
+
+  it('should use the per-document source override and reset it on document change', () => {
+    settingsService.set(SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST, false)
+    const getSetting = settingsService.get.bind(settingsService)
+    jest
+      .spyOn(settingsService, 'get')
+      .mockImplementation((key) =>
+        key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
+      )
+    initNormally()
+    expect(component.suggestionSource).toEqual(SuggestionSource.AI)
+    component.suggestionSourceOverride.set(SuggestionSource.ML)
+    expect(component.suggestionSource).toEqual(SuggestionSource.ML)
+
+    jest
+      .spyOn(documentService, 'get')
+      .mockReturnValueOnce(of(Object.assign({}, doc)))
+    ;(component as any).loadDocument(doc.id, true)
+    expect(component.suggestionSourceOverride()).toBeNull()
+    expect(component.fetchedSuggestionSources()).toEqual([])
+  })
+
+  it('should keep suggestions from one source if the other fails', () => {
+    settingsService.set(
+      SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
+      SuggestionSource.Both
+    )
+    const getSetting = settingsService.get.bind(settingsService)
+    jest
+      .spyOn(settingsService, 'get')
+      .mockImplementation((key) =>
+        key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
+      )
+    const errorSpy = jest.spyOn(toastService, 'showError')
+    jest
+      .spyOn(documentService, 'getSuggestions')
+      .mockReturnValue(of({ tags: [42] }))
+    jest
+      .spyOn(documentService, 'getAiSuggestions')
+      .mockReturnValue(throwError(() => new Error('failed')))
+    initNormally()
+    expect(errorSpy).toHaveBeenCalled()
+    expect(component.suggestions().tags).toEqual([42])
+    expect(component.fetchedSuggestionSources()).toEqual([SuggestionSource.ML])
+  })
+
   it('should show error if needed for get suggestions', () => {
     const suggestionsSpy = jest.spyOn(documentService, 'getSuggestions')
     const errorSpy = jest.spyOn(toastService, 'showError')

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -28,7 +28,7 @@ import {
 import { dirtyCheck, DirtyComponent } from '@ngneat/dirty-check-forms'
 import { NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
 import { DeviceDetectorService } from 'ngx-device-detector'
-import { BehaviorSubject, Observable, of, Subject, timer } from 'rxjs'
+import { BehaviorSubject, merge, Observable, of, Subject, timer } from 'rxjs'
 import {
   catchError,
   debounceTime,
@@ -48,7 +48,10 @@ import { DataType } from 'src/app/data/datatype'
 import { Document, DocumentVersionInfo } from 'src/app/data/document'
 import { DocumentMetadata } from 'src/app/data/document-metadata'
 import { DocumentNote } from 'src/app/data/document-note'
-import { DocumentSuggestions } from 'src/app/data/document-suggestions'
+import {
+  DocumentSuggestions,
+  mergeSuggestions,
+} from 'src/app/data/document-suggestions'
 import { DocumentType } from 'src/app/data/document-type'
 import { FilterRule } from 'src/app/data/filter-rule'
 import {
@@ -63,7 +66,7 @@ import {
 import { ObjectWithId } from 'src/app/data/object-with-id'
 import { StoragePath } from 'src/app/data/storage-path'
 import { Tag } from 'src/app/data/tag'
-import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
+import { SETTINGS_KEYS, SuggestionSource } from 'src/app/data/ui-settings'
 import { User } from 'src/app/data/user'
 import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
 import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
@@ -240,6 +243,10 @@ export class DocumentDetailComponent
   private readonly autoSuggestSetting = this.settings.getSignal<boolean>(
     SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST
   )
+  private readonly suggestionSourceSetting =
+    this.settings.getSignal<SuggestionSource>(
+      SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE
+    )
   private readonly hiddenFieldsSetting = this.settings.getSignal<
     DocumentDetailFieldID[]
   >(SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS)
@@ -261,6 +268,9 @@ export class DocumentDetailComponent
   readonly metadata = signal<DocumentMetadata>(undefined)
   readonly suggestions = signal<DocumentSuggestions>(undefined)
   readonly suggestionsLoading = signal(false)
+  // per-document, resets on navigation
+  readonly suggestionSourceOverride = signal<SuggestionSource>(null)
+  readonly fetchedSuggestionSources = signal<SuggestionSource[]>([])
   readonly users = signal<User[]>(undefined)
 
   readonly title = signal<string>(undefined)
@@ -365,6 +375,15 @@ export class DocumentDetailComponent
     return this.autoSuggestSetting()
   }
 
+  get defaultSuggestionSource(): SuggestionSource {
+    return this.aiEnabled ? this.suggestionSourceSetting() : SuggestionSource.ML
+  }
+
+  get suggestionSource(): SuggestionSource {
+    if (!this.aiEnabled) return SuggestionSource.ML
+    return this.suggestionSourceOverride() ?? this.defaultSuggestionSource
+  }
+
   get archiveContentRenderType(): ContentRenderType {
     const hasArchiveVersion =
       this.metadata()?.has_archive_version ??
@@ -590,6 +609,8 @@ export class DocumentDetailComponent
           }
           this.documentId.set(doc.id)
           this.suggestions.set(null)
+          this.suggestionSourceOverride.set(null)
+          this.fetchedSuggestionSources.set([])
           const openDocument = this.openDocumentService.getOpenDocument(
             this.documentId()
           )
@@ -1077,29 +1098,44 @@ export class DocumentDetailComponent
     return this.documentForm.get('custom_fields') as FormArray
   }
 
-  getSuggestions() {
+  getSuggestions(source: SuggestionSource = this.suggestionSource) {
+    const sources = (
+      source === SuggestionSource.Both
+        ? [SuggestionSource.ML, SuggestionSource.AI]
+        : [source]
+    ).filter((s) => !this.fetchedSuggestionSources().includes(s))
+    if (!sources.length) return
+
     this.suggestionsLoading.set(true)
-    const suggestionsObservable = this.aiEnabled
+    merge(
-      ? this.documentsService.getAiSuggestions(this.documentId())
+      ...sources.map((s) =>
-      : this.documentsService.getSuggestions(this.documentId())
+        (s === SuggestionSource.AI
-    suggestionsObservable
+          ? this.documentsService.getAiSuggestions(this.documentId())
+          : this.documentsService.getSuggestions(this.documentId())
+        ).pipe(
+          first(),
+          map((result) => ({ source: s, result })),
+          catchError((error) => {
+            this.toastService.showError(
+              $localize`Error retrieving suggestions.`,
+              error
+            )
+            return of(null)
+          })
+        )
+      )
+    )
       .pipe(
-        first(),
         takeUntil(this.unsubscribeNotifier),
         takeUntil(this.docChangeNotifier),
         finalize(() => this.suggestionsLoading.set(false))
       )
-      .subscribe({
+      .subscribe((response) => {
-        next: (result) => {
+        if (!response) return
-          this.suggestions.set(result)
+        this.fetchedSuggestionSources.update((f) => [...f, response.source])
-        },
+        this.suggestions.set(
-        error: (error) => {
+          mergeSuggestions(this.suggestions(), response.result)
-          this.suggestions.set(null)
+        )
-          this.toastService.showError(
-            $localize`Error retrieving suggestions.`,
-            error
-          )
-        },
       })
   }
 

src-ui/src/app/components/document-list/document-list.component.spec.ts

@@ -146,6 +146,19 @@ describe('DocumentListComponent', () => {
     expect(reloadSpy).toHaveBeenCalled()
   })
 
+  it('should stop reloading on document deleted after destroy', () => {
+    const reloadSpy = jest.spyOn(documentListService, 'reload')
+    const documentDeletedSubject = new Subject<boolean>()
+    jest
+      .spyOn(websocketStatusService, 'onDocumentDeleted')
+      .mockReturnValue(documentDeletedSubject)
+    fixture.detectChanges()
+    fixture.destroy()
+    reloadSpy.mockClear()
+    documentDeletedSubject.next(true)
+    expect(reloadSpy).not.toHaveBeenCalled()
+  })
+
   it('should show score sort fields on fulltext queries', () => {
     documentListService.setFilterRules([
       {

src-ui/src/app/components/document-list/document-list.component.ts

@@ -270,9 +270,12 @@ export class DocumentListComponent
         this.list.reload()
       })
 
-    this.websocketStatusService.onDocumentDeleted().subscribe(() => {
+    this.websocketStatusService
-      this.list.reload()
+      .onDocumentDeleted()
-    })
+      .pipe(takeUntil(this.unsubscribeNotifier))
+      .subscribe(() => {
+        this.list.reload()
+      })
 
     this.route.paramMap
       .pipe(

src-ui/src/app/data/document-suggestions.ts

@@ -15,3 +15,33 @@ export interface DocumentSuggestions {
 
   dates?: string[] // ISO-formatted date string e.g. 2022-11-03
 }
+
+const union = <T>(a: T[] = [], b: T[] = []): T[] => [...new Set([...a, ...b])]
+
+export function mergeSuggestions(
+  a: DocumentSuggestions,
+  b: DocumentSuggestions
+): DocumentSuggestions {
+  if (!a) return b
+  return {
+    title: a.title || b.title,
+    tags: union(a.tags, b.tags),
+    suggested_tags: union(a.suggested_tags, b.suggested_tags),
+    correspondents: union(a.correspondents, b.correspondents),
+    suggested_correspondents: union(
+      a.suggested_correspondents,
+      b.suggested_correspondents
+    ),
+    document_types: union(a.document_types, b.document_types),
+    suggested_document_types: union(
+      a.suggested_document_types,
+      b.suggested_document_types
+    ),
+    storage_paths: union(a.storage_paths, b.storage_paths),
+    suggested_storage_paths: union(
+      a.suggested_storage_paths,
+      b.suggested_storage_paths
+    ),
+    dates: union(a.dates, b.dates),
+  }
+}

src-ui/src/app/data/paperless-config.ts

@@ -353,6 +353,14 @@ export const PaperlessConfigOptions: ConfigOption[] = [
     config_key: 'PAPERLESS_AI_LLM_EMBEDDING_MODEL',
     category: ConfigCategory.AI,
   },
+  {
+    key: 'llm_embedding_api_key',
+    title: $localize`LLM Embedding API Key`,
+    type: ConfigOptionType.Password,
+    note: $localize`Used for embeddings when set, otherwise LLM API key is used.`,
+    config_key: 'PAPERLESS_AI_LLM_EMBEDDING_API_KEY',
+    category: ConfigCategory.AI,
+  },
   {
     key: 'llm_embedding_endpoint',
     title: $localize`LLM Embedding Endpoint`,
@@ -457,6 +465,7 @@ export interface PaperlessConfig extends ObjectWithId {
   ai_enabled: boolean
   llm_embedding_backend: string
   llm_embedding_model: string
+  llm_embedding_api_key: string
   llm_embedding_endpoint: string
   llm_embedding_chunk_size: number
   llm_context_size: number

src-ui/src/app/data/ui-settings.ts

@@ -20,6 +20,12 @@ export enum GlobalSearchType {
   TITLE_CONTENT = 'title-content',
 }
 
+export enum SuggestionSource {
+  ML = 'ml',
+  AI = 'ai',
+  Both = 'both',
+}
+
 export enum CollapsibleSection {
   ATTRIBUTES = 'attributes',
 }
@@ -98,6 +104,8 @@ export const SETTINGS_KEYS = {
     'general-settings:document-editing:overlay-thumbnail',
   DOCUMENT_EDITING_AUTO_SUGGEST:
     'general-settings:document-editing:auto-suggest',
+  DOCUMENT_EDITING_SUGGESTION_SOURCE:
+    'general-settings:document-editing:suggestion-source',
   DOCUMENT_DETAILS_HIDDEN_FIELDS:
     'general-settings:document-details:hidden-fields',
   SEARCH_DB_ONLY: 'general-settings:search:db-only',
@@ -326,6 +334,11 @@ export const SETTINGS: UiSetting[] = [
     type: 'boolean',
     default: true,
   },
+  {
+    key: SETTINGS_KEYS.DOCUMENT_EDITING_SUGGESTION_SOURCE,
+    type: 'string',
+    default: SuggestionSource.AI,
+  },
   {
     key: SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS,
     type: 'array',

src-ui/src/main.ts

@@ -74,6 +74,7 @@ import {
   clipboardCheckFill,
   clipboardFill,
   clockHistory,
+  cpu,
   creditCard,
   dash,
   dashCircle,
@@ -118,6 +119,7 @@ import {
   infoCircle,
   journalBookmarkFill,
   journals,
+  lightbulb,
   link,
   list,
   listNested,
@@ -322,6 +324,7 @@ const icons = {
   clipboardCheckFill,
   clipboardFill,
   clockHistory,
+  cpu,
   cash,
   creditCard,
   dash,
@@ -367,6 +370,7 @@ const icons = {
   infoCircle,
   journalBookmarkFill,
   journals,
+  lightbulb,
   link,
   list,
   listNested,

src-ui/src/styles.scss

@@ -292,6 +292,7 @@ a.btn-link:active,
 a.btn-link:focus-visible,
 .btn-link:hover,
 .btn-link:active,
+.btn-link.show,
 .btn-link:focus-visible {
   color: var(--pngx-primary-lighten-10) !important;
   .primary-light & {

src-ui/src/test/mocks/pdfjs-web-pdf_viewer.ts

@@ -25,10 +25,20 @@ export class PDFFindController {
   onIsPageVisible?: () => boolean
 }
 
+export const LinkTarget = {
+  NONE: 0,
+  SELF: 1,
+  BLANK: 2,
+  PARENT: 3,
+  TOP: 4,
+}
+
 export class PDFLinkService {
   private document?: unknown
   private viewer?: unknown
 
+  constructor(readonly options: Record<string, unknown> = {}) {}
+
   setDocument(document: unknown): void {
     this.document = document
   }

src/conftest.py

@@ -0,0 +1,151 @@
+"""Fixtures available to every Paperless-ngx app.
+
+Loaded automatically for every test path. Keep module-scope imports minimal:
+this file is imported for every session,  so anything heavy belongs inside
+the fixture body that needs it.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+    from pathlib import Path
+
+    from django.contrib.auth.models import User
+    from pytest_django.fixtures import Settings
+    from rest_framework.test import APIClient
+
+    from paperless_testing.dirs import PaperlessDirs
+    from paperless_testing.fakes.progress import FakeProgressManager
+
+
+@pytest.fixture(scope="session", autouse=True)
+def faker_session_locale() -> str:
+    """Pin Faker's locale so generated data does not follow the host locale.
+
+    The seed itself is left to pytest-randomly, which derives one per run.
+    """
+    return "en_US"
+
+
+@pytest.fixture(autouse=True)
+def _fast_password_hasher(settings: Settings) -> None:
+    """Hash test passwords with MD5 instead of Django's default PBKDF2.
+
+    PBKDF2 is deliberately slow, and every ``admin_user`` or
+    ``create_superuser`` call pays for it: about 600 ms each. No test depends
+    on the hash format, only on ``check_password`` and on the stored value
+    changing when the password does.
+    """
+    settings.PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
+
+
+@pytest.fixture(autouse=True)
+def _clear_content_type_caches() -> None:
+    """Clear Django's ContentType cache and guardian's lru_cache before each test.
+
+    Tests that delete and reinsert ContentType/Permission rows (e.g. the
+    importer) corrupt both caches. Without this fixture a subsequent test on
+    the same xdist worker sees stale ContentType objects and guardian raises
+    MixedContentTypeError.
+    """
+    from django.contrib.contenttypes.models import ContentType
+    from guardian.shortcuts import clear_ct_cache
+
+    ContentType.objects.clear_cache()
+    clear_ct_cache()
+
+
+@pytest.fixture(autouse=True)
+def _clear_django_caches() -> None:
+    """Clear every configured cache before each test.
+
+    Cached values outlive the test that wrote them: the classifier keys its
+    vectorized content on a hash of the content itself, so a second test
+    generating the same fixture data takes the cache-hit path and never calls
+    the code it is asserting against.
+    """
+    from django.core.cache import caches
+
+    for cache in caches.all(initialized_only=False):
+        cache.clear()
+
+
+@pytest.fixture
+def paperless_dirs(
+    tmp_path: Path,
+    settings: Settings,
+) -> Generator[PaperlessDirs, None, None]:
+    """The standard temp directory layout, applied to Django settings."""
+    from documents.search import reset_backend
+    from paperless_testing.dirs import build_paperless_dirs
+    from paperless_testing.dirs import dirs_settings
+
+    dirs = build_paperless_dirs(tmp_path)
+    for name, value in dirs_settings(dirs).items():
+        setattr(settings, name, value)
+
+    # Not directory settings, but they are needed alongside the layout by the
+    # sanity checker tests.
+    settings.IGNORABLE_FILES = {".DS_Store", "Thumbs.db", "desktop.ini"}
+    settings.APP_LOGO = ""
+
+    reset_backend()
+    yield dirs
+    reset_backend()
+
+
+@pytest.fixture
+def rest_api_client() -> APIClient:
+    """The basic DRF APIClient, unauthenticated."""
+    from rest_framework.test import APIClient
+
+    return APIClient()
+
+
+@pytest.fixture
+def regular_user(db: None) -> User:
+    """Unprivileged user for permission boundary tests."""
+    from paperless_testing.factories import UserFactory
+
+    return UserFactory(username="regular")
+
+
+@pytest.fixture
+def admin_client(rest_api_client: APIClient, admin_user: User) -> APIClient:
+    """Admin client pre-authenticated and sending the v10 Accept header."""
+    rest_api_client.force_authenticate(user=admin_user)
+    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
+    return rest_api_client
+
+
+@pytest.fixture
+def v9_client(rest_api_client: APIClient, admin_user: User) -> APIClient:
+    """Admin client pre-authenticated and sending the v9 Accept header."""
+    rest_api_client.force_authenticate(user=admin_user)
+    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=9")
+    return rest_api_client
+
+
+@pytest.fixture
+def user_client(rest_api_client: APIClient, regular_user: User) -> APIClient:
+    """Regular-user client pre-authenticated and sending the v10 Accept header."""
+    rest_api_client.force_authenticate(user=regular_user)
+    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
+    return rest_api_client
+
+
+@pytest.fixture
+def fake_progress_manager(
+    monkeypatch: pytest.MonkeyPatch,
+) -> type[FakeProgressManager]:
+    """Replace documents.tasks.ProgressManager with the fake, so consuming a file
+    in a test never tries to reach a broker."""
+    from paperless_testing.fakes.progress import FakeProgressManager
+
+    monkeypatch.setattr("documents.tasks.ProgressManager", FakeProgressManager)
+    return FakeProgressManager

src/documents/apps.py

@@ -26,7 +26,6 @@ class DocumentsConfig(AppConfig):
         document_consumption_finished.connect(set_document_type)
         document_consumption_finished.connect(set_tags)
         document_consumption_finished.connect(set_storage_path)
-        document_consumption_finished.connect(add_to_index)
         document_consumption_finished.connect(run_workflows_added)
         document_consumption_finished.connect(add_to_index)
         document_consumption_finished.connect(add_or_update_document_in_llm_index)

src/documents/consumer.py

@@ -857,8 +857,9 @@ class ConsumerPlugin(
             self.log.debug(f"Creation date from parse_date: {create_date}")
         else:
             stats = Path(self.input_doc.original_file).stat()
-            create_date = timezone.make_aware(
+            create_date = datetime.datetime.fromtimestamp(
-                datetime.datetime.fromtimestamp(stats.st_mtime),
+                stats.st_mtime,
+                tz=timezone.get_current_timezone(),
             )
             self.log.debug(f"Creation date from st_mtime: {create_date}")
 

src/documents/search/_backend.py

@@ -196,52 +196,49 @@ class WriteBatch:
         return self._raw_writer
 
     def __enter__(self) -> Self:
-        if self._backend._path is not None:
+        lock_path = self._backend._path / ".tantivy.lock"
-            lock_path = self._backend._path / ".tantivy.lock"
+        self._lock = filelock.FileLock(str(lock_path))
-            self._lock = filelock.FileLock(str(lock_path))
+        for attempt in range(_LOCK_RETRY_ATTEMPTS):
-            for attempt in range(_LOCK_RETRY_ATTEMPTS):
+            try:
-                try:
+                self._lock.acquire(timeout=self._lock_timeout)
-                    self._lock.acquire(timeout=self._lock_timeout)
+                break
-                    break
+            except filelock.Timeout:
-                except filelock.Timeout:
+                if attempt == _LOCK_RETRY_ATTEMPTS - 1:
-                    if attempt == _LOCK_RETRY_ATTEMPTS - 1:
+                    raise SearchIndexLockError(
-                        raise SearchIndexLockError(
+                        f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} "
-                            f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} "
+                        f"attempts (timeout={self._lock_timeout}s each)",
-                            f"attempts (timeout={self._lock_timeout}s each)",
-                        )
-                    sleep_s = random.uniform(
-                        0,
-                        min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)),
                     )
-                    logger.debug(
+                sleep_s = random.uniform(
-                        "Index lock contention; retrying in %.2fs (attempt %d/%d)",
+                    0,
-                        sleep_s,
+                    min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)),
-                        attempt + 1,
+                )
-                        _LOCK_RETRY_ATTEMPTS,
+                logger.debug(
-                    )
+                    "Index lock contention; retrying in %.2fs (attempt %d/%d)",
-                    time.sleep(sleep_s)
+                    sleep_s,
+                    attempt + 1,
+                    _LOCK_RETRY_ATTEMPTS,
+                )
+                time.sleep(sleep_s)
 
-            # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)
+        # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)
-            # for the write, rather than reusing the process-local cached
+        # for the write, rather than reusing the process-local cached
-            # index. ManagedDirectory loads its GC bookkeeping (.managed.json)
+        # index. ManagedDirectory loads its GC bookkeeping (.managed.json)
-            # once, at construction, and never re-reads it; paperless runs
+        # once, at construction, and never re-reads it; paperless runs
-            # several long-lived processes (Granian workers, Celery workers)
+        # several long-lived processes (Granian workers, Celery workers)
-            # that take turns writing under the file lock above. A cached,
+        # that take turns writing under the file lock above. A cached,
-            # long-lived writer index would carry a stale managed-files view
+        # long-lived writer index would carry a stale managed-files view
-            # and, on commit, overwrite .managed.json with that stale view -
+        # and, on commit, overwrite .managed.json with that stale view -
-            # permanently losing track of segment files other processes
+        # permanently losing track of segment files other processes
-            # registered in the meantime, so they can never be garbage
+        # registered in the meantime, so they can never be garbage
-            # collected. Reopening fresh here always picks up the current
+        # collected. Reopening fresh here always picks up the current
-            # on-disk state. The long-lived self._backend._index is used for
+        # on-disk state. The long-lived self._backend._index is used for
-            # reads only and is reloaded (not reopened) after commit below.
+        # reads only and is reloaded (not reopened) after commit below.
-            write_index = tantivy.Index(
+        write_index = tantivy.Index(
-                build_schema(),
+            build_schema(),
-                path=str(self._backend._path),
+            path=str(self._backend._path),
-            )
+        )
-            register_tokenizers(write_index, settings.SEARCH_LANGUAGE)
+        register_tokenizers(write_index, settings.SEARCH_LANGUAGE)
-            self._raw_writer = write_index.writer()
+        self._raw_writer = write_index.writer()
-        else:
-            self._raw_writer = self._backend._index.writer()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -372,9 +369,8 @@ class TantivyBackend:
     Tantivy search backend with explicit lifecycle management.
 
     Provides full-text search capabilities using the Tantivy search engine.
-    Supports in-memory indexes (for testing) and persistent on-disk indexes
+    Keeps a persistent on-disk index. Handles document indexing, search queries,
-    (for production use). Handles document indexing, search queries, autocompletion,
+    autocompletion, and "more like this" functionality.
-    and "more like this" functionality.
 
     The backend manages its own connection lifecycle and can be reset when
     the underlying index directory changes (e.g., during test isolation).
@@ -408,9 +404,7 @@ class TantivyBackend:
         },
     )
 
-    def __init__(self, path: Path | None = None):
+    def __init__(self, path: Path):
-        # path=None → in-memory index (for tests)
-        # path=some_dir → on-disk index (for production)
         self._path = path
         self._raw_index: tantivy.Index | None = None
         self._raw_schema: tantivy.Schema | None = None
@@ -429,16 +423,13 @@ class TantivyBackend:
         """
         Open or rebuild the index as needed.
 
-        For disk-based indexes, checks if rebuilding is needed due to schema
+        Checks if rebuilding is needed due to schema version or language
-        version or language changes. Registers custom tokenizers after opening.
+        changes. Registers custom tokenizers after opening.
         Safe to call multiple times - subsequent calls are no-ops.
         """
         if self._raw_index is not None:
             return  # pragma: no cover
-        if self._path is not None:
+        self._raw_index = open_or_rebuild_index(self._path)
-            self._raw_index = open_or_rebuild_index(self._path)
-        else:
-            self._raw_index = tantivy.Index(build_schema())
         register_tokenizers(self._raw_index, settings.SEARCH_LANGUAGE)
         self._raw_schema = self._raw_index.schema
 
@@ -1102,13 +1093,9 @@ class TantivyBackend:
                 writer's threads). Larger values buffer more docs in RAM before
                 flushing a segment, deferring merge work; they do not avoid it.
         """
-        # Create new index (on-disk or in-memory)
+        wipe_index(self._path)
-        if self._path is not None:
+        new_index = tantivy.Index(build_schema(), path=str(self._path))
-            wipe_index(self._path)
+        _write_sentinels(self._path)
-            new_index = tantivy.Index(build_schema(), path=str(self._path))
-            _write_sentinels(self._path)
-        else:
-            new_index = tantivy.Index(build_schema())
         register_tokenizers(new_index, settings.SEARCH_LANGUAGE)
 
         # Point instance at the new index so _build_tantivy_doc uses it

src/documents/serialisers/base.py

@@ -0,0 +1,568 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import Literal
+
+from django.contrib.auth.models import Group
+from django.contrib.auth.models import User
+from django.contrib.contenttypes.models import ContentType
+from django.utils.text import slugify
+from django.utils.translation import gettext as _
+from drf_spectacular.utils import extend_schema_field
+from guardian.core import ObjectPermissionChecker
+from guardian.shortcuts import get_users_with_perms
+from guardian.utils import get_group_obj_perms_model
+from guardian.utils import get_user_obj_perms_model
+from rest_framework import serializers
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.fields import SerializerMethodField
+from rest_framework.utils import model_meta
+
+from documents import bulk_edit
+from documents.models import Document
+from documents.models import MatchingModel
+from documents.models import Note
+from documents.permissions import get_groups_with_only_permission
+from documents.permissions import set_permissions_for_object
+from documents.regex import validate_regex_pattern
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+
+logger = logging.getLogger("paperless.serializers")
+
+
+# https://www.django-rest-framework.org/api-guide/serializers/#example
+class DynamicFieldsModelSerializer(serializers.ModelSerializer[Any]):
+    """
+    A ModelSerializer that takes an additional `fields` argument that
+    controls which fields should be displayed.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        # Don't pass the 'fields' arg up to the superclass
+        fields = kwargs.pop("fields", None)
+
+        # Instantiate the superclass normally
+        super().__init__(*args, **kwargs)
+
+        if fields is not None:
+            # Drop any fields that are not specified in the `fields` argument.
+            allowed = set(fields)
+            existing = set(self.fields)
+            for field_name in existing - allowed:
+                self.fields.pop(field_name)
+
+
+class DocumentUpdateFieldsModelSerializer(DynamicFieldsModelSerializer):
+    stale_update_excluded_fields = frozenset({"filename", "archive_filename"})
+
+    def _get_update_fields(self, validated_data) -> list[str]:
+        model_fields = {
+            field.name
+            for field in self.Meta.model._meta.concrete_fields
+            if field.name not in self.stale_update_excluded_fields
+        }
+        update_fields = [
+            field_name for field_name in validated_data if field_name in model_fields
+        ]
+        if "modified" in model_fields and "modified" not in update_fields:
+            update_fields.append("modified")
+        return update_fields
+
+    def update(self, instance, validated_data):
+        serializers.raise_errors_on_nested_writes("update", self, validated_data)
+        info = model_meta.get_field_info(instance)
+
+        m2m_fields = []
+        for attr, value in validated_data.items():
+            if attr in info.relations and info.relations[attr].to_many:
+                m2m_fields.append((attr, value))
+            else:
+                setattr(instance, attr, value)
+
+        # File names are managed by post-save file handling.  Saving only the
+        # serializer-updated fields prevents stale in-memory path values from
+        # overwriting a concurrent move.
+        instance.save(update_fields=self._get_update_fields(validated_data))
+
+        for attr, value in m2m_fields:
+            field = getattr(instance, attr)
+            field.set(value)
+
+        return instance
+
+
+class MatchingModelSerializer(serializers.ModelSerializer[Any]):
+    document_count = serializers.IntegerField(read_only=True)
+
+    def get_slug(self, obj) -> str:
+        return slugify(obj.name)
+
+    slug = SerializerMethodField()
+
+    def validate(self, data):
+        # TODO: remove pending https://github.com/encode/django-rest-framework/issues/7173
+        name = data.get(
+            "name",
+            self.instance.name if hasattr(self.instance, "name") else None,
+        )
+        owner = (
+            data["owner"]
+            if "owner" in data
+            else self.user
+            if hasattr(self, "user")
+            else None
+        )
+        pk = self.instance.pk if hasattr(self.instance, "pk") else None
+        if ("name" in data or "owner" in data) and self.Meta.model.objects.filter(
+            name=name,
+            owner=owner,
+        ).exclude(pk=pk).exists():
+            raise serializers.ValidationError(
+                {"error": "Object violates owner / name unique constraint"},
+            )
+        return data
+
+    def validate_match(self, match):
+        if (
+            "matching_algorithm" in self.initial_data
+            and self.initial_data["matching_algorithm"] == MatchingModel.MATCH_REGEX
+        ):
+            try:
+                validate_regex_pattern(match)
+            except ValueError as e:
+                logger.debug(f"Invalid regular expression: {e!s}")
+                raise serializers.ValidationError(
+                    "Invalid regular expression, see log for details.",
+                )
+        return match
+
+
+PERMISSION_ACTIONS = ("view", "change")
+
+
+class SetPermissionsMixin:
+    def _validate_user_ids(self, user_ids):
+        users = User.objects.none()
+        if user_ids is not None:
+            users = User.objects.filter(id__in=user_ids)
+            if not users.count() == len(user_ids):
+                raise serializers.ValidationError(
+                    "Some users in don't exist or were specified twice.",
+                )
+        return users
+
+    def _validate_group_ids(self, group_ids):
+        groups = Group.objects.none()
+        if group_ids is not None:
+            groups = Group.objects.filter(id__in=group_ids)
+            if not groups.count() == len(group_ids):
+                raise serializers.ValidationError(
+                    "Some groups in don't exist or were specified twice.",
+                )
+        return groups
+
+    def validate_set_permissions(self, set_permissions=None):
+        permissions_dict = {action: {} for action in PERMISSION_ACTIONS}
+        if set_permissions is not None:
+            for action in PERMISSION_ACTIONS:
+                if action in set_permissions:
+                    if "users" in set_permissions[action]:
+                        users = set_permissions[action]["users"]
+                        permissions_dict[action]["users"] = self._validate_user_ids(
+                            users,
+                        )
+                    if "groups" in set_permissions[action]:
+                        groups = set_permissions[action]["groups"]
+                        permissions_dict[action]["groups"] = self._validate_group_ids(
+                            groups,
+                        )
+                else:
+                    del permissions_dict[action]
+        return permissions_dict
+
+    def _set_permissions(self, permissions, object) -> None:
+        set_permissions_for_object(permissions, object)
+
+
+class SerializerWithPerms(serializers.Serializer[dict[str, Any]]):
+    def __init__(self, *args, **kwargs) -> None:
+        self.user = kwargs.pop("user", None)
+        self.full_perms = kwargs.pop("full_perms", False)
+        self.all_fields = kwargs.pop("all_fields", False)
+        super().__init__(*args, **kwargs)
+
+
+class PermissionSetSerializer(serializers.Serializer[dict[str, Any]]):
+    users = serializers.ListField(
+        child=serializers.IntegerField(),
+        required=False,
+        allow_null=True,
+    )
+    groups = serializers.ListField(
+        child=serializers.IntegerField(),
+        required=False,
+        allow_null=True,
+    )
+
+
+class SetPermissionsSerializer(serializers.Serializer[dict[str, Any]]):
+    view = PermissionSetSerializer(required=False)
+    change = PermissionSetSerializer(required=False)
+
+    def to_internal_value(self, data):
+        if isinstance(data, dict):
+            unknown_keys = set(data) - set(PERMISSION_ACTIONS)
+            if unknown_keys:
+                raise serializers.ValidationError(
+                    {key: "Unknown permission action." for key in sorted(unknown_keys)},
+                )
+        return super().to_internal_value(data)
+
+
+class OwnedObjectSerializer(
+    SerializerWithPerms,
+    serializers.ModelSerializer[Any],
+    SetPermissionsMixin,
+):
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        if not self.all_fields:
+            try:
+                if self.full_perms:
+                    self.fields.pop("user_can_change")
+                    self.fields.pop("is_shared_by_requester")
+                else:
+                    self.fields.pop("permissions")
+            except KeyError:
+                pass
+
+    def _get_perms(self, obj, codename: str, target: Literal["users", "groups"]):
+        """
+        Get the given permissions from context or from django-guardian.
+
+        :param codename: The permission codename, e.g. 'view' or 'change'
+        :param target: 'users' or 'groups'
+        """
+        key = f"{target}_{codename}_perms"
+        cached = self.context.get(key, {}).get(obj.pk)
+        if cached is not None:
+            return list(cached)
+
+        # Permission not found in the context, get it from guardian
+        if target == "users":
+            return list(
+                get_users_with_perms(
+                    obj,
+                    only_with_perms_in=[f"{codename}_{obj.__class__.__name__.lower()}"],
+                    with_group_users=False,
+                ).values_list("id", flat=True),
+            )
+        else:  # groups
+            return list(
+                get_groups_with_only_permission(
+                    obj,
+                    codename=f"{codename}_{obj.__class__.__name__.lower()}",
+                ).values_list("id", flat=True),
+            )
+
+    @extend_schema_field(
+        field={
+            "type": "object",
+            "properties": {
+                "view": {
+                    "type": "object",
+                    "properties": {
+                        "users": {
+                            "type": "array",
+                            "items": {"type": "integer"},
+                        },
+                        "groups": {
+                            "type": "array",
+                            "items": {"type": "integer"},
+                        },
+                    },
+                },
+                "change": {
+                    "type": "object",
+                    "properties": {
+                        "users": {
+                            "type": "array",
+                            "items": {"type": "integer"},
+                        },
+                        "groups": {
+                            "type": "array",
+                            "items": {"type": "integer"},
+                        },
+                    },
+                },
+            },
+        },
+    )
+    def get_permissions(self, obj) -> dict:
+        return {
+            "view": {
+                "users": self._get_perms(obj, "view", "users"),
+                "groups": self._get_perms(obj, "view", "groups"),
+            },
+            "change": {
+                "users": self._get_perms(obj, "change", "users"),
+                "groups": self._get_perms(obj, "change", "groups"),
+            },
+        }
+
+    def get_user_can_change(self, obj) -> bool:
+        if obj.owner is None or obj.owner == self.user:
+            return True
+        if self.user is None:
+            return False
+        if self.user.is_active and self.user.is_superuser:
+            # Mirrors guardian's own ObjectPermissionChecker.has_perm() shortcut --
+            # superusers aren't necessarily granted explicit object permissions,
+            # so the batched context below would otherwise incorrectly say no.
+            return True
+
+        # Prefer the page-level batch computed by BulkPermissionMixin
+        # (get_serializer_context) over a fresh per-object guardian check,
+        # which would otherwise query the permission tables once per row.
+        users_change_perms = self.context.get("users_change_perms")
+        groups_change_perms = self.context.get("groups_change_perms")
+        if users_change_perms is not None and groups_change_perms is not None:
+            if self.user.pk in users_change_perms.get(obj.pk, []):
+                return True
+            user_group_ids = getattr(self, "_user_group_ids", None)
+            if user_group_ids is None:
+                user_group_ids = set(self.user.groups.values_list("id", flat=True))
+                self._user_group_ids = user_group_ids
+            return bool(
+                user_group_ids.intersection(groups_change_perms.get(obj.pk, [])),
+            )
+
+        checker = ObjectPermissionChecker(self.user)
+        return checker.has_perm(f"change_{obj.__class__.__name__.lower()}", obj)
+
+    @staticmethod
+    def get_shared_object_pks(objects: Iterable):
+        """
+        Return the primary keys of the subset of objects that are shared.
+        """
+        try:
+            first_obj = next(iter(objects))
+        except StopIteration:
+            return set()
+
+        ctype = ContentType.objects.get_for_model(first_obj)
+        object_pks = list(obj.pk for obj in objects)
+        pk_type = type(first_obj.pk)
+
+        def get_pks_for_permission_type(model):
+            return map(
+                pk_type,  # coerce the pk to be the same type of the provided objects
+                model.objects.filter(
+                    content_type=ctype,
+                    object_pk__in=object_pks,
+                )
+                .values_list("object_pk", flat=True)
+                .distinct(),
+            )
+
+        UserObjectPermission = get_user_obj_perms_model()
+        GroupObjectPermission = get_group_obj_perms_model()
+        user_permission_pks = get_pks_for_permission_type(UserObjectPermission)
+        group_permission_pks = get_pks_for_permission_type(GroupObjectPermission)
+
+        return set(user_permission_pks) | set(group_permission_pks)
+
+    def get_is_shared_by_requester(self, obj: Document) -> bool:
+        # First check the context to see if `shared_object_pks` is set by the parent.
+        shared_object_pks = self.context.get("shared_object_pks")
+        # If not just check if the current object is shared.
+        if shared_object_pks is None:
+            shared_object_pks = self.get_shared_object_pks([obj])
+        return obj.owner == self.user and obj.id in shared_object_pks
+
+    permissions = SerializerMethodField(read_only=True, required=False)
+    user_can_change = SerializerMethodField(read_only=True, required=False)
+    is_shared_by_requester = SerializerMethodField(read_only=True, required=False)
+
+    set_permissions = SetPermissionsSerializer(
+        label="Set permissions",
+        required=False,
+        write_only=True,
+    )
+    # other methods in mixin
+
+    def validate_unique_together(self, validated_data, instance=None) -> None:
+        # workaround for https://github.com/encode/django-rest-framework/issues/9358
+        if "owner" in validated_data and "name" in self.Meta.fields:
+            name = validated_data.get("name", instance.name if instance else None)
+            objects = (
+                self.Meta.model.objects.exclude(pk=instance.pk)
+                if instance
+                else self.Meta.model.objects.all()
+            )
+            not_unique = objects.filter(
+                owner=validated_data["owner"],
+                name=name,
+            ).exists()
+            if not_unique:
+                raise serializers.ValidationError(
+                    {"error": "Object violates owner / name unique constraint"},
+                )
+
+    def create(self, validated_data):
+        # default to current user if not set
+        request = self.context.get("request")
+        if (
+            "owner" not in validated_data
+            or (request is not None and "owner" not in request.data)
+        ) and self.user:
+            validated_data["owner"] = self.user
+        permissions = None
+        if "set_permissions" in validated_data:
+            permissions = validated_data.pop("set_permissions")
+        self.validate_unique_together(validated_data)
+        instance = super().create(validated_data)
+        if permissions is not None:
+            self._set_permissions(permissions, instance)
+        return instance
+
+    def update(self, instance, validated_data):
+        user = getattr(self, "user", None)
+        is_superuser = user.is_superuser if user is not None else False
+        is_owner = instance.owner == user if user is not None else False
+        is_unowned = instance.owner is None
+
+        if (
+            ("owner" in validated_data and validated_data["owner"] != instance.owner)
+            or "set_permissions" in validated_data
+        ) and not (is_superuser or is_owner or is_unowned):
+            raise PermissionDenied(
+                _("Insufficient permissions."),
+            )
+
+        if "set_permissions" in validated_data:
+            self._set_permissions(validated_data["set_permissions"], instance)
+        self.validate_unique_together(validated_data, instance)
+        return super().update(instance, validated_data)
+
+
+class OwnedObjectListSerializer(serializers.ListSerializer[Any]):
+    def to_representation(self, documents):
+        self.child.context["shared_object_pks"] = self.child.get_shared_object_pks(
+            documents,
+        )
+        return super().to_representation(documents)
+
+
+class ReadWriteSerializerMethodField(serializers.SerializerMethodField):
+    """
+    Based on https://stackoverflow.com/a/62579804
+    """
+
+    def __init__(self, method_name=None, *args, **kwargs) -> None:
+        self.method_name = method_name
+        kwargs["source"] = "*"
+        super(serializers.SerializerMethodField, self).__init__(*args, **kwargs)
+
+    def to_internal_value(self, data):
+        return {self.field_name: data}
+
+
+class DocumentListSerializer(serializers.Serializer[dict[str, list[int]]]):
+    documents = serializers.ListField(
+        required=True,
+        label="Documents",
+        write_only=True,
+        child=serializers.IntegerField(),
+    )
+
+    def _validate_document_id_list(self, documents, name="documents") -> None:
+        if not isinstance(documents, list):
+            raise serializers.ValidationError(f"{name} must be a list")
+        if not all(isinstance(i, int) for i in documents):
+            raise serializers.ValidationError(f"{name} must be a list of integers")
+        count = Document.objects.filter(id__in=documents).count()
+        if not count == len(documents):
+            raise serializers.ValidationError(
+                f"Some documents in {name} don't exist or were specified twice.",
+            )
+
+    def validate_documents(self, documents):
+        self._validate_document_id_list(documents)
+        return documents
+
+
+class DocumentSelectionSerializer(DocumentListSerializer):
+    documents = serializers.ListField(
+        required=False,
+        label="Documents",
+        write_only=True,
+        child=serializers.IntegerField(),
+    )
+
+    all = serializers.BooleanField(
+        default=False,
+        required=False,
+        write_only=True,
+    )
+
+    filters = serializers.DictField(
+        required=False,
+        allow_empty=True,
+        write_only=True,
+    )
+
+    excluded_documents = serializers.ListField(
+        required=False,
+        default=list,
+        write_only=True,
+        child=serializers.IntegerField(),
+    )
+
+    def validate(self, attrs):
+        if attrs.get("all", False):
+            attrs.setdefault("documents", [])
+            return attrs
+
+        if attrs["excluded_documents"]:
+            raise serializers.ValidationError(
+                "excluded_documents is only supported when all is true.",
+            )
+
+        if "documents" not in attrs:
+            raise serializers.ValidationError(
+                "documents is required unless all is true.",
+            )
+
+        documents = attrs["documents"]
+        self._validate_document_id_list(documents)
+        return attrs
+
+
+class SourceModeValidationMixin:
+    def validate_source_mode(self, source_mode: str) -> str:
+        if source_mode not in bulk_edit.SourceModeChoices.__dict__.values():
+            raise serializers.ValidationError("Invalid source_mode")
+        return source_mode
+
+
+class BasicUserSerializer(serializers.ModelSerializer[User]):
+    # Different than paperless.serializers.UserSerializer
+    class Meta:
+        model = User
+        fields = ["id", "username", "first_name", "last_name"]
+
+
+class NotesSerializer(serializers.ModelSerializer[Note]):
+    user = BasicUserSerializer(read_only=True)
+
+    class Meta:
+        model = Note
+        fields = ["id", "note", "created", "user"]
+        ordering = ["-created"]

src/documents/serialisers/bulk_edit.py

@@ -0,0 +1,750 @@
+from __future__ import annotations
+
+import logging
+
+from django.contrib.auth.models import User
+from rest_framework import serializers
+
+from documents import bulk_edit
+from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import Document
+from documents.models import DocumentType
+from documents.models import StoragePath
+from documents.models import Tag
+
+from .base import DocumentListSerializer
+from .base import DocumentSelectionSerializer
+from .base import SerializerWithPerms
+from .base import SetPermissionsMixin
+from .base import SetPermissionsSerializer
+from .base import SourceModeValidationMixin
+from .metadata import CustomFieldInstanceSerializer
+
+logger = logging.getLogger("paperless.serializers")
+
+
+def _validate_rotation_degrees(degrees: int, field: str = "degrees") -> int:
+    # QPDF refuses any other angle, which would otherwise fail inside the task
+    if degrees % 90 != 0:
+        raise serializers.ValidationError(f"{field} must be a multiple of 90")
+    return degrees
+
+
+class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidationMixin):
+    degrees = serializers.IntegerField(required=True)
+    source_mode = serializers.CharField(
+        required=False,
+        default=bulk_edit.SourceModeChoices.LATEST_VERSION,
+    )
+    from_webui = serializers.BooleanField(required=False, default=False)
+
+    def validate_degrees(self, value: int) -> int:
+        return _validate_rotation_degrees(value)
+
+
+class MergeDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin):
+    metadata_document_id = serializers.IntegerField(
+        required=False,
+        allow_null=True,
+    )
+    delete_originals = serializers.BooleanField(required=False, default=False)
+    archive_fallback = serializers.BooleanField(required=False, default=False)
+    source_mode = serializers.CharField(
+        required=False,
+        default=bulk_edit.SourceModeChoices.LATEST_VERSION,
+    )
+    from_webui = serializers.BooleanField(required=False, default=False)
+
+
+class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
+    root_document_id = serializers.IntegerField(required=True)
+    version_label = serializers.CharField(
+        required=False,
+        allow_blank=True,
+        allow_null=True,
+        max_length=64,
+    )
+
+    def validate_version_label(self, value):
+        if value is None:
+            return None
+        normalized = value.strip()
+        return normalized or None
+
+    def validate(self, attrs):
+        documents = attrs["documents"]
+        if len(documents) < 2:
+            raise serializers.ValidationError(
+                "At least two documents are required.",
+            )
+        if attrs.get("version_label") is not None and len(documents) != 2:
+            raise serializers.ValidationError(
+                "version_label can only be used when merging one source document.",
+            )
+        if attrs["root_document_id"] not in documents:
+            raise serializers.ValidationError(
+                "root_document_id must be one of the selected documents.",
+            )
+
+        selected_documents = Document.objects.filter(id__in=documents)
+        if selected_documents.filter(root_document__isnull=False).exists():
+            raise serializers.ValidationError(
+                "Only top-level documents can be merged as versions.",
+            )
+
+        source_document_ids = set(documents) - {attrs["root_document_id"]}
+        if Document.global_objects.filter(
+            root_document_id__in=source_document_ids,
+        ).exists():
+            raise serializers.ValidationError(
+                "Documents with existing versions cannot be merged into another document.",
+            )
+        return attrs
+
+
+class PdfEditOperationSerializer(serializers.Serializer[dict[str, int]]):
+    page = serializers.IntegerField(min_value=1)
+    rotate = serializers.IntegerField(required=False)
+    doc = serializers.IntegerField(required=False, min_value=0)
+
+    def validate_rotate(self, value: int) -> int:
+        return _validate_rotation_degrees(value, field="rotate")
+
+
+class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin):
+    operations = serializers.ListField(
+        child=PdfEditOperationSerializer(),
+        required=True,
+        allow_empty=False,
+    )
+    delete_original = serializers.BooleanField(required=False, default=False)
+    update_document = serializers.BooleanField(required=False, default=False)
+    include_metadata = serializers.BooleanField(required=False, default=True)
+    source_mode = serializers.CharField(
+        required=False,
+        default=bulk_edit.SourceModeChoices.LATEST_VERSION,
+    )
+    from_webui = serializers.BooleanField(required=False, default=False)
+
+    def validate(self, attrs):
+        documents = attrs["documents"]
+        if len(documents) > 1:
+            raise serializers.ValidationError(
+                "Edit PDF method only supports one document",
+            )
+
+        operations = attrs["operations"]
+
+        if any(op.get("doc", 0) >= len(operations) for op in operations):
+            raise serializers.ValidationError("doc index is out of bounds")
+
+        if attrs["update_document"]:
+            max_idx = max(op.get("doc", 0) for op in operations)
+            if max_idx > 0:
+                raise serializers.ValidationError(
+                    "update_document only allowed with a single output document",
+                )
+
+        doc = Document.objects.get(id=documents[0])
+        if doc.page_count:
+            for op in operations:
+                if op["page"] > doc.page_count:
+                    raise serializers.ValidationError(
+                        f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.",
+                    )
+        return attrs
+
+
+class RemovePasswordDocumentsSerializer(
+    DocumentListSerializer,
+    SourceModeValidationMixin,
+):
+    password = serializers.CharField(required=True)
+    update_document = serializers.BooleanField(required=False, default=False)
+    delete_original = serializers.BooleanField(required=False, default=False)
+    include_metadata = serializers.BooleanField(required=False, default=True)
+    source_mode = serializers.CharField(
+        required=False,
+        default=bulk_edit.SourceModeChoices.LATEST_VERSION,
+    )
+    from_webui = serializers.BooleanField(required=False, default=False)
+
+
+class DeleteDocumentsSerializer(DocumentSelectionSerializer):
+    pass
+
+
+class ReprocessDocumentsSerializer(DocumentSelectionSerializer):
+    remote_ocr = serializers.BooleanField(required=False, default=False)
+
+
+class BulkEditSerializer(
+    SerializerWithPerms,
+    DocumentSelectionSerializer,
+    SetPermissionsMixin,
+    SourceModeValidationMixin,
+):
+    # TODO: remove this and related backwards compatibility code when API v9 is dropped
+    # split, delete_pages can be removed entirely
+    MOVED_DOCUMENT_ACTION_ENDPOINTS = {
+        "delete": "/api/documents/delete/",
+        "reprocess": "/api/documents/reprocess/",
+        "rotate": "/api/documents/rotate/",
+        "merge": "/api/documents/merge/",
+        "edit_pdf": "/api/documents/edit_pdf/",
+        "remove_password": "/api/documents/remove_password/",
+        "split": "/api/documents/edit_pdf/",
+        "delete_pages": "/api/documents/edit_pdf/",
+    }
+    LEGACY_DOCUMENT_ACTION_METHODS = tuple(MOVED_DOCUMENT_ACTION_ENDPOINTS.keys())
+
+    method = serializers.ChoiceField(
+        choices=[
+            "set_correspondent",
+            "set_document_type",
+            "set_storage_path",
+            "add_tag",
+            "remove_tag",
+            "modify_tags",
+            "modify_custom_fields",
+            "set_permissions",
+            *LEGACY_DOCUMENT_ACTION_METHODS,
+        ],
+        label="Method",
+        write_only=True,
+    )
+
+    parameters = serializers.DictField(allow_empty=True, default={}, write_only=True)
+    from_webui = serializers.BooleanField(required=False, default=False)
+
+    def _validate_tag_id_list(self, tags, name="tags") -> None:
+        if not isinstance(tags, list):
+            raise serializers.ValidationError(f"{name} must be a list")
+        if not all(isinstance(i, int) for i in tags):
+            raise serializers.ValidationError(f"{name} must be a list of integers")
+        count = Tag.objects.filter(id__in=tags).count()
+        if not count == len(tags):
+            raise serializers.ValidationError(
+                f"Some tags in {name} don't exist or were specified twice.",
+            )
+
+    def _validate_custom_field_id_list_or_dict(
+        self,
+        custom_fields,
+        name="custom_fields",
+    ) -> None:
+        ids = custom_fields
+        if isinstance(custom_fields, dict):
+            try:
+                ids = [int(i[0]) for i in custom_fields.items()]
+            except Exception as e:
+                logger.exception(f"Error validating custom fields: {e}")
+                raise serializers.ValidationError(
+                    f"{name} must be a list of integers or a dict of id:value pairs, see the log for details",
+                )
+        elif not isinstance(custom_fields, list) or not all(
+            isinstance(i, int) for i in ids
+        ):
+            raise serializers.ValidationError(
+                f"{name} must be a list of integers or a dict of id:value pairs",
+            )
+        count = CustomField.objects.filter(id__in=ids).count()
+        if not count == len(ids):
+            raise serializers.ValidationError(
+                f"Some custom fields in {name} don't exist or were specified twice.",
+            )
+
+    def _validate_custom_field_values(self, custom_fields, name):
+        if not isinstance(custom_fields, dict):
+            return custom_fields
+
+        validated = {}
+        errors = {}
+        for raw_field_id, value in custom_fields.items():
+            field_id = int(raw_field_id)
+            validator = CustomFieldInstanceSerializer(
+                data={"field": field_id, "value": value},
+                context=self.context,
+            )
+            if validator.is_valid():
+                validated[field_id] = validator.validated_data["value"]
+            else:
+                errors[str(field_id)] = validator.errors
+
+        if errors:
+            raise serializers.ValidationError({name: errors})
+
+        return validated
+
+    def validate_method(self, method):
+        if method == "set_correspondent":
+            return bulk_edit.set_correspondent
+        elif method == "set_document_type":
+            return bulk_edit.set_document_type
+        elif method == "set_storage_path":
+            return bulk_edit.set_storage_path
+        elif method == "add_tag":
+            return bulk_edit.add_tag
+        elif method == "remove_tag":
+            return bulk_edit.remove_tag
+        elif method == "modify_tags":
+            return bulk_edit.modify_tags
+        elif method == "modify_custom_fields":
+            return bulk_edit.modify_custom_fields
+        elif method == "delete":
+            return bulk_edit.delete
+        elif method == "redo_ocr" or method == "reprocess":
+            return bulk_edit.reprocess
+        elif method == "set_permissions":
+            return bulk_edit.set_permissions
+        elif method == "rotate":
+            return bulk_edit.rotate
+        elif method == "merge":
+            return bulk_edit.merge
+        elif method == "split":
+            return bulk_edit.split
+        elif method == "delete_pages":
+            return bulk_edit.delete_pages
+        elif method == "edit_pdf":
+            return bulk_edit.edit_pdf
+        elif method == "remove_password":
+            return bulk_edit.remove_password
+        else:
+            raise serializers.ValidationError("Unsupported method.")
+
+    def _validate_parameters_tags(self, parameters) -> None:
+        if "tag" in parameters:
+            tag_id = parameters["tag"]
+            try:
+                Tag.objects.get(id=tag_id)
+            except Tag.DoesNotExist:
+                raise serializers.ValidationError("Tag does not exist")
+        else:
+            raise serializers.ValidationError("tag not specified")
+
+    def _validate_parameters_document_type(self, parameters) -> None:
+        if "document_type" in parameters:
+            document_type_id = parameters["document_type"]
+            if document_type_id is None:
+                # None is ok
+                return
+            try:
+                DocumentType.objects.get(id=document_type_id)
+            except DocumentType.DoesNotExist:
+                raise serializers.ValidationError("Document type does not exist")
+        else:
+            raise serializers.ValidationError("document_type not specified")
+
+    def _validate_parameters_correspondent(self, parameters) -> None:
+        if "correspondent" in parameters:
+            correspondent_id = parameters["correspondent"]
+            if correspondent_id is None:
+                return
+            try:
+                Correspondent.objects.get(id=correspondent_id)
+            except Correspondent.DoesNotExist:
+                raise serializers.ValidationError("Correspondent does not exist")
+        else:
+            raise serializers.ValidationError("correspondent not specified")
+
+    def _validate_storage_path(self, parameters) -> None:
+        if "storage_path" in parameters:
+            storage_path_id = parameters["storage_path"]
+            if storage_path_id is None:
+                return
+            try:
+                StoragePath.objects.get(id=storage_path_id)
+            except StoragePath.DoesNotExist:
+                raise serializers.ValidationError(
+                    "Storage path does not exist",
+                )
+        else:
+            raise serializers.ValidationError("storage path not specified")
+
+    def _validate_parameters_modify_tags(self, parameters) -> None:
+        if "add_tags" in parameters:
+            self._validate_tag_id_list(parameters["add_tags"], "add_tags")
+        else:
+            raise serializers.ValidationError("add_tags not specified")
+
+        if "remove_tags" in parameters:
+            self._validate_tag_id_list(parameters["remove_tags"], "remove_tags")
+        else:
+            raise serializers.ValidationError("remove_tags not specified")
+
+    def _validate_parameters_modify_custom_fields(self, parameters) -> None:
+        if "add_custom_fields" in parameters:
+            self._validate_custom_field_id_list_or_dict(
+                parameters["add_custom_fields"],
+                "add_custom_fields",
+            )
+            parameters["add_custom_fields"] = self._validate_custom_field_values(
+                parameters["add_custom_fields"],
+                "add_custom_fields",
+            )
+        else:
+            raise serializers.ValidationError("add_custom_fields not specified")
+
+        if "remove_custom_fields" in parameters:
+            self._validate_custom_field_id_list_or_dict(
+                parameters["remove_custom_fields"],
+                "remove_custom_fields",
+            )
+        else:
+            raise serializers.ValidationError("remove_custom_fields not specified")
+
+    def _validate_owner(self, owner) -> User:
+        owner_field = serializers.PrimaryKeyRelatedField(queryset=User.objects.all())
+        try:
+            return owner_field.run_validation(owner)
+        except serializers.ValidationError as e:
+            raise serializers.ValidationError(
+                "Specified owner cannot be found",
+            ) from e
+
+    def _validate_parameters_set_permissions(self, parameters) -> None:
+        if "set_permissions" not in parameters:
+            raise serializers.ValidationError("set_permissions not specified")
+        set_permissions = parameters["set_permissions"]
+        if set_permissions is not None:
+            set_permissions = SetPermissionsSerializer().run_validation(
+                set_permissions,
+            )
+        parameters["set_permissions"] = self.validate_set_permissions(
+            set_permissions,
+        )
+        if "owner" in parameters and parameters["owner"] is not None:
+            parameters["owner"] = self._validate_owner(parameters["owner"]).pk
+        if "merge" not in parameters:
+            parameters["merge"] = False
+
+    def _validate_parameters_rotate(self, parameters) -> None:
+        if "degrees" not in parameters:
+            raise serializers.ValidationError("invalid rotation degrees")
+        try:
+            degrees = serializers.IntegerField().run_validation(parameters["degrees"])
+        except serializers.ValidationError as e:
+            raise serializers.ValidationError("invalid rotation degrees") from e
+        parameters["degrees"] = _validate_rotation_degrees(degrees)
+
+    def _validate_source_mode(self, parameters) -> None:
+        source_mode = parameters.get(
+            "source_mode",
+            bulk_edit.SourceModeChoices.LATEST_VERSION,
+        )
+        parameters["source_mode"] = self.validate_source_mode(source_mode)
+
+    def _validate_parameters_split(self, parameters, document_id) -> None:
+        if "pages" not in parameters:
+            raise serializers.ValidationError("pages not specified")
+        if not isinstance(parameters["pages"], str):
+            raise serializers.ValidationError("invalid pages specified")
+        page_count = Document.objects.get(id=document_id).page_count
+        if not page_count:
+            raise serializers.ValidationError("document page count is unknown")
+        pages = []
+        for group in parameters["pages"].split(","):
+            start, is_range, end = group.partition("-")
+            try:
+                first = int(start)
+                last = int(end) if is_range else first
+            except ValueError as e:
+                raise serializers.ValidationError("invalid pages specified") from e
+            # Bound the range before building it, a huge one would exhaust memory
+            if not 1 <= first <= last <= page_count:
+                raise serializers.ValidationError("invalid pages specified")
+            pages.append(list(range(first, last + 1)))
+        parameters["pages"] = pages
+
+        if "delete_originals" in parameters:
+            if not isinstance(parameters["delete_originals"], bool):
+                raise serializers.ValidationError("delete_originals must be a boolean")
+        else:
+            parameters["delete_originals"] = False
+
+    def _validate_parameters_delete_pages(self, parameters) -> None:
+        if "pages" not in parameters:
+            raise serializers.ValidationError("pages not specified")
+        if not isinstance(parameters["pages"], list):
+            raise serializers.ValidationError("pages must be a list")
+        if not all(isinstance(i, int) for i in parameters["pages"]):
+            raise serializers.ValidationError("pages must be a list of integers")
+
+    def _validate_parameters_merge(self, parameters) -> None:
+        if "delete_originals" in parameters:
+            if not isinstance(parameters["delete_originals"], bool):
+                raise serializers.ValidationError("delete_originals must be a boolean")
+        else:
+            parameters["delete_originals"] = False
+        if "archive_fallback" in parameters:
+            if not isinstance(parameters["archive_fallback"], bool):
+                raise serializers.ValidationError("archive_fallback must be a boolean")
+        else:
+            parameters["archive_fallback"] = False
+
+    def _validate_parameters_edit_pdf(self, parameters, document_id) -> None:
+        if "operations" not in parameters:
+            raise serializers.ValidationError("operations not specified")
+        operations_field = serializers.ListField(
+            child=PdfEditOperationSerializer(),
+            allow_empty=False,
+        )
+        try:
+            operations = operations_field.run_validation(parameters["operations"])
+        except serializers.ValidationError as e:
+            # Key the errors under "operations" so they match what the
+            # dedicated edit_pdf endpoint returns
+            raise serializers.ValidationError({"operations": e.detail}) from e
+        parameters["operations"] = operations
+
+        if "update_document" in parameters:
+            if not isinstance(parameters["update_document"], bool):
+                raise serializers.ValidationError("update_document must be a boolean")
+        else:
+            parameters["update_document"] = False
+        if "include_metadata" in parameters:
+            if not isinstance(parameters["include_metadata"], bool):
+                raise serializers.ValidationError("include_metadata must be a boolean")
+        else:
+            parameters["include_metadata"] = True
+
+        if any(op.get("doc", 0) >= len(operations) for op in operations):
+            raise serializers.ValidationError("doc index is out of bounds")
+
+        if parameters["update_document"]:
+            max_idx = max(op.get("doc", 0) for op in operations)
+            if max_idx > 0:
+                raise serializers.ValidationError(
+                    "update_document only allowed with a single output document",
+                )
+
+        doc = Document.objects.get(id=document_id)
+        # doc existence is already validated
+        if doc.page_count:
+            for op in operations:
+                if op["page"] > doc.page_count:
+                    raise serializers.ValidationError(
+                        f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.",
+                    )
+
+    def _validate_parameters_reprocess(self, parameters) -> None:
+        if "remote_ocr" in parameters:
+            if not isinstance(parameters["remote_ocr"], bool):
+                raise serializers.ValidationError("remote_ocr must be a boolean")
+        else:
+            parameters["remote_ocr"] = False
+
+    def validate_parameters_remove_password(self, parameters):
+        if "password" not in parameters:
+            raise serializers.ValidationError("password not specified")
+        if not isinstance(parameters["password"], str):
+            raise serializers.ValidationError("password must be a string")
+
+    def validate(self, attrs):
+        attrs = super().validate(attrs)
+
+        if attrs.get("all", False) and attrs["method"] in [
+            bulk_edit.merge,
+            bulk_edit.split,
+            bulk_edit.delete_pages,
+            bulk_edit.edit_pdf,
+            bulk_edit.remove_password,
+        ]:
+            raise serializers.ValidationError(
+                "This method does not support all=true.",
+            )
+
+        method = attrs["method"]
+        parameters = attrs["parameters"]
+
+        if "source_mode" in parameters:
+            self._validate_source_mode(parameters)
+
+        if method == bulk_edit.set_correspondent:
+            self._validate_parameters_correspondent(parameters)
+        elif method == bulk_edit.set_document_type:
+            self._validate_parameters_document_type(parameters)
+        elif method == bulk_edit.add_tag or method == bulk_edit.remove_tag:
+            self._validate_parameters_tags(parameters)
+        elif method == bulk_edit.modify_tags:
+            self._validate_parameters_modify_tags(parameters)
+        elif method == bulk_edit.set_storage_path:
+            self._validate_storage_path(parameters)
+        elif method == bulk_edit.modify_custom_fields:
+            self._validate_parameters_modify_custom_fields(parameters)
+        elif method == bulk_edit.set_permissions:
+            self._validate_parameters_set_permissions(parameters)
+        elif method == bulk_edit.rotate:
+            self._validate_parameters_rotate(parameters)
+        elif method == bulk_edit.split:
+            if len(attrs["documents"]) > 1:
+                raise serializers.ValidationError(
+                    "Split method only supports one document",
+                )
+            self._validate_parameters_split(parameters, attrs["documents"][0])
+        elif method == bulk_edit.delete_pages:
+            if len(attrs["documents"]) > 1:
+                raise serializers.ValidationError(
+                    "Delete pages method only supports one document",
+                )
+            self._validate_parameters_delete_pages(parameters)
+        elif method == bulk_edit.merge:
+            self._validate_parameters_merge(parameters)
+        elif method == bulk_edit.edit_pdf:
+            if len(attrs["documents"]) > 1:
+                raise serializers.ValidationError(
+                    "Edit PDF method only supports one document",
+                )
+            self._validate_parameters_edit_pdf(parameters, attrs["documents"][0])
+        elif method == bulk_edit.remove_password:
+            self.validate_parameters_remove_password(parameters)
+        elif method == bulk_edit.reprocess:
+            self._validate_parameters_reprocess(parameters)
+
+        return attrs
+
+
+class BulkDownloadSerializer(DocumentSelectionSerializer):
+    content = serializers.ChoiceField(
+        choices=["archive", "originals", "both"],
+        default="archive",
+    )
+
+    compression = serializers.ChoiceField(
+        choices=["none", "deflated", "bzip2", "lzma"],
+        default="none",
+    )
+
+    follow_formatting = serializers.BooleanField(
+        default=False,
+    )
+
+    def validate_compression(self, compression):
+        import zipfile
+
+        return {
+            "none": zipfile.ZIP_STORED,
+            "deflated": zipfile.ZIP_DEFLATED,
+            "bzip2": zipfile.ZIP_BZIP2,
+            "lzma": zipfile.ZIP_LZMA,
+        }[compression]
+
+
+class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
+    objects = serializers.ListField(
+        required=False,
+        allow_empty=True,
+        label="Objects",
+        write_only=True,
+        child=serializers.IntegerField(),
+    )
+
+    all = serializers.BooleanField(
+        default=False,
+        required=False,
+        write_only=True,
+    )
+
+    filters = serializers.DictField(
+        required=False,
+        allow_empty=True,
+        write_only=True,
+    )
+
+    object_type = serializers.ChoiceField(
+        choices=[
+            "tags",
+            "correspondents",
+            "document_types",
+            "storage_paths",
+        ],
+        label="Object Type",
+        write_only=True,
+    )
+
+    operation = serializers.ChoiceField(
+        choices=[
+            "set_permissions",
+            "delete",
+        ],
+        label="Operation",
+        required=True,
+        write_only=True,
+    )
+
+    owner = serializers.PrimaryKeyRelatedField(
+        queryset=User.objects.all(),
+        required=False,
+        allow_null=True,
+    )
+
+    permissions = SetPermissionsSerializer(
+        label="Set permissions",
+        required=False,
+        write_only=True,
+    )
+
+    merge = serializers.BooleanField(
+        default=False,
+        write_only=True,
+        required=False,
+    )
+
+    def get_object_class(self, object_type):
+        object_class = None
+        if object_type == "tags":
+            object_class = Tag
+        elif object_type == "correspondents":
+            object_class = Correspondent
+        elif object_type == "document_types":
+            object_class = DocumentType
+        elif object_type == "storage_paths":
+            object_class = StoragePath
+        return object_class
+
+    def _validate_objects(self, objects, object_type):
+        if not isinstance(objects, list):
+            raise serializers.ValidationError("objects must be a list")
+        if not all(isinstance(i, int) for i in objects):
+            raise serializers.ValidationError("objects must be a list of integers")
+        object_class = self.get_object_class(object_type)
+        count = object_class.objects.filter(id__in=objects).count()
+        if not count == len(objects):
+            raise serializers.ValidationError(
+                "Some ids in objects don't exist or were specified twice.",
+            )
+        return objects
+
+    def _validate_permissions(self, permissions) -> dict:
+        return self.validate_set_permissions(
+            permissions,
+        )
+
+    def validate(self, attrs):
+        object_type = attrs["object_type"]
+        objects = attrs.get("objects")
+        apply_to_all = attrs.get("all", False)
+        operation = attrs.get("operation")
+
+        if apply_to_all:
+            attrs.setdefault("objects", [])
+        else:
+            if objects is None:
+                raise serializers.ValidationError(
+                    "objects is required unless all is true.",
+                )
+            if len(objects) == 0:
+                raise serializers.ValidationError("objects must not be empty")
+            self._validate_objects(objects, object_type)
+
+        if operation == "set_permissions":
+            permissions = attrs.get("permissions")
+            if permissions is not None:
+                if not permissions:
+                    raise serializers.ValidationError(
+                        "permissions must not be empty",
+                    )
+                attrs["permissions"] = self._validate_permissions(permissions)
+
+        return attrs

src/documents/serialisers/documents.py

@@ -0,0 +1,484 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+from typing import Any
+from typing import TypedDict
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.db.models import Q
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import get_current_timezone
+from django.utils.timezone import is_naive
+from django.utils.timezone import make_aware
+from drf_spectacular.utils import extend_schema_field
+from drf_spectacular.utils import extend_schema_serializer
+from drf_writable_nested.serializers import NestedUpdateMixin
+from rest_framework import serializers
+from rest_framework.fields import SerializerMethodField
+
+if settings.AUDIT_LOG_ENABLED:
+    from auditlog.context import set_actor
+
+
+from documents import bulk_edit
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import Document
+from documents.models import Tag
+from documents.permissions import permitted_document_ids
+from documents.versioning import has_prefetched_effective_content
+from documents.versioning import sort_versions_newest_first
+
+from .base import DocumentUpdateFieldsModelSerializer
+from .base import NotesSerializer
+from .base import OwnedObjectListSerializer
+from .base import OwnedObjectSerializer
+from .metadata import CorrespondentField
+from .metadata import CustomFieldInstanceSerializer
+from .metadata import DocumentTypeField
+from .metadata import StoragePathField
+from .metadata import TagsField
+from .upload import PostDocumentSerializer
+
+if TYPE_CHECKING:
+    from datetime import datetime
+
+    from django.db.models.query import QuerySet
+    from rest_framework.relations import ManyRelatedField
+    from rest_framework.relations import RelatedField
+
+
+logger = logging.getLogger("paperless.serializers")
+
+
+def _get_viewable_duplicates(
+    document: Document,
+    user: User | None,
+) -> QuerySet[Document]:
+    checksums = {document.checksum}
+    if document.archive_checksum:
+        checksums.add(document.archive_checksum)
+    duplicates = Document.global_objects.filter(
+        Q(checksum__in=checksums) | Q(archive_checksum__in=checksums),
+    ).exclude(pk=document.pk)
+    duplicates = duplicates.filter(root_document__isnull=True)
+    duplicates = duplicates.order_by("-created")
+    allowed_ids = permitted_document_ids(user, include_deleted=True)
+    return duplicates.filter(id__in=allowed_ids)
+
+
+class DuplicateDocumentSummarySerializer(serializers.Serializer[dict[str, Any]]):
+    id = serializers.IntegerField()
+    title = serializers.CharField()
+    deleted_at = serializers.DateTimeField(allow_null=True)
+
+
+class _DocumentVersionInfo(TypedDict):
+    id: int
+    added: datetime
+    version_label: str | None
+    checksum: str | None
+    is_root: bool
+
+
+class DocumentVersionInfoSerializer(serializers.Serializer[_DocumentVersionInfo]):
+    id = serializers.IntegerField()
+    added = serializers.DateTimeField()
+    version_label = serializers.CharField(required=False, allow_null=True)
+    checksum = serializers.CharField(required=False, allow_null=True)
+    is_root = serializers.BooleanField()
+
+
+@extend_schema_serializer(
+    deprecate_fields=["created_date"],
+)
+class DocumentSerializer(
+    OwnedObjectSerializer,
+    NestedUpdateMixin,
+    DocumentUpdateFieldsModelSerializer,
+):
+    correspondent = CorrespondentField(allow_null=True)
+    tags = TagsField(many=True)
+    document_type = DocumentTypeField(allow_null=True)
+    storage_path = StoragePathField(allow_null=True)
+
+    original_file_name = SerializerMethodField()
+    archived_file_name = SerializerMethodField()
+    created_date = serializers.DateField(required=False)
+    page_count = SerializerMethodField()
+    duplicate_documents = SerializerMethodField()
+
+    notes = NotesSerializer(many=True, required=False, read_only=True)
+    root_document: RelatedField[Document, Document, Any] | ManyRelatedField = (
+        serializers.PrimaryKeyRelatedField(read_only=True)
+    )
+    versions = SerializerMethodField()
+
+    custom_fields = CustomFieldInstanceSerializer(
+        many=True,
+        allow_null=False,
+        required=False,
+    )
+
+    owner = serializers.PrimaryKeyRelatedField(
+        queryset=User.objects.all(),
+        required=False,
+        allow_null=True,
+    )
+
+    remove_inbox_tags = serializers.BooleanField(
+        default=False,
+        write_only=True,
+        allow_null=True,
+        required=False,
+    )
+
+    def get_page_count(self, obj) -> int | None:
+        # Like content versions get their own page count from the newest version,
+        # use the prefetched versions cache to avoid an extra query
+        prefetched_cache = getattr(obj, "_prefetched_objects_cache", None)
+        prefetched_versions = (
+            prefetched_cache.get("versions")
+            if isinstance(prefetched_cache, dict)
+            else None
+        )
+        if obj.root_document_id is None and prefetched_versions:
+            return sort_versions_newest_first(prefetched_versions)[0].page_count
+        return obj.page_count
+
+    @extend_schema_field(DuplicateDocumentSummarySerializer(many=True))
+    def get_duplicate_documents(self, obj):
+        view = self.context.get("view")
+        if view and getattr(view, "action", None) != "retrieve":
+            return []
+        request = self.context.get("request")
+        user = request.user if request else None
+        duplicates = _get_viewable_duplicates(obj, user)
+        return list(duplicates.values("id", "title", "deleted_at"))
+
+    @extend_schema_field(DocumentVersionInfoSerializer(many=True))
+    def get_versions(self, obj):
+        root_doc = obj if obj.root_document_id is None else obj.root_document
+        if root_doc is None:
+            return []
+
+        prefetched_cache = getattr(obj, "_prefetched_objects_cache", None)
+        prefetched_versions = (
+            prefetched_cache.get("versions")
+            if isinstance(prefetched_cache, dict)
+            else None
+        )
+
+        versions: list[Document]
+        if prefetched_versions is not None:
+            versions = [*prefetched_versions, root_doc]
+        else:
+            versions_qs = Document.objects.filter(root_document=root_doc).only(
+                "id",
+                "added",
+                "checksum",
+                "version_label",
+                "root_document_id",
+                "version_index",
+            )
+            versions = [*versions_qs, root_doc]
+
+        versions = sort_versions_newest_first(versions)
+
+        def build_info(doc: Document) -> _DocumentVersionInfo:
+            return {
+                "id": doc.id,
+                "added": doc.added,
+                "version_label": doc.version_label,
+                "checksum": doc.checksum,
+                "is_root": doc.id == root_doc.id,
+            }
+
+        return [build_info(doc) for doc in versions]
+
+    def get_original_file_name(self, obj) -> str | None:
+        return obj.original_filename
+
+    def get_archived_file_name(self, obj) -> str | None:
+        if obj.has_archive_version:
+            return obj.get_public_filename(archive=True)
+        else:
+            return None
+
+    def to_representation(self, instance):
+        doc = super().to_representation(instance)
+        if "content" in self.fields and has_prefetched_effective_content(instance):
+            # Only resolve version-aware content when it's cheap: an SQL
+            # annotation or a versions prefetch is already on the instance.
+            # A caller that set up neither (e.g. TrashView, GlobalSearchView,
+            # which build their own querysets) gets the document's own,
+            # unresolved content instead of paying for an extra per-instance
+            # query -- same as before effective_content resolution existed.
+            doc["content"] = instance.get_effective_content() or ""
+        if self.truncate_content and "content" in self.fields:
+            doc["content"] = doc.get("content")[0:550]
+        return doc
+
+    def to_internal_value(self, data):
+        if (
+            "created" in data
+            and isinstance(data["created"], str)
+            and ":" in data["created"]
+        ):
+            # Handle old format of isoformat datetime string
+            parsed = parse_datetime(data["created"])
+            if parsed:
+                if is_naive(parsed):
+                    parsed = make_aware(parsed, get_current_timezone())
+                data["created"] = parsed.astimezone().date()
+        return super().to_internal_value(data)
+
+    def validate(self, attrs):
+        if (
+            "archive_serial_number" in attrs
+            and attrs["archive_serial_number"] is not None
+            and len(str(attrs["archive_serial_number"])) > 0
+            and Document.deleted_objects.filter(
+                archive_serial_number=attrs["archive_serial_number"],
+            ).exists()
+        ):
+            raise serializers.ValidationError(
+                {
+                    "archive_serial_number": [
+                        "Document with this Archive Serial Number already exists in the trash.",
+                    ],
+                },
+            )
+        return super().validate(attrs)
+
+    def update(self, instance: Document, validated_data):
+        if "created_date" in validated_data:
+            if "created" not in validated_data:
+                validated_data["created"] = validated_data["created_date"]
+            logger.warning(
+                "created_date is deprecated, use created instead",
+            )
+            validated_data.pop("created_date")
+        if instance.custom_fields.count() > 0 and "custom_fields" in validated_data:
+            incoming_custom_fields = [
+                field["field"] for field in validated_data["custom_fields"]
+            ]
+            for custom_field_instance in instance.custom_fields.filter(
+                field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
+            ):
+                if (
+                    custom_field_instance.field not in incoming_custom_fields
+                    and custom_field_instance.value is not None
+                ):
+                    # Doc link field is being removed entirely
+                    for doc_id in custom_field_instance.value:
+                        bulk_edit.remove_doclink(
+                            instance,
+                            custom_field_instance.field,
+                            doc_id,
+                        )
+        if "tags" in validated_data:
+            # Respect tag hierarchy on updates:
+            # - Adding a child adds its ancestors
+            # - Removing a parent removes all its descendants
+            prev_tags = set(instance.tags.all())
+            requested_tags = set(validated_data["tags"])
+
+            # Tags newly added in this update and the ancestors they require
+            added_tags = requested_tags - prev_tags
+            required_by_add_tags = set(added_tags)
+            for t in added_tags:
+                required_by_add_tags.update(t.get_ancestors())
+
+            # Tags being removed in this update and all descendants, except
+            # those required by a tag that is being added in this same update
+            removed_tags = prev_tags - requested_tags
+            blocked_tags = set(removed_tags)
+            for t in removed_tags:
+                blocked_tags.update(t.get_descendants())
+            blocked_tags.difference_update(required_by_add_tags)
+
+            # Add all parent tags
+            final_tags = set(requested_tags)
+            for t in requested_tags:
+                final_tags.update(t.get_ancestors())
+
+            # Drop removed parents and their descendants
+            final_tags.difference_update(blocked_tags)
+
+            validated_data["tags"] = list(final_tags)
+        if validated_data.get("remove_inbox_tags"):
+            current_tag_ids = {t.pk for t in instance.tags.all()}
+            tags = (
+                validated_data["tags"]
+                if "tags" in validated_data
+                else list(instance.tags.all())
+            )
+
+            # Tags newly added in this update, plus their ancestors, are kept
+            keep_ids: set[int] = set()
+            for tag in tags:
+                if tag.pk not in current_tag_ids:
+                    keep_ids.add(tag.pk)
+                    keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
+
+            # Remove inbox tags and their descendants, except those being kept
+            remove_ids: set[int] = set()
+            for inbox_tag in (
+                Tag.objects.filter(is_inbox_tag=True)
+                .exclude(pk__in=keep_ids)
+                .only("pk", "tn_descendants_pks")
+            ):
+                remove_ids.add(inbox_tag.pk)
+                remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
+
+            validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
+
+        if settings.AUDIT_LOG_ENABLED:
+            with set_actor(self.user):
+                super().update(instance, validated_data)
+        else:
+            super().update(instance, validated_data)
+
+        # hard delete custom field instances that were soft deleted
+        CustomFieldInstance.deleted_objects.filter(document=instance).delete()
+        return instance
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.truncate_content = kwargs.pop("truncate_content", False)
+
+        # return full permissions if we're doing a PATCH or PUT
+        context = kwargs.get("context")
+        if context is not None and (
+            context.get("request").method == "PATCH"
+            or context.get("request").method == "PUT"
+        ):
+            kwargs["full_perms"] = True
+
+        super().__init__(*args, **kwargs)
+
+    class Meta:
+        model = Document
+        fields = (
+            "id",
+            "correspondent",
+            "document_type",
+            "storage_path",
+            "title",
+            "content",
+            "tags",
+            "created",
+            "created_date",
+            "modified",
+            "added",
+            "deleted_at",
+            "archive_serial_number",
+            "original_file_name",
+            "archived_file_name",
+            "duplicate_documents",
+            "owner",
+            "permissions",
+            "user_can_change",
+            "is_shared_by_requester",
+            "set_permissions",
+            "notes",
+            "custom_fields",
+            "remove_inbox_tags",
+            "page_count",
+            "mime_type",
+            "root_document",
+            "versions",
+        )
+        read_only_fields = ("deleted_at",)
+        list_serializer_class = OwnedObjectListSerializer
+
+
+class SearchResultListSerializer(serializers.ListSerializer[Document]):
+    def to_representation(self, hits):
+        document_ids = [hit["id"] for hit in hits]
+        # Fetch all Document objects in the list in one SQL query.
+        documents = self.child.fetch_documents(document_ids)
+        self.child.context["documents"] = documents
+        # Also check if they are shared with other users / groups.
+        self.child.context["shared_object_pks"] = self.child.get_shared_object_pks(
+            documents.values(),
+        )
+
+        return super().to_representation(hits)
+
+
+class SearchResultSerializer(DocumentSerializer):
+    @staticmethod
+    def fetch_documents(ids):
+        """
+        Return a dict that maps given document IDs to Document objects.
+        """
+        return {
+            document.id: document
+            for document in Document.objects.select_related(
+                "correspondent",
+                "storage_path",
+                "document_type",
+                "owner",
+            )
+            .prefetch_related("tags", "custom_fields", "notes")
+            .filter(id__in=ids)
+        }
+
+    def to_representation(self, hit):
+        # Again we first check if the parent has already fetched the documents.
+        documents = self.context.get("documents")
+        # Otherwise we fetch this document.
+        if documents is None:  # pragma: no cover
+            # In practice we only serialize **lists** of SearchHit dicts.
+            # Keeping this check for completeness but marking it no cover for now.
+            documents = self.fetch_documents([hit["id"]])
+        document = documents[hit["id"]]
+
+        highlights = hit.get("highlights", {})
+        r = super().to_representation(document)
+        r["__search_hit__"] = {
+            "score": hit["score"],
+            "highlights": highlights.get("content", ""),
+            "note_highlights": highlights.get("notes") or None,
+            "rank": hit["rank"],
+        }
+
+        return r
+
+    class Meta(DocumentSerializer.Meta):
+        list_serializer_class = SearchResultListSerializer
+
+
+class DocumentVersionSerializer(serializers.Serializer[dict[str, Any]]):
+    document = serializers.FileField(
+        label="Document",
+        write_only=True,
+    )
+    version_label = serializers.CharField(
+        label="Version label",
+        required=False,
+        allow_blank=True,
+        allow_null=True,
+        max_length=64,
+    )
+
+    validate_document = PostDocumentSerializer().validate_document
+
+
+class DocumentVersionLabelSerializer(serializers.Serializer[dict[str, str | None]]):
+    version_label = serializers.CharField(
+        label="Version label",
+        required=True,
+        allow_blank=True,
+        allow_null=True,
+        max_length=64,
+    )
+
+    def validate_version_label(self, value):
+        if value is None:
+            return None
+        normalized = value.strip()
+        return normalized or None

src/documents/serialisers/metadata.py

@@ -0,0 +1,557 @@
+from __future__ import annotations
+
+import logging
+import math
+import re
+from decimal import Decimal
+from typing import Any
+
+from django.core.exceptions import ValidationError
+from django.core.validators import DecimalValidator
+from django.core.validators import MaxLengthValidator
+from django.core.validators import MaxValueValidator
+from django.core.validators import MinValueValidator
+from django.core.validators import RegexValidator
+from django.core.validators import integer_validator
+from django.db.models import Count
+from django.db.models.functions import Lower
+from django.utils.crypto import get_random_string
+from django.utils.translation import gettext as _
+from drf_spectacular.utils import extend_schema_field
+from rest_framework import serializers
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.filters import OrderingFilter
+
+from documents import bulk_edit
+from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import Document
+from documents.models import DocumentType
+from documents.models import PaperlessTask
+from documents.models import StoragePath
+from documents.models import Tag
+from documents.permissions import get_document_count_filter_for_user
+from documents.permissions import permitted_document_ids
+from documents.permissions import restrict_queryset_to_visible
+from documents.templating.filepath import validate_filepath_template_and_render
+from documents.templating.utils import convert_format_str_to_template_format
+from documents.validators import uri_validator
+
+from .base import MatchingModelSerializer
+from .base import OwnedObjectSerializer
+from .base import ReadWriteSerializerMethodField
+from .base import SerializerWithPerms
+
+logger = logging.getLogger("paperless.serializers")
+
+
+class CorrespondentSerializer(MatchingModelSerializer, OwnedObjectSerializer):
+    last_correspondence = serializers.DateField(read_only=True, required=False)
+
+    class Meta:
+        model = Correspondent
+        fields = (
+            "id",
+            "slug",
+            "name",
+            "match",
+            "matching_algorithm",
+            "is_insensitive",
+            "document_count",
+            "last_correspondence",
+            "owner",
+            "permissions",
+            "user_can_change",
+            "set_permissions",
+        )
+
+
+class DocumentTypeSerializer(MatchingModelSerializer, OwnedObjectSerializer):
+    class Meta:
+        model = DocumentType
+        fields = (
+            "id",
+            "slug",
+            "name",
+            "match",
+            "matching_algorithm",
+            "is_insensitive",
+            "document_count",
+            "owner",
+            "permissions",
+            "user_can_change",
+            "set_permissions",
+        )
+
+
+class DeprecatedColors:
+    COLOURS = (
+        (1, "#a6cee3"),
+        (2, "#1f78b4"),
+        (3, "#b2df8a"),
+        (4, "#33a02c"),
+        (5, "#fb9a99"),
+        (6, "#e31a1c"),
+        (7, "#fdbf6f"),
+        (8, "#ff7f00"),
+        (9, "#cab2d6"),
+        (10, "#6a3d9a"),
+        (11, "#b15928"),
+        (12, "#000000"),
+        (13, "#cccccc"),
+    )
+
+
+@extend_schema_field(
+    serializers.ChoiceField(
+        choices=DeprecatedColors.COLOURS,
+    ),
+)
+class ColorField(serializers.Field):
+    def to_internal_value(self, data):
+        for id, color in DeprecatedColors.COLOURS:
+            if id == data:
+                return color
+        raise serializers.ValidationError
+
+    def to_representation(self, value):
+        for id, color in DeprecatedColors.COLOURS:
+            if color == value:
+                return id
+        return 1
+
+
+class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
+    def get_text_color(self, obj) -> str:
+        try:
+            h = obj.color.lstrip("#")
+            rgb = tuple(int(h[i : i + 2], 16) / 256 for i in (0, 2, 4))
+            luminance = math.sqrt(
+                0.299 * math.pow(rgb[0], 2)
+                + 0.587 * math.pow(rgb[1], 2)
+                + 0.114 * math.pow(rgb[2], 2),
+            )
+            return "#ffffff" if luminance < 0.53 else "#000000"
+        except ValueError:
+            return "#000000"
+
+    text_color = serializers.SerializerMethodField()
+
+    # map to treenode's tn_parent
+    parent = serializers.PrimaryKeyRelatedField(
+        queryset=Tag.objects.all(),
+        allow_null=True,
+        required=False,
+        source="tn_parent",
+    )
+
+    @extend_schema_field(
+        field=serializers.ListSerializer(
+            child=serializers.PrimaryKeyRelatedField(
+                queryset=Tag.objects.all(),
+            ),
+        ),
+    )
+    def get_children(self, obj):
+        children_map = self.context.get("children_map")
+        if children_map is not None:
+            children = children_map.get(obj.pk, [])
+        else:
+            filter_q = self.context.get("document_count_filter")
+            request = self.context.get("request")
+            if filter_q is None:
+                user = getattr(request, "user", None) if request else None
+                filter_q = get_document_count_filter_for_user(user)
+                self.context["document_count_filter"] = filter_q
+
+            children = (
+                obj.get_children_queryset()
+                .select_related("owner")
+                .annotate(document_count=Count("documents", filter=filter_q))
+            )
+            user = getattr(request, "user", None) if request else self.user
+            children = restrict_queryset_to_visible(children, user, "view_tag")
+
+            view = self.context.get("view")
+            ordering = (
+                OrderingFilter().get_ordering(request, children, view)
+                if request and view
+                else None
+            )
+            ordering = ordering or (Lower("name"),)
+            children = children.order_by(*ordering)
+
+        if not children:
+            return []
+
+        serializer = TagSerializer(
+            children,
+            many=True,
+            user=self.user,
+            full_perms=self.full_perms,
+            all_fields=self.all_fields,
+            context=self.context,
+        )
+        return serializer.data
+
+    # children as nested Tag objects
+    children = serializers.SerializerMethodField()
+
+    class Meta:
+        model = Tag
+        fields = (
+            "id",
+            "slug",
+            "name",
+            "color",
+            "text_color",
+            "match",
+            "matching_algorithm",
+            "is_insensitive",
+            "is_inbox_tag",
+            "document_count",
+            "owner",
+            "permissions",
+            "user_can_change",
+            "set_permissions",
+            "parent",
+            "children",
+        )
+
+    def validate_color(self, color):
+        regex = r"#[0-9a-fA-F]{6}"
+        if not re.match(regex, color):
+            raise serializers.ValidationError(_("Invalid color."))
+        return color
+
+    def validate(self, attrs):
+        # Validate when changing parent
+        parent = attrs.get(
+            "tn_parent",
+            self.instance.get_parent() if self.instance else None,
+        )
+
+        if self.instance:
+            # Temporarily set parent on the instance if updating and use model clean()
+            original_parent = self.instance.get_parent()
+            try:
+                # Temporarily set tn_parent in-memory to validate clean()
+                self.instance.tn_parent = parent
+                self.instance.clean()
+            except ValidationError as e:
+                logger.debug("Tag parent validation failed: %s", e)
+                raise e
+            finally:
+                self.instance.tn_parent = original_parent
+        else:
+            # For new instances, create a transient Tag and validate
+            temp = Tag(tn_parent=parent)
+            try:
+                temp.clean()
+            except ValidationError as e:
+                logger.debug("Tag parent validation failed: %s", e)
+                raise e
+
+        return super().validate(attrs)
+
+
+class CorrespondentField(serializers.PrimaryKeyRelatedField[Correspondent]):
+    def get_queryset(self):
+        return Correspondent.objects.all()
+
+
+class TagsField(serializers.PrimaryKeyRelatedField[Tag]):
+    def get_queryset(self):
+        return Tag.objects.all()
+
+
+class DocumentTypeField(serializers.PrimaryKeyRelatedField[DocumentType]):
+    def get_queryset(self):
+        return DocumentType.objects.all()
+
+
+class StoragePathField(serializers.PrimaryKeyRelatedField[StoragePath]):
+    def get_queryset(self):
+        return StoragePath.objects.all()
+
+
+class CustomFieldSerializer(serializers.ModelSerializer[CustomField]):
+    data_type = serializers.ChoiceField(
+        choices=CustomField.FieldDataType,
+        read_only=False,
+    )
+
+    document_count = serializers.IntegerField(read_only=True)
+
+    class Meta:
+        model = CustomField
+        fields = [
+            "id",
+            "name",
+            "data_type",
+            "extra_data",
+            "document_count",
+        ]
+
+    def validate(self, attrs):
+        # TODO: remove pending https://github.com/encode/django-rest-framework/issues/7173
+        name = attrs.get(
+            "name",
+            self.instance.name if hasattr(self.instance, "name") else None,
+        )
+        objects = (
+            self.Meta.model.objects.exclude(
+                pk=self.instance.pk,
+            )
+            if self.instance is not None
+            else self.Meta.model.objects.all()
+        )
+        if ("name" in attrs) and objects.filter(
+            name=name,
+        ).exists():
+            raise serializers.ValidationError(
+                {"error": "Object violates name unique constraint"},
+            )
+        if (
+            "data_type" in attrs
+            and attrs["data_type"] == CustomField.FieldDataType.SELECT
+        ) or (
+            self.instance
+            and self.instance.data_type == CustomField.FieldDataType.SELECT
+        ):
+            if (
+                "extra_data" not in attrs
+                or "select_options" not in attrs["extra_data"]
+                or not isinstance(attrs["extra_data"]["select_options"], list)
+                or len(attrs["extra_data"]["select_options"]) == 0
+                or not all(
+                    len(option.get("label", "")) > 0
+                    for option in attrs["extra_data"]["select_options"]
+                )
+            ):
+                raise serializers.ValidationError(
+                    {"error": "extra_data.select_options must be a valid list"},
+                )
+            # labels are valid, generate ids if not present
+            for option in attrs["extra_data"]["select_options"]:
+                if option.get("id") is None:
+                    option["id"] = get_random_string(length=16)
+        elif (
+            "data_type" in attrs
+            and attrs["data_type"] == CustomField.FieldDataType.MONETARY
+            and "extra_data" in attrs
+            and "default_currency" in attrs["extra_data"]
+            and attrs["extra_data"]["default_currency"] is not None
+            and (
+                not isinstance(attrs["extra_data"]["default_currency"], str)
+                or (
+                    len(attrs["extra_data"]["default_currency"]) > 0
+                    and len(attrs["extra_data"]["default_currency"]) != 3
+                )
+            )
+        ):
+            raise serializers.ValidationError(
+                {"error": "extra_data.default_currency must be a 3-character string"},
+            )
+        return super().validate(attrs)
+
+
+def validate_documentlink_targets(user, doc_ids):
+    if Document.objects.filter(id__in=doc_ids).count() != len(doc_ids):
+        raise serializers.ValidationError(
+            "Some documents in value don't exist or were specified twice.",
+        )
+
+    if user is None:
+        return
+
+    if (
+        Document.objects.filter(id__in=doc_ids)
+        .exclude(id__in=permitted_document_ids(user, perm="change_document"))
+        .exists()
+    ):
+        raise PermissionDenied(
+            _("Insufficient permissions."),
+        )
+
+
+class CustomFieldInstanceSerializer(serializers.ModelSerializer[CustomFieldInstance]):
+    field = serializers.PrimaryKeyRelatedField(queryset=CustomField.objects.all())
+    value = ReadWriteSerializerMethodField(allow_null=True)
+
+    def create(self, validated_data):
+        # An instance is attached to a document
+        document: Document = validated_data["document"]
+        # And to a CustomField
+        custom_field: CustomField = validated_data["field"]
+        # This key must exist, as it is validated
+        data_store_name = CustomFieldInstance.get_value_field_name(
+            custom_field.data_type,
+        )
+
+        if custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
+            # prior to update so we can look for any docs that are going to be removed
+            bulk_edit.reflect_doclinks(document, custom_field, validated_data["value"])
+
+        # Actually update or create the instance, providing the value
+        # to fill in the correct attribute based on the type
+        instance, _ = CustomFieldInstance.objects.update_or_create(
+            document=document,
+            field=custom_field,
+            defaults={data_store_name: validated_data["value"]},
+        )
+        return instance
+
+    def get_value(self, obj: CustomFieldInstance) -> str | int | float | dict | None:
+        return obj.value
+
+    def validate(self, data):
+        """
+        Probably because we're kind of doing it odd, validation from the model
+        doesn't run against the field "value", so we have to re-create it here.
+
+        Don't like it, but it is better than returning an HTTP 500 when the database
+        hates the value
+        """
+        data = super().validate(data)
+        field: CustomField = data["field"]
+        if "value" in data and data["value"] is not None:
+            if (
+                field.data_type == CustomField.FieldDataType.URL
+                and len(data["value"]) > 0
+            ):
+                uri_validator(data["value"])
+            elif field.data_type == CustomField.FieldDataType.INT:
+                integer_validator(data["value"])
+                try:
+                    value_int = int(data["value"])
+                except (TypeError, ValueError):
+                    raise serializers.ValidationError("Enter a valid integer.")
+                # Keep values within the PostgreSQL integer range
+                MinValueValidator(-2147483648)(value_int)
+                MaxValueValidator(2147483647)(value_int)
+            elif (
+                field.data_type == CustomField.FieldDataType.MONETARY
+                and data["value"] != ""
+            ):
+                try:
+                    # First try to validate as a number from legacy format
+                    DecimalValidator(max_digits=12, decimal_places=2)(
+                        Decimal(str(data["value"])),
+                    )
+                except Exception:
+                    # If that fails, try to validate as a monetary string
+                    RegexValidator(
+                        regex=r"^[A-Z]{3}-?\d+(\.\d{1,2})$",
+                        message="Must be a two-decimal number with optional currency code e.g. GBP123.45",
+                    )(data["value"])
+            elif field.data_type == CustomField.FieldDataType.STRING:
+                MaxLengthValidator(limit_value=128)(data["value"])
+            elif field.data_type == CustomField.FieldDataType.SELECT:
+                select_options = field.extra_data["select_options"]
+                try:
+                    next(
+                        option
+                        for option in select_options
+                        if option["id"] == data["value"]
+                    )
+                except Exception:
+                    raise serializers.ValidationError(
+                        f"Value must be an id of an element in {select_options}",
+                    )
+            elif field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
+                if not (isinstance(data["value"], list) or data["value"] is None):
+                    raise serializers.ValidationError(
+                        "Value must be a list",
+                    )
+                doc_ids = data["value"]
+                request = self.context.get("request")
+                validate_documentlink_targets(
+                    getattr(request, "user", None) if request is not None else None,
+                    doc_ids,
+                )
+            elif field.data_type == CustomField.FieldDataType.DATE:
+                data["value"] = serializers.DateField().to_internal_value(data["value"])
+
+        return data
+
+    class Meta:
+        model = CustomFieldInstance
+        fields = [
+            "value",
+            "field",
+        ]
+
+
+class StoragePathSerializer(MatchingModelSerializer, OwnedObjectSerializer):
+    class Meta:
+        model = StoragePath
+        fields = (
+            "id",
+            "slug",
+            "name",
+            "path",
+            "match",
+            "matching_algorithm",
+            "is_insensitive",
+            "document_count",
+            "owner",
+            "permissions",
+            "user_can_change",
+            "set_permissions",
+        )
+
+    def validate_path(self, path: str):
+        converted_path = convert_format_str_to_template_format(path)
+        if converted_path != path:
+            logger.warning(
+                f"Storage path {path} is not using the new style format, consider updating",
+            )
+        result = validate_filepath_template_and_render(converted_path)
+
+        if result is None:
+            raise serializers.ValidationError(_("Invalid variable detected."))
+
+        return converted_path
+
+    def update(self, instance, validated_data):
+        """
+        When a storage path is updated, see if documents
+        using it require a rename/move
+        """
+        doc_ids = [doc.id for doc in instance.documents.all()]
+        if doc_ids:
+            bulk_edit.bulk_update_documents.apply_async(
+                kwargs={"document_ids": doc_ids},
+                headers={"trigger_source": PaperlessTask.TriggerSource.SYSTEM},
+            )
+
+        return super().update(instance, validated_data)
+
+
+class StoragePathTestSerializer(SerializerWithPerms):
+    path = serializers.CharField(
+        required=True,
+        label="Path",
+        write_only=True,
+    )
+
+    document = serializers.PrimaryKeyRelatedField(
+        queryset=Document.objects.none(),
+        required=True,
+        label="Document",
+        write_only=True,
+    )
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        request = self.context.get("request")
+        user = getattr(request, "user", None) if request else None
+        if user is not None and user.is_authenticated:
+            document_field = self.fields.get("document")
+            if not isinstance(document_field, serializers.PrimaryKeyRelatedField):
+                return
+            document_field.queryset = Document.objects.filter(
+                id__in=permitted_document_ids(user),
+            )

src/documents/serialisers/saved_views.py

@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import logging
+import re
+
+from django.conf import settings
+from rest_framework import serializers
+
+from documents.models import CustomField
+from documents.models import SavedView
+from documents.models import SavedViewFilterRule
+from documents.models import UiSettings
+
+from .base import OwnedObjectSerializer
+
+logger = logging.getLogger("paperless.serializers")
+
+
+class SavedViewFilterRuleSerializer(serializers.ModelSerializer[SavedViewFilterRule]):
+    class Meta:
+        model = SavedViewFilterRule
+        fields = ["rule_type", "value"]
+
+
+class SavedViewSerializer(OwnedObjectSerializer):
+    filter_rules = SavedViewFilterRuleSerializer(many=True)
+
+    class Meta:
+        model = SavedView
+        fields = [
+            "id",
+            "name",
+            "icon",
+            "sort_field",
+            "sort_reverse",
+            "filter_rules",
+            "page_size",
+            "display_mode",
+            "display_fields",
+            "owner",
+            "permissions",
+            "user_can_change",
+            "set_permissions",
+        ]
+
+    def _get_api_version(self) -> int:
+        request = self.context.get("request")
+        return int(
+            request.version if request else settings.REST_FRAMEWORK["DEFAULT_VERSION"],
+        )
+
+    def _update_legacy_visibility_preferences(
+        self,
+        saved_view_id: int,
+        *,
+        show_on_dashboard: bool | None,
+        show_in_sidebar: bool | None,
+    ) -> UiSettings | None:
+        if show_on_dashboard is None and show_in_sidebar is None:
+            return None
+
+        request = self.context.get("request")
+        user = request.user if request else self.user
+        if user is None:
+            return None
+
+        ui_settings, _ = UiSettings.objects.get_or_create(
+            user=user,
+            defaults={"settings": {}},
+        )
+        current_settings = (
+            ui_settings.settings if isinstance(ui_settings.settings, dict) else {}
+        )
+        current_settings = dict(current_settings)
+
+        saved_views_settings = current_settings.get("saved_views")
+        if isinstance(saved_views_settings, dict):
+            saved_views_settings = dict(saved_views_settings)
+        else:
+            saved_views_settings = {}
+
+        dashboard_ids = {
+            int(raw_id)
+            for raw_id in saved_views_settings.get("dashboard_views_visible_ids", [])
+            if str(raw_id).isdigit()
+        }
+        sidebar_ids = {
+            int(raw_id)
+            for raw_id in saved_views_settings.get("sidebar_views_visible_ids", [])
+            if str(raw_id).isdigit()
+        }
+
+        if show_on_dashboard is not None:
+            if show_on_dashboard:
+                dashboard_ids.add(saved_view_id)
+            else:
+                dashboard_ids.discard(saved_view_id)
+        if show_in_sidebar is not None:
+            if show_in_sidebar:
+                sidebar_ids.add(saved_view_id)
+            else:
+                sidebar_ids.discard(saved_view_id)
+
+        saved_views_settings["dashboard_views_visible_ids"] = sorted(dashboard_ids)
+        saved_views_settings["sidebar_views_visible_ids"] = sorted(sidebar_ids)
+        current_settings["saved_views"] = saved_views_settings
+        ui_settings.settings = current_settings
+        ui_settings.save(update_fields=["settings"])
+        return ui_settings
+
+    def to_representation(self, instance):
+        # TODO: remove this and related backwards compatibility code when API v9 is dropped
+        ret = super().to_representation(instance)
+        request = self.context.get("request")
+        api_version = self._get_api_version()
+
+        if api_version < 10:
+            dashboard_ids = set()
+            sidebar_ids = set()
+            user = request.user if request else None
+            if user is not None and hasattr(user, "ui_settings"):
+                ui_settings = user.ui_settings.settings or None
+                saved_views = None
+                if isinstance(ui_settings, dict):
+                    saved_views = ui_settings.get("saved_views", {})
+                if isinstance(saved_views, dict):
+                    dashboard_ids = set(
+                        saved_views.get("dashboard_views_visible_ids", []),
+                    )
+                    sidebar_ids = set(
+                        saved_views.get("sidebar_views_visible_ids", []),
+                    )
+            ret["show_on_dashboard"] = instance.id in dashboard_ids
+            ret["show_in_sidebar"] = instance.id in sidebar_ids
+
+        return ret
+
+    def to_internal_value(self, data):
+        # TODO: remove this and related backwards compatibility code when API v9 is dropped
+        api_version = self._get_api_version()
+        if api_version >= 10:
+            return super().to_internal_value(data)
+
+        normalized_data = data.copy()
+        legacy_visibility_fields = {}
+        boolean_field = serializers.BooleanField()
+
+        for field_name in ("show_on_dashboard", "show_in_sidebar"):
+            if field_name in normalized_data:
+                try:
+                    legacy_visibility_fields[field_name] = (
+                        boolean_field.to_internal_value(
+                            normalized_data.get(field_name),
+                        )
+                    )
+                except serializers.ValidationError as exc:
+                    raise serializers.ValidationError({field_name: exc.detail})
+                del normalized_data[field_name]
+
+        ret = super().to_internal_value(normalized_data)
+        ret.update(legacy_visibility_fields)
+        return ret
+
+    def validate(self, attrs):
+        attrs = super().validate(attrs)
+        if "display_fields" in attrs and attrs["display_fields"] is not None:
+            for field in attrs["display_fields"]:
+                if (
+                    SavedView.DisplayFields.CUSTOM_FIELD[:-2] in field
+                ):  # i.e. check for 'custom_field_' prefix
+                    field_id = int(re.search(r"\d+", field)[0])
+                    if not CustomField.objects.filter(id=field_id).exists():
+                        raise serializers.ValidationError(
+                            f"Invalid field: {field}",
+                        )
+                elif field not in SavedView.DisplayFields.values:
+                    raise serializers.ValidationError(
+                        f"Invalid field: {field}",
+                    )
+        return attrs
+
+    def update(self, instance, validated_data):
+        request = self.context.get("request")
+        show_on_dashboard = validated_data.pop("show_on_dashboard", None)
+        show_in_sidebar = validated_data.pop("show_in_sidebar", None)
+        if "filter_rules" in validated_data:
+            rules_data = validated_data.pop("filter_rules")
+        else:
+            rules_data = None
+        if "user" in validated_data:
+            # backwards compatibility
+            validated_data["owner"] = validated_data.pop("user")
+        if (
+            "display_fields" in validated_data
+            and isinstance(
+                validated_data["display_fields"],
+                list,
+            )
+            and len(validated_data["display_fields"]) == 0
+        ):
+            validated_data["display_fields"] = None
+        instance = super().update(instance, validated_data)
+        if rules_data is not None:
+            SavedViewFilterRule.objects.filter(saved_view=instance).delete()
+            for rule_data in rules_data:
+                SavedViewFilterRule.objects.create(saved_view=instance, **rule_data)
+        ui_settings = self._update_legacy_visibility_preferences(
+            instance.id,
+            show_on_dashboard=show_on_dashboard,
+            show_in_sidebar=show_in_sidebar,
+        )
+        if request is not None and ui_settings is not None:
+            request.user.ui_settings = ui_settings
+        return instance
+
+    def create(self, validated_data):
+        request = self.context.get("request")
+        show_on_dashboard = validated_data.pop("show_on_dashboard", None)
+        show_in_sidebar = validated_data.pop("show_in_sidebar", None)
+        rules_data = validated_data.pop("filter_rules")
+        if "user" in validated_data:
+            # backwards compatibility
+            validated_data["owner"] = validated_data.pop("user")
+        saved_view = super().create(validated_data)
+        for rule_data in rules_data:
+            SavedViewFilterRule.objects.create(saved_view=saved_view, **rule_data)
+        ui_settings = self._update_legacy_visibility_preferences(
+            saved_view.id,
+            show_on_dashboard=show_on_dashboard,
+            show_in_sidebar=show_in_sidebar,
+        )
+        if request is not None and ui_settings is not None:
+            request.user.ui_settings = ui_settings
+        return saved_view

src/documents/serialisers/sharing.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+
+from django.core.exceptions import ValidationError
+from django.core.validators import EmailValidator
+from django.utils import timezone
+from django.utils.crypto import get_random_string
+from django.utils.translation import gettext as _
+from rest_framework import serializers
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.fields import SerializerMethodField
+
+from documents.models import Document
+from documents.models import ShareLink
+from documents.models import ShareLinkBundle
+from documents.permissions import has_perms_owner_aware
+
+from .base import DocumentListSerializer
+from .base import OwnedObjectSerializer
+
+logger = logging.getLogger("paperless.serializers")
+
+
+class EmailSerializer(DocumentListSerializer):
+    addresses = serializers.CharField(
+        required=True,
+        label="Email addresses",
+        help_text="Comma-separated email addresses",
+    )
+
+    subject = serializers.CharField(
+        required=True,
+        label="Email subject",
+    )
+
+    message = serializers.CharField(
+        required=True,
+        label="Email message",
+    )
+
+    use_archive_version = serializers.BooleanField(
+        default=True,
+        label="Use archive version",
+        help_text="Use archive version of documents if available",
+    )
+
+    def validate_addresses(self, addresses):
+        address_list = [addr.strip() for addr in addresses.split(",")]
+        if not address_list:
+            raise serializers.ValidationError("At least one email address is required")
+
+        email_validator = EmailValidator()
+        try:
+            for address in address_list:
+                email_validator(address)
+        except ValidationError:
+            raise serializers.ValidationError(f"Invalid email address: {address}")
+
+        return ",".join(address_list)
+
+    def validate_documents(self, documents):
+        super().validate_documents(documents)
+        if not documents:
+            raise serializers.ValidationError("At least one document is required")
+
+        return documents
+
+
+class ShareLinkSerializer(OwnedObjectSerializer):
+    document_title = serializers.CharField(
+        source="document.title",
+        read_only=True,
+    )
+
+    class Meta:
+        model = ShareLink
+        fields = (
+            "id",
+            "created",
+            "expiration",
+            "slug",
+            "document",
+            "document_title",
+            "file_version",
+        )
+
+    def create(self, validated_data):
+        validated_data["slug"] = get_random_string(50)
+        return super().create(validated_data)
+
+    def validate_document(self, document):
+        if (
+            self.user is not None
+            and self.user.has_perm("documents.view_document")
+            and has_perms_owner_aware(
+                self.user,
+                "view_document",
+                document,
+            )
+        ):
+            return document
+        raise PermissionDenied(
+            _("Insufficient permissions."),
+        )
+
+
+class ShareLinkBundleSerializer(OwnedObjectSerializer):
+    document_ids = serializers.ListField(
+        child=serializers.IntegerField(min_value=1),
+        allow_empty=False,
+        write_only=True,
+    )
+    expiration_days = serializers.IntegerField(
+        required=False,
+        allow_null=True,
+        min_value=1,
+        write_only=True,
+    )
+    documents = serializers.PrimaryKeyRelatedField(
+        many=True,
+        read_only=True,
+    )
+    document_count = SerializerMethodField()
+
+    class Meta:
+        model = ShareLinkBundle
+        fields = (
+            "id",
+            "created",
+            "expiration",
+            "expiration_days",
+            "slug",
+            "file_version",
+            "status",
+            "size_bytes",
+            "last_error",
+            "built_at",
+            "documents",
+            "document_ids",
+            "document_count",
+        )
+        read_only_fields = (
+            "id",
+            "created",
+            "expiration",
+            "slug",
+            "status",
+            "size_bytes",
+            "last_error",
+            "built_at",
+            "documents",
+            "document_count",
+        )
+
+    def validate_document_ids(self, value):
+        unique_ids = set(value)
+        if len(unique_ids) != len(value):
+            raise serializers.ValidationError(
+                _("Duplicate document identifiers are not allowed."),
+            )
+        return value
+
+    def create(self, validated_data):
+        document_ids = validated_data.pop("document_ids")
+        expiration_days = validated_data.pop("expiration_days", None)
+        validated_data["slug"] = get_random_string(50)
+        if expiration_days:
+            validated_data["expiration"] = timezone.now() + timedelta(
+                days=expiration_days,
+            )
+        else:
+            validated_data["expiration"] = None
+
+        share_link_bundle = super().create(validated_data)
+
+        documents = list(
+            Document.objects.filter(pk__in=document_ids).only(
+                "pk",
+            ),
+        )
+        documents_by_id = {doc.pk: doc for doc in documents}
+        missing = [
+            str(doc_id) for doc_id in document_ids if doc_id not in documents_by_id
+        ]
+        if missing:
+            raise serializers.ValidationError(
+                {
+                    "document_ids": _(
+                        "Documents not found: %(ids)s",
+                    )
+                    % {"ids": ", ".join(missing)},
+                },
+            )
+
+        ordered_documents = [documents_by_id[doc_id] for doc_id in document_ids]
+        share_link_bundle.documents.set(ordered_documents)
+        share_link_bundle.document_total = len(ordered_documents)
+
+        return share_link_bundle
+
+    def get_document_count(self, obj: ShareLinkBundle) -> int:
+        return getattr(obj, "document_total") or obj.documents.count()

src/documents/serialisers/system.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import logging
+
+from rest_framework import serializers
+
+from documents.models import Document
+from documents.models import UiSettings
+
+from .base import SerializerWithPerms
+
+logger = logging.getLogger("paperless.serializers")
+
+
+class UiSettingsViewSerializer(serializers.ModelSerializer[UiSettings]):
+    settings = serializers.DictField(required=False, allow_null=True)
+
+    class Meta:
+        model = UiSettings
+        depth = 1
+        fields = [
+            "id",
+            "settings",
+        ]
+
+    def validate_settings(self, settings):
+        # we never save update checking backend setting
+        if "update_checking" in settings:
+            try:
+                settings["update_checking"].pop("backend_setting")
+            except KeyError:
+                pass
+        return settings
+
+    def create(self, validated_data):
+        ui_settings = UiSettings.objects.update_or_create(
+            user=validated_data.get("user"),
+            defaults={"settings": validated_data.get("settings", None)},
+        )
+        return ui_settings
+
+
+class TrashSerializer(SerializerWithPerms):
+    documents = serializers.ListField(
+        required=False,
+        label="Documents",
+        write_only=True,
+        child=serializers.IntegerField(),
+    )
+
+    action = serializers.ChoiceField(
+        choices=["restore", "empty"],
+        label="Action",
+        write_only=True,
+    )
+
+    def validate_documents(self, documents: list[int]) -> list[int]:
+        count = Document.deleted_objects.filter(id__in=documents).count()
+        if not count == len(documents):
+            raise serializers.ValidationError(
+                "Some documents in the list have not yet been deleted.",
+            )
+        return documents

src/documents/serialisers/tasks.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from rest_framework import serializers
+
+from documents.models import Document
+from documents.models import PaperlessTask
+from documents.permissions import permitted_document_ids
+
+from .base import OwnedObjectSerializer
+
+logger = logging.getLogger("paperless.serializers")
+
+
+class TaskSerializerV10(OwnedObjectSerializer):
+    """Task serializer for API v10+ using new field names."""
+
+    related_document_ids = serializers.ListField(
+        child=serializers.IntegerField(),
+        read_only=True,
+    )
+    task_type_display = serializers.CharField(
+        source="get_task_type_display",
+        read_only=True,
+    )
+    trigger_source_display = serializers.CharField(
+        source="get_trigger_source_display",
+        read_only=True,
+    )
+    status_display = serializers.CharField(
+        source="get_status_display",
+        read_only=True,
+    )
+
+    class Meta:
+        model = PaperlessTask
+        fields = (
+            "id",
+            "task_id",
+            "task_type",
+            "task_type_display",
+            "trigger_source",
+            "trigger_source_display",
+            "status",
+            "status_display",
+            "date_created",
+            "date_started",
+            "date_done",
+            "duration_seconds",
+            "wait_time_seconds",
+            "input_data",
+            "result_data",
+            "related_document_ids",
+            "acknowledged",
+            "owner",
+        )
+        read_only_fields = fields
+
+
+class TaskSerializerV9(serializers.ModelSerializer[PaperlessTask]):
+    """Task serializer for API v9 backwards compatibility.
+
+    Maps old field names to the new model fields so existing clients continue
+    to work unchanged.
+    """
+
+    # v9 field: task_name -> task_type (with value remapping for renamed tasks)
+    task_name = serializers.SerializerMethodField()
+
+    # v9 field: task_file_name -> input_data.filename
+    task_file_name = serializers.SerializerMethodField()
+
+    # v9 field: type -> trigger_source (mapped to old enum labels)
+    type = serializers.SerializerMethodField()
+
+    # v9 field: status -> uppercase Celery state strings
+    status = serializers.SerializerMethodField()
+
+    # v9 field: result -> derived from result_data
+    result = serializers.SerializerMethodField()
+
+    # v9 field: related_document -> first document ID from result_data
+    related_document = serializers.SerializerMethodField()
+
+    # v9 field: duplicate_documents -> list of duplicate IDs from result_data
+    duplicate_documents = serializers.SerializerMethodField()
+
+    class Meta:
+        model = PaperlessTask
+        fields = (
+            "id",
+            "task_id",
+            "task_name",
+            "task_file_name",
+            "type",
+            "status",
+            "date_created",
+            "date_done",
+            "result",
+            "acknowledged",
+            "related_document",
+            "duplicate_documents",
+            "owner",
+        )
+        read_only_fields = fields
+
+    _TASK_TYPE_TO_V9_NAME = {
+        PaperlessTask.TaskType.SANITY_CHECK: "check_sanity",
+        PaperlessTask.TaskType.LLM_INDEX: "llmindex_update",
+    }
+
+    def get_result(self, obj: PaperlessTask) -> str | None:
+        """Reconstruct a human-readable result string from result_data for v9 clients."""
+        if not obj.result_data:
+            return None
+        if doc_id := obj.result_data.get("document_id"):
+            return f"Success. New document id {doc_id} created"
+        if reason := obj.result_data.get("reason"):
+            return reason
+        if dup_id := obj.result_data.get("duplicate_of"):
+            return f"Not consuming: It is a duplicate of document #{dup_id}"
+        if error := obj.result_data.get("error_message"):
+            return error
+        return None
+
+    def get_task_name(self, obj: PaperlessTask) -> str:
+        return self._TASK_TYPE_TO_V9_NAME.get(obj.task_type, obj.task_type)
+
+    def get_task_file_name(self, obj: PaperlessTask) -> str | None:
+        if not obj.input_data:
+            return None
+        return obj.input_data.get("filename")
+
+    _STATUS_TO_V9 = {
+        PaperlessTask.Status.PENDING: "PENDING",
+        PaperlessTask.Status.STARTED: "STARTED",
+        PaperlessTask.Status.SUCCESS: "SUCCESS",
+        PaperlessTask.Status.FAILURE: "FAILURE",
+        PaperlessTask.Status.REVOKED: "REVOKED",
+    }
+
+    def get_status(self, obj: PaperlessTask) -> str:
+        return self._STATUS_TO_V9.get(obj.status, obj.status.upper())
+
+    _TRIGGER_SOURCE_TO_V9_TYPE = {
+        PaperlessTask.TriggerSource.SCHEDULED: "scheduled_task",
+        PaperlessTask.TriggerSource.SYSTEM: "auto_task",
+        # Email and folder-consumer documents are system-initiated, not manually triggered
+        PaperlessTask.TriggerSource.EMAIL_CONSUME: "auto_task",
+        PaperlessTask.TriggerSource.FOLDER_CONSUME: "auto_task",
+    }
+
+    def get_type(self, obj: PaperlessTask) -> str:
+        return self._TRIGGER_SOURCE_TO_V9_TYPE.get(obj.trigger_source, "manual_task")
+
+    def get_related_document(self, obj: PaperlessTask) -> int | None:
+        ids = obj.related_document_ids
+        return ids[0] if ids else None
+
+    def get_duplicate_documents(
+        self,
+        obj: PaperlessTask,
+    ) -> list[dict[str, Any]]:
+        if not obj.result_data:
+            return []
+        dup_of = obj.result_data.get("duplicate_of")
+        if dup_of is None:
+            return []
+        request = self.context.get("request")
+        if request is None:
+            return []
+        user = request.user
+        qs = Document.global_objects.filter(pk=dup_of)
+        if not user.is_staff:
+            allowed_ids = permitted_document_ids(user, include_deleted=True)
+            qs = qs.filter(pk__in=allowed_ids)
+        return list(qs.values("id", "title", "deleted_at"))
+
+
+class TaskSummarySerializer(serializers.Serializer[dict[str, Any]]):
+    task_type = serializers.CharField()
+    total_count = serializers.IntegerField()
+    pending_count = serializers.IntegerField()
+    success_count = serializers.IntegerField()
+    failure_count = serializers.IntegerField()
+    avg_duration_seconds = serializers.FloatField(allow_null=True)
+    avg_wait_time_seconds = serializers.FloatField(allow_null=True)
+    last_run = serializers.DateTimeField(allow_null=True)
+    last_success = serializers.DateTimeField(allow_null=True)
+    last_failure = serializers.DateTimeField(allow_null=True)
+
+
+class RunTaskSerializer(serializers.Serializer[dict[str, str]]):
+    task_type = serializers.ChoiceField(
+        choices=PaperlessTask.TaskType.choices,
+        label="Task Type",
+        write_only=True,
+    )
+
+
+class AcknowledgeTasksViewSerializer(serializers.Serializer[dict[str, Any]]):
+    tasks = serializers.ListField(
+        required=False,
+        label="Tasks",
+        write_only=True,
+        child=serializers.IntegerField(),
+    )
+    all = serializers.BooleanField(
+        required=False,
+        default=False,
+        label="All",
+        write_only=True,
+    )
+
+    def _validate_task_id_list(self, tasks, name="tasks") -> None:
+        if not isinstance(tasks, list):
+            raise serializers.ValidationError(f"{name} must be a list")
+        if not all(isinstance(i, int) for i in tasks):
+            raise serializers.ValidationError(f"{name} must be a list of integers")
+        queryset = self.context.get("queryset", PaperlessTask.objects.all())
+        count = queryset.filter(id__in=tasks).count()
+        if not count == len(tasks):
+            raise serializers.ValidationError(
+                f"Some tasks in {name} don't exist or were specified twice.",
+            )
+
+    def validate_tasks(self, tasks):
+        self._validate_task_id_list(tasks)
+        return tasks
+
+    def validate(self, attrs):
+        acknowledge_all = attrs.get("all", False)
+        task_ids = attrs.get("tasks")
+
+        if acknowledge_all and task_ids is not None:
+            raise serializers.ValidationError(
+                "Set either all or tasks, not both.",
+            )
+        if not acknowledge_all and task_ids is None:
+            raise serializers.ValidationError(
+                "Either all must be true or tasks must be provided.",
+            )
+
+        return attrs

src/documents/serialisers/upload.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Any
+
+import magic
+from django.conf import settings
+from django.utils.translation import gettext as _
+from rest_framework import serializers
+
+from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import Document
+from documents.models import DocumentType
+from documents.models import StoragePath
+from documents.models import Tag
+from documents.parsers import is_mime_type_supported
+
+from .metadata import CustomFieldInstanceSerializer
+
+logger = logging.getLogger("paperless.serializers")
+
+
+class PostDocumentSerializer(serializers.Serializer[dict[str, Any]]):
+    created = serializers.DateTimeField(
+        label="Created",
+        allow_null=True,
+        write_only=True,
+        required=False,
+    )
+
+    document = serializers.FileField(
+        label="Document",
+        write_only=True,
+    )
+
+    title = serializers.CharField(
+        label="Title",
+        write_only=True,
+        required=False,
+    )
+
+    correspondent = serializers.PrimaryKeyRelatedField(
+        queryset=Correspondent.objects.all(),
+        label="Correspondent",
+        allow_null=True,
+        write_only=True,
+        required=False,
+    )
+
+    document_type = serializers.PrimaryKeyRelatedField(
+        queryset=DocumentType.objects.all(),
+        label="Document type",
+        allow_null=True,
+        write_only=True,
+        required=False,
+    )
+
+    storage_path = serializers.PrimaryKeyRelatedField(
+        queryset=StoragePath.objects.all(),
+        label="Storage path",
+        allow_null=True,
+        write_only=True,
+        required=False,
+    )
+
+    tags = serializers.PrimaryKeyRelatedField(
+        many=True,
+        queryset=Tag.objects.all(),
+        label="Tags",
+        write_only=True,
+        required=False,
+    )
+
+    archive_serial_number = serializers.IntegerField(
+        label="ASN",
+        write_only=True,
+        required=False,
+        min_value=Document.ARCHIVE_SERIAL_NUMBER_MIN,
+        max_value=Document.ARCHIVE_SERIAL_NUMBER_MAX,
+    )
+
+    # Accept either a list of custom field ids or a dict mapping id -> value
+    custom_fields = serializers.JSONField(
+        label="Custom fields",
+        write_only=True,
+        required=False,
+    )
+
+    from_webui = serializers.BooleanField(
+        label="Documents are from Paperless-ngx WebUI",
+        write_only=True,
+        required=False,
+    )
+
+    def validate_document(self, document):
+        document_data = document.file.read()
+        mime_type = magic.from_buffer(document_data, mime=True)
+
+        if not is_mime_type_supported(mime_type):
+            if (
+                mime_type in settings.CONSUMER_PDF_RECOVERABLE_MIME_TYPES
+                and document.name.endswith(
+                    ".pdf",
+                )
+            ):
+                # If the file is an invalid PDF, we can try to recover it later in the consumer
+                mime_type = "application/pdf"
+            else:
+                raise serializers.ValidationError(
+                    _("File type %(type)s not supported") % {"type": mime_type},
+                )
+
+        return document.name, document_data
+
+    def validate_correspondent(self, correspondent):
+        if correspondent:
+            return correspondent.id
+        else:
+            return None
+
+    def validate_document_type(self, document_type):
+        if document_type:
+            return document_type.id
+        else:
+            return None
+
+    def validate_storage_path(self, storage_path):
+        if storage_path:
+            return storage_path.id
+        else:
+            return None
+
+    def validate_tags(self, tags):
+        if tags:
+            return [tag.id for tag in tags]
+        else:
+            return None
+
+    def validate_custom_fields(self, custom_fields):
+        if not custom_fields:
+            return None
+
+        # Normalize single values to a list
+        if isinstance(custom_fields, int):
+            custom_fields = [custom_fields]
+        if isinstance(custom_fields, dict):
+            custom_field_serializer = CustomFieldInstanceSerializer()
+            normalized = {}
+            for field_id, value in custom_fields.items():
+                try:
+                    field_id_int = int(field_id)
+                except (TypeError, ValueError):
+                    raise serializers.ValidationError(
+                        _("Custom field id must be an integer: %(id)s")
+                        % {"id": field_id},
+                    )
+                try:
+                    field = CustomField.objects.get(id=field_id_int)
+                except CustomField.DoesNotExist:
+                    raise serializers.ValidationError(
+                        _("Custom field with id %(id)s does not exist")
+                        % {"id": field_id_int},
+                    )
+                custom_field_serializer.validate(
+                    {
+                        "field": field,
+                        "value": value,
+                    },
+                )
+                normalized[field_id_int] = value
+            return normalized
+        elif isinstance(custom_fields, list):
+            try:
+                ids = [int(i) for i in custom_fields]
+            except (TypeError, ValueError):
+                raise serializers.ValidationError(
+                    _(
+                        "Custom fields must be a list of integers or an object mapping ids to values.",
+                    ),
+                )
+            if CustomField.objects.filter(id__in=ids).count() != len(set(ids)):
+                raise serializers.ValidationError(
+                    _("Some custom fields don't exist or were specified twice."),
+                )
+            return ids
+        raise serializers.ValidationError(
+            _(
+                "Custom fields must be a list of integers or an object mapping ids to values.",
+            ),
+        )
+
+    # custom_fields_w_values handled via validate_custom_fields
+
+    def validate_created(self, created):
+        # support datetime format for created for backwards compatibility
+        if isinstance(created, datetime):
+            return created.date()

src/documents/serialisers/workflows.py

@@ -0,0 +1,584 @@
+from __future__ import annotations
+
+import logging
+
+from django.db.models import Count
+from rest_framework import fields
+from rest_framework import serializers
+
+from documents.data_models import DocumentSource
+from documents.filters import CustomFieldQueryParser
+from documents.models import Workflow
+from documents.models import WorkflowAction
+from documents.models import WorkflowActionEmail
+from documents.models import WorkflowActionWebhook
+from documents.models import WorkflowTrigger
+from documents.templating.workflows import validate_workflow_template
+from documents.validators import url_validator
+
+from .metadata import CorrespondentField
+from .metadata import DocumentTypeField
+from .metadata import StoragePathField
+from .metadata import TagsField
+
+logger = logging.getLogger("paperless.serializers")
+
+
+class WorkflowTriggerSerializer(serializers.ModelSerializer[WorkflowTrigger]):
+    id = serializers.IntegerField(required=False, allow_null=True)
+    sources = fields.MultipleChoiceField(
+        choices=WorkflowTrigger.DocumentSourceChoices.choices,
+        allow_empty=True,
+        default={
+            DocumentSource.ConsumeFolder,
+            DocumentSource.ApiUpload,
+            DocumentSource.MailFetch,
+        },
+    )
+
+    type = serializers.ChoiceField(
+        choices=WorkflowTrigger.WorkflowTriggerType.choices,
+        label="Trigger Type",
+    )
+
+    class Meta:
+        model = WorkflowTrigger
+        fields = [
+            "id",
+            "sources",
+            "type",
+            "filter_path",
+            "filter_filename",
+            "filter_mailrule",
+            "matching_algorithm",
+            "match",
+            "is_insensitive",
+            "filter_has_tags",
+            "filter_has_all_tags",
+            "filter_has_not_tags",
+            "filter_custom_field_query",
+            "filter_has_any_correspondents",
+            "filter_has_not_correspondents",
+            "filter_has_any_document_types",
+            "filter_has_not_document_types",
+            "filter_has_any_storage_paths",
+            "filter_has_not_storage_paths",
+            "filter_has_correspondent",
+            "filter_has_document_type",
+            "filter_has_storage_path",
+            "schedule_offset_days",
+            "schedule_is_recurring",
+            "schedule_recurring_interval_days",
+            "schedule_date_field",
+            "schedule_date_custom_field",
+        ]
+
+    def validate(self, attrs):
+        # Empty strings treated as None to avoid unexpected behavior
+        if (
+            "filter_filename" in attrs
+            and attrs["filter_filename"] is not None
+            and len(attrs["filter_filename"]) == 0
+        ):
+            attrs["filter_filename"] = None
+        if (
+            "filter_path" in attrs
+            and attrs["filter_path"] is not None
+            and len(attrs["filter_path"]) == 0
+        ):
+            attrs["filter_path"] = None
+
+        if (
+            "filter_custom_field_query" in attrs
+            and attrs["filter_custom_field_query"] is not None
+            and len(attrs["filter_custom_field_query"]) == 0
+        ):
+            attrs["filter_custom_field_query"] = None
+
+        if (
+            "filter_custom_field_query" in attrs
+            and attrs["filter_custom_field_query"] is not None
+        ):
+            parser = CustomFieldQueryParser("filter_custom_field_query")
+            parser.parse(attrs["filter_custom_field_query"])
+
+        trigger_type = attrs.get("type", getattr(self.instance, "type", None))
+        if (
+            trigger_type == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION
+            and "filter_mailrule" not in attrs
+            and ("filter_filename" not in attrs or attrs["filter_filename"] is None)
+            and ("filter_path" not in attrs or attrs["filter_path"] is None)
+        ):
+            raise serializers.ValidationError(
+                "File name, path or mail rule filter are required",
+            )
+
+        return attrs
+
+    @staticmethod
+    def normalize_workflow_trigger_sources(trigger) -> None:
+        """
+        Convert sources to strings to handle django-multiselectfield v1.0 changes
+        """
+        if trigger and "sources" in trigger:
+            trigger["sources"] = [
+                str(s.value if hasattr(s, "value") else s) for s in trigger["sources"]
+            ]
+
+    def create(self, validated_data):
+        WorkflowTriggerSerializer.normalize_workflow_trigger_sources(validated_data)
+        return super().create(validated_data)
+
+    def update(self, instance, validated_data):
+        WorkflowTriggerSerializer.normalize_workflow_trigger_sources(validated_data)
+        return super().update(instance, validated_data)
+
+
+class WorkflowActionEmailSerializer(serializers.ModelSerializer[WorkflowActionEmail]):
+    id = serializers.IntegerField(allow_null=True, required=False)
+
+    class Meta:
+        model = WorkflowActionEmail
+        fields = [
+            "id",
+            "subject",
+            "body",
+            "to",
+            "include_document",
+        ]
+
+
+class WorkflowActionWebhookSerializer(
+    serializers.ModelSerializer[WorkflowActionWebhook],
+):
+    id = serializers.IntegerField(allow_null=True, required=False)
+
+    def validate_url(self, url):
+        url_validator(url)
+        return url
+
+    class Meta:
+        model = WorkflowActionWebhook
+        fields = [
+            "id",
+            "url",
+            "use_params",
+            "as_json",
+            "params",
+            "body",
+            "headers",
+            "include_document",
+        ]
+
+
+class WorkflowActionSerializer(serializers.ModelSerializer[WorkflowAction]):
+    id = serializers.IntegerField(required=False, allow_null=True)
+    assign_correspondent = CorrespondentField(allow_null=True, required=False)
+    assign_tags = TagsField(many=True, allow_null=True, required=False)
+    assign_document_type = DocumentTypeField(allow_null=True, required=False)
+    assign_storage_path = StoragePathField(allow_null=True, required=False)
+    email = WorkflowActionEmailSerializer(allow_null=True, required=False)
+    webhook = WorkflowActionWebhookSerializer(allow_null=True, required=False)
+
+    class Meta:
+        model = WorkflowAction
+        fields = [
+            "id",
+            "type",
+            "assign_title",
+            "assign_tags",
+            "assign_correspondent",
+            "assign_document_type",
+            "assign_storage_path",
+            "assign_owner",
+            "assign_view_users",
+            "assign_view_groups",
+            "assign_change_users",
+            "assign_change_groups",
+            "assign_custom_fields",
+            "assign_custom_fields_values",
+            "remove_all_tags",
+            "remove_tags",
+            "remove_all_correspondents",
+            "remove_correspondents",
+            "remove_all_document_types",
+            "remove_document_types",
+            "remove_all_storage_paths",
+            "remove_storage_paths",
+            "remove_custom_fields",
+            "remove_all_custom_fields",
+            "remove_all_owners",
+            "remove_owners",
+            "remove_all_permissions",
+            "remove_view_users",
+            "remove_view_groups",
+            "remove_change_users",
+            "remove_change_groups",
+            "email",
+            "webhook",
+            "passwords",
+            "ai_suggestion_fields",
+            "ai_create_missing",
+            "ai_overwrite_existing",
+        ]
+
+    def validate(self, attrs):
+        if "assign_title" in attrs and attrs["assign_title"] is not None:
+            if len(attrs["assign_title"]) == 0:
+                # Empty strings treated as None to avoid unexpected behavior
+                attrs["assign_title"] = None
+            else:
+                try:
+                    validate_workflow_template(attrs["assign_title"])
+                except (ValueError, KeyError) as e:
+                    raise serializers.ValidationError(
+                        {"assign_title": f"{e.args[0]}"},
+                    )
+
+        if attrs.get("assign_custom_fields_values"):
+            # Empty strings treated as None to avoid unexpected behavior
+            attrs["assign_custom_fields_values"] = {
+                field_id: (None if value == "" else value)
+                for field_id, value in attrs["assign_custom_fields_values"].items()
+            }
+
+        if (
+            "type" in attrs
+            and attrs["type"] == WorkflowAction.WorkflowActionType.EMAIL
+            and "email" not in attrs
+        ):
+            raise serializers.ValidationError(
+                "Email data is required for email actions",
+            )
+
+        if (
+            "type" in attrs
+            and attrs["type"] == WorkflowAction.WorkflowActionType.WEBHOOK
+            and "webhook" not in attrs
+        ):
+            raise serializers.ValidationError(
+                "Webhook data is required for webhook actions",
+            )
+
+        if (
+            "type" in attrs
+            and attrs["type"] == WorkflowAction.WorkflowActionType.PASSWORD_REMOVAL
+        ):
+            passwords = attrs.get("passwords")
+            # ensure passwords is a non-empty list of non-empty strings
+            if (
+                passwords is None
+                or not isinstance(passwords, list)
+                or len(passwords) == 0
+                or any(not isinstance(pw, str) for pw in passwords)
+                or any(len(pw.strip()) == 0 for pw in passwords)
+            ):
+                raise serializers.ValidationError(
+                    "Passwords are required for password removal actions",
+                )
+
+        if (
+            "type" in attrs
+            and attrs["type"] == WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS
+        ):
+            fields = attrs.get("ai_suggestion_fields")
+            valid_fields = set(WorkflowAction.AISuggestionField.values)
+            if (
+                fields is None
+                or not isinstance(fields, list)
+                or len(fields) == 0
+                or any(field not in valid_fields for field in fields)
+            ):
+                raise serializers.ValidationError(
+                    "At least one valid field is required for apply AI "
+                    f"suggestions actions, options are: {sorted(valid_fields)}",
+                )
+
+        return attrs
+
+
+class WorkflowSerializer(serializers.ModelSerializer[Workflow]):
+    order = serializers.IntegerField(required=False)
+
+    triggers = WorkflowTriggerSerializer(many=True)
+    actions = WorkflowActionSerializer(many=True)
+
+    class Meta:
+        model = Workflow
+        fields = [
+            "id",
+            "name",
+            "order",
+            "enabled",
+            "triggers",
+            "actions",
+        ]
+
+    def validate(self, attrs):
+        attrs = super().validate(attrs)
+
+        if "actions" in attrs:
+            has_remote_ocr_action = any(
+                action.get("type") == WorkflowAction.WorkflowActionType.REMOTE_OCR
+                for action in attrs["actions"]
+            )
+            has_ai_suggestions_action = any(
+                action.get("type")
+                == WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS
+                for action in attrs["actions"]
+            )
+        else:
+            has_remote_ocr_action = self.instance is not None and (
+                self.instance.actions.filter(
+                    type=WorkflowAction.WorkflowActionType.REMOTE_OCR,
+                ).exists()
+            )
+            has_ai_suggestions_action = self.instance is not None and (
+                self.instance.actions.filter(
+                    type=WorkflowAction.WorkflowActionType.APPLY_AI_SUGGESTIONS,
+                ).exists()
+            )
+
+        if "triggers" in attrs:
+            has_consumption_trigger = any(
+                trigger.get("type") == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION
+                for trigger in attrs["triggers"]
+            )
+            has_non_consumption_trigger = any(
+                trigger.get("type") != WorkflowTrigger.WorkflowTriggerType.CONSUMPTION
+                for trigger in attrs["triggers"]
+            )
+        else:
+            has_consumption_trigger = self.instance is not None and (
+                self.instance.triggers.filter(
+                    type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
+                ).exists()
+            )
+            has_non_consumption_trigger = self.instance is not None and (
+                self.instance.triggers.exclude(
+                    type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
+                ).exists()
+            )
+
+        # Remote OCR can only work with consumption triggers
+        if has_remote_ocr_action and not has_consumption_trigger:
+            raise serializers.ValidationError(
+                "Remote OCR actions require a consumption started trigger",
+            )
+
+        # Suggestions are made from the document content, which does not exist
+        # until after consumption has finished
+        if has_ai_suggestions_action and not has_non_consumption_trigger:
+            raise serializers.ValidationError(
+                "Apply AI suggestions actions require a trigger other than "
+                "consumption started",
+            )
+
+        return attrs
+
+    def update_triggers_and_actions(
+        self,
+        instance: Workflow,
+        triggers,
+        actions,
+    ) -> None:
+        set_triggers = []
+        set_actions = []
+
+        if triggers is not None and triggers is not serializers.empty:
+            for trigger in triggers:
+                filter_has_tags = trigger.pop("filter_has_tags", None)
+                filter_has_all_tags = trigger.pop("filter_has_all_tags", None)
+                filter_has_not_tags = trigger.pop("filter_has_not_tags", None)
+                filter_has_any_correspondents = trigger.pop(
+                    "filter_has_any_correspondents",
+                    None,
+                )
+                filter_has_not_correspondents = trigger.pop(
+                    "filter_has_not_correspondents",
+                    None,
+                )
+                filter_has_any_document_types = trigger.pop(
+                    "filter_has_any_document_types",
+                    None,
+                )
+                filter_has_not_document_types = trigger.pop(
+                    "filter_has_not_document_types",
+                    None,
+                )
+                filter_has_any_storage_paths = trigger.pop(
+                    "filter_has_any_storage_paths",
+                    None,
+                )
+                filter_has_not_storage_paths = trigger.pop(
+                    "filter_has_not_storage_paths",
+                    None,
+                )
+                # Convert sources to strings to handle django-multiselectfield v1.0 changes
+                WorkflowTriggerSerializer.normalize_workflow_trigger_sources(trigger)
+                trigger_instance, _ = WorkflowTrigger.objects.update_or_create(
+                    id=trigger.get("id"),
+                    defaults=trigger,
+                )
+                if filter_has_tags is not None:
+                    trigger_instance.filter_has_tags.set(filter_has_tags)
+                if filter_has_all_tags is not None:
+                    trigger_instance.filter_has_all_tags.set(filter_has_all_tags)
+                if filter_has_not_tags is not None:
+                    trigger_instance.filter_has_not_tags.set(filter_has_not_tags)
+                if filter_has_any_correspondents is not None:
+                    trigger_instance.filter_has_any_correspondents.set(
+                        filter_has_any_correspondents,
+                    )
+                if filter_has_not_correspondents is not None:
+                    trigger_instance.filter_has_not_correspondents.set(
+                        filter_has_not_correspondents,
+                    )
+                if filter_has_any_document_types is not None:
+                    trigger_instance.filter_has_any_document_types.set(
+                        filter_has_any_document_types,
+                    )
+                if filter_has_not_document_types is not None:
+                    trigger_instance.filter_has_not_document_types.set(
+                        filter_has_not_document_types,
+                    )
+                if filter_has_any_storage_paths is not None:
+                    trigger_instance.filter_has_any_storage_paths.set(
+                        filter_has_any_storage_paths,
+                    )
+                if filter_has_not_storage_paths is not None:
+                    trigger_instance.filter_has_not_storage_paths.set(
+                        filter_has_not_storage_paths,
+                    )
+                set_triggers.append(trigger_instance)
+
+        if actions is not None and actions is not serializers.empty:
+            for index, action in enumerate(actions):
+                action["order"] = index
+                assign_tags = action.pop("assign_tags", None)
+                assign_view_users = action.pop("assign_view_users", None)
+                assign_view_groups = action.pop("assign_view_groups", None)
+                assign_change_users = action.pop("assign_change_users", None)
+                assign_change_groups = action.pop("assign_change_groups", None)
+                assign_custom_fields = action.pop("assign_custom_fields", None)
+                remove_tags = action.pop("remove_tags", None)
+                remove_correspondents = action.pop("remove_correspondents", None)
+                remove_document_types = action.pop("remove_document_types", None)
+                remove_storage_paths = action.pop("remove_storage_paths", None)
+                remove_custom_fields = action.pop("remove_custom_fields", None)
+                remove_owners = action.pop("remove_owners", None)
+                remove_view_users = action.pop("remove_view_users", None)
+                remove_view_groups = action.pop("remove_view_groups", None)
+                remove_change_users = action.pop("remove_change_users", None)
+                remove_change_groups = action.pop("remove_change_groups", None)
+
+                email_data = action.pop("email", None)
+                webhook_data = action.pop("webhook", None)
+
+                action_instance, _ = WorkflowAction.objects.update_or_create(
+                    id=action.get("id"),
+                    defaults=action,
+                )
+
+                if email_data is not None:
+                    serializer = WorkflowActionEmailSerializer(data=email_data)
+                    serializer.is_valid(raise_exception=True)
+                    email, _ = WorkflowActionEmail.objects.update_or_create(
+                        id=email_data.get("id"),
+                        defaults=serializer.validated_data,
+                    )
+                    action_instance.email = email
+                    action_instance.save()
+
+                if webhook_data is not None:
+                    serializer = WorkflowActionWebhookSerializer(data=webhook_data)
+                    serializer.is_valid(raise_exception=True)
+                    webhook, _ = WorkflowActionWebhook.objects.update_or_create(
+                        id=webhook_data.get("id"),
+                        defaults=serializer.validated_data,
+                    )
+                    action_instance.webhook = webhook
+                    action_instance.save()
+
+                if assign_tags is not None:
+                    action_instance.assign_tags.set(assign_tags)
+                if assign_view_users is not None:
+                    action_instance.assign_view_users.set(assign_view_users)
+                if assign_view_groups is not None:
+                    action_instance.assign_view_groups.set(assign_view_groups)
+                if assign_change_users is not None:
+                    action_instance.assign_change_users.set(assign_change_users)
+                if assign_change_groups is not None:
+                    action_instance.assign_change_groups.set(assign_change_groups)
+                if assign_custom_fields is not None:
+                    action_instance.assign_custom_fields.set(assign_custom_fields)
+                if remove_tags is not None:
+                    action_instance.remove_tags.set(remove_tags)
+                if remove_correspondents is not None:
+                    action_instance.remove_correspondents.set(remove_correspondents)
+                if remove_document_types is not None:
+                    action_instance.remove_document_types.set(remove_document_types)
+                if remove_storage_paths is not None:
+                    action_instance.remove_storage_paths.set(remove_storage_paths)
+                if remove_custom_fields is not None:
+                    action_instance.remove_custom_fields.set(remove_custom_fields)
+                if remove_owners is not None:
+                    action_instance.remove_owners.set(remove_owners)
+                if remove_view_users is not None:
+                    action_instance.remove_view_users.set(remove_view_users)
+                if remove_view_groups is not None:
+                    action_instance.remove_view_groups.set(remove_view_groups)
+                if remove_change_users is not None:
+                    action_instance.remove_change_users.set(remove_change_users)
+                if remove_change_groups is not None:
+                    action_instance.remove_change_groups.set(remove_change_groups)
+
+                set_actions.append(action_instance)
+
+        if triggers is not serializers.empty:
+            instance.triggers.set(set_triggers)
+        if actions is not serializers.empty:
+            instance.actions.set(set_actions)
+        instance.save()
+
+    def prune_triggers_and_actions(self) -> None:
+        """
+        ManyToMany fields dont support e.g. on_delete so we need to discard unattached
+        triggers and actions manually
+        """
+        WorkflowTrigger.objects.annotate(
+            workflow_count=Count("workflows"),
+        ).filter(workflow_count=0).delete()
+
+        WorkflowAction.objects.annotate(
+            workflow_count=Count("workflows"),
+        ).filter(workflow_count=0).delete()
+
+        WorkflowActionEmail.objects.filter(action=None).delete()
+        WorkflowActionWebhook.objects.filter(action=None).delete()
+
+    def create(self, validated_data) -> Workflow:
+        if "triggers" in validated_data:
+            triggers = validated_data.pop("triggers")
+
+        if "actions" in validated_data:
+            actions = validated_data.pop("actions")
+            for action in actions:
+                action.pop("id", None)
+
+        instance = super().create(validated_data)
+
+        self.update_triggers_and_actions(instance, triggers, actions)
+
+        return instance
+
+    def update(self, instance: Workflow, validated_data) -> Workflow:
+        triggers = validated_data.pop("triggers", serializers.empty)
+        actions = validated_data.pop("actions", serializers.empty)
+
+        instance = super().update(instance, validated_data)
+
+        self.update_triggers_and_actions(instance, triggers, actions)
+        self.prune_triggers_and_actions()
+
+        return instance

src/documents/signals/handlers.py

@@ -56,6 +56,7 @@ from documents.permissions import get_objects_for_user_owner_aware
 from documents.plugins.helpers import DocumentsStatusManager
 from documents.templating.utils import convert_format_str_to_template_format
 from documents.utils import compute_checksum
+from documents.utils import copy_file_with_basic_stats
 from documents.workflows.actions import build_workflow_action_context
 from documents.workflows.actions import execute_email_action
 from documents.workflows.actions import execute_move_to_trash_action
@@ -363,7 +364,11 @@ def cleanup_document_deletion(sender, instance, **kwargs) -> None:
 
             logger.debug(f"Moving {instance.source_path} to trash at {new_file_path}")
             try:
-                shutil.move(instance.source_path, new_file_path)
+                shutil.move(
+                    instance.source_path,
+                    new_file_path,
+                    copy_function=copy_file_with_basic_stats,
+                )
             except OSError as e:
                 logger.error(
                     f"Failed to move {instance.source_path} to trash at "

src/documents/templates/index.html

@@ -12,7 +12,7 @@
 	<meta name="robots" content="noindex,nofollow">
 	<meta name="author" content="The Paperless-ngx Team">
 	<link rel="icon" type="image/x-icon" href="favicon.ico">
-	<link rel="manifest" href="{% static webmanifest %}">
+	<link rel="manifest" href="{% static webmanifest %}" crossorigin="use-credentials">
 	<link rel="stylesheet" href="{% static styles_css %}">
 	<link rel="apple-touch-icon" href="{% static apple_touch_icon %}">
 </head>

src/documents/tests/conftest.py

@@ -1,88 +1,41 @@
 import shutil
-import zoneinfo
-from collections.abc import Generator
-from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import filelock
 import pytest
-from django.contrib.auth import get_user_model
-from django.contrib.contenttypes.models import ContentType
-from guardian.shortcuts import clear_ct_cache
-from pytest_django.fixtures import Settings
-from rest_framework.test import APIClient
 
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
-
-UserModelT = get_user_model()
 
 if TYPE_CHECKING:
     from documents.models import Document
-
+    from paperless_testing.dirs import PaperlessDirs
-
-@dataclass(frozen=True, slots=True)
-class PaperlessDirs:
-    """Standard Paperless-ngx directory layout for tests."""
-
-    media: Path
-    originals: Path
-    archive: Path
-    thumbnails: Path
 
 
 @pytest.fixture(scope="session")
-def samples_dir() -> Path:
+def document_samples_dir() -> Path:
     """Path to the shared test sample documents."""
     return Path(__file__).parent / "samples" / "documents"
 
 
-@pytest.fixture()
-def paperless_dirs(tmp_path: Path) -> PaperlessDirs:
-    """Create and return the directory structure for testing."""
-    media = tmp_path / "media"
-    dirs = PaperlessDirs(
-        media=media,
-        originals=media / "documents" / "originals",
-        archive=media / "documents" / "archive",
-        thumbnails=media / "documents" / "thumbnails",
-    )
-    for d in (dirs.originals, dirs.archive, dirs.thumbnails):
-        d.mkdir(parents=True)
-    return dirs
-
-
-@pytest.fixture()
-def _media_settings(paperless_dirs: PaperlessDirs, settings) -> None:
-    """Configure Django settings to point at temp directories."""
-    settings.MEDIA_ROOT = paperless_dirs.media
-    settings.ORIGINALS_DIR = paperless_dirs.originals
-    settings.ARCHIVE_DIR = paperless_dirs.archive
-    settings.THUMBNAIL_DIR = paperless_dirs.thumbnails
-    settings.MEDIA_LOCK = paperless_dirs.media / "media.lock"
-    settings.IGNORABLE_FILES = {".DS_Store", "Thumbs.db", "desktop.ini"}
-    settings.APP_LOGO = ""
-
-
 @pytest.fixture()
 def sample_doc(
-    paperless_dirs: PaperlessDirs,
+    paperless_dirs: "PaperlessDirs",
-    _media_settings: None,
+    document_samples_dir: Path,
-    samples_dir: Path,
 ) -> "Document":
     """Create a document with valid files and matching checksums."""
-    with filelock.FileLock(paperless_dirs.media / "media.lock"):
+    with filelock.FileLock(paperless_dirs.media_lock):
         shutil.copy(
-            samples_dir / "originals" / "0000001.pdf",
+            document_samples_dir / "originals" / "0000001.pdf",
-            paperless_dirs.originals / "0000001.pdf",
+            paperless_dirs.originals_dir / "0000001.pdf",
         )
         shutil.copy(
-            samples_dir / "archive" / "0000001.pdf",
+            document_samples_dir / "archive" / "0000001.pdf",
-            paperless_dirs.archive / "0000001.pdf",
+            paperless_dirs.archive_dir / "0000001.pdf",
         )
         shutil.copy(
-            samples_dir / "thumbnails" / "0000001.webp",
+            document_samples_dir / "thumbnails" / "0000001.webp",
-            paperless_dirs.thumbnails / "0000001.webp",
+            paperless_dirs.thumbnail_dir / "0000001.webp",
         )
 
     return DocumentFactory(
@@ -97,95 +50,17 @@ def sample_doc(
     )
 
 
-@pytest.fixture()
+@pytest.fixture
-def _search_index(
+def _search_index(paperless_dirs: "PaperlessDirs") -> None:
-    tmp_path: Path,
+    """Point the search backend at a fresh, empty index directory.
-    settings: Settings,
-) -> Generator[None, None, None]:
-    """Create a temp index directory and point INDEX_DIR at it.
 
-    Resets the backend singleton before and after so each test gets a clean
+    paperless_dirs owns INDEX_DIR and resets the backend singleton on both
-    index rather than reusing a stale singleton from another test.
+    sides of the test, so requesting it is all that is needed.
     """
-    from documents.search import reset_backend
-
-    index_dir = tmp_path / "index"
-    index_dir.mkdir()
-    settings.INDEX_DIR = index_dir
-    reset_backend()
-    yield
-    reset_backend()
-
-
-@pytest.fixture()
-def settings_timezone(settings: Settings) -> zoneinfo.ZoneInfo:
-    return zoneinfo.ZoneInfo(settings.TIME_ZONE)
 
 
 @pytest.fixture
-def rest_api_client():
+def searchable_document(_search_index: None) -> "Document":
-    """
-    The basic DRF ApiClient
-    """
-    yield APIClient()
-
-
-@pytest.fixture()
-def regular_user(django_user_model: type[UserModelT]) -> UserModelT:
-    """Unprivileged authenticated user for permission boundary tests."""
-    return django_user_model.objects.create_user(username="regular", password="regular")
-
-
-@pytest.fixture()
-def admin_client(rest_api_client: APIClient, admin_user: UserModelT) -> APIClient:
-    """Admin client pre-authenticated and sending the v10 Accept header."""
-    rest_api_client.force_authenticate(user=admin_user)
-    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
-    return rest_api_client
-
-
-@pytest.fixture()
-def v9_client(rest_api_client: APIClient, admin_user: UserModelT) -> APIClient:
-    """Admin client pre-authenticated and sending the v9 Accept header."""
-    rest_api_client.force_authenticate(user=admin_user)
-    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=9")
-    return rest_api_client
-
-
-@pytest.fixture()
-def user_client(rest_api_client: APIClient, regular_user: UserModelT) -> APIClient:
-    """Regular-user client pre-authenticated and sending the v10 Accept header."""
-    rest_api_client.force_authenticate(user=regular_user)
-    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
-    return rest_api_client
-
-
-@pytest.fixture(autouse=True)
-def _clear_content_type_caches() -> None:
-    """Clear Django's ContentType cache and guardian's lru_cache before each test.
-
-    Tests that delete and reinsert ContentType/Permission rows (e.g. the
-    importer) corrupt both caches. Without this fixture a subsequent test on
-    the same xdist worker sees stale ContentType objects and guardian raises
-    MixedContentTypeError.
-    """
-    ContentType.objects.clear_cache()
-    clear_ct_cache()
-
-
-@pytest.fixture(scope="session", autouse=True)
-def faker_session_locale():
-    """Set Faker locale for reproducibility."""
-    return "en_US"
-
-
-@pytest.fixture(scope="session", autouse=True)
-def faker_seed():
-    return 12345
-
-
-@pytest.fixture
-def indexed_document(_search_index: None) -> "Document":
     """One searchable document, for tests about what the search endpoint
     returns rather than about what it finds.
     """

src/documents/tests/helpers.py

@@ -0,0 +1,10 @@
+import re
+
+
+def dummy_preprocess(content: str) -> str:
+    """
+    Simpler, faster pre-processing for testing purposes
+    """
+    content = content.lower().strip()
+    content = re.sub(r"\s+", " ", content)
+    return content

src/documents/tests/management/test_management_sanity_checker.py

@@ -15,11 +15,11 @@ from rich.console import Console
 
 from documents.management.commands.document_sanity_checker import Command
 from documents.sanity_checker import SanityCheckMessages
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
     from documents.models import Document
-    from documents.tests.conftest import PaperlessDirs
+    from paperless_testing.dirs import PaperlessDirs
 
 
 def _render_to_string(messages: SanityCheckMessages) -> str:
@@ -71,7 +71,7 @@ class TestRenderResultsWithIssues:
         assert "INFO" in output
         assert "No OCR data" in output
 
-    @pytest.mark.usefixtures("_media_settings")
+    @pytest.mark.usefixtures("paperless_dirs")
     def test_global_message(self) -> None:
         msgs = SanityCheckMessages()
         msgs.warning(None, "Orphaned file: /tmp/stray.pdf")
@@ -87,7 +87,7 @@ class TestRenderResultsWithIssues:
         assert "Thumbnail missing" in output
         assert "Checksum mismatch" in output
 
-    @pytest.mark.usefixtures("_media_settings")
+    @pytest.mark.usefixtures("paperless_dirs")
     def test_unknown_doc_pk(self) -> None:
         msgs = SanityCheckMessages()
         msgs.error(99999, "Ghost document")
@@ -184,7 +184,6 @@ class TestDocumentSanityCheckerCommand:
         assert "ERROR" in output
         assert "Original of document does not exist" in output
 
-    @pytest.mark.usefixtures("_media_settings")
     def test_checksum_mismatch(self, paperless_dirs: PaperlessDirs) -> None:
         """Lightweight document with zero-byte files triggers checksum mismatch."""
         doc = DocumentFactory(

src/documents/tests/search/conftest.py

@@ -9,29 +9,21 @@ from documents.search._backend import TantivyBackend
 from documents.search._backend import reset_backend
 from documents.search._schema import build_schema
 from documents.search._tokenizer import register_tokenizers
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
     from collections.abc import Callable
     from collections.abc import Generator
-    from pathlib import Path
 
     from pytest_django.fixtures import Settings
 
     from documents.models import Document
+    from paperless_testing.dirs import PaperlessDirs
 
 
 @pytest.fixture
-def index_dir(tmp_path: Path, settings: Settings) -> Path:
+def backend(paperless_dirs: PaperlessDirs) -> Generator[TantivyBackend, None, None]:
-    path = tmp_path / "index"
+    b = TantivyBackend(path=paperless_dirs.index_dir)
-    path.mkdir()
-    settings.INDEX_DIR = path
-    return path
-
-
-@pytest.fixture
-def backend() -> Generator[TantivyBackend, None, None]:
-    b = TantivyBackend()  # path=None → in-memory index
     b.open()
     try:
         yield b

src/documents/tests/search/test_acceptance.py

@@ -14,7 +14,6 @@ from typing import TYPE_CHECKING
 
 import pytest
 import time_machine
-from django.contrib.auth.models import User
 
 from documents.models import CustomField
 from documents.models import CustomFieldInstance
@@ -22,7 +21,8 @@ from documents.models import DocumentType
 from documents.models import Note
 from documents.models import StoragePath
 from documents.search._query import parse_user_query
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import UserFactory
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -148,7 +148,7 @@ class TestJsonSubpaths:
         THEN:
             - Only the document with alice's note matches
         """
-        alice = User.objects.create_user(username="alice")
+        alice = UserFactory(username="alice")
         doc_with_note = DocumentFactory(
             title="Has note",
             content="x",

src/documents/tests/search/test_backend.py

@@ -3,10 +3,8 @@ from pathlib import Path
 
 import pytest
 from django.contrib.auth.models import Group
-from django.contrib.auth.models import User
 from django.db import connection
 from django.test.utils import CaptureQueriesContext
-from guardian.shortcuts import assign_perm
 from pytest_mock import MockerFixture
 
 from documents.models import CustomField
@@ -19,11 +17,12 @@ from documents.search._backend import WriteBatch
 from documents.search._backend import get_backend
 from documents.search._backend import reset_backend
 from documents.signals.handlers import add_to_index
-from documents.tests.factories import CorrespondentFactory
+from paperless_testing.factories import CorrespondentFactory
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
-from documents.tests.factories import DocumentTypeFactory
+from paperless_testing.factories import DocumentTypeFactory
-from documents.tests.factories import TagFactory
+from paperless_testing.factories import TagFactory
-from documents.tests.factories import UserFactory
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_object
 
 pytestmark = [pytest.mark.search, pytest.mark.django_db]
 
@@ -189,7 +188,7 @@ class TestAddOrUpdateIds:
             pk=1,
             owner=owner,
         )
-        assign_perm("view_document", user, doc)
+        grant_object(user, doc, "view_document")
 
         with backend.batch_update() as batch:
             batch.add_or_update_ids([doc.pk])
@@ -209,7 +208,7 @@ class TestAddOrUpdateIds:
             pk=1,
             owner=owner,
         )
-        assign_perm("view_document", group, doc)
+        grant_object(group, doc, "view_document")
 
         with backend.batch_update() as batch:
             batch.add_or_update_ids([doc.pk])
@@ -763,8 +762,8 @@ class TestSearchIds:
 
     def test_respects_permission_filter(self, backend: TantivyBackend) -> None:
         """search_ids must respect user permission filtering."""
-        owner = User.objects.create_user("ids_owner")
+        owner = UserFactory(username="ids_owner")
-        other = User.objects.create_user("ids_other")
+        other = UserFactory(username="ids_other")
         doc = Document.objects.create(
             title="private doc",
             content="secret keyword",
@@ -843,7 +842,7 @@ class TestRebuild:
             content="group secret keyword",
             owner=owner,
         )
-        assign_perm("view_document", group, doc)
+        grant_object(group, doc, "view_document")
 
         backend.rebuild(Document.objects.all())
 
@@ -948,7 +947,8 @@ class TestSingleton:
         yield
         reset_backend()
 
-    def test_returns_same_instance_on_repeated_calls(self, index_dir) -> None:
+    @pytest.mark.usefixtures("paperless_dirs")
+    def test_returns_same_instance_on_repeated_calls(self) -> None:
         """Singleton pattern: repeated calls to get_backend() must return the same instance."""
         assert get_backend() is get_backend()
 
@@ -965,7 +965,8 @@ class TestSingleton:
         assert b1 is not b2
         assert b2._path == tmp_path / "b"
 
-    def test_reset_forces_new_instance(self, index_dir) -> None:
+    @pytest.mark.usefixtures("paperless_dirs")
+    def test_reset_forces_new_instance(self) -> None:
         """reset_backend() must force creation of a new backend instance on next get_backend() call."""
         b1 = get_backend()
         reset_backend()
@@ -1071,7 +1072,7 @@ class TestFieldHandling:
 
     def test_notes_include_user_information(self, backend: TantivyBackend) -> None:
         """Notes must be indexed with user information when available for structured queries."""
-        user = User.objects.create_user("notewriter")
+        user = UserFactory(username="notewriter")
         doc = Document.objects.create(
             title="Doc with notes",
             content="test",
@@ -1173,7 +1174,7 @@ class TestHighlightHits:
         notes.note: prefix so the query targets notes content directly, but
         the snippet is generated from notes_text which stores the same text.
         """
-        user = User.objects.create_user("hl_noteuser")
+        user = UserFactory(username="hl_noteuser")
         doc = Document.objects.create(
             title="Doc with matching note",
             content="unrelated content",

src/documents/tests/search/test_documented_syntax.py

@@ -27,7 +27,7 @@ import time_machine
 from documents.models import Note
 from documents.models import Tag
 from documents.search._errors import InvalidDateQuery
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -269,7 +269,7 @@ class TestDocumentedDateForms:
             yield
 
     @pytest.fixture
-    def dated(self, index_document: Callable[..., Document]) -> dict[str, int]:
+    def dated(self, backend: TantivyBackend) -> dict[str, int]:
         stamps = {
             "today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),
             "yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),
@@ -279,14 +279,14 @@ class TestDocumentedDateForms:
             "january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC),
             "old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
         }
-        return {
+        docs = {
-            label: index_document(
+            label: DocumentFactory(title=label, content="dated body", added=stamp)
-                title=label,
-                content="dated body",
-                added=stamp,
-            ).pk
             for label, stamp in stamps.items()
         }
+        with backend.batch_update() as batch:
+            for doc in docs.values():
+                batch.add_or_update(doc)
+        return {label: doc.pk for label, doc in docs.items()}
 
     @pytest.mark.parametrize(
         ("query", "label"),

src/documents/tests/search/test_highlight_query_guard.py

@@ -29,7 +29,7 @@ from rest_framework import status
 
 from documents.search._backend import SearchMode
 from documents.search._query import parse_simple_text_highlight_query
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
     from rest_framework.test import APIClient

src/documents/tests/search/test_json_field_prefixes.py

@@ -17,12 +17,12 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 import pytest
-from django.contrib.auth.models import User
 
 from documents.models import CustomField
 from documents.models import CustomFieldInstance
 from documents.models import Note
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import UserFactory
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -53,7 +53,7 @@ class TestBareJsonFieldPrefixes:
               decoy's content match does not resurface through a demoted
               text search
         """
-        alice = User.objects.create_user(username="alice")
+        alice = UserFactory(username="alice")
         with_note = DocumentFactory(title="Has note", content="x")
         Note.objects.create(document=with_note, user=alice, note="crocodile")
         backend.add_or_update(with_note)
@@ -116,7 +116,7 @@ class TestBareJsonFieldPrefixes:
               document; the default-subpath resolution for the bare
               prefix does not interfere with explicit subpath addressing
         """
-        bob = User.objects.create_user(username="bob")
+        bob = UserFactory(username="bob")
         doc = DocumentFactory(title="Bob note", content="x")
         Note.objects.create(document=doc, user=bob, note="remark")
         backend.add_or_update(doc)

src/documents/tests/search/test_json_subpath_completeness.py

@@ -20,7 +20,6 @@ from typing import TYPE_CHECKING
 
 import pytest
 import tantivy
-from django.contrib.auth.models import User
 from whoosh_compat import FieldKind
 
 from documents.models import CustomField
@@ -28,6 +27,7 @@ from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import Note
 from documents.search._fields import PUBLIC_FIELDS
+from paperless_testing.factories import UserFactory
 
 if TYPE_CHECKING:
     from documents.search._backend import TantivyBackend
@@ -49,7 +49,7 @@ class TestJsonSubpathsAreWrittenAtIndexTime:
             - Every subpath PUBLIC_FIELDS declares for notes/custom_fields
               is present as a key in the document's stored JSON payload
         """
-        user = User.objects.create_user(username="completeness-user")
+        user = UserFactory(username="completeness-user")
         field = CustomField.objects.create(
             name="Completeness Field",
             data_type=CustomField.FieldDataType.STRING,

src/documents/tests/search/test_lock_backoff.py

@@ -15,7 +15,7 @@ from documents.search._backend import SearchIndexLockError
 from documents.search._backend import TantivyBackend
 from documents.tasks import index_document
 from documents.tasks import remove_document_from_index
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
     from collections.abc import Generator

src/documents/tests/search/test_migration_fulltext_query_field_prefixes.py

@@ -1,6 +1,6 @@
 import pytest
 
-from documents.tests.utils import TestMigrations
+from paperless_testing.migrations import TestMigrations
 
 pytestmark = pytest.mark.search
 

src/documents/tests/search/test_permission_field_isolation.py

@@ -18,13 +18,14 @@ from typing import TYPE_CHECKING
 import pytest
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
-from guardian.shortcuts import assign_perm
 
 from documents.models import Correspondent
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.models import Tag
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_object
 
 if TYPE_CHECKING:
     from documents.search._backend import TantivyBackend
@@ -34,22 +35,22 @@ pytestmark = [pytest.mark.search, pytest.mark.django_db]
 
 @pytest.fixture
 def owner() -> User:
-    return User.objects.create_user(username="owner")
+    return UserFactory(username="owner")
 
 
 @pytest.fixture
 def stranger() -> User:
-    return User.objects.create_user(username="stranger")
+    return UserFactory(username="stranger")
 
 
 @pytest.fixture
 def viewer() -> User:
-    return User.objects.create_user(username="viewer")
+    return UserFactory(username="viewer")
 
 
 @pytest.fixture
 def group_member() -> User:
-    user = User.objects.create_user(username="group_member")
+    user = UserFactory(username="group_member")
     user.groups.add(Group.objects.create(name="accounting"))
     return user
 
@@ -127,7 +128,7 @@ class TestPermissionFilteringOnIndexedDocuments:
             checksum="perm-shared-user",
             owner=owner,
         )
-        assign_perm("view_document", viewer, doc)
+        grant_object(viewer, doc, "view_document")
         backend.add_or_update(doc)
 
         assert backend.search_ids("invoice", user=viewer) == [doc.pk]
@@ -157,7 +158,7 @@ class TestPermissionFilteringOnIndexedDocuments:
             checksum="perm-shared-group",
             owner=owner,
         )
-        assign_perm("view_document", group_member.groups.first(), doc)
+        grant_object(group_member.groups.first(), doc, "view_document")
         backend.add_or_update(doc)
 
         assert backend.search_ids("invoice", user=group_member) == [doc.pk]

src/documents/tests/search/test_schema.py

@@ -13,11 +13,11 @@ from documents.search._schema import needs_rebuild
 from documents.search._schema import schema_fingerprint
 
 if TYPE_CHECKING:
-    from pathlib import Path
-
     import tantivy
     from pytest_django.fixtures import Settings
 
+    from paperless_testing.dirs import PaperlessDirs
+
 
 pytestmark = pytest.mark.search
 
@@ -25,16 +25,19 @@ pytestmark = pytest.mark.search
 class TestNeedsRebuild:
     """needs_rebuild covers all sentinel-file states that require a full reindex."""
 
-    def test_returns_true_when_settings_file_missing(self, index_dir: Path) -> None:
+    def test_returns_true_when_settings_file_missing(
-        assert needs_rebuild(index_dir) is True
+        self,
+        paperless_dirs: PaperlessDirs,
+    ) -> None:
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_returns_false_when_version_and_language_match(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = "en"
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps(
                 {
                     "schema_version": SCHEMA_VERSION,
@@ -43,51 +46,51 @@ class TestNeedsRebuild:
                 },
             ),
         )
-        assert needs_rebuild(index_dir) is False
+        assert needs_rebuild(paperless_dirs.index_dir) is False
 
     def test_returns_true_on_schema_version_mismatch(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = None
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": SCHEMA_VERSION - 1, "language": None}),
         )
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_returns_true_when_version_is_not_an_integer(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = None
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": "not-a-number", "language": None}),
         )
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_returns_true_when_language_key_missing(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = "en"
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": SCHEMA_VERSION}),
         )
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_returns_true_when_language_differs(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = "de"
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
         )
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
 
 def _schema_fields(schema: tantivy.Schema) -> dict[str, dict]:

src/documents/tests/search/test_schema_fingerprint.py

@@ -35,6 +35,8 @@ if TYPE_CHECKING:
 
     from pytest_django.fixtures import SettingsWrapper
 
+    from paperless_testing.dirs import PaperlessDirs
+
 pytestmark = pytest.mark.search
 
 # The on-disk field layout of a v2 index, pinned as data. Any edit here is an
@@ -469,7 +471,7 @@ def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str:
 class TestNeedsRebuildOnFingerprint:
     def test_matching_fingerprint_does_not_rebuild(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: SettingsWrapper,
     ) -> None:
         """
@@ -482,13 +484,13 @@ class TestNeedsRebuildOnFingerprint:
             - It returns False
         """
         settings.SEARCH_LANGUAGE = None
-        _sentinels(index_dir)
+        _sentinels(paperless_dirs.index_dir)
 
-        assert needs_rebuild(index_dir) is False
+        assert needs_rebuild(paperless_dirs.index_dir) is False
 
     def test_stale_fingerprint_rebuilds_despite_a_matching_version(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: SettingsWrapper,
         monkeypatch: pytest.MonkeyPatch,
     ) -> None:
@@ -505,7 +507,7 @@ class TestNeedsRebuildOnFingerprint:
               every subsequent write would raise
         """
         settings.SEARCH_LANGUAGE = None
-        _sentinels(index_dir)
+        _sentinels(paperless_dirs.index_dir)
         extended = [
             *field_descriptors(),
             FieldDescriptor(
@@ -519,11 +521,11 @@ class TestNeedsRebuildOnFingerprint:
         ]
         monkeypatch.setattr(_schema, "field_descriptors", lambda: extended)
 
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_reordered_schema_rebuilds(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: SettingsWrapper,
         monkeypatch: pytest.MonkeyPatch,
     ) -> None:
@@ -538,16 +540,16 @@ class TestNeedsRebuildOnFingerprint:
             - It returns True
         """
         settings.SEARCH_LANGUAGE = None
-        _sentinels(index_dir)
+        _sentinels(paperless_dirs.index_dir)
         reordered = field_descriptors()
         reordered[1], reordered[2] = reordered[2], reordered[1]
         monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered)
 
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_missing_fingerprint_rebuilds(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: SettingsWrapper,
     ) -> None:
         """
@@ -561,15 +563,15 @@ class TestNeedsRebuildOnFingerprint:
               is rebuilt rather than trusted
         """
         settings.SEARCH_LANGUAGE = None
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": SCHEMA_VERSION, "language": None}),
         )
 
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_written_sentinels_satisfy_the_check(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: SettingsWrapper,
     ) -> None:
         """
@@ -582,6 +584,6 @@ class TestNeedsRebuildOnFingerprint:
             - It returns False
         """
         settings.SEARCH_LANGUAGE = "en"
-        _write_sentinels(index_dir)
+        _write_sentinels(paperless_dirs.index_dir)
 
-        assert needs_rebuild(index_dir) is False
+        assert needs_rebuild(paperless_dirs.index_dir) is False

src/documents/tests/test_admin.py

@@ -16,11 +16,11 @@ from documents.models import Document
 from documents.models import Tag
 from documents.search import get_backend
 from documents.search import reset_backend
-from documents.tests.factories import DocumentFactory
-from documents.tests.factories import TagFactory
-from documents.tests.factories import UserFactory
-from documents.tests.utils import DirectoriesMixin
 from paperless.admin import PaperlessUserAdmin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import TagFactory
+from paperless_testing.factories import UserFactory
 
 
 @pytest.fixture

src/documents/tests/test_api_app_config.py

@@ -3,7 +3,6 @@ from io import BytesIO
 from pathlib import Path
 from unittest.mock import patch
 
-from django.contrib.auth.models import User
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import override_settings
 from PIL import Image
@@ -11,10 +10,11 @@ from PIL.PngImagePlugin import PngInfo
 from rest_framework import status
 from rest_framework.test import APITestCase
 
-from documents.tests.utils import DirectoriesMixin
-from documents.tests.utils import read_streaming_response
 from paperless.models import ApplicationConfiguration
 from paperless.models import ColorConvertChoices
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.http import read_streaming_response
 
 
 class TestApiAppConfig(DirectoriesMixin, APITestCase):
@@ -23,7 +23,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
+        user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=user)
 
     def test_api_get_config(self) -> None:
@@ -81,6 +81,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
                 "ai_enabled": None,
                 "llm_embedding_backend": None,
                 "llm_embedding_model": None,
+                "llm_embedding_api_key": None,
                 "llm_embedding_endpoint": None,
                 "llm_embedding_chunk_size": None,
                 "llm_context_size": None,
@@ -267,7 +268,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
         THEN:
             - old app_logo file is deleted
         """
-        admin = User.objects.create_superuser(username="admin")
+        admin = UserFactory(username="admin", superuser=True)
         self.client.force_login(user=admin)
         response = self.client.get("/logo/")
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
@@ -922,6 +923,49 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
         self.assertEqual(ApplicationConfiguration.objects.count(), 1)
 
+    def test_update_llm_embedding_api_key(self) -> None:
+        """
+        GIVEN:
+            - Existing config with llm_embedding_api_key specified
+        WHEN:
+            - API to update llm_embedding_api_key is called with all *s
+            - API to update llm_embedding_api_key is called with empty string
+        THEN:
+            - llm_embedding_api_key is unchanged
+            - llm_embedding_api_key is set to None
+        """
+        config = ApplicationConfiguration.objects.first()
+        assert config is not None
+        config.llm_embedding_api_key = "1234567890"
+        config.save()
+
+        # Test with all *
+        response = self.client.patch(
+            f"{self.ENDPOINT}1/",
+            json.dumps(
+                {
+                    "llm_embedding_api_key": "*" * 32,
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        config.refresh_from_db()
+        self.assertEqual(config.llm_embedding_api_key, "1234567890")
+        # Test with empty string
+        response = self.client.patch(
+            f"{self.ENDPOINT}1/",
+            json.dumps(
+                {
+                    "llm_embedding_api_key": "",
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        config.refresh_from_db()
+        self.assertEqual(config.llm_embedding_api_key, None)
+
     def test_update_llm_api_key(self) -> None:
         """
         GIVEN:

src/documents/tests/test_api_bulk_download.py

@@ -4,8 +4,6 @@ import json
 import shutil
 import zipfile
 
-from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.test import override_settings
 from django.utils import timezone
 from rest_framework import status
@@ -14,9 +12,11 @@ from rest_framework.test import APITestCase
 from documents.models import Correspondent
 from documents.models import Document
 from documents.models import DocumentType
-from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import SampleDirMixin
-from documents.tests.utils import read_streaming_response
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.http import read_streaming_response
+from paperless_testing.permissions import grant_global
 
 
 class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
@@ -25,7 +25,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
         self.doc1 = Document.objects.create(title="unrelated", checksum="A")
@@ -166,7 +166,15 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
             ),
             content_type="application/json",
         )
-        response.close()
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response["Content-Type"], "application/zip")
+
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
+            self.assertEqual(zipf.infolist()[0].compress_type, zipfile.ZIP_LZMA)
+
+            with self.doc2.source_file as f:
+                self.assertEqual(f.read(), zipf.read("2021-01-01 document A.pdf"))
 
     @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
     def test_formatted_download_originals(self) -> None:
@@ -326,10 +334,8 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
                 )
 
     def test_download_insufficient_permissions(self) -> None:
-        user = User.objects.create_user(username="temp_user")
+        user = UserFactory(username="temp_user")
-        user.user_permissions.add(
+        grant_global(user, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         self.client.force_authenticate(user=user)
 
         self.doc2.owner = self.user

src/documents/tests/test_api_bulk_edit.py

@@ -2,27 +2,30 @@ import json
 from unittest import mock
 
 from auditlog.models import LogEntry
-from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from django.test import override_settings
-from guardian.shortcuts import assign_perm
 from rest_framework import status
 from rest_framework.test import APITestCase
 
 from documents.models import Correspondent
 from documents.models import CustomField
+from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.models import Tag
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_all_global
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestBulkEditAPI(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
+        user = UserFactory(username="temp_admin", superuser=True)
         self.user = user
         self.client.force_authenticate(user=user)
 
@@ -200,7 +203,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.doc1.refresh_from_db()
         self.assertFalse(self.doc1.tags.filter(pk=self.t1.pk).exists())
 
-    @mock.patch("documents.serialisers.bulk_edit.modify_tags")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.modify_tags")
     def test_api_modify_tags(self, m) -> None:
         self.setup_mock(m, "modify_tags")
         response = self.client.post(
@@ -224,7 +227,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(kwargs["add_tags"], [self.t1.id])
         self.assertEqual(kwargs["remove_tags"], [self.t2.id])
 
-    @mock.patch("documents.serialisers.bulk_edit.modify_tags")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.modify_tags")
     def test_api_modify_tags_not_provided(self, m) -> None:
         """
         GIVEN:
@@ -252,7 +255,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.modify_custom_fields")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.modify_custom_fields")
     def test_api_modify_custom_fields(self, m) -> None:
         self.setup_mock(m, "modify_custom_fields")
         response = self.client.post(
@@ -278,15 +281,15 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(kwargs["add_custom_fields"], [self.cf1.id])
         self.assertEqual(kwargs["remove_custom_fields"], [self.cf2.id])
 
-    @mock.patch("documents.serialisers.bulk_edit.modify_custom_fields")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.modify_custom_fields")
     def test_api_modify_custom_fields_documentlink_forbidden_for_unpermitted_target(
         self,
         m,
     ) -> None:
         self.setup_mock(m, "modify_custom_fields")
-        user = User.objects.create_user(username="doc-owner")
+        user = UserFactory(username="doc-owner")
-        user.user_permissions.add(Permission.objects.get(codename="change_document"))
+        grant_global(user, "change_document")
-        other_user = User.objects.create_user(username="other-user")
+        other_user = UserFactory(username="other-user")
         source_doc = Document.objects.create(
             checksum="source",
             title="Source",
@@ -322,7 +325,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.modify_custom_fields")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.modify_custom_fields")
     def test_api_modify_custom_fields_with_values(self, m) -> None:
         self.setup_mock(m, "modify_custom_fields")
         response = self.client.post(
@@ -346,7 +349,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(kwargs["add_custom_fields"], {self.cf1.id: "foo"})
         self.assertEqual(kwargs["remove_custom_fields"], [self.cf2.id])
 
-    @mock.patch("documents.serialisers.bulk_edit.modify_custom_fields")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.modify_custom_fields")
     def test_api_modify_custom_fields_rejects_invalid_value(self, m) -> None:
         self.setup_mock(m, "modify_custom_fields")
 
@@ -370,7 +373,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertIn(str(self.cf1.id), response.data["add_custom_fields"])
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.modify_custom_fields")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.modify_custom_fields")
     def test_api_modify_custom_fields_invalid_params(self, m) -> None:
         """
         GIVEN:
@@ -490,7 +493,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.delete")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.delete")
     def test_api_delete(self, m) -> None:
         self.setup_mock(m, "delete")
         response = self.client.post(
@@ -506,7 +509,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(args[0], [self.doc1.id])
         self.assertEqual(len(kwargs), 0)
 
-    @mock.patch("documents.views.bulk_edit.delete")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.delete")
     def test_delete_documents_endpoint(self, m) -> None:
         self.setup_mock(m, "delete")
         response = self.client.post(
@@ -520,7 +523,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(args[0], [self.doc1.id])
         self.assertEqual(len(kwargs), 0)
 
-    @mock.patch("documents.views.bulk_edit.delete")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.delete")
     def test_delete_documents_endpoint_with_excluded_documents(self, m) -> None:
         self.setup_mock(m, "delete")
         response = self.client.post(
@@ -544,7 +547,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(args[0], [self.doc1.id])
         self.assertEqual(len(kwargs), 0)
 
-    @mock.patch("documents.views.bulk_edit.reprocess")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.reprocess")
     def test_reprocess_documents_endpoint(self, m) -> None:
         self.setup_mock(m, "reprocess")
         response = self.client.post(
@@ -558,7 +561,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(args[0], [self.doc1.id])
         self.assertEqual(kwargs, {"remote_ocr": False})
 
-    @mock.patch("documents.views.bulk_edit.reprocess")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.reprocess")
     def test_reprocess_documents_endpoint_remote_ocr(self, m) -> None:
         """
         GIVEN:
@@ -580,7 +583,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(args[0], [self.doc1.id])
         self.assertEqual(kwargs, {"remote_ocr": True})
 
-    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_storage_path")
     def test_api_set_storage_path(self, m) -> None:
         """
         GIVEN:
@@ -610,7 +613,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertListEqual(args[0], [self.doc1.id])
         self.assertEqual(kwargs["storage_path"], self.sp1.id)
 
-    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_storage_path")
     def test_api_unset_storage_path(self, m) -> None:
         """
         GIVEN:
@@ -735,7 +738,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             response.content,
         )
 
-    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_storage_path")
     def test_api_bulk_edit_with_all_true_resolves_documents_from_filters(
         self,
         m,
@@ -761,7 +764,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(args[0], [self.doc2.id])
         self.assertEqual(kwargs["storage_path"], self.sp1.id)
 
-    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_storage_path")
     def test_api_bulk_edit_with_all_true_excludes_documents(self, m) -> None:
         self.setup_mock(m, "set_storage_path")
 
@@ -784,13 +787,11 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertCountEqual(args[0], [self.doc1.id, self.doc3.id, self.doc5.id])
         self.assertEqual(kwargs["storage_path"], self.sp1.id)
 
-    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_storage_path")
     def test_api_bulk_edit_with_all_true_resolves_owned_duplicates(self, m) -> None:
         self.setup_mock(m, "set_storage_path")
-        user = User.objects.create_user(username="duplicate-owner")
+        user = UserFactory(username="duplicate-owner")
-        user.user_permissions.add(
+        grant_global(user, "change_document")
-            Permission.objects.get(codename="change_document"),
-        )
         first_duplicate = Document.objects.create(
             checksum="owned-duplicate",
             title="First duplicate",
@@ -823,7 +824,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(kwargs["storage_path"], self.sp1.id)
 
     @mock.patch("documents.search.get_backend")
-    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_storage_path")
     def test_api_bulk_edit_with_all_true_resolves_documents_from_search_filters(
         self,
         m,
@@ -1178,7 +1179,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         user1 = User.objects.create(username="user1")
         self.client.force_authenticate(user=user1)
 
-        assign_perm("view_document", user1, self.doc2)
+        grant_object(user1, self.doc2, "view_document")
 
         response = self.client.post(
             "/api/documents/selection_data/",
@@ -1188,9 +1189,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
-        user1.user_permissions.add(
+        grant_global(user1, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         user1 = User.objects.get(pk=user1.pk)
         self.client.force_authenticate(user=user1)
         response = self.client.post(
@@ -1200,7 +1199,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(response.status_code, status.HTTP_200_OK)
 
-    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_permissions")
     def test_set_permissions(self, m) -> None:
         self.setup_mock(m, "set_permissions")
         user1 = User.objects.create(username="user1")
@@ -1235,7 +1234,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertCountEqual(args[0], [self.doc2.id, self.doc3.id])
         self.assertEqual(len(kwargs["set_permissions"]["view"]["users"]), 2)
 
-    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_permissions")
     def test_set_permissions_requires_set_permissions_parameter(self, m) -> None:
         self.setup_mock(m, "set_permissions")
 
@@ -1259,7 +1258,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertIn(b"set_permissions not specified", response.content)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_permissions")
     def test_set_permissions_rejects_malformed_set_permissions(self, m) -> None:
         """
         GIVEN:
@@ -1294,7 +1293,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
                 self.assertIn(expected_message, response.content)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_permissions")
     def test_set_permissions_rejects_invalid_owner(self, m) -> None:
         """
         GIVEN:
@@ -1336,7 +1335,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
                 self.assertIn(b"Specified owner cannot be found", response.content)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_permissions")
     def test_set_permissions_null_is_a_noop(self, m) -> None:
         """
         GIVEN:
@@ -1372,7 +1371,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             {"view": {}, "change": {}},
         )
 
-    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_permissions")
     def test_set_permissions_passes_validated_owner_id(self, m) -> None:
         """
         GIVEN:
@@ -1403,7 +1402,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         m.assert_called_once()
         self.assertEqual(m.call_args.kwargs["owner"], self.user.id)
 
-    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_permissions")
     def test_set_permissions_merge(self, m) -> None:
         self.setup_mock(m, "set_permissions")
         user1 = User.objects.create(username="user1")
@@ -1455,8 +1454,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         _, kwargs = m.call_args
         self.assertEqual(kwargs["merge"], True)
 
-    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_storage_path")
-    @mock.patch("documents.views.bulk_edit.merge")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.merge")
     def test_insufficient_global_perms(self, mock_merge, mock_set_storage) -> None:
         """
         GIVEN:
@@ -1519,7 +1518,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         mock_merge.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_permissions")
     def test_insufficient_permissions_ownership(self, m) -> None:
         """
         GIVEN:
@@ -1533,7 +1532,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1573,7 +1572,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         m.assert_called_once()
 
-    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.set_storage_path")
     def test_insufficient_permissions_edit(self, m) -> None:
         """
         GIVEN:
@@ -1587,8 +1586,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        assign_perm("view_document", user1, self.doc1)
+        grant_object(user1, self.doc1, "view_document")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1609,7 +1608,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         m.assert_not_called()
         self.assertEqual(response.content, b"Insufficient permissions")
 
-        assign_perm("change_document", user1, self.doc1)
+        grant_object(user1, self.doc1, "change_document")
 
         response = self.client.post(
             "/api/documents/bulk_edit/",
@@ -1627,7 +1626,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
 
         m.assert_called_once()
 
-    @mock.patch("documents.views.bulk_edit.rotate")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.rotate")
     def test_rotate(self, m) -> None:
         self.setup_mock(m, "rotate")
         response = self.client.post(
@@ -1649,7 +1648,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(kwargs["source_mode"], "latest_version")
         self.assertEqual(kwargs["user"], self.user)
 
-    @mock.patch("documents.views.bulk_edit.rotate")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.rotate")
     def test_rotate_invalid_params(self, m) -> None:
         response = self.client.post(
             "/api/documents/rotate/",
@@ -1689,7 +1688,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertIn(b"degrees must be a multiple of 90", response.content)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.rotate")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.rotate")
     def test_bulk_edit_rotate_rejects_invalid_degrees(self, m) -> None:
         """
         GIVEN:
@@ -1726,7 +1725,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
                 self.assertIn(expected_message, response.content)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.rotate")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.rotate")
     def test_bulk_edit_rotate_passes_integer_degrees(self, m) -> None:
         """
         GIVEN:
@@ -1754,7 +1753,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         m.assert_called_once()
         self.assertEqual(m.call_args.kwargs["degrees"], -90)
 
-    @mock.patch("documents.serialisers.bulk_edit.split")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.split")
     def test_bulk_edit_split_rejects_invalid_pages(self, m) -> None:
         """
         GIVEN:
@@ -1786,7 +1785,37 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
                 self.assertIn(b"invalid pages specified", response.content)
         m.assert_not_called()
 
-    @mock.patch("documents.serialisers.bulk_edit.split")
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.split")
+    def test_bulk_edit_split_rejects_unknown_page_count(self, m) -> None:
+        """
+        GIVEN:
+            - A legacy split bulk edit of a document without a page count
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - API returns HTTP 400
+            - split is not called
+        """
+        self.setup_mock(m, "split")
+
+        for pages in ("1", "1-5000000"):
+            with self.subTest(pages=pages):
+                response = self.client.post(
+                    "/api/documents/bulk_edit/",
+                    json.dumps(
+                        {
+                            "documents": [self.doc1.id],
+                            "method": "split",
+                            "parameters": {"pages": pages},
+                        },
+                    ),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(b"document page count is unknown", response.content)
+        m.assert_not_called()
+
+    @mock.patch("documents.serialisers.bulk_edit.bulk_edit.split")
     def test_bulk_edit_split_parses_pages(self, m) -> None:
         """
         GIVEN:
@@ -1814,12 +1843,12 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         m.assert_called_once()
         self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]])
 
-    @mock.patch("documents.views.bulk_edit.rotate")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.rotate")
     def test_rotate_insufficient_permissions(self, m) -> None:
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1853,7 +1882,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         m.assert_called_once()
 
-    @mock.patch("documents.views.bulk_edit.merge")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.merge")
     def test_merge(self, m) -> None:
         self.setup_mock(m, "merge")
         response = self.client.post(
@@ -1875,12 +1904,12 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(kwargs["source_mode"], "latest_version")
         self.assertEqual(kwargs["user"], self.user)
 
-    @mock.patch("documents.views.bulk_edit.merge")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.merge")
     def test_merge_and_delete_insufficient_permissions(self, m) -> None:
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1916,14 +1945,11 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         m.assert_called_once()
 
-    @mock.patch("documents.views.bulk_edit.merge")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.merge")
     def test_merge_and_delete_requires_change_permission(self, m) -> None:
         self.setup_mock(m, "merge")
-        user = User.objects.create_user(username="no-change")
+        user = UserFactory(username="no-change")
-        user.user_permissions.add(
+        grant_global(user, "add_document", "delete_document")
-            Permission.objects.get(codename="add_document"),
-            Permission.objects.get(codename="delete_document"),
-        )
         self.client.force_authenticate(user=user)
 
         response = self.client.post(
@@ -1940,7 +1966,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         m.assert_not_called()
 
-    @mock.patch("documents.views.bulk_edit.merge")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.merge")
     def test_merge_invalid_parameters(self, m) -> None:
         self.setup_mock(m, "merge")
         response = self.client.post(
@@ -1973,7 +1999,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             for method, parameters in method_payloads.items():
                 with self.subTest(method=method, version=version):
                     with mock.patch(
-                        f"documents.views.bulk_edit.{method}",
+                        f"documents.views.bulk_edit.bulk_edit.{method}",
                     ) as mocked_method:
                         self.setup_mock(mocked_method, method)
                         with self.assertLogs("paperless.api", level="WARNING") as logs:
@@ -2062,7 +2088,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             {"operations": ["This list may not be empty."]},
         )
 
-    @mock.patch("documents.views.bulk_edit.edit_pdf")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.edit_pdf")
     def test_edit_pdf(self, m) -> None:
         self.setup_mock(m, "edit_pdf")
         response = self.client.post(
@@ -2222,7 +2248,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertIn(b"Invalid source_mode", response.content)
 
-    @mock.patch("documents.views.bulk_edit.edit_pdf")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.edit_pdf")
     def test_edit_pdf_rejects_invalid_operation_values(self, m) -> None:
         """
         GIVEN:
@@ -2288,7 +2314,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             {"operations": {"0": {"rotate": ["rotate must be a multiple of 90"]}}},
         )
 
-    @mock.patch("documents.views.bulk_edit.edit_pdf")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.edit_pdf")
     def test_edit_pdf_page_out_of_bounds(self, m) -> None:
         self.setup_mock(m, "edit_pdf")
         response = self.client.post(
@@ -2305,12 +2331,12 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertIn(b"out of bounds", response.content)
         m.assert_not_called()
 
-    @mock.patch("documents.views.bulk_edit.edit_pdf")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.edit_pdf")
     def test_edit_pdf_insufficient_permissions(self, m) -> None:
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -2342,10 +2368,10 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         m.assert_called_once()
 
-    @mock.patch("documents.views.bulk_edit.edit_pdf")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.edit_pdf")
     def test_edit_pdf_update_requires_change_permission(self, m) -> None:
         self.setup_mock(m, "edit_pdf")
-        user = User.objects.create_user(username="no-change")
+        user = UserFactory(username="no-change")
         self.client.force_authenticate(user=user)
 
         response = self.client.post(
@@ -2363,8 +2389,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         m.assert_not_called()
 
-    @mock.patch("documents.views.bulk_edit.remove_password")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.remove_password")
-    @mock.patch("documents.views.bulk_edit.edit_pdf")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.edit_pdf")
     def test_delete_original_requires_delete_permission(
         self,
         edit_pdf_mock,
@@ -2372,11 +2398,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
     ) -> None:
         self.setup_mock(edit_pdf_mock, "edit_pdf")
         self.setup_mock(remove_password_mock, "remove_password")
-        user = User.objects.create_user(username="no-delete")
+        user = UserFactory(username="no-delete")
-        user.user_permissions.add(
+        grant_global(user, "add_document", "change_document")
-            Permission.objects.get(codename="add_document"),
-            Permission.objects.get(codename="change_document"),
-        )
         self.client.force_authenticate(user=user)
 
         cases = [
@@ -2410,7 +2433,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
                 self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
                 operation_mock.assert_not_called()
 
-    @mock.patch("documents.views.bulk_edit.remove_password")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.remove_password")
     def test_remove_password(self, m) -> None:
         self.setup_mock(m, "remove_password")
         response = self.client.post(
@@ -2458,12 +2481,12 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
 
-    @mock.patch("documents.views.bulk_edit.remove_password")
+    @mock.patch("documents.views.bulk_edit.bulk_edit.remove_password")
     def test_remove_password_insufficient_permissions(self, m) -> None:
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -2503,7 +2526,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         WHEN:
             - API to bulk edit documents is called
         THEN:
-            - Audit log is created
+            - Audit log is created with the old and new correspondent
         """
         LogEntry.objects.all().delete()
         response = self.client.post(
@@ -2519,7 +2542,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 1)
+        entry = LogEntry.objects.get_for_object(self.doc1).get()
+        self.assertEqual(entry.changes, {"correspondent": [None, self.c2.id]})
 
     @override_settings(AUDIT_LOG_ENABLED=True)
     def test_bulk_edit_audit_log_enabled_tags(self) -> None:
@@ -2527,16 +2551,18 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         GIVEN:
             - Audit log is enabled
         WHEN:
-            - API to bulk edit tags is called
+            - API to bulk edit tags is called on an untagged document and a
+              document with several tags
         THEN:
-            - Audit log is created
+            - Audit log is created for each document with its full tag list
+              before and after the edit
         """
         LogEntry.objects.all().delete()
         response = self.client.post(
             "/api/documents/bulk_edit/",
             json.dumps(
                 {
-                    "documents": [self.doc1.id],
+                    "documents": [self.doc1.id, self.doc4.id],
                     "method": "modify_tags",
                     "parameters": {
                         "add_tags": [self.t1.id],
@@ -2548,18 +2574,32 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 1)
+        entry = LogEntry.objects.get_for_object(self.doc1).get()
+        self.assertEqual(entry.changes, {"tags": [[], [self.t1.id]]})
+        entry = LogEntry.objects.get_for_object(self.doc4).get()
+        self.assertEqual(
+            entry.changes,
+            {"tags": [[self.t1.id, self.t2.id], [self.t1.id]]},
+        )
 
     @override_settings(AUDIT_LOG_ENABLED=True)
     def test_bulk_edit_audit_log_enabled_custom_fields(self) -> None:
         """
         GIVEN:
             - Audit log is enabled
+            - A document with two custom fields
         WHEN:
-            - API to bulk edit custom fields is called
+            - API to bulk edit custom fields is called to add a third
         THEN:
-            - Audit log is created
+            - Audit log is created with every custom field instance before and
+              after the edit
+            - Audit log is created for the new custom field instance
         """
+        cf3 = CustomField.objects.create(name="cf3", data_type="string")
+        existing = [
+            CustomFieldInstance.objects.create(document=self.doc1, field=field)
+            for field in (self.cf2, cf3)
+        ]
         LogEntry.objects.all().delete()
         response = self.client.post(
             "/api/documents/bulk_edit/",
@@ -2577,7 +2617,14 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 2)
+        added = CustomFieldInstance.objects.get(document=self.doc1, field=self.cf1)
+        existing_ids = [instance.id for instance in existing]
+        entry = LogEntry.objects.get_for_object(self.doc1).get()
+        self.assertEqual(
+            entry.changes,
+            {"custom_fields": [existing_ids, [*existing_ids, added.id]]},
+        )
+        self.assertEqual(LogEntry.objects.get_for_object(added).count(), 1)
 
     def test_api_bulk_edit_with_bad_search_query_returns_400(self) -> None:
         """

src/documents/tests/test_api_chat.py

@@ -4,20 +4,22 @@ from typing import TYPE_CHECKING
 from unittest import mock
 
 import pytest
-from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from rest_framework import status
 from rest_framework.test import APIClient
 from rest_framework.test import APITestCase
 
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
+
 if TYPE_CHECKING:
+    from django.contrib.auth.models import User
     from pytest_mock import MockerFixture
 
 
 class TestChatStreamingViewInputValidation(APITestCase):
     def setUp(self) -> None:
         super().setUp()
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def _mock_ai_enabled(self) -> mock.MagicMock:
@@ -28,7 +30,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
 
     def test_oversized_question_is_rejected(self) -> None:
         with mock.patch(
-            "documents.views.AIConfig",
+            "documents.views.chat.AIConfig",
             return_value=self._mock_ai_enabled(),
         ):
             resp = self.client.post(
@@ -55,11 +57,11 @@ class TestChatStreamingViewInputValidation(APITestCase):
         chunks = [f"token{i} " for i in range(40)]
         with (
             mock.patch(
-                "documents.views.AIConfig",
+                "documents.views.chat.AIConfig",
                 return_value=self._mock_ai_enabled(),
             ),
             mock.patch(
-                "documents.views.stream_chat_with_documents",
+                "documents.views.chat.stream_chat_with_documents",
                 return_value=iter(chunks),
             ),
         ):
@@ -76,7 +78,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
 
     def test_missing_question_is_rejected(self) -> None:
         with mock.patch(
-            "documents.views.AIConfig",
+            "documents.views.chat.AIConfig",
             return_value=self._mock_ai_enabled(),
         ):
             resp = self.client.post(
@@ -100,9 +102,11 @@ class TestChatStreamingViewUnrestrictedFlag:
         never touches the real vector store; returns the patched callable so
         tests can inspect how it was called.
         """
-        mocker.patch("documents.views.AIConfig").return_value.ai_enabled = True
+        mocker.patch(
+            "documents.views.chat.AIConfig",
+        ).return_value.ai_enabled = True
         return mocker.patch(
-            "documents.views.stream_chat_with_documents",
+            "documents.views.chat.stream_chat_with_documents",
             return_value=iter(()),
         )
 
@@ -113,9 +117,7 @@ class TestChatStreamingViewUnrestrictedFlag:
         needs to reach the view at all. Model-level only: says nothing
         about which documents (if any) this user can actually see.
         """
-        regular_user.user_permissions.add(
+        grant_global(regular_user, "view_document")
-            *Permission.objects.filter(codename="view_document"),
-        )
         return user_client
 
     @pytest.mark.parametrize(

src/documents/tests/test_api_custom_fields.py

@@ -4,23 +4,24 @@ from unittest import mock
 from unittest.mock import ANY
 
 from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.test import override_settings
-from guardian.shortcuts import assign_perm
 from rest_framework import status
 from rest_framework.test import APITestCase
 
 from documents.models import CustomField
 from documents.models import CustomFieldInstance
 from documents.models import Document
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
     ENDPOINT = "/api/custom_fields/"
 
     def setUp(self) -> None:
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
         return super().setUp()
 
@@ -1174,11 +1175,9 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
     def test_documentlink_patch_requires_change_permission_on_target_documents(
         self,
     ) -> None:
-        source_owner = User.objects.create_user(username="source-owner")
+        source_owner = UserFactory(username="source-owner")
-        source_owner.user_permissions.add(
+        grant_global(source_owner, "change_document")
-            Permission.objects.get(codename="change_document"),
+        other_user = UserFactory(username="other-user")
-        )
-        other_user = User.objects.create_user(username="other-user")
 
         source_doc = Document.objects.create(
             title="Source",
@@ -1221,11 +1220,9 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
     def test_documentlink_patch_allowed_with_change_permission_on_target_documents(
         self,
     ) -> None:
-        source_owner = User.objects.create_user(username="source-owner")
+        source_owner = UserFactory(username="source-owner")
-        source_owner.user_permissions.add(
+        grant_global(source_owner, "change_document")
-            Permission.objects.get(codename="change_document"),
+        other_user = UserFactory(username="other-user")
-        )
-        other_user = User.objects.create_user(username="other-user")
 
         source_doc = Document.objects.create(
             title="Source",
@@ -1244,7 +1241,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
             data_type=CustomField.FieldDataType.DOCUMENTLINK,
         )
 
-        assign_perm("change_document", source_owner, target_doc)
+        grant_object(source_owner, target_doc, "change_document")
         self.client.force_authenticate(user=source_owner)
 
         resp = self.client.patch(
@@ -1337,7 +1334,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(results[0]["document_count"], 1)
 
         # Test as user without access to the document
-        non_superuser = User.objects.create_user(username="non_superuser")
+        non_superuser = UserFactory(username="non_superuser")
         non_superuser.user_permissions.add(
             *Permission.objects.exclude(codename="view_global_statistics"),
         )

src/documents/tests/test_api_document_versions.py

@@ -5,8 +5,6 @@ from typing import TYPE_CHECKING
 from unittest import mock
 
 from auditlog.models import LogEntry  # type: ignore[import-untyped]
-from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase as DjangoTestCase
@@ -18,10 +16,13 @@ from documents.data_models import DocumentSource
 from documents.filters import EffectiveContentFilter
 from documents.filters import TitleContentFilter
 from documents.models import Document
-from documents.tests.utils import DirectoriesMixin
-from documents.tests.utils import read_streaming_response
 from documents.versioning import annotate_effective_content
-from documents.views import DocumentSelectionMixin
+from documents.views.base import DocumentSelectionMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import UserFactory
+from paperless_testing.http import read_streaming_response
+from paperless_testing.permissions import grant_global
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -31,7 +32,7 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def _make_pdf_upload(self, name: str = "version.pdf") -> SimpleUploadedFile:
@@ -89,11 +90,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
 
     def test_root_endpoint_returns_403_when_user_lacks_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        viewer = UserFactory(username="viewer")
-        viewer.user_permissions.add(
+        grant_global(viewer, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -283,11 +282,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(mock_backend.add_or_update.call_args[0][0].id, root.id)
 
     def test_delete_version_returns_403_without_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        other = User.objects.create_user(username="other")
+        other = UserFactory(username="other")
-        other.user_permissions.add(
+        grant_global(other, "delete_document")
-            Permission.objects.get(codename="delete_document"),
-        )
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -371,11 +368,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertTrue(resp.data["is_root"])
 
     def test_update_version_label_returns_403_without_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        other = User.objects.create_user(username="other")
+        other = UserFactory(username="other")
-        other.user_permissions.add(
+        grant_global(other, "change_document")
-            Permission.objects.get(codename="change_document"),
-        )
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -521,7 +516,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
             root_document=root,
         )
 
-        with mock.patch("documents.views.DocumentViewSet.get_metadata") as metadata:
+        with mock.patch(
+            "documents.views.documents.DocumentViewSet.get_metadata",
+        ) as metadata:
             metadata.return_value = []
             resp = self.client.get(
                 f"/api/documents/{root.id}/metadata/?version={version.id}",
@@ -553,11 +550,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
 
     def test_metadata_returns_403_when_user_lacks_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        other = User.objects.create_user(username="other")
+        other = UserFactory(username="other")
-        other.user_permissions.add(
+        grant_global(other, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         doc = Document.objects.create(
             title="root",
             checksum="root",
@@ -581,7 +576,7 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         async_task = mock.Mock()
         async_task.id = "task-123"
 
-        with mock.patch("documents.views.consume_file") as consume_mock:
+        with mock.patch("documents.views.documents.consume_file") as consume_mock:
             consume_mock.apply_async.return_value = async_task
             resp = self.client.post(
                 f"/api/documents/{root.id}/update_version/",
@@ -617,7 +612,7 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         async_task = mock.Mock()
         async_task.id = "task-123"
 
-        with mock.patch("documents.views.consume_file") as consume_mock:
+        with mock.patch("documents.views.documents.consume_file") as consume_mock:
             consume_mock.apply_async.return_value = async_task
             resp = self.client.post(
                 f"/api/documents/{version.id}/update_version/",
@@ -642,7 +637,7 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         )
         upload = self._make_pdf_upload()
 
-        with mock.patch("documents.views.consume_file") as consume_mock:
+        with mock.patch("documents.views.documents.consume_file") as consume_mock:
             consume_mock.apply_async.side_effect = Exception("boom")
             resp = self.client.post(
                 f"/api/documents/{root.id}/update_version/",
@@ -653,8 +648,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR)
 
     def test_update_version_returns_403_without_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        other = User.objects.create_user(username="other")
+        other = UserFactory(username="other")
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -672,8 +667,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
     def test_update_version_requires_global_change_permission(self) -> None:
-        user = User.objects.create_user(username="add-only")
+        user = UserFactory(username="add-only")
-        user.user_permissions.add(Permission.objects.get(codename="add_document"))
+        grant_global(user, "add_document")
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -681,7 +676,7 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         )
         self.client.force_authenticate(user=user)
 
-        with mock.patch("documents.views.consume_file") as consume_mock:
+        with mock.patch("documents.views.documents.consume_file") as consume_mock:
             resp = self.client.post(
                 f"/api/documents/{root.id}/update_version/",
                 {"document": self._make_pdf_upload()},
@@ -829,6 +824,26 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
         self.assertEqual(resp.data["content"], "v1-content")
 
+    def test_page_count_resolves_to_latest_version(self) -> None:
+        root = DocumentFactory(page_count=2)
+        DocumentFactory(root_document=root, version_index=1, page_count=1)
+        unversioned = DocumentFactory(page_count=5)
+
+        resp = self.client.get("/api/documents/?fields=id,page_count")
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            {doc["id"]: doc["page_count"] for doc in resp.data["results"]},
+            {root.id: 1, unversioned.id: 5},
+        )
+
+        resp = self.client.get(f"/api/documents/{root.id}/")
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(resp.data["page_count"], 1)
+
+        resp = self.client.get(f"/api/documents/{root.id}/?version={root.id}")
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(resp.data["page_count"], 2)
+
     def _make_root_with_out_of_order_versions(self) -> tuple[Document, ...]:
         """
         A root whose newest version has a *lower* id than an older one, which is
@@ -978,7 +993,7 @@ class TestVersionAwareFilters(DjangoTestCase):
         superseded content -- selecting documents the list view, filtered by
         the same term, does not show.
         """
-        user = User.objects.create_superuser(username="bulk_selection")
+        user = UserFactory(username="bulk_selection", superuser=True)
 
         selected = DocumentSelectionMixin()._resolve_document_ids(
             user=user,
@@ -1005,7 +1020,7 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
         "Select all matching" reconstructs the document list, which never
         contains version documents as rows of their own.
         """
-        user = User.objects.create_superuser(username="bulk_versions")
+        user = UserFactory(username="bulk_versions", superuser=True)
         root = Document.objects.create(
             title="shared-title root",
             checksum="bulk-root",

src/documents/tests/test_api_documents.py

@@ -23,7 +23,6 @@ from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import DataError
 from django.test import override_settings
 from django.utils import timezone
-from guardian.shortcuts import assign_perm
 from rest_framework import status
 from rest_framework.test import APITestCase
 
@@ -48,18 +47,22 @@ from documents.models import Workflow
 from documents.models import WorkflowAction
 from documents.models import WorkflowTrigger
 from documents.signals.handlers import run_workflows
-from documents.tests.factories import DocumentFactory
-from documents.tests.factories import TagFactory
 from documents.tests.utils import ConsumeTaskMixin
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
-from documents.tests.utils import read_streaming_response
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import TagFactory
+from paperless_testing.factories import UserFactory
+from paperless_testing.http import read_streaming_response
+from paperless_testing.permissions import grant_all_global
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
         cache.clear()
 
@@ -357,10 +360,10 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         with Path(filename).open("wb") as f:
             f.write(content)
 
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(user1, "view_document")
-        user2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(user2, "view_document")
 
         self.client.force_authenticate(user2)
 
@@ -383,7 +386,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         response = self.client.get(f"/api/documents/{doc.pk}/thumb/")
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
-        assign_perm("view_document", user2, doc)
+        grant_object(user2, doc, "view_document")
 
         response = self.client.get(f"/api/documents/{doc.pk}/download/")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -760,8 +763,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - History is returned
         """
         # No auditlog permissions
-        user = User.objects.create_user(username="test")
+        user = UserFactory(username="test")
-        user.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(user, "view_document")
         self.client.force_authenticate(user=user)
         doc = Document.objects.create(
             title="First title",
@@ -776,7 +779,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         # superuser
         user.is_superuser = True
         user.save()
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         doc2 = Document.objects.create(
             title="Second title",
             checksum="456",
@@ -1073,11 +1076,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.data["duplicate_documents"], [])
 
     def test_has_duplicates_filter_respects_document_permissions(self) -> None:
-        owner = User.objects.create_user(username="duplicate-owner")
+        owner = UserFactory(username="duplicate-owner")
-        requester = User.objects.create_user(username="duplicate-requester")
+        requester = UserFactory(username="duplicate-requester")
-        requester.user_permissions.add(
+        grant_global(requester, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         visible_document = Document.objects.create(
             title="visible document",
             checksum="permission-match",
@@ -1096,7 +1097,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             [document["id"] for document in response.data["results"]],
         )
 
-        assign_perm("view_document", requester, hidden_duplicate)
+        grant_object(requester, hidden_duplicate, "view_document")
         response = self.client.get("/api/documents/?has_duplicates=true")
         self.assertIn(
             visible_document.id,
@@ -1317,10 +1318,10 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Owner filters work correctly but still respect permissions
         """
-        u1 = User.objects.create_user("user1")
+        u1 = UserFactory(username="user1")
-        u2 = User.objects.create_user("user2")
+        u2 = UserFactory(username="user2")
-        u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(u1, "view_document")
-        u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(u2, "view_document")
 
         u1_doc1 = Document.objects.create(
             title="none1",
@@ -1353,7 +1354,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         )
 
         self.client.force_authenticate(user=u1)
-        assign_perm("view_document", u1, u2_doc2)
+        grant_object(u1, u2_doc2, "view_document")
 
         # Will not show any u1 docs or u2_doc1 which isn't shared
         response = self.client.get(f"/api/documents/?owner__id__none={u1.id}")
@@ -1400,7 +1401,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             [u1_doc1.id, u1_doc2.id, u2_doc2.id],
         )
 
-        assign_perm("view_document", u2, u1_doc1)
+        grant_object(u2, u1_doc1, "view_document")
 
         # Will show only documents shared by user
         response = self.client.get(f"/api/documents/?shared_by__id={u1.id}")
@@ -1424,8 +1425,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - The document is returned exactly once, not once per permission path
             (regression test for https://github.com/paperless-ngx/paperless-ngx/issues/13331)
         """
-        user = User.objects.create_user("user1")
+        user = UserFactory(username="user1")
-        user.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(user, "view_document")
         group = Group.objects.create(name="group1")
         user.groups.add(group)
 
@@ -1433,7 +1434,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         tag2 = TagFactory()
         doc = DocumentFactory(title="shared", owner=user)
         doc.tags.add(tag1, tag2)
-        assign_perm("view_document", group, doc)
+        grant_object(group, doc, "view_document")
 
         self.client.force_authenticate(user=user)
         response = self.client.get(
@@ -1452,11 +1453,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - The document does not appear in their results
         """
-        owner = User.objects.create_user("owner1")
+        owner = UserFactory(username="owner1")
-        stranger = User.objects.create_user("stranger1")
+        stranger = UserFactory(username="stranger1")
-        stranger.user_permissions.add(
+        grant_global(stranger, "view_document")
-            *Permission.objects.filter(codename="view_document"),
-        )
 
         DocumentFactory(title="private", owner=owner)
 
@@ -1474,17 +1473,17 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Only the group member sees the document
         """
-        owner = User.objects.create_user("owner2")
+        owner = UserFactory(username="owner2")
-        member = User.objects.create_user("member1")
+        member = UserFactory(username="member1")
-        non_member = User.objects.create_user("nonmember1")
+        non_member = UserFactory(username="nonmember1")
         for u in (member, non_member):
-            u.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+            grant_global(u, "view_document")
 
         group = Group.objects.create(name="group2")
         member.groups.add(group)
 
         doc = DocumentFactory(title="shared2", owner=owner)
-        assign_perm("view_document", group, doc)
+        grant_object(group, doc, "view_document")
 
         self.client.force_authenticate(user=member)
         response = self.client.get("/api/documents/")
@@ -1785,8 +1784,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Statistics only include inbox counts for tags accessible by the user
         """
-        u1 = User.objects.create_user("user1")
+        u1 = UserFactory(username="user1")
-        u2 = User.objects.create_user("user2")
+        u2 = UserFactory(username="user2")
         inbox_tag_u1 = Tag.objects.create(name="inbox_u1", is_inbox_tag=True, owner=u1)
         Tag.objects.create(name="inbox_u2", is_inbox_tag=True, owner=u2)
         doc_u1 = Document.objects.create(
@@ -1816,11 +1815,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.data["documents_inbox"], 0)
 
     def test_statistics_with_statistics_permission(self) -> None:
-        owner = User.objects.create_user("owner")
+        owner = UserFactory(username="owner")
-        stats_user = User.objects.create_user("stats-user")
+        stats_user = UserFactory(username="stats-user")
-        stats_user.user_permissions.add(
+        grant_global(stats_user, "view_global_statistics")
-            Permission.objects.get(codename="view_global_statistics"),
-        )
 
         inbox_tag = Tag.objects.create(
             name="stats_inbox",
@@ -1986,7 +1983,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
 
     def test_upload_insufficient_permissions(self) -> None:
-        self.client.force_authenticate(user=User.objects.create_user("testuser2"))
+        self.client.force_authenticate(user=UserFactory(username="testuser2"))
 
         with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
             response = self.client.post(
@@ -2608,7 +2605,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         response = self.client.get("/api/documents/34676/suggestions/")
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
 
-    @mock.patch("documents.views.get_ai_document_classification")
+    @mock.patch("documents.views.documents.get_ai_document_classification")
     @override_settings(AI_ENABLED=True)
     def test_suggestions_still_uses_classifier_when_ai_enabled(
         self,
@@ -2631,10 +2628,10 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         )
         mock_get_ai_classification.assert_not_called()
 
-    @mock.patch("documents.views.match_storage_paths")
+    @mock.patch("documents.views.documents.match_storage_paths")
-    @mock.patch("documents.views.match_document_types")
+    @mock.patch("documents.views.documents.match_document_types")
-    @mock.patch("documents.views.match_tags")
+    @mock.patch("documents.views.documents.match_tags")
-    @mock.patch("documents.views.match_correspondents")
+    @mock.patch("documents.views.documents.match_correspondents")
     @override_settings(NUMBER_OF_SUGGESTED_DATES=10)
     def test_get_suggestions(
         self,
@@ -2666,11 +2663,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             },
         )
 
-    @mock.patch("documents.views.load_classifier")
+    @mock.patch("documents.views.documents.load_classifier")
-    @mock.patch("documents.views.match_storage_paths")
+    @mock.patch("documents.views.documents.match_storage_paths")
-    @mock.patch("documents.views.match_document_types")
+    @mock.patch("documents.views.documents.match_document_types")
-    @mock.patch("documents.views.match_tags")
+    @mock.patch("documents.views.documents.match_tags")
-    @mock.patch("documents.views.match_correspondents")
+    @mock.patch("documents.views.documents.match_correspondents")
     @override_settings(NUMBER_OF_SUGGESTED_DATES=10)
     def test_get_suggestions_cached(
         self,
@@ -2757,7 +2754,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         response = self.client.get(f"/api/documents/{doc.pk}/suggestions/")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
 
-    @mock.patch("documents.views.get_date_parser")
+    @mock.patch("documents.views.documents.get_date_parser")
     @override_settings(NUMBER_OF_SUGGESTED_DATES=0)
     def test_get_suggestions_dates_disabled(
         self,
@@ -2782,9 +2779,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         mock_get_date_parser.assert_not_called()
 
     def test_saved_views(self) -> None:
-        u1 = User.objects.create_user("user1")
+        u1 = UserFactory(username="user1")
-        u2 = User.objects.create_user("user2")
+        u2 = UserFactory(username="user2")
-        u3 = User.objects.create_user("user3")
+        u3 = UserFactory(username="user3")
 
         view_perm = Permission.objects.get(codename="view_savedview")
         change_perm = Permission.objects.get(codename="change_savedview")
@@ -2807,9 +2804,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             sort_field="",
         )
 
-        assign_perm("view_savedview", u1, v2)
+        grant_object(u1, v2, "view_savedview")
-        assign_perm("change_savedview", u1, v2)
+        grant_object(u1, v2, "change_savedview")
-        assign_perm("view_savedview", u1, v3)
+        grant_object(u1, v3, "view_savedview")
 
         self.client.force_authenticate(user=u1)
 
@@ -3064,7 +3061,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertListEqual(saved_view_settings["sidebar_views_visible_ids"], [v2.id])
 
     def test_saved_view_create_update_patch(self) -> None:
-        User.objects.create_user("user1")
+        UserFactory(username="user1")
 
         view = {
             "name": "test",
@@ -3127,7 +3124,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - Display options are updated
             - Display fields are validated
         """
-        User.objects.create_user("user1")
+        UserFactory(username="user1")
 
         view = {
             "name": "test",
@@ -3568,11 +3565,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Notes are neither created nor deleted
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         user2.save()
 
         doc = Document.objects.create(
@@ -3592,7 +3589,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(resp.content, b"Insufficient permissions to view notes")
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
-        assign_perm("view_document", user1, doc)
+        grant_object(user1, doc, "view_document")
 
         resp = self.client.post(
             f"/api/documents/{doc.pk}/notes/",
@@ -3616,12 +3613,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
     def test_notes_require_global_document_permissions(self) -> None:
-        user = User.objects.create_user(username="note_editor")
+        user = UserFactory(username="note_editor")
-        user.user_permissions.add(
+        grant_global(user, "view_note", "add_note", "delete_note")
-            *Permission.objects.filter(
-                codename__in=["view_note", "add_note", "delete_note"],
-            ),
-        )
         doc = Document.objects.create(
             title="test",
             mime_type="application/pdf",
@@ -3634,9 +3627,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         response = self.client.get(f"/api/documents/{doc.pk}/notes/")
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
-        user.user_permissions.add(
+        grant_global(user, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         user = User.objects.get(pk=user.pk)
         self.client.force_authenticate(user)
         response = self.client.get(f"/api/documents/{doc.pk}/notes/")
@@ -3648,9 +3639,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         )
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
-        user.user_permissions.add(
+        grant_global(user, "change_document")
-            Permission.objects.get(codename="change_document"),
-        )
         user = User.objects.get(pk=user.pk)
         self.client.force_authenticate(user)
         response = self.client.post(
@@ -3797,12 +3786,12 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - Unique items are created
             - Non-unique items are not allowed
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="add_tag"))
+        grant_global(user1, "add_tag")
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
-        user2.user_permissions.add(*Permission.objects.filter(codename="add_tag"))
+        grant_global(user2, "add_tag")
         user2.save()
 
         # User 1 creates tag 1 owned by user 1 by default
@@ -3857,12 +3846,12 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - Unique items are created
             - Non-unique items are not allowed on update
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="change_tag"))
+        grant_global(user1, "change_tag")
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
-        user2.user_permissions.add(*Permission.objects.filter(codename="change_tag"))
+        grant_global(user2, "change_tag")
         user2.save()
 
         # Create name tag 1 owned by user 1
@@ -3993,11 +3982,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Links only shown if user has permissions
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         user2.save()
 
         doc = Document.objects.create(
@@ -4017,7 +4006,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(resp.content, b"Insufficient permissions to add share link")
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
-        assign_perm("change_document", user1, doc)
+        grant_object(user1, doc, "change_document")
 
         resp = self.client.get(
             f"/api/documents/{doc.pk}/share_links/",
@@ -4034,11 +4023,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Share link creation is denied until view permission is granted
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="add_sharelink"))
+        grant_global(user1, "add_sharelink")
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         user2.save()
 
         doc = Document.objects.create(
@@ -4060,7 +4049,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         )
         self.assertEqual(create_resp.status_code, status.HTTP_403_FORBIDDEN)
 
-        assign_perm("view_document", user1, doc)
+        grant_object(user1, doc, "view_document")
 
         create_resp = self.client.post(
             "/api/share_links/",
@@ -4072,9 +4061,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         )
         self.assertEqual(create_resp.status_code, status.HTTP_403_FORBIDDEN)
 
-        user1.user_permissions.add(
+        grant_global(user1, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         user1 = User.objects.get(pk=user1.pk)
         self.client.force_authenticate(user1)
         create_resp = self.client.post(
@@ -4097,11 +4084,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - ASN +1 from user2's doc is returned for user1
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         user2.save()
 
         doc1 = Document.objects.create(
@@ -4141,8 +4128,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - ASN 1 is returned
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
 
         doc1 = Document.objects.create(
@@ -4170,7 +4157,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Explicit error is returned
         """
-        user1 = User.objects.create_superuser(username="test1")
+        user1 = UserFactory(username="test1", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -4348,8 +4335,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Error response is returned
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
 
         doc = Document.objects.create(
@@ -4454,7 +4441,7 @@ class TestDocumentApiTagColors(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
 
         self.client.force_authenticate(user=self.user)
 
@@ -4534,7 +4521,7 @@ class TestDocumentApiCustomFieldsSorting(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
         self.doc1 = Document.objects.create(

src/documents/tests/test_api_email.py

@@ -2,16 +2,16 @@ import json
 import shutil
 from unittest import mock
 
-from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.core import mail
 from django.test import override_settings
 from rest_framework import status
 from rest_framework.test import APITestCase
 
 from documents.models import Document
-from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import SampleDirMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
 
 
 class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
@@ -20,7 +20,7 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
         self.doc1 = Document.objects.create(
@@ -303,8 +303,8 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
         THEN:
             - Forbidden response is returned
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(user1, "view_document")
 
         doc_owned = Document.objects.create(
             title="owned_doc",
@@ -338,8 +338,8 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
         THEN:
             - Request succeeds
         """
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(user1, "view_document")
 
         self.client.force_authenticate(user1)
 

src/documents/tests/test_api_filter_by_custom_fields.py

@@ -4,7 +4,6 @@ from collections.abc import Callable
 from datetime import date
 from urllib.parse import quote
 
-from django.contrib.auth.models import User
 from rest_framework.test import APITestCase
 
 from documents.models import CustomField
@@ -12,8 +11,9 @@ from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import SavedView
 from documents.models import SavedViewFilterRule
-from documents.serialisers import DocumentSerializer
+from documents.serialisers.documents import DocumentSerializer
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
 
 
 class DocumentWrapper:
@@ -35,7 +35,7 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
         # Create one custom field per type. The fields are called f"{type}_field".

src/documents/tests/test_api_objects.py

@@ -3,12 +3,10 @@ import json
 from unittest import mock
 
 from django.contrib.auth.models import Group
-from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from django.db import connection
 from django.test import override_settings
 from django.test.utils import CaptureQueriesContext
-from guardian.shortcuts import assign_perm
 from guardian.shortcuts import get_groups_with_perms
 from guardian.shortcuts import get_users_with_perms
 from rest_framework import status
@@ -21,14 +19,17 @@ from documents.models import Document
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.models import Tag
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestApiObjects(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
+        user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=user)
 
         self.tag1 = Tag.objects.create(name="t1", is_inbox_tag=True)
@@ -153,7 +154,7 @@ class TestApiObjects(DirectoriesMixin, APITestCase):
 
         # A newer document owned by another user must not leak through the
         # aggregate for a non-superuser who cannot view it
-        other = User.objects.create_user(username="other")
+        other = UserFactory(username="other")
         Document.objects.create(
             mime_type="application/pdf",
             correspondent=self.c1,
@@ -162,10 +163,8 @@ class TestApiObjects(DirectoriesMixin, APITestCase):
             owner=other,
         )
 
-        user = User.objects.create_user(username="regular")
+        user = UserFactory(username="regular")
-        user.user_permissions.add(
+        grant_global(user, "view_correspondent")
-            Permission.objects.get(codename="view_correspondent"),
-        )
         self.client.force_authenticate(user=user)
 
         response = self.client.get("/api/correspondents/?last_correspondence=true")
@@ -200,7 +199,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
+        user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=user)
 
         self.sp1 = StoragePath.objects.create(name="sp1", path="Something/{checksum}")
@@ -455,11 +454,9 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         self.assertEqual(response.data, "folder/Something.pdf")
 
     def test_test_storage_path_requires_document_view_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        unprivileged = User.objects.create_user(username="unprivileged")
+        unprivileged = UserFactory(username="unprivileged")
-        unprivileged.user_permissions.add(
+        grant_global(unprivileged, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         document = Document.objects.create(
             mime_type="application/pdf",
             owner=owner,
@@ -481,15 +478,15 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         self.assertIn("document", response.data)
 
     def test_test_storage_path_allows_shared_document_view_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        viewer = UserFactory(username="viewer")
         document = Document.objects.create(
             mime_type="application/pdf",
             owner=owner,
             title="Shared",
             checksum="123",
         )
-        assign_perm("view_document", viewer, document)
+        grant_object(viewer, document, "view_document")
 
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
@@ -504,9 +501,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
-        viewer.user_permissions.add(
+        grant_global(viewer, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         viewer = User.objects.get(pk=viewer.pk)
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
@@ -545,14 +540,12 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def test_test_storage_path_exposes_basic_document_context_but_not_sensitive_owner_data(
         self,
     ) -> None:
-        owner = User.objects.create_user(
+        owner = UserFactory(
             username="owner",
             password="password",
             email="owner@example.com",
         )
-        owner.user_permissions.add(
+        grant_global(owner, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
         document = Document.objects.create(
             mime_type="application/pdf",
             owner=owner,
@@ -614,8 +607,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def test_test_storage_path_includes_related_objects_for_visible_document(
         self,
     ) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        viewer = UserFactory(username="viewer")
         private_correspondent = Correspondent.objects.create(
             name="Private Correspondent",
             owner=owner,
@@ -627,10 +620,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
             title="Document",
             checksum="123",
         )
-        assign_perm("view_document", viewer, document)
+        grant_object(viewer, document, "view_document")
-        viewer.user_permissions.add(
+        grant_global(viewer, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
 
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
@@ -662,7 +653,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         self.assertEqual(response.data, "Private Correspondent.pdf")
 
     def test_test_storage_path_superuser_can_view_private_related_objects(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
         private_correspondent = Correspondent.objects.create(
             name="Private Correspondent",
             owner=owner,
@@ -693,8 +684,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def test_test_storage_path_includes_doc_type_storage_path_and_tags(
         self,
     ) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        viewer = UserFactory(username="viewer")
         private_document_type = DocumentType.objects.create(
             name="Private Type",
             owner=owner,
@@ -717,10 +708,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
             checksum="123",
         )
         document.tags.add(private_tag)
-        assign_perm("view_document", viewer, document)
+        grant_object(viewer, document, "view_document")
-        viewer.user_permissions.add(
+        grant_global(viewer, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
 
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
@@ -756,8 +745,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def test_test_storage_path_includes_custom_fields_for_visible_document(
         self,
     ) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        viewer = UserFactory(username="viewer")
         document = Document.objects.create(
             mime_type="application/pdf",
             owner=owner,
@@ -773,10 +762,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
             field=custom_field,
             value_int=42,
         )
-        assign_perm("view_document", viewer, document)
+        grant_object(viewer, document, "view_document")
-        viewer.user_permissions.add(
+        grant_global(viewer, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
 
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
@@ -798,7 +785,7 @@ class TestBulkEditObjects(APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.temp_admin = User.objects.create_superuser(username="temp_admin")
+        self.temp_admin = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.temp_admin)
 
         self.t1 = Tag.objects.create(name="t1")
@@ -1030,9 +1017,7 @@ class TestBulkEditObjects(APITestCase):
         THEN:
             - User is able to delete objects
         """
-        self.user1.user_permissions.add(
+        grant_global(self.user1, "delete_tag")
-            *Permission.objects.filter(codename="delete_tag"),
-        )
         self.user1.save()
         self.client.force_authenticate(user=self.user1)
 
@@ -1062,9 +1047,7 @@ class TestBulkEditObjects(APITestCase):
         self.t2.owner = User.objects.get(username="temp_admin")
         self.t2.save()
 
-        self.user1.user_permissions.add(
+        grant_global(self.user1, "delete_tag")
-            *Permission.objects.filter(codename="delete_tag"),
-        )
         self.user1.save()
         self.client.force_authenticate(user=self.user1)
 
@@ -1097,9 +1080,7 @@ class TestBulkEditObjects(APITestCase):
         self.t2.owner = User.objects.get(username="temp_admin")
         self.t2.save()
 
-        self.user1.user_permissions.add(
+        grant_global(self.user1, "delete_tag")
-            *Permission.objects.filter(codename="delete_tag"),
-        )
         self.user1.save()
         self.client.force_authenticate(user=self.user1)
 

src/documents/tests/test_api_permissions.py

@@ -5,9 +5,7 @@ from unittest import mock
 from allauth.mfa.models import Authenticator
 from allauth.mfa.totp.internal import auth as totp_auth
 from django.contrib.auth.models import Group
-from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
-from guardian.shortcuts import assign_perm
 from guardian.shortcuts import get_perms
 from guardian.shortcuts import get_users_with_perms
 from rest_framework import status
@@ -19,7 +17,11 @@ from documents.models import DocumentType
 from documents.models import MatchingModel
 from documents.models import StoragePath
 from documents.models import Tag
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_all_global
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestApiAuth(DirectoriesMixin, APITestCase):
@@ -93,14 +95,14 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         self.assertNotIn("X-Version", response)
 
     def test_api_version_with_auth(self) -> None:
-        user = User.objects.create_superuser(username="test")
+        user = UserFactory(username="test", superuser=True)
         self.client.force_authenticate(user)
         response = self.client.get("/api/documents/")
         self.assertIn("X-Api-Version", response)
         self.assertIn("X-Version", response)
 
     def test_api_insufficient_permissions(self) -> None:
-        user = User.objects.create_user(username="test")
+        user = UserFactory(username="test")
         self.client.force_authenticate(user)
 
         Document.objects.create(title="Test")
@@ -137,8 +139,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         )
 
     def test_api_sufficient_permissions(self) -> None:
-        user = User.objects.create_user(username="test")
+        user = UserFactory(username="test")
-        user.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user)
         user.is_staff = True
         self.client.force_authenticate(user)
 
@@ -166,9 +168,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         )
 
     def test_api_get_object_permissions(self) -> None:
-        user1 = User.objects.create_user(username="test1")
+        user1 = UserFactory(username="test1")
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(user1, "view_document")
         self.client.force_authenticate(user1)
 
         self.assertEqual(
@@ -205,7 +207,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - Object created with current user as owner
         """
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -234,7 +236,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - Object created with no owner
         """
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -265,7 +267,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - Object permissions are set appropriately
         """
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
         user2 = User.objects.create(username="user2")
         group1 = Group.objects.create(name="group1")
 
@@ -313,7 +315,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - Object permissions are set appropriately
         """
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
         user2 = User.objects.create(username="user2")
         group1 = Group.objects.create(name="group1")
 
@@ -363,7 +365,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             mime_type="application/pdf",
             content="this is a document",
         )
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
         user2 = User.objects.create(username="user2")
         group1 = Group.objects.create(name="group1")
 
@@ -413,16 +415,16 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             mime_type="application/pdf",
             content="this is a document",
         )
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
         user2 = User.objects.create(username="user2")
         group1 = Group.objects.create(name="group1")
         doc.owner = user1
         doc.save()
 
-        assign_perm("view_document", user2, doc)
+        grant_object(user2, doc, "view_document")
-        assign_perm("change_document", user2, doc)
+        grant_object(user2, doc, "change_document")
-        assign_perm("view_document", group1, doc)
+        grant_object(group1, doc, "view_document")
-        assign_perm("change_document", group1, doc)
+        grant_object(group1, doc, "change_document")
 
         self.client.force_authenticate(user1)
 
@@ -446,11 +448,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         self.assertIn("change_document", get_perms(group1, doc))
 
     def test_document_permissions_change_requires_owner(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
-        editor = User.objects.create_user(username="editor")
+        editor = UserFactory(username="editor")
-        editor.user_permissions.add(
+        grant_all_global(editor)
-            *Permission.objects.all(),
-        )
 
         doc = Document.objects.create(
             title="Ownered doc",
@@ -460,8 +460,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             owner=owner,
         )
 
-        assign_perm("view_document", editor, doc)
+        grant_object(editor, doc, "view_document")
-        assign_perm("change_document", editor, doc)
+        grant_object(editor, doc, "change_document")
 
         self.client.force_authenticate(editor)
         response = self.client.patch(
@@ -499,9 +499,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
     def test_dynamic_permissions_fields(self) -> None:
-        user1 = User.objects.create_user(username="user1")
+        user1 = UserFactory(username="user1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(user1, "view_document")
-        user2 = User.objects.create_user(username="user2")
+        user2 = UserFactory(username="user2")
 
         Document.objects.create(title="Test", content="content 1", checksum="1")
         doc2 = Document.objects.create(
@@ -523,10 +523,10 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             owner=user1,
         )
 
-        assign_perm("view_document", user1, doc2)
+        grant_object(user1, doc2, "view_document")
-        assign_perm("view_document", user1, doc3)
+        grant_object(user1, doc3, "view_document")
-        assign_perm("change_document", user1, doc3)
+        grant_object(user1, doc3, "change_document")
-        assign_perm("view_document", user2, doc4)
+        grant_object(user2, doc4, "view_document")
 
         self.client.force_authenticate(user1)
 
@@ -574,8 +574,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         owned by someone else with no explicit guardian grant -- mirrors
         guardian's own ObjectPermissionChecker.has_perm() superuser shortcut.
         """
-        superuser = User.objects.create_superuser(username="admin")
+        superuser = UserFactory(username="admin", superuser=True)
-        other_user = User.objects.create_user(username="user2")
+        other_user = UserFactory(username="user2")
         Document.objects.create(
             title="Test",
             content="content",
@@ -602,7 +602,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - MFA required error is returned
         """
-        user1 = User.objects.create_user(username="user1")
+        user1 = UserFactory(username="user1")
         user1.set_password("password")
         user1.save()
 
@@ -626,7 +626,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - MFA code is required
         """
-        user1 = User.objects.create_user(username="user1")
+        user1 = UserFactory(username="user1")
         user1.set_password("password")
         user1.save()
 
@@ -688,7 +688,7 @@ class TestApiUser(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def test_get_users(self) -> None:
@@ -858,10 +858,8 @@ class TestApiUser(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
 
-        regular_user = User.objects.create_user(username="regular_user")
+        regular_user = UserFactory(username="regular_user")
-        regular_user.user_permissions.add(
+        grant_all_global(regular_user)
-            *Permission.objects.all(),
-        )
         self.client.force_authenticate(regular_user)
         Authenticator.objects.create(
             user=user1,
@@ -885,9 +883,9 @@ class TestApiUser(DirectoriesMixin, APITestCase):
             - Only superusers can change superuser status
         """
 
-        user1 = User.objects.create_user(username="user1")
+        user1 = UserFactory(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
-        user2 = User.objects.create_superuser(username="user2")
+        user2 = UserFactory(username="user2", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -972,9 +970,9 @@ class TestApiUser(DirectoriesMixin, APITestCase):
             - Only superusers can change staff status
         """
 
-        user1 = User.objects.create_user(username="user1")
+        user1 = UserFactory(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
-        user2 = User.objects.create_superuser(username="user2")
+        user2 = UserFactory(username="user2", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -1027,7 +1025,7 @@ class TestApiGroup(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def test_get_groups(self) -> None:
@@ -1128,7 +1126,7 @@ class TestBulkEditObjectPermissions(APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.temp_admin = User.objects.create_superuser(username="temp_admin")
+        self.temp_admin = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.temp_admin)
 
         self.t1 = Tag.objects.create(name="t1")
@@ -1276,7 +1274,7 @@ class TestBulkEditObjectPermissions(APITestCase):
             },
         }
 
-        assign_perm("view_tag", self.user3, self.t1)
+        grant_object(self.user3, self.t1, "view_tag")
         self.t1.owner = self.user3
         self.t1.save()
 
@@ -1373,13 +1371,9 @@ class TestBulkEditObjectPermissions(APITestCase):
         """
         self.t1.owner = self.user2
         self.t1.save()
-        assign_perm("view_tag", self.user1, self.t1)
+        grant_object(self.user1, self.t1, "view_tag")
-        assign_perm("change_tag", self.user1, self.t1)
+        grant_object(self.user1, self.t1, "change_tag")
-        self.user1.user_permissions.add(
+        grant_global(self.user1, "view_tag", "change_tag")
-            *Permission.objects.filter(
-                codename__in=["view_tag", "change_tag"],
-            ),
-        )
         user1 = User.objects.get(pk=self.user1.pk)
         self.client.force_authenticate(user=user1)
 
@@ -1426,13 +1420,9 @@ class TestBulkEditObjectPermissions(APITestCase):
         """
         owned = Tag.objects.create(name="owned", owner=self.user1)
         shared = Tag.objects.create(name="shared", owner=self.user2)
-        assign_perm("view_tag", self.user1, shared)
+        grant_object(self.user1, shared, "view_tag")
-        assign_perm("change_tag", self.user1, shared)
+        grant_object(self.user1, shared, "change_tag")
-        self.user1.user_permissions.add(
+        grant_global(self.user1, "view_tag", "change_tag")
-            *Permission.objects.filter(
-                codename__in=["view_tag", "change_tag"],
-            ),
-        )
         user1 = User.objects.get(pk=self.user1.pk)
         self.client.force_authenticate(user=user1)
 
@@ -1473,14 +1463,10 @@ class TestBulkEditObjectPermissions(APITestCase):
         """
         self.t1.owner = self.user2
         self.t1.save()
-        assign_perm("view_tag", self.user1, self.t1)
+        grant_object(self.user1, self.t1, "view_tag")
-        assign_perm("change_tag", self.user1, self.t1)
+        grant_object(self.user1, self.t1, "change_tag")
-        assign_perm("delete_tag", self.user1, self.t1)
+        grant_object(self.user1, self.t1, "delete_tag")
-        self.user1.user_permissions.add(
+        grant_global(self.user1, "view_tag", "change_tag", "delete_tag")
-            *Permission.objects.filter(
-                codename__in=["view_tag", "change_tag", "delete_tag"],
-            ),
-        )
         user1 = User.objects.get(pk=self.user1.pk)
         self.client.force_authenticate(user=user1)
 
@@ -1585,7 +1571,7 @@ class TestBulkEditObjectPermissions(APITestCase):
             - Request succeeds and null is treated as an empty user list,
               so the existing view permission is removed
         """
-        assign_perm("view_tag", self.user1, self.t1)
+        grant_object(self.user1, self.t1, "view_tag")
 
         response = self.client.post(
             "/api/bulk_edit_objects/",
@@ -1680,7 +1666,7 @@ class TestFullPermissionsFlag(APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.admin = User.objects.create_superuser(username="admin")
+        self.admin = UserFactory(username="admin", superuser=True)
 
     def test_full_perms_flag(self) -> None:
         """

src/documents/tests/test_api_post_document_nfc.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import unicodedata
 from typing import TYPE_CHECKING
 from unittest import mock
@@ -7,8 +9,11 @@ import pytest
 from django.core.files.uploadedfile import SimpleUploadedFile
 
 if TYPE_CHECKING:
+    from rest_framework.test import APIClient
+
     from documents.data_models import ConsumableDocument
     from documents.data_models import DocumentMetadataOverrides
+    from paperless_testing.dirs import PaperlessDirs
 
 
 @pytest.fixture()
@@ -18,22 +23,14 @@ def consume_file_mock():
         yield m
 
 
-@pytest.fixture()
-def directories(tmp_path, settings, _media_settings):
-    scratch = tmp_path / "scratch"
-    scratch.mkdir()
-    settings.SCRATCH_DIR = scratch
-    return scratch
-
-
 @pytest.mark.django_db
 class TestPostDocumentNFCNormalization:
     def test_nfd_filename_normalized_to_nfc(
         self,
-        admin_client,
+        admin_client: APIClient,
         consume_file_mock: mock.MagicMock,
-        directories,
+        paperless_dirs: PaperlessDirs,
-    ):
+    ) -> None:
         """Uploaded file with NFD filename must have its name stored as NFC."""
         nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
         nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
@@ -69,10 +66,10 @@ class TestPostDocumentNFCNormalization:
 
     def test_already_nfc_filename_unchanged(
         self,
-        admin_client,
+        admin_client: APIClient,
         consume_file_mock: mock.MagicMock,
-        directories,
+        paperless_dirs: PaperlessDirs,
-    ):
+    ) -> None:
         """Uploaded file with already-NFC filename must pass through unchanged."""
         nfc = unicodedata.normalize("NFC", "Invoice_2024.pdf")
 

src/documents/tests/test_api_profile.py

@@ -8,7 +8,8 @@ from rest_framework import status
 from rest_framework.authtoken.models import Token
 from rest_framework.test import APITestCase
 
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
 
 
 # see allauth.socialaccount.providers.openid.provider.OpenIDProvider
@@ -55,23 +56,23 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(
+        self.user = UserFactory(
             username="temp_admin",
             first_name="firstname",
             last_name="surname",
+            superuser=True,
         )
         self.client.force_authenticate(user=self.user)
 
-    def setupSocialAccount(self) -> None:
+    def setupSocialAccount(self) -> SocialAccount:
         SocialApp.objects.create(
             name="Keycloak",
             provider="openid_connect",
             provider_id="keycloak-test",
         )
-        self.user.socialaccount_set.add(
+        social_account = SocialAccount(uid="123456789", provider="keycloak-test")
-            SocialAccount(uid="123456789", provider="keycloak-test"),
+        self.user.socialaccount_set.add(social_account, bulk=False)
-            bulk=False,
+        return social_account
-        )
 
     def test_get_profile(self) -> None:
         """
@@ -109,19 +110,17 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
         THEN:
             - Profile is returned with social accounts
         """
-        self.setupSocialAccount()
+        social_account = self.setupSocialAccount()
 
-        openid_provider = (
+        openid_provider = MockOpenIDConnectProvider(
-            MockOpenIDConnectProvider(
+            app=SocialApp.objects.get(provider_id="keycloak-test"),
-                app=SocialApp.objects.get(provider_id="keycloak-test"),
-            ),
         )
         mock_list_providers.return_value = [
             openid_provider,
         ]
         mock_get_provider_account.return_value = MockOpenIDConnectProviderAccount(
             mock_social_account_dict={
-                "name": openid_provider[0].name,
+                "name": openid_provider.name,
             },
         )
 
@@ -133,7 +132,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
             response.data["social_accounts"],
             [
                 {
-                    "id": 1,
+                    "id": social_account.pk,
                     "provider": "keycloak-test",
                     "name": "Keycloak",
                 },
@@ -150,7 +149,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
         THEN:
             - Profile is returned with "Unknown App" as name
         """
-        self.setupSocialAccount()
+        social_account = self.setupSocialAccount()
 
         # Remove the social app
         SocialApp.objects.get(provider_id="keycloak-test").delete()
@@ -163,7 +162,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
             response.data["social_accounts"],
             [
                 {
-                    "id": 1,
+                    "id": social_account.pk,
                     "provider": "keycloak-test",
                     "name": "Unknown App",
                 },
@@ -401,7 +400,7 @@ class TestApiTOTPViews(APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def test_get_totp(self) -> None:

src/documents/tests/test_api_search.py

@@ -7,10 +7,8 @@ import time_machine
 from dateutil.relativedelta import relativedelta
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.test import override_settings
 from django.utils import timezone
-from guardian.shortcuts import assign_perm
 from rest_framework import status
 from rest_framework.test import APITestCase
 
@@ -27,10 +25,13 @@ from documents.models import Tag
 from documents.models import Workflow
 from documents.search import get_backend
 from documents.search import reset_backend
-from documents.tests.factories import DocumentFactory
-from documents.tests.utils import DirectoriesMixin
 from paperless_mail.models import MailAccount
 from paperless_mail.models import MailRule
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 pytestmark = pytest.mark.search
 
@@ -40,7 +41,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         super().setUp()
         reset_backend()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def tearDown(self) -> None:
@@ -949,9 +950,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         THEN:
             - Terms only within docs user has access to are returned
         """
-        u1 = User.objects.create_user("user1")
+        u1 = UserFactory(username="user1")
-        u2 = User.objects.create_user("user2")
+        u2 = UserFactory(username="user2")
-        u1.user_permissions.add(Permission.objects.get(codename="view_document"))
+        grant_global(u1, "view_document")
 
         self.client.force_authenticate(user=u1)
 
@@ -991,7 +992,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data, ["applebaum", "apples"])
 
-        assign_perm("view_document", u1, d3)
+        grant_object(u1, d3, "view_document")
         backend.add_or_update(d3)
 
         response = self.client.get("/api/search/autocomplete/?term=app")
@@ -999,10 +1000,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.data, ["applebaum", "apples", "appletini"])
 
     def test_search_autocomplete_group_revocation_is_immediate(self) -> None:
-        user = User.objects.create_user("group-user")
+        user = UserFactory(username="group-user")
-        owner = User.objects.create_user("document-owner")
+        owner = UserFactory(username="document-owner")
         group = Group.objects.create(name="temporary-viewers")
-        user.user_permissions.add(Permission.objects.get(codename="view_document"))
+        grant_global(user, "view_document")
         user.groups.add(group)
 
         document = Document.objects.create(
@@ -1011,7 +1012,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             checksum="group-revocation",
             owner=owner,
         )
-        assign_perm("view_document", group, document)
+        grant_object(group, document, "view_document")
         get_backend().add_or_update(document)
         self.client.force_authenticate(user=user)
 
@@ -1091,11 +1092,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertIsNone(response.data["corrected_query"])
 
     def test_search_spelling_suggestion_suppressed_for_private_terms(self) -> None:
-        owner = User.objects.create_user("owner")
+        owner = UserFactory(username="owner")
-        attacker = User.objects.create_user("attacker")
+        attacker = UserFactory(username="attacker")
-        attacker.user_permissions.add(
+        grant_global(attacker, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
 
         backend = get_backend()
         for i in range(5):
@@ -1222,11 +1221,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         THEN:
             - The request is rejected
         """
-        owner = User.objects.create_user("owner")
+        owner = UserFactory(username="owner")
-        attacker = User.objects.create_user("attacker")
+        attacker = UserFactory(username="attacker")
-        attacker.user_permissions.add(
+        grant_global(attacker, "view_document")
-            Permission.objects.get(codename="view_document"),
-        )
 
         private_seed = Document.objects.create(
             title="private bank statement",
@@ -1534,11 +1531,11 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             - Only owned docs are returned for regular users
             - All docs are returned for superuser
         """
-        superuser = User.objects.create_superuser("superuser")
+        superuser = UserFactory(username="superuser", superuser=True)
-        u1 = User.objects.create_user("user1")
+        u1 = UserFactory(username="user1")
-        u2 = User.objects.create_user("user2")
+        u2 = UserFactory(username="user2")
-        u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(u1, "view_document")
-        u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(u2, "view_document")
 
         Document.objects.create(checksum="1", content="test 1", owner=u1)
         Document.objects.create(checksum="2", content="test 2", owner=u2)
@@ -1588,10 +1585,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         THEN:
             - Only docs with granted view permissions are returned
         """
-        u1 = User.objects.create_user("user1")
+        u1 = UserFactory(username="user1")
-        u2 = User.objects.create_user("user2")
+        u2 = UserFactory(username="user2")
-        u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(u1, "view_document")
-        u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        grant_global(u2, "view_document")
 
         d1 = Document.objects.create(checksum="1", content="test 1", owner=u1)
         d2 = Document.objects.create(checksum="2", content="test 2", owner=u2)
@@ -1616,9 +1613,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         r = self.client.get("/api/documents/?query=test&owner__isnull=true")
         self.assertEqual(r.data["count"], 1)
 
-        assign_perm("view_document", u1, d2)
+        grant_object(u1, d2, "view_document")
-        assign_perm("view_document", u1, d3)
+        grant_object(u1, d3, "view_document")
-        assign_perm("view_document", u2, d1)
+        grant_object(u2, d1, "view_document")
 
         backend.add_or_update(d1)
         backend.add_or_update(d2)
@@ -1641,8 +1638,8 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(r.data["count"], 1)
 
     def test_search_sorting(self) -> None:
-        u1 = User.objects.create_user("user1")
+        u1 = UserFactory(username="user1")
-        u2 = User.objects.create_user("user2")
+        u2 = UserFactory(username="user2")
         c1 = Correspondent.objects.create(name="corres Ax")
         c2 = Correspondent.objects.create(name="corres Cx")
         c3 = Correspondent.objects.create(name="corres Bx")
@@ -1892,8 +1889,8 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             pk=5,
         )
 
-        user1 = User.objects.create_user("bank user1")
+        user1 = UserFactory(username="bank user1")
-        user2 = User.objects.create_superuser("user2")
+        user2 = UserFactory(username="user2", superuser=True)
         group1 = Group.objects.create(name="bank group1")
         Group.objects.create(name="group2")
 
@@ -1925,7 +1922,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             sort_field="",
             owner=user2,
         )
-        assign_perm("view_savedview", user1, shared_view)
+        grant_object(user1, shared_view, "view_savedview")
         mail_account1 = MailAccount.objects.create(name="bank mail account 1")
         mail_account2 = MailAccount.objects.create(name="mail account 2")
         mail_rule1 = MailRule.objects.create(
@@ -2018,12 +2015,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(returned.get(root.id), "latest content")
 
     def test_global_search_filters_owned_mail_objects(self) -> None:
-        user1 = User.objects.create_user("mail-search-user")
+        user1 = UserFactory(username="mail-search-user")
-        user2 = User.objects.create_user("other-mail-search-user")
+        user2 = UserFactory(username="other-mail-search-user")
-        user1.user_permissions.add(
+        grant_global(user1, "view_mailaccount", "view_mailrule")
-            Permission.objects.get(codename="view_mailaccount"),
-            Permission.objects.get(codename="view_mailrule"),
-        )
 
         own_account = MailAccount.objects.create(
             name="bank owned account",

src/documents/tests/test_api_search_errors.py

@@ -33,7 +33,7 @@ class TestSearchQueryErrorStillBecomesA400:
         self,
         admin_client: APIClient,
         monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -68,7 +68,7 @@ class TestLibraryDefectsPropagate:
         self,
         admin_client: APIClient,
         monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -98,7 +98,7 @@ class TestLibraryDefectsPropagate:
         self,
         admin_client: APIClient,
         monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -141,7 +141,7 @@ class TestSelectionPathsAgreeWithSearch:
         self,
         admin_client: APIClient,
         monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -181,7 +181,7 @@ class TestSelectionPathsAgreeWithSearch:
         self,
         admin_client: APIClient,
         monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -221,7 +221,7 @@ class TestSelectionPathsAgreeWithSearch:
         self,
         admin_client: APIClient,
         monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -259,7 +259,7 @@ class TestSelectionPathsAgreeWithSearch:
         self,
         admin_client: APIClient,
         monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -287,7 +287,7 @@ class TestSelectionPathsAgreeWithSearch:
             {
                 "documents": [],
                 "all": True,
-                "filters": {"more_like_id": indexed_document.pk},
+                "filters": {"more_like_id": searchable_document.pk},
             },
             format="json",
         )
@@ -298,7 +298,7 @@ class TestSelectionPathsAgreeWithSearch:
         self,
         admin_client: APIClient,
         monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -328,7 +328,7 @@ class TestSelectionPathsAgreeWithSearch:
                 {
                     "documents": [],
                     "all": True,
-                    "filters": {"more_like_id": indexed_document.pk},
+                    "filters": {"more_like_id": searchable_document.pk},
                 },
                 format="json",
             )

src/documents/tests/test_api_search_query_length.py

@@ -22,7 +22,7 @@ import pytest
 from rest_framework import status
 
 import documents.search._backend
-from documents.views import _MAX_QUERY_LENGTH
+from documents.views.base import _MAX_QUERY_LENGTH
 
 if TYPE_CHECKING:
     from rest_framework.test import APIClient
@@ -36,7 +36,7 @@ class TestGetSearchEndpointEnforcesTheCap:
     def test_query_one_over_the_cap_is_a_400(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -66,7 +66,7 @@ class TestGetSearchEndpointEnforcesTheCap:
     def test_query_at_exactly_the_cap_is_accepted(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -86,7 +86,7 @@ class TestGetSearchEndpointEnforcesTheCap:
     def test_an_ordinary_query_is_unaffected(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -112,7 +112,7 @@ class TestPostSelectionPathsEnforceTheCap:
     def test_bulk_edit_query_one_over_the_cap_is_a_400(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -154,7 +154,7 @@ class TestPostSelectionPathsEnforceTheCap:
         self,
         bulk_update_task_mock: mock.MagicMock,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -187,7 +187,7 @@ class TestPostSelectionPathsEnforceTheCap:
     def test_bulk_download_query_one_over_the_cap_is_a_400(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -236,7 +236,7 @@ class TestGlobalSearchEnforcesTheCapToo:
     def test_query_one_over_the_cap_is_a_400(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:
@@ -260,7 +260,7 @@ class TestGlobalSearchEnforcesTheCapToo:
     def test_query_at_exactly_the_cap_is_accepted(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN:

src/documents/tests/test_api_search_unterminated_date_range.py

@@ -38,7 +38,7 @@ class TestUnterminatedBracketReturnsA400:
     def test_unterminated_bracket_is_a_400(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
         query: str,
     ) -> None:
         """
@@ -59,7 +59,7 @@ class TestUnterminatedBracketReturnsA400:
     def test_properly_closed_bracket_still_searches_cleanly(
         self,
         admin_client: APIClient,
-        indexed_document: Document,
+        searchable_document: Document,
     ) -> None:
         """
         GIVEN: