Compare commits

..
Author SHA1 Message Date
stumpylog d81a761f51 Fixes the E2E test that depended on the old location 2026-10-22 10:45:33 -07:00
Trenton HandClaude Sonnet 5.5 1aa792df09 Chore: Drop the sample duplicate guard test
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 10:45:33 -07:00
Trenton HandClaude Sonnet 5.5 99320c2aea Chore: Use shared samples in tests that arrived from dev
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 10:45:33 -07:00
Trenton HandClaude Sonnet 5.5 82c4a8d397 Chore: Tidy leftover parentheses and the samples module docstring
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 10:40:15 -07:00
Trenton HandClaude Sonnet 5.5 24c4bf780b Chore: Remove duplicate and unreferenced test sample files
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 10:40:15 -07:00
Trenton HandClaude Sonnet 5.5 b376b44d44 Chore: Materialise duplicated media-tree samples from the shared copies
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 10:40:14 -07:00
Trenton HandClaude Sonnet 5.5 215cd2c99a Chore: Use the shared simple-digital.pdf in documents tests and drop the unreferenced simple.txt
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 10:40:08 -07:00
Trenton HandClaude Sonnet 5.5 28255668b9 Chore: Move the four duplicated parser sample PDFs into a shared paperless_testing location
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-22 10:40:01 -07:00
67 changed files with 844 additions and 1625 deletions

No files matched your search

+34 -43
View File
@@ -1,62 +1,53 @@
name: Install apt packages
description: >
Caches apt packages and their dependencies for an exact hosted runner image. Cache hits install offline; cache misses download from Ubuntu mirrors with bounded retries.
Installs apt packages with a per-attempt timeout and retries. The hosted runner's Azure apt mirror intermittently serves archives at a crawl, which apt's own timeout does not catch. A stuck attempt is killed and retried, which usually lands on a healthy mirror node.
inputs:
packages:
description: Space separated list of package names to install
description: Space separated list of packages to install
required: true
attempts:
description: Maximum number of download attempts
description: Maximum number of attempts
default: "3"
attempt-timeout:
description: Seconds before a single download attempt is killed
description: Seconds before a single attempt is killed
default: "180"
runs:
using: composite
steps:
- name: Identify apt cache
id: cache-key
shell: bash
env:
PACKAGES: ${{ inputs.packages }}
run: |
# The bundle contains only dependencies missing from this runner. Never
# restore it onto a different image or initial package installation.
# Without an image version (e.g. self-hosted), download normally.
if [ -n "${ImageVersion:-}" ]; then
fingerprint=$({
printf '%s\n' "$PACKAGES"
dpkg --print-architecture
dpkg-query -W -f='${binary:Package}=${Version} ${db:Status-Abbrev}\n'
sha256sum "$GITHUB_ACTION_PATH/action.yml" "$GITHUB_ACTION_PATH/install.sh" | cut -d ' ' -f 1
} | sha256sum | cut -d ' ' -f 1)
echo "key=apt-v1-${ImageOS}-${ImageVersion}-${fingerprint}" >> "$GITHUB_OUTPUT"
fi
- name: Restore apt packages
id: cache
if: steps.cache-key.outputs.key != ''
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
${{ runner.temp }}/apt-packages/*.deb
${{ runner.temp }}/apt-packages/complete
key: ${{ steps.cache-key.outputs.key }}
- name: Install ${{ inputs.packages }}
shell: bash
env:
PACKAGES: ${{ inputs.packages }}
ATTEMPTS: ${{ inputs.attempts }}
ATTEMPT_TIMEOUT: ${{ inputs.attempt-timeout }}
CACHE_DIR: ${{ runner.temp }}/apt-packages
CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
run: bash "$GITHUB_ACTION_PATH/install.sh"
# Save immediately: a later test failure must not discard a good apt bundle.
- name: Save apt packages
if: steps.cache-key.outputs.key != '' && steps.cache.outputs.cache-hit != 'true'
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
${{ runner.temp }}/apt-packages/*.deb
${{ runner.temp }}/apt-packages/complete
key: ${{ steps.cache-key.outputs.key }}
run: |
SUDO=""
if [ "$(id -u)" -ne 0 ]; then
SUDO="sudo"
fi
attempt_install() {
$SUDO apt-get update -q
# Intentionally unquoted, this is a list of package names
$SUDO apt-get install -q --yes --no-install-recommends $PACKAGES
}
export -f attempt_install
export SUDO PACKAGES
for attempt in $(seq 1 "${ATTEMPTS}"); do
echo "::group::apt install attempt ${attempt}/${ATTEMPTS}"
start=$(date +%s)
if timeout --kill-after=10 "${ATTEMPT_TIMEOUT}" bash -c attempt_install; then
echo "::endgroup::"
echo "Installed in $(( $(date +%s) - start ))s"
exit 0
fi
echo "::endgroup::"
echo "::warning::apt install attempt ${attempt}/${ATTEMPTS} failed or exceeded ${ATTEMPT_TIMEOUT}s"
# A killed attempt can leave dpkg half configured
$SUDO dpkg --configure -a || true
done
echo "::error::apt install failed after ${ATTEMPTS} attempts"
exit 1
-67
View File
@@ -1,67 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
SUDO=()
if [ "$(id -u)" -ne 0 ]; then
SUDO=(sudo)
fi
# Inputs are package names, separated by whitespace (not shell expressions).
read -r -a requested_packages <<< "$PACKAGES"
mkdir -p "$CACHE_DIR"
install_local() {
local archives=("$CACHE_DIR"/*.deb)
if [ -e "${archives[0]}" ]; then
# Local .deb arguments pin the downloaded versions. No package lists or
# network access are needed, and apt still runs all package maintainer scripts.
"${SUDO[@]}" chmod a+r "${archives[@]}"
"${SUDO[@]}" apt-get install -q --yes --no-install-recommends \
--no-download -o "Dir::Cache::archives=$CACHE_DIR" "${archives[@]}"
fi
# An empty bundle is valid when the runner already includes every package.
local package
for package in "${requested_packages[@]}"; do
[ "$(dpkg-query -W -f='${Status}' "$package")" = "install ok installed" ]
done
}
if [ "$CACHE_HIT" = true ]; then
echo "Installing cached apt packages without contacting package repositories"
install_local
exit 0
fi
# The hosted runner gives Azure priority 1. A slow transfer never triggers its
# fallback mirrors, so retrying alone keeps selecting the same unhealthy mirror.
if [ -f /etc/apt/apt-mirrors.txt ]; then
"${SUDO[@]}" sed -i '\|azure\.archive\.ubuntu\.com|d' /etc/apt/apt-mirrors.txt
fi
# Download first so the bundle contains every missing transitive dependency.
# Keep downloads outside apt's default cache, which Docker cleanup hooks may wipe.
download_packages() {
"${SUDO[@]}" apt-get update -q --error-on=any
"${SUDO[@]}" apt-get install -q --yes --no-install-recommends \
--download-only -o "Dir::Cache::archives=$CACHE_DIR" "${requested_packages[@]}"
}
# Recreate the sudo and package arrays in the timed child process.
if [ "${1:-}" = download ]; then
download_packages
exit 0
fi
for attempt in $(seq 1 "$ATTEMPTS"); do
echo "::group::apt download attempt ${attempt}/${ATTEMPTS}"
if timeout --kill-after=10 "$ATTEMPT_TIMEOUT" bash "$0" download; then
echo "::endgroup::"
install_local
touch "$CACHE_DIR/complete"
exit 0
fi
echo "::endgroup::"
echo "::warning::apt download attempt ${attempt}/${ATTEMPTS} failed or exceeded ${ATTEMPT_TIMEOUT}s"
done
echo "::error::apt download failed after ${ATTEMPTS} attempts"
exit 1
+3 -4
View File
@@ -11,7 +11,7 @@ concurrency:
group: backend-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
DEFAULT_UV_VERSION: "0.13.x"
DEFAULT_UV_VERSION: "0.12.x"
# Match the Docker image: nltk refuses to read hardlinked data files, such as
# the copy bundled with llama-index when uv links packages from its cache
UV_LINK_MODE: copy
@@ -76,7 +76,6 @@ jobs:
- 'uv.lock'
- 'docker/compose/docker-compose.ci-test.yml'
- '.github/workflows/ci-backend.yml'
- '.github/actions/apt-install/**'
test:
needs: changes
if: needs.changes.outputs.backend_changed == 'true'
@@ -103,7 +102,7 @@ jobs:
with:
python-version: "${{ matrix.python-version }}"
- name: Install uv
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
version: ${{ env.DEFAULT_UV_VERSION }}
enable-cache: true
@@ -173,7 +172,7 @@ jobs:
with:
python-version: "${{ env.DEFAULT_PYTHON }}"
- name: Install uv
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
version: ${{ env.DEFAULT_UV_VERSION }}
enable-cache: true
+2 -2
View File
@@ -11,7 +11,7 @@ concurrency:
permissions:
contents: read
env:
DEFAULT_UV_VERSION: "0.13.x"
DEFAULT_UV_VERSION: "0.12.x"
DEFAULT_PYTHON_VERSION: "3.12"
jobs:
changes:
@@ -78,7 +78,7 @@ jobs:
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
- name: Install uv
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
version: ${{ env.DEFAULT_UV_VERSION }}
enable-cache: true
+2 -3
View File
@@ -68,7 +68,6 @@ jobs:
frontend:
- 'src-ui/**'
- '.github/workflows/ci-frontend.yml'
- '.github/actions/apt-install/**'
install-dependencies:
needs: changes
if: needs.changes.outputs.frontend_changed == 'true'
@@ -217,9 +216,9 @@ jobs:
with:
python-version: '3.12'
- name: Install uv
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
version: '0.13.x'
version: '0.12.x'
enable-cache: false
python-version: ${{ steps.setup-python.outputs.python-version }}
- name: Cache frontend dependencies
+3 -3
View File
@@ -8,7 +8,7 @@ concurrency:
group: release-${{ github.ref }}
cancel-in-progress: false
env:
DEFAULT_UV_VERSION: "0.13.x"
DEFAULT_UV_VERSION: "0.12.x"
DEFAULT_PYTHON_VERSION: "3.12"
permissions: {}
jobs:
@@ -59,7 +59,7 @@ jobs:
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
- name: Install uv
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
version: ${{ env.DEFAULT_UV_VERSION }}
enable-cache: false
@@ -213,7 +213,7 @@ jobs:
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
- name: Install uv
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
version: ${{ env.DEFAULT_UV_VERSION }}
enable-cache: false
+2 -2
View File
@@ -4,7 +4,7 @@ on:
branches:
- dev
env:
DEFAULT_UV_VERSION: "0.13.x"
DEFAULT_UV_VERSION: "0.12.x"
jobs:
generate-translate-strings:
name: Generate Translation Strings
@@ -30,7 +30,7 @@ jobs:
with:
packages: gettext
- name: Install uv
uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
version: ${{ env.DEFAULT_UV_VERSION }}
enable-cache: true
+1
View File
@@ -38,6 +38,7 @@ src/documents/bulk_edit.py:0: error: Incompatible types in assignment (expressio
src/documents/bulk_edit.py:0: error: Invalid index type "str" for "dict[FieldDataType, str]"; expected type "FieldDataType" [index]
src/documents/bulk_edit.py:0: error: List comprehension has incompatible type List[tuple[int, Any]]; expected List[int] [misc]
src/documents/bulk_edit.py:0: error: List comprehension has incompatible type List[tuple[int, None]]; expected List[int] [misc]
src/documents/bulk_edit.py:0: error: Missing named argument "p" for "remove" of "PageList" [call-arg]
src/documents/bulk_edit.py:0: error: Missing type arguments for generic type "dict" [type-arg]
src/documents/bulk_edit.py:0: error: Missing type arguments for generic type "dict" [type-arg]
src/documents/bulk_edit.py:0: error: Need type annotation for "to_create" (hint: "to_create: list[<type>] = ...") [var-annotated]
+7
View File
@@ -91,6 +91,13 @@
"concise_description": "Argument `list[int]` is not assignable to parameter `args` with type `tuple[Any, ...] | None` in function `celery.app.task.Task.apply_async`",
"severity": "error"
},
{
"column": 33,
"path": "src/documents/bulk_edit.py",
"name": "missing-argument",
"concise_description": "Missing argument `p` in function `pikepdf._core.PageList.remove`",
"severity": "error"
},
{
"column": 25,
"path": "src/documents/caching.py",
+1 -1
View File
@@ -30,7 +30,7 @@ RUN set -eux \
# Purpose: Installs s6-overlay and rootfs
# Comments:
# - Don't leave anything extra in here either
FROM ghcr.io/astral-sh/uv:0.13.5-python3.14-trixie-slim@sha256:d8e732839dfc4225a1ff698803b478b2223c05636c055d3add41ab9dbec0f315 AS s6-overlay-base
FROM ghcr.io/astral-sh/uv:0.12.23-python3.14-trixie-slim AS s6-overlay-base
WORKDIR /usr/src/s6
-7
View File
@@ -1,12 +1,5 @@
# Changelog
## paperless-ngx 3.3.4
### All App Changes
- Chore: Update certifi for eMudhra CA incident [@shamoon](https://github.com/shamoon) ([#14533](https://github.com/paperless-ngx/paperless-ngx/pull/14533))
- Chore: Update uv to 0.13 for the eMudhra CA fixes [@stumpylog](https://github.com/stumpylog) ([#14532](https://github.com/paperless-ngx/paperless-ngx/pull/14532))
## paperless-ngx 3.3.3
### Bug Fixes
-9
View File
@@ -370,15 +370,6 @@ instead of being removed from the filesystem. Only the original version is kept.
This setting was previously named PAPERLESS_TRASH_DIR.
#### [`PAPERLESS_SCRATCH_DIR=<path>`](#PAPERLESS_SCRATCH_DIR) {#PAPERLESS_SCRATCH_DIR}
: This folder is used as a temporary workspace for multiple paperless operations, including
consuming documents, editing, and the document exporter.
This must be writeable by the user running paperless.
Defaults to "/tmp/paperless".
#### [`PAPERLESS_MEDIA_ROOT=<path>`](#PAPERLESS_MEDIA_ROOT) {#PAPERLESS_MEDIA_ROOT}
: This is where your documents and thumbnails are stored.
-1
View File
@@ -16,7 +16,6 @@
#PAPERLESS_CONSUMPTION_DIR=../consume
#PAPERLESS_DATA_DIR=../data
#PAPERLESS_EMPTY_TRASH_DIR=
#PAPERLESS_SCRATCH_DIR=/tmp/paperless
#PAPERLESS_MEDIA_ROOT=../media
#PAPERLESS_STATICDIR=../static
#PAPERLESS_FILENAME_FORMAT=
+3 -3
View File
@@ -1,6 +1,6 @@
[project]
name = "paperless-ngx"
version = "3.3.4"
version = "3.3.3"
description = """\
A community-supported supercharged document management system: scan, index and archive all your physical documents\
"""
@@ -256,8 +256,8 @@ isort.force-single-line = true
[tool.codespell]
ignore-words-list = "criterias,afterall,valeu,ureue,equest,ure,assertIn,Oktober,commitish,NIN,nin,reprot"
skip = """\
src-ui/src/locale/*,src-ui/pnpm-lock.yaml,src-ui/e2e/*,src/paperless_mail/tests/samples/*,src/paperless/tests/samples\
/mail/*,src/documents/tests/samples/*,*.po,*.json\
src-ui/src/locale/*,src-ui/pnpm-lock.yaml,src-ui/e2e/*,src/paperless/tests/samples/mail/*,src/documents/tests/samples\
/*,src/paperless_testing/sample_files/*,*.po,*.json\
"""
write-changes = true
+3 -1
View File
@@ -134,7 +134,9 @@ def seed_database() -> None:
originals.mkdir(parents=True)
thumbnails = Path(os.environ["PAPERLESS_MEDIA_ROOT"]) / "documents" / "thumbnails"
thumbnails.mkdir(parents=True)
sample_pdf = SOURCE_ROOT / "documents" / "tests" / "samples" / "simple.pdf"
sample_pdf = (
SOURCE_ROOT / "paperless_testing" / "sample_files" / "simple-digital.pdf"
)
for document in documents:
shutil.copyfile(sample_pdf, originals / document.filename)
shutil.copyfile(
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "paperless-ngx-ui",
"version": "3.3.4",
"version": "3.3.3",
"scripts": {
"preinstall": "npx only-allow pnpm",
"ng": "ng",
+1 -1
View File
@@ -8,7 +8,7 @@ export const environment = {
apiVersion: '10', // match src/paperless/settings.py
appTitle: DEFAULT_APP_TITLE,
tag: 'prod',
version: '3.3.4',
version: '3.3.3',
webSocketHost: window.location.host,
webSocketProtocol: window.location.protocol == 'https:' ? 'wss:' : 'ws:',
webSocketBaseUrl: base_url.pathname + 'ws/',
+38
View File
@@ -11,6 +11,8 @@ from typing import TYPE_CHECKING
import pytest
from paperless_testing import samples
if TYPE_CHECKING:
from collections.abc import Generator
from pathlib import Path
@@ -149,3 +151,39 @@ def fake_progress_manager(
monkeypatch.setattr("documents.tasks.ProgressManager", FakeProgressManager)
return FakeProgressManager
@pytest.fixture(scope="session")
def shared_samples_dir() -> Path:
"""Directory of sample files used by more than one app's tests."""
return samples.SHARED_SAMPLES_DIR
@pytest.fixture(scope="session")
def simple_digital_pdf_file() -> Path:
"""One-page PDF with a text layer."""
return samples.SIMPLE_DIGITAL_PDF
@pytest.fixture(scope="session")
def multi_page_digital_pdf_file() -> Path:
"""Three-page PDF with a text layer."""
return samples.MULTI_PAGE_DIGITAL_PDF
@pytest.fixture(scope="session")
def with_form_pdf_file() -> Path:
"""PDF containing a fillable form."""
return samples.WITH_FORM_PDF
@pytest.fixture(scope="session")
def multi_page_images_pdf_file() -> Path:
"""Multi-page PDF of scanned images, no text layer."""
return samples.MULTI_PAGE_IMAGES_PDF
@pytest.fixture(scope="session")
def thumbnail_webp_file() -> Path:
"""Small WebP thumbnail."""
return samples.THUMBNAIL_WEBP
+219 -184
View File
@@ -3,7 +3,6 @@ from __future__ import annotations
import logging
import tempfile
import uuid
from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Literal
@@ -24,7 +23,6 @@ from django.db.models import When
from django.db.models.functions import Coalesce
from django.utils import timezone
from documents import pdf_ops
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
@@ -125,11 +123,6 @@ def _resolve_root_and_source_doc(
)
def _scratch_path(name: str) -> Path:
"""A path inside a fresh directory under SCRATCH_DIR."""
return Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) / name
def set_correspondent(
doc_ids: list[int],
correspondent: Correspondent,
@@ -506,6 +499,8 @@ def rotate(
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
docs_by_root_id.setdefault(pair.root_doc.id, pair)
import pikepdf
for pair in docs_by_root_id.values():
if pair.source_doc.mime_type != "application/pdf":
logger.warning(
@@ -518,7 +513,11 @@ def rotate(
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
/ f"{pair.root_doc.id}_rotated.pdf"
)
pdf_ops.rotate_pdf(pair.source_doc.source_path, filepath, degrees)
with pikepdf.open(pair.source_doc.source_path) as pdf:
for page in pdf.pages:
page.rotate(degrees, relative=True)
pdf.remove_unreferenced_resources()
pdf.save(filepath)
# Preserve metadata/permissions via overrides; mark as new version
overrides = DocumentMetadataOverrides().from_document(pair.root_doc)
@@ -561,45 +560,48 @@ def merge(
qs = Document.objects.select_related("root_document").filter(id__in=doc_ids)
docs_by_id = {doc.id: doc for doc in qs}
affected_docs: list[int] = []
handoff_asn: int | None = None
with pdf_ops.PdfMerger() as merger:
# use doc_ids to preserve order
for doc_id in doc_ids:
doc = docs_by_id.get(doc_id)
if doc is None:
continue
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
try:
# archive_path is None when there is no archive version
archive_path = (
pair.source_doc.archive_path
if archive_fallback
and pair.source_doc.mime_type != "application/pdf"
else None
)
merger.add(
archive_path
if archive_path is not None
else pair.source_doc.source_path,
)
affected_docs.append(doc.id)
if handoff_asn is None and doc.archive_serial_number is not None:
handoff_asn = doc.archive_serial_number
except Exception as e:
logger.exception(
f"Error merging document {doc.id}, it will not be included in the merge: {e}",
)
if len(affected_docs) == 0:
logger.warning("No documents were merged")
return "OK"
import pikepdf
filepath = (
Path(
tempfile.mkdtemp(dir=settings.SCRATCH_DIR),
merged_pdf = pikepdf.new()
version: str = merged_pdf.pdf_version
handoff_asn: int | None = None
# use doc_ids to preserve order
for doc_id in doc_ids:
doc = docs_by_id.get(doc_id)
if doc is None:
continue
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
try:
doc_path = (
pair.source_doc.archive_path
if archive_fallback
and pair.source_doc.mime_type != "application/pdf"
and pair.source_doc.has_archive_version
else pair.source_doc.source_path
)
/ f"{'_'.join([str(doc_id) for doc_id in affected_docs])[:100]}_merged.pdf"
with pikepdf.open(str(doc_path)) as pdf:
version = max(version, pdf.pdf_version)
merged_pdf.pages.extend(pdf.pages)
affected_docs.append(doc.id)
if handoff_asn is None and doc.archive_serial_number is not None:
handoff_asn = doc.archive_serial_number
except Exception as e:
logger.exception(
f"Error merging document {doc.id}, it will not be included in the merge: {e}",
)
if len(affected_docs) == 0:
logger.warning("No documents were merged")
return "OK"
filepath = (
Path(
tempfile.mkdtemp(dir=settings.SCRATCH_DIR),
)
merger.save(filepath)
/ f"{'_'.join([str(doc_id) for doc_id in affected_docs])[:100]}_merged.pdf"
)
merged_pdf.remove_unreferenced_resources()
merged_pdf.save(filepath, min_version=version)
merged_pdf.close()
if metadata_document_id:
metadata_document = qs.get(id=metadata_document_id)
@@ -775,60 +777,64 @@ def split(
)
doc = Document.objects.select_related("root_document").get(id=doc_ids[0])
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
import pikepdf
consume_tasks = []
try:
outputs = [
(
[pdf_ops.PageSpec(page) for page in split_doc],
partial(_scratch_path, f"{doc.id}_{split_doc[0]}-{split_doc[-1]}.pdf"),
)
for split_doc in pages
]
filepaths = pdf_ops.build_pdfs(pair.source_doc.source_path, outputs)
with pikepdf.open(pair.source_doc.source_path) as pdf:
for idx, split_doc in enumerate(pages):
dst: pikepdf.Pdf = pikepdf.new()
for page in split_doc:
dst.pages.append(pdf.pages[page - 1])
filepath: Path = (
Path(
tempfile.mkdtemp(dir=settings.SCRATCH_DIR),
)
/ f"{doc.id}_{split_doc[0]}-{split_doc[-1]}.pdf"
)
dst.remove_unreferenced_resources()
dst.save(filepath)
dst.close()
for idx, (split_doc, filepath) in enumerate(
zip(pages, filepaths, strict=True),
):
overrides: DocumentMetadataOverrides = (
DocumentMetadataOverrides().from_document(doc)
)
overrides.title = f"{doc.title} (split {idx + 1})"
if user is not None:
overrides.owner_id = user.id
if not delete_originals:
overrides.skip_asn_if_exists = True
logger.info(
f"Adding split document with pages {split_doc} to the task queue.",
)
consume_tasks.append(
consume_file.s(
input_doc=ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=filepath,
),
overrides=overrides,
).set(headers={"trigger_source": trigger_source}),
)
overrides: DocumentMetadataOverrides = (
DocumentMetadataOverrides().from_document(doc)
)
overrides.title = f"{doc.title} (split {idx + 1})"
if user is not None:
overrides.owner_id = user.id
if not delete_originals:
overrides.skip_asn_if_exists = True
logger.info(
f"Adding split document with pages {split_doc} to the task queue.",
)
consume_tasks.append(
consume_file.s(
input_doc=ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=filepath,
),
overrides=overrides,
).set(headers={"trigger_source": trigger_source}),
)
if delete_originals:
backup = release_archive_serial_numbers([doc.id])
logger.info(
"Queueing removal of original document after consumption of the split documents",
)
try:
chord(
header=consume_tasks,
body=delete.si([doc.id]),
).on_error(
restore_archive_serial_numbers_task.s(backup),
).apply_async()
except Exception:
restore_archive_serial_numbers(backup)
raise
else:
group(consume_tasks).delay()
if delete_originals:
backup = release_archive_serial_numbers([doc.id])
logger.info(
"Queueing removal of original document after consumption of the split documents",
)
try:
chord(
header=consume_tasks,
body=delete.si([doc.id]),
).on_error(
restore_archive_serial_numbers_task.s(backup),
).apply_async()
except Exception:
restore_archive_serial_numbers(backup)
raise
else:
group(consume_tasks).delay()
except Exception as e:
logger.exception(f"Error splitting document {doc.id}: {e}")
@@ -849,6 +855,8 @@ def delete_pages(
)
doc = Document.objects.select_related("root_document").get(id=doc_ids[0])
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
pages = sorted(pages) # sort pages to avoid index issues
import pikepdf
try:
# Produce edited PDF to a temp file and create a new version
@@ -856,7 +864,13 @@ def delete_pages(
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
/ f"{pair.root_doc.id}_pages_deleted.pdf"
)
pdf_ops.remove_pages(pair.source_doc.source_path, filepath, pages)
with pikepdf.open(pair.source_doc.source_path) as pdf:
offset = 1 # pages are 1-indexed
for page_num in pages:
pdf.pages.remove(pdf.pages[page_num - offset])
offset += 1 # remove() changes the index of the pages
pdf.remove_unreferenced_resources()
pdf.save(filepath)
overrides = DocumentMetadataOverrides().from_document(pair.root_doc)
if user is not None:
@@ -905,28 +919,47 @@ def edit_pdf(
)
doc = Document.objects.select_related("root_document").get(id=doc_ids[0])
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
import pikepdf
pdf_docs: list[pikepdf.Pdf] = []
try:
output_count = pdf_ops.validate_page_operations(
operations,
single_output=update_document,
)
page_specs: list[list[pdf_ops.PageSpec]] = [[] for _ in range(output_count)]
for op in operations:
page_specs[op.get("doc", 0)].append(
pdf_ops.PageSpec(op["page"], op.get("rotate", 0)),
if not operations:
raise ValueError("Output document index is out of bounds")
max_idx = max(op.get("doc", 0) for op in operations)
if update_document and max_idx > 0:
logger.error(
"Update requested but multiple output documents specified",
)
raise ValueError("Multiple output documents specified")
if any(
op.get("doc", 0) < 0 or op.get("doc", 0) >= len(operations)
for op in operations
):
raise ValueError("Output document index is out of bounds")
with pikepdf.open(pair.source_doc.source_path) as src:
# prepare output documents
pdf_docs = [pikepdf.new() for _ in range(max_idx + 1)]
for op in operations:
dst = pdf_docs[op.get("doc", 0)]
page = src.pages[op["page"] - 1]
dst.pages.append(page)
if op.get("rotate"):
dst.pages[-1].rotate(op["rotate"], relative=True)
if update_document:
# Create a new version from the edited PDF rather than replacing in-place
(filepath,) = pdf_ops.build_pdfs(
pair.source_doc.source_path,
[
(
page_specs[0],
partial(_scratch_path, f"{pair.root_doc.id}_edited.pdf"),
),
],
pdf = pdf_docs[0]
pdf.remove_unreferenced_resources()
filepath: Path = (
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
/ f"{pair.root_doc.id}_edited.pdf"
)
pdf.save(filepath)
overrides = (
DocumentMetadataOverrides().from_document(pair.root_doc)
if include_metadata
@@ -947,19 +980,6 @@ def edit_pdf(
headers={"trigger_source": trigger_source},
)
else:
version_filepaths = pdf_ops.build_pdfs(
pair.source_doc.source_path,
[
(
specs,
partial(
_scratch_path,
f"{pair.root_doc.id}_edit_{idx}.pdf",
),
)
for idx, specs in enumerate(page_specs, start=1)
],
)
consume_tasks = []
overrides = (
DocumentMetadataOverrides().from_document(pair.root_doc)
@@ -971,9 +991,15 @@ def edit_pdf(
overrides.actor_id = user.id
if not delete_original:
overrides.skip_asn_if_exists = True
if delete_original and output_count == 1:
if delete_original and len(pdf_docs) == 1:
overrides.asn = pair.root_doc.archive_serial_number
for version_filepath in version_filepaths:
for idx, pdf in enumerate(pdf_docs, start=1):
version_filepath: Path = (
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
/ f"{pair.root_doc.id}_edit_{idx}.pdf"
)
pdf.remove_unreferenced_resources()
pdf.save(version_filepath)
consume_tasks.append(
consume_file.s(
input_doc=ConsumableDocument(
@@ -1023,6 +1049,8 @@ def remove_password(
"""
Remove password protection from PDF documents.
"""
import pikepdf
for doc_id in doc_ids:
doc = Document.objects.select_related("root_document").get(id=doc_id)
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
@@ -1036,69 +1064,76 @@ def remove_password(
doc.id,
pair.source_doc.source_path,
)
if not pdf_ops.needs_decrypt(source_path):
logger.info(
"Skipping password removal for document %s because the "
"source PDF is not encrypted",
pair.root_doc.id,
)
continue
try:
with pikepdf.open(source_path) as pdf:
if not pdf.is_encrypted:
logger.info(
"Skipping password removal for document %s because the "
"source PDF is not encrypted",
pair.root_doc.id,
)
continue
except pikepdf.PasswordError:
# Password-protected PDFs need the supplied password below.
pass
filepath = pdf_ops.decrypt_pdf(
source_path,
partial(_scratch_path, f"{pair.root_doc.id}_unprotected.pdf"),
password,
)
with pikepdf.open(source_path, password=password) as pdf:
filepath: Path = (
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
/ f"{pair.root_doc.id}_unprotected.pdf"
)
pdf.remove_unreferenced_resources()
pdf.save(filepath)
if update_document:
# Create a new version rather than modifying the root/original in place.
overrides = (
DocumentMetadataOverrides().from_document(pair.root_doc)
if include_metadata
else DocumentMetadataOverrides()
)
if user is not None:
overrides.owner_id = user.id
overrides.actor_id = user.id
consume_file.apply_async(
kwargs={
"input_doc": ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=filepath,
root_document_id=pair.root_doc.id,
),
"overrides": overrides,
},
headers={"trigger_source": trigger_source},
)
else:
consume_tasks = []
overrides = (
DocumentMetadataOverrides().from_document(pair.root_doc)
if include_metadata
else DocumentMetadataOverrides()
)
if user is not None:
overrides.owner_id = user.id
overrides.actor_id = user.id
consume_tasks.append(
consume_file.s(
input_doc=ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=filepath,
),
overrides=overrides,
).set(headers={"trigger_source": trigger_source}),
)
if delete_original:
chord(
header=consume_tasks,
body=delete.si([doc.id]),
).delay()
if update_document:
# Create a new version rather than modifying the root/original in place.
overrides = (
DocumentMetadataOverrides().from_document(pair.root_doc)
if include_metadata
else DocumentMetadataOverrides()
)
if user is not None:
overrides.owner_id = user.id
overrides.actor_id = user.id
consume_file.apply_async(
kwargs={
"input_doc": ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=filepath,
root_document_id=pair.root_doc.id,
),
"overrides": overrides,
},
headers={"trigger_source": trigger_source},
)
else:
group(consume_tasks).delay()
consume_tasks = []
overrides = (
DocumentMetadataOverrides().from_document(pair.root_doc)
if include_metadata
else DocumentMetadataOverrides()
)
if user is not None:
overrides.owner_id = user.id
overrides.actor_id = user.id
consume_tasks.append(
consume_file.s(
input_doc=ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=filepath,
),
overrides=overrides,
).set(headers={"trigger_source": trigger_source}),
)
if delete_original:
chord(
header=consume_tasks,
body=delete.si([doc.id]),
).delay()
else:
group(consume_tasks).delay()
except Exception as e:
logger.exception(
-3
View File
@@ -193,9 +193,6 @@ class DirectoryExportSink(ExportSink):
raise RuntimeError("A stream is already open on this sink")
target = (self._target / arcname).resolve()
tmp = target.with_suffix(target.suffix + ".tmp")
# A stale tmp from an interrupted run is overwritten and consumed here,
# so it must not be pruned as a leftover at finalize.
self._snapshot.discard(tmp)
target.parent.mkdir(parents=True, exist_ok=True)
handle = tmp.open("w", encoding="utf-8")
self._stream_open = True
-194
View File
@@ -1,194 +0,0 @@
"""
Pure PDF page operations used by documents.bulk_edit.
This module deliberately knows nothing about Django, Celery or the documents
app: callers resolve documents, choose output paths and queue work. Every
function that writes a PDF removes unreferenced resources before saving.
pikepdf is always called as ``pikepdf.open(...)`` / ``pikepdf.new()`` (never
``from pikepdf import open``) so tests can patch those module attributes.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import NamedTuple
import pikepdf
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Iterable
from collections.abc import Mapping
from collections.abc import Sequence
from pathlib import Path
from types import TracebackType
class PageSpec(NamedTuple):
"""One page of an output PDF: a 1-indexed source page, optionally rotated."""
page: int
rotate: int = 0 # relative degrees, 0 leaves the page alone
def _require_positive(pages: Iterable[int]) -> None:
for page in pages:
if page < 1:
raise ValueError(f"Page numbers start at 1, got {page}")
def rotate_pdf(src: Path, dst: Path, degrees: int) -> None:
"""
Rotate every page relatively on the opened document, not a rebuild, so Info,
XMP and outlines are kept. ``src`` is not modified.
"""
with pikepdf.open(src) as pdf:
for page in pdf.pages:
page.rotate(degrees, relative=True)
pdf.remove_unreferenced_resources()
pdf.save(dst)
def remove_pages(src: Path, dst: Path, pages: Iterable[int]) -> None:
"""
Remove 1-indexed pages from the opened document, not a rebuild, so Info, XMP
and outlines are kept. ``src`` is not modified.
Duplicates are ignored. Pages are removed highest first so earlier removals
never shift the index of later ones.
"""
unique = sorted(set(pages))
_require_positive(unique)
with pikepdf.open(src) as pdf:
for page_num in reversed(unique):
del pdf.pages[page_num - 1]
pdf.remove_unreferenced_resources()
pdf.save(dst)
def build_pdfs(
src: Path,
outputs: Sequence[tuple[Sequence[PageSpec], Callable[[], Path]]],
) -> list[Path]:
"""
Build one new PDF per output from pages of ``src``, opening ``src`` once.
Each output is ``(page_specs, make_dst)``. Every page number is checked against
``src`` before any output is built, and ``make_dst`` is called after that
output's pages are copied and immediately before it is saved, so a bad page
number in any output never leaves a destination behind. Document-level data
(Info, XMP, outlines) is not carried over. Returns the written paths in output
order.
"""
for specs, _ in outputs:
_require_positive(spec.page for spec in specs)
written: list[Path] = []
with pikepdf.open(src) as source:
page_count = len(source.pages)
for specs, _ in outputs:
for spec in specs:
if spec.page > page_count:
raise IndexError(
f"Page {spec.page} is out of range, the PDF has "
f"{page_count} pages",
)
for specs, make_dst in outputs:
dst = pikepdf.new()
for spec in specs:
dst.pages.append(source.pages[spec.page - 1])
if spec.rotate:
dst.pages[-1].rotate(spec.rotate, relative=True)
dst.remove_unreferenced_resources()
path = make_dst()
dst.save(path)
dst.close()
written.append(path)
return written
def validate_page_operations(
operations: Sequence[Mapping[str, int]],
*,
single_output: bool,
) -> int:
"""
Validate ``edit_pdf`` style operations and return the output document count.
Each operation has ``page`` and optionally ``rotate`` and ``doc`` (the output
document index, default 0). The bounds rule is kept as it was: a ``doc`` index
must be below the number of operations.
"""
if not operations:
raise ValueError("Output document index is out of bounds")
max_idx = max(op.get("doc", 0) for op in operations)
if single_output and max_idx > 0:
raise ValueError("Multiple output documents specified")
if any(
op.get("doc", 0) < 0 or op.get("doc", 0) >= len(operations) for op in operations
):
raise ValueError("Output document index is out of bounds")
return max_idx + 1
def needs_decrypt(src: Path) -> bool:
"""
True if ``src`` is encrypted. A PDF that needs a password to open at all
counts as encrypted.
"""
try:
with pikepdf.open(src) as pdf:
return bool(pdf.is_encrypted)
except pikepdf.PasswordError:
return True
def decrypt_pdf(src: Path, make_dst: Callable[[], Path], password: str) -> Path:
"""
Write an unencrypted copy of ``src`` and return its path.
``make_dst`` is only called once the password has been accepted, so a wrong
password never leaves a destination behind.
"""
with pikepdf.open(src, password=password) as pdf:
pdf.remove_unreferenced_resources()
dst = make_dst()
pdf.save(dst)
return dst
class PdfMerger:
"""
Accumulates the pages of several PDFs into one new PDF.
``add`` raises if a source cannot be read; deciding whether to skip it is the
caller's policy. Use as a context manager so the merged PDF is closed.
"""
def __init__(self) -> None:
self._pdf = pikepdf.new()
self._version: str = self._pdf.pdf_version
def __enter__(self) -> PdfMerger:
return self
def __exit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
self._pdf.close()
def add(self, path: Path) -> None:
with pikepdf.open(str(path)) as pdf:
self._version = max(self._version, pdf.pdf_version)
self._pdf.pages.extend(pdf.pages)
def save(self, dst: Path) -> None:
self._pdf.remove_unreferenced_resources()
self._pdf.save(dst, min_version=self._version)
-2
View File
@@ -2144,8 +2144,6 @@ class BulkEditSerializer(
raise serializers.ValidationError("pages must be a list")
if not all(isinstance(i, int) for i in parameters["pages"]):
raise serializers.ValidationError("pages must be a list of integers")
if any(i < 1 for i in parameters["pages"]):
raise serializers.ValidationError("pages must be positive integers")
def _validate_parameters_merge(self, parameters) -> None:
if "delete_originals" in parameters:
+6 -10
View File
@@ -12,29 +12,25 @@ if TYPE_CHECKING:
from paperless_testing.dirs import PaperlessDirs
@pytest.fixture(scope="session")
def document_samples_dir() -> Path:
"""Path to the shared test sample documents."""
return Path(__file__).parent / "samples" / "documents"
@pytest.fixture()
def sample_doc(
paperless_dirs: "PaperlessDirs",
document_samples_dir: Path,
simple_digital_pdf_file: Path,
multi_page_images_pdf_file: Path,
thumbnail_webp_file: Path,
) -> "Document":
"""Create a document with valid files and matching checksums."""
with filelock.FileLock(paperless_dirs.media_lock):
shutil.copy(
document_samples_dir / "originals" / "0000001.pdf",
simple_digital_pdf_file,
paperless_dirs.originals_dir / "0000001.pdf",
)
shutil.copy(
document_samples_dir / "archive" / "0000001.pdf",
multi_page_images_pdf_file,
paperless_dirs.archive_dir / "0000001.pdf",
)
shutil.copy(
document_samples_dir / "thumbnails" / "0000001.webp",
thumbnail_webp_file,
paperless_dirs.thumbnail_dir / "0000001.webp",
)
Binary file not shown.

Before

Width:  |  Height:  |  Size: 32 KiB

-22
View File
@@ -157,28 +157,6 @@ class TestDirectoryExportSink:
assert not stale.exists()
assert (target / "originals" / "doc.pdf").exists()
def test_delete_with_stale_stream_tmp_from_interrupted_run(
self,
tmp_path: Path,
) -> None:
# A previous run killed mid-stream leaves manifest.json.tmp behind; the
# next run reuses that path, so pruning must not trip over it.
target: Path = tmp_path / "out"
target.mkdir()
(target / "manifest.json.tmp").write_text("[", encoding="utf-8")
with DirectoryExportSink(
target,
compare_checksums=False,
compare_json=False,
delete=True,
) as sink:
with sink.stream("manifest.json") as handle:
writer: StreamingManifestWriter = StreamingManifestWriter(handle)
writer.write_record({"pk": 1})
writer.close()
assert json.loads((target / "manifest.json").read_text()) == [{"pk": 1}]
assert not (target / "manifest.json.tmp").exists()
def test_no_delete_keeps_unwritten_files(
self,
tmp_path: Path,
Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.
-1
View File
@@ -1 +0,0 @@
This is a test file.
@@ -53,7 +53,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
archive_checksum="D",
)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", self.doc2.source_path)
shutil.copy(self.SIMPLE_PDF, self.doc2.source_path)
shutil.copy(self.SAMPLE_DIR / "simple.png", self.doc2b.source_path)
shutil.copy(self.SAMPLE_DIR / "simple.jpg", self.doc3.source_path)
shutil.copy(self.SAMPLE_DIR / "test_with_bom.pdf", self.doc3.archive_path)
-30
View File
@@ -1844,36 +1844,6 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
m.assert_called_once()
self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]])
@mock.patch("documents.serialisers.bulk_edit.delete_pages")
def test_bulk_edit_delete_pages_rejects_pages_below_one(self, m) -> None:
"""
GIVEN:
- A legacy delete_pages bulk edit
WHEN:
- API to bulk edit is called with a page number below 1
THEN:
- API returns HTTP 400
- delete_pages is not called
"""
self.setup_mock(m, "delete_pages")
for pages in ([0], [-1], [1, 0]):
with self.subTest(pages=pages):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "delete_pages",
"parameters": {"pages": pages},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"pages must be positive integers", response.content)
m.assert_not_called()
@mock.patch("documents.views.bulk_edit.rotate")
def test_rotate_insufficient_permissions(self, m) -> None:
self.doc1.owner = User.objects.get(username="temp_admin")
+49 -33
View File
@@ -56,6 +56,8 @@ from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_all_global
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
from paperless_testing.samples import THUMBNAIL_WEBP
class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
@@ -1852,7 +1854,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SimpleUploadedFile(
"simple.pdf",
SIMPLE_DIGITAL_PDF.read_bytes(),
content_type="application/pdf",
) as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f},
@@ -1880,7 +1886,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
payload = SimpleUploadedFile(
"../../outside.pdf",
(Path(__file__).parent / "samples" / "simple.pdf").read_bytes(),
SIMPLE_DIGITAL_PDF.read_bytes(),
content_type="application/pdf",
)
@@ -1909,7 +1915,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
pdf_bytes = (Path(__file__).parent / "samples" / "simple.pdf").read_bytes()
pdf_bytes = SIMPLE_DIGITAL_PDF.read_bytes()
boundary = "paperless-boundary"
payload = (
(
@@ -1985,7 +1991,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
def test_upload_insufficient_permissions(self) -> None:
self.client.force_authenticate(user=UserFactory(username="testuser2"))
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f},
@@ -1998,7 +2004,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SimpleUploadedFile(
"simple.pdf",
SIMPLE_DIGITAL_PDF.read_bytes(),
content_type="application/pdf",
) as f:
response = self.client.post(
"/api/documents/post_document/",
{
@@ -2031,7 +2041,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"documenst": f},
@@ -2057,7 +2067,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "title": "my custom title"},
@@ -2077,7 +2087,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
)
c = Correspondent.objects.create(name="test-corres")
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "correspondent": c.id},
@@ -2096,7 +2106,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "correspondent": 3456},
@@ -2111,7 +2121,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
)
dt = DocumentType.objects.create(name="invoice")
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "document_type": dt.id},
@@ -2130,7 +2140,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "document_type": 34578},
@@ -2145,7 +2155,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
)
sp = StoragePath.objects.create(name="invoices")
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "storage_path": sp.id},
@@ -2164,7 +2174,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "storage_path": 34578},
@@ -2180,7 +2190,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
t1 = Tag.objects.create(name="tag1")
t2 = Tag.objects.create(name="tag2")
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "tags": [t2.id, t1.id]},
@@ -2201,7 +2211,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
t1 = Tag.objects.create(name="tag1")
t2 = Tag.objects.create(name="tag2")
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "tags": [t2.id, t1.id, 734563]},
@@ -2225,7 +2235,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
0,
tzinfo=zoneinfo.ZoneInfo("America/Los_Angeles"),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "created": created},
@@ -2241,7 +2251,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SimpleUploadedFile(
"simple.pdf",
SIMPLE_DIGITAL_PDF.read_bytes(),
content_type="application/pdf",
) as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "archive_serial_number": 500},
@@ -2268,7 +2282,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
data_type=CustomField.FieldDataType.STRING,
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SimpleUploadedFile(
"simple.pdf",
SIMPLE_DIGITAL_PDF.read_bytes(),
content_type="application/pdf",
) as f:
response = self.client.post(
"/api/documents/post_document/",
{
@@ -2323,7 +2341,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
w1.actions.add(action1)
w1.save()
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{
@@ -2364,7 +2382,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
data_type=CustomField.FieldDataType.INT,
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SimpleUploadedFile(
"simple.pdf",
SIMPLE_DIGITAL_PDF.read_bytes(),
content_type="application/pdf",
) as f:
response = self.client.post(
"/api/documents/post_document/",
{
@@ -2413,7 +2435,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
]
for payload in error_payloads:
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
data = {"document": f, **payload}
response = self.client.post(
"/api/documents/post_document/",
@@ -2471,7 +2493,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
id=str(uuid.uuid4()),
)
with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
with SIMPLE_DIGITAL_PDF.open("rb") as f:
response = self.client.post(
"/api/documents/post_document/",
{"document": f, "from_webui": True},
@@ -2510,14 +2532,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
archive_filename="archive.pdf",
)
source_file: Path = (
Path(__file__).parent
/ "samples"
/ "documents"
/ "thumbnails"
/ "0000001.webp"
)
archive_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
source_file: Path = THUMBNAIL_WEBP
archive_file: Path = SIMPLE_DIGITAL_PDF
shutil.copy(source_file, doc.source_path)
shutil.copy(archive_file, doc.archive_path)
@@ -2550,7 +2566,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
mime_type="application/pdf",
)
shutil.copy(Path(__file__).parent / "samples" / "simple.pdf", doc.source_path)
shutil.copy(SIMPLE_DIGITAL_PDF, doc.source_path)
response = self.client.get(f"/api/documents/{doc.pk}/metadata/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -4316,8 +4332,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
filename="test2.pdf",
)
archive_file = Path(__file__).parent / "samples" / "simple.pdf"
source_file = Path(__file__).parent / "samples" / "simple.pdf"
archive_file = SIMPLE_DIGITAL_PDF
source_file = SIMPLE_DIGITAL_PDF
shutil.copy(archive_file, doc.archive_path)
shutil.copy(source_file, doc2.source_path)
+8 -5
View File
@@ -12,6 +12,9 @@ from documents.tests.utils import SampleDirMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
from paperless_testing.samples import MULTI_PAGE_DIGITAL_PDF
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
from paperless_testing.samples import WITH_FORM_PDF
class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
@@ -42,15 +45,15 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
# Copy sample files to document paths (using different files to distinguish versions)
shutil.copy(
self.SAMPLE_DIR / "documents" / "originals" / "0000001.pdf",
SIMPLE_DIGITAL_PDF,
self.doc1.archive_path,
)
shutil.copy(
self.SAMPLE_DIR / "documents" / "originals" / "0000002.pdf",
MULTI_PAGE_DIGITAL_PDF,
self.doc1.source_path,
)
shutil.copy(
self.SAMPLE_DIR / "documents" / "originals" / "0000003.pdf",
WITH_FORM_PDF,
self.doc2.source_path,
)
@@ -377,7 +380,7 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
checksum="3",
filename="test3.pdf",
)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", doc3.source_path)
shutil.copy(self.SIMPLE_PDF, doc3.source_path)
doc4 = Document.objects.create(
title="test1",
@@ -386,7 +389,7 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
checksum="4",
filename="test4.pdf",
)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", doc4.source_path)
shutil.copy(self.SIMPLE_PDF, doc4.source_path)
response = self.client.post(
self.ENDPOINT,
+2 -2
View File
@@ -169,7 +169,7 @@ class TestBarcode(
- No barcodes detected
- No pages to split on
"""
test_file = self.SAMPLE_DIR / "simple.pdf"
test_file = self.SIMPLE_PDF
with self.get_reader(test_file) as reader:
reader.detect()
separator_page_numbers = reader.get_separation_pages()
@@ -461,7 +461,7 @@ class TestBarcode(
THEN:
- Nothing happens
"""
test_file = self.SAMPLE_DIR / "simple.pdf"
test_file = self.SIMPLE_PDF
with self.get_reader(test_file) as reader:
try:
+240 -182
View File
@@ -1,9 +1,9 @@
import shutil
from collections.abc import Callable
from datetime import date
from pathlib import Path
from unittest import mock
import pikepdf
import pytest
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
@@ -23,11 +23,13 @@ from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.models import Tag
from documents.pdf_ops import PageSpec
from documents.permissions import set_permissions_for_objects
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.permissions import grant_object
from paperless_testing.samples import MULTI_PAGE_DIGITAL_PDF
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
from paperless_testing.samples import WITH_FORM_PDF
class TestBulkEdit(DirectoriesMixin, TestCase):
@@ -705,47 +707,27 @@ class TestPDFActions(DirectoriesMixin, TestCase):
super().setUp()
sample1 = self.dirs.scratch_dir / "sample.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf",
SIMPLE_DIGITAL_PDF,
sample1,
)
sample1_archive = self.dirs.archive_dir / "sample_archive.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf",
SIMPLE_DIGITAL_PDF,
sample1_archive,
)
sample2 = self.dirs.scratch_dir / "sample2.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000002.pdf",
MULTI_PAGE_DIGITAL_PDF,
sample2,
)
sample2_archive = self.dirs.archive_dir / "sample2_archive.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000002.pdf",
MULTI_PAGE_DIGITAL_PDF,
sample2_archive,
)
sample3 = self.dirs.scratch_dir / "sample3.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000003.pdf",
WITH_FORM_PDF,
sample3,
)
self.doc1 = Document.objects.create(
@@ -779,11 +761,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
img_doc_archive = self.dirs.archive_dir / "sample_image.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf",
SIMPLE_DIGITAL_PDF,
img_doc_archive,
)
self.img_doc = Document.objects.create(
@@ -797,14 +775,16 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.img_doc.save()
@staticmethod
def fake_decrypt(
src: Path,
make_dst: Callable[[], Path],
password: str,
) -> Path:
dst = make_dst()
dst.write_bytes(b"password removed")
return dst
def mock_password_required_pdf(
mock_open: mock.Mock,
fake_pdf: mock.Mock,
) -> None:
password_context = mock.MagicMock()
password_context.__enter__.return_value = fake_pdf
mock_open.side_effect = [
pikepdf.PasswordError("password required"),
password_context,
]
@mock.patch("documents.tasks.consume_file.s")
def test_merge(self, mock_consume_file) -> None:
@@ -849,12 +829,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK")
@mock.patch("documents.pdf_ops.PdfMerger")
@mock.patch("pikepdf.open")
@mock.patch("documents.tasks.consume_file.s")
def test_merge_uses_latest_version_source_for_root_selection(
self,
mock_consume_file,
mock_merger,
mock_open_pdf,
) -> None:
version_file = self.dirs.scratch_dir / "sample2_version_merge.pdf"
shutil.copy(self.doc2.source_path, version_file)
@@ -865,14 +845,16 @@ class TestPDFActions(DirectoriesMixin, TestCase):
filename=version_file,
mime_type="application/pdf",
)
merger = mock_merger.return_value.__enter__.return_value
merger.save.side_effect = lambda dst: shutil.copy(version.source_path, dst)
fake_pdf = mock.MagicMock()
fake_pdf.pdf_version = "1.7"
fake_pdf.pages = [mock.Mock()]
mock_open_pdf.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.merge([self.doc2.id])
self.assertEqual(result, "OK")
merger.add.assert_called_once_with(version.source_path)
mock_consume_file.assert_called_once()
mock_open_pdf.assert_called_once_with(str(version.source_path))
mock_consume_file.assert_not_called()
@mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s")
@@ -1034,18 +1016,18 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
@mock.patch("documents.tasks.consume_file.delay")
@mock.patch("documents.pdf_ops.PdfMerger.add")
def test_merge_with_errors(self, mock_add, mock_consume_file) -> None:
@mock.patch("pikepdf.open")
def test_merge_with_errors(self, mock_open_pdf, mock_consume_file) -> None:
"""
GIVEN:
- Existing documents
WHEN:
- Merge action is called with 2 documents
- Error occurs when adding both files
- Error occurs when opening both files
THEN:
- Consume file should not be called
"""
mock_add.side_effect = Exception("Error opening PDF")
mock_open_pdf.side_effect = Exception("Error opening PDF")
doc_ids = [self.doc2.id, self.doc3.id]
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
@@ -1082,12 +1064,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK")
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.pdf_ops.build_pdfs")
@mock.patch("pikepdf.open")
@mock.patch("documents.tasks.consume_file.s")
def test_split_uses_latest_version_source_for_root_selection(
self,
mock_consume_file,
mock_build_pdfs,
mock_open_pdf,
mock_group,
) -> None:
version_file = self.dirs.scratch_dir / "sample2_version_split.pdf"
@@ -1099,15 +1081,17 @@ class TestPDFActions(DirectoriesMixin, TestCase):
filename=version_file,
mime_type="application/pdf",
)
mock_build_pdfs.return_value = [version.source_path, version.source_path]
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock()]
mock_open_pdf.return_value.__enter__.return_value = fake_pdf
mock_group.return_value.delay.return_value = None
result = bulk_edit.split([self.doc2.id], [[1], [2]])
self.assertEqual(result, "OK")
self.assertEqual(mock_build_pdfs.call_args.args[0], version.source_path)
self.assertEqual(mock_consume_file.call_count, 2)
mock_group.return_value.delay.assert_called_once()
mock_open_pdf.assert_called_once_with(version.source_path)
mock_consume_file.assert_not_called()
mock_group.return_value.delay.assert_not_called()
@mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s")
@@ -1195,18 +1179,18 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(self.doc2.archive_serial_number, 222)
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.build_pdfs")
def test_split_with_errors(self, mock_build_pdfs, mock_consume_file) -> None:
@mock.patch("pikepdf.Pdf.save")
def test_split_with_errors(self, mock_save_pdf, mock_consume_file) -> None:
"""
GIVEN:
- Existing documents
WHEN:
- Split action is called with 1 document and 2 page groups
- Error occurs when building the files
- Error occurs when saving the files
THEN:
- Consume file should not be called
"""
mock_build_pdfs.side_effect = Exception("Error building PDFs")
mock_save_pdf.side_effect = Exception("Error saving PDF")
doc_ids = [self.doc2.id]
pages = [[1, 2], [3]]
@@ -1241,10 +1225,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK")
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.rotate_pdf")
@mock.patch("pikepdf.Pdf.save")
def test_rotate_with_error(
self,
mock_rotate_pdf,
mock_pdf_save,
mock_consume_delay,
) -> None:
"""
@@ -1252,11 +1236,11 @@ class TestPDFActions(DirectoriesMixin, TestCase):
- Existing documents
WHEN:
- Rotate action is called with 2 documents
- Rotating the PDF raises an error
- PikePDF raises an error
THEN:
- Rotate action should be called 0 times
"""
mock_rotate_pdf.side_effect = Exception("Error rotating PDF")
mock_pdf_save.side_effect = Exception("Error saving PDF")
doc_ids = [self.doc2.id, self.doc3.id]
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
@@ -1291,10 +1275,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.rotate_pdf")
@mock.patch("pikepdf.open")
def test_rotate_explicit_selection_uses_root_source_when_root_selected(
self,
mock_rotate_pdf,
mock_open,
mock_consume_delay,
mock_magic,
) -> None:
@@ -1303,6 +1287,9 @@ class TestPDFActions(DirectoriesMixin, TestCase):
title="B version 1",
root_document=self.doc2,
)
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock()]
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.rotate(
[self.doc2.id],
@@ -1311,35 +1298,26 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
self.assertEqual(mock_rotate_pdf.call_args.args[0], self.doc2.source_path)
mock_open.assert_called_once_with(self.doc2.source_path)
mock_consume_delay.assert_called_once()
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.remove_pages")
@mock.patch("pikepdf.Pdf.save")
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
def test_delete_pages(
self,
mock_magic,
mock_remove_pages,
mock_consume_delay,
) -> None:
def test_delete_pages(self, mock_magic, mock_pdf_save, mock_consume_delay) -> None:
"""
GIVEN:
- Existing documents
WHEN:
- Delete pages action is called with 1 document and 2 pages
THEN:
- The pages are removed from the document's source PDF
- Save should be called once
- A new version should be enqueued via consume_file
"""
doc_ids = [self.doc2.id]
pages = [1, 3]
result = bulk_edit.delete_pages(doc_ids, pages)
mock_remove_pages.assert_called_once_with(
self.doc2.source_path,
mock.ANY,
[1, 3],
)
mock_pdf_save.assert_called_once()
mock_consume_delay.assert_called_once()
task_kwargs = mock_consume_delay.call_args.kwargs["kwargs"]
self.assertEqual(task_kwargs["input_doc"].root_document_id, self.doc2.id)
@@ -1351,10 +1329,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.remove_pages")
@mock.patch("pikepdf.open")
def test_delete_pages_explicit_selection_uses_root_source_when_root_selected(
self,
mock_remove_pages,
mock_open,
mock_consume_delay,
mock_magic,
) -> None:
@@ -1363,6 +1341,9 @@ class TestPDFActions(DirectoriesMixin, TestCase):
title="B version 1",
root_document=self.doc2,
)
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock()]
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.delete_pages(
[self.doc2.id],
@@ -1371,26 +1352,23 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
self.assertEqual(mock_remove_pages.call_args.args[0], self.doc2.source_path)
mock_open.assert_called_once_with(self.doc2.source_path)
mock_consume_delay.assert_called_once()
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.remove_pages")
def test_delete_pages_with_error(
self,
mock_remove_pages,
mock_consume_delay,
) -> None:
@mock.patch("pikepdf.Pdf.save")
def test_delete_pages_with_error(self, mock_pdf_save, mock_consume_delay) -> None:
"""
GIVEN:
- Existing documents
WHEN:
- Delete pages action is called with 1 document and 2 pages
- Removing the pages raises an error
- PikePDF raises an error
THEN:
- Save should be called once
- No new version should be enqueued
"""
mock_remove_pages.side_effect = Exception("Error removing pages")
mock_pdf_save.side_effect = Exception("Error saving PDF")
doc_ids = [self.doc2.id]
pages = [1, 3]
@@ -1420,41 +1398,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK")
mock_group.return_value.delay.assert_called_once()
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.pdf_ops.build_pdfs")
@mock.patch("documents.tasks.consume_file.s")
def test_edit_pdf_maps_operations_to_outputs(
self,
mock_consume_file: mock.Mock,
mock_build_pdfs: mock.Mock,
mock_group: mock.Mock,
) -> None:
"""
GIVEN:
- Existing document
WHEN:
- edit_pdf is called with operations interleaved across two outputs,
some of them rotated
THEN:
- Each output is built from its own operations, in operation order,
with the requested rotation
"""
mock_build_pdfs.return_value = [self.doc2.source_path, self.doc2.source_path]
mock_group.return_value.delay.return_value = None
operations = [
{"page": 3, "doc": 1},
{"page": 1, "doc": 0, "rotate": 90},
{"page": 2, "doc": 1, "rotate": 180},
]
bulk_edit.edit_pdf([self.doc2.id], operations)
outputs = mock_build_pdfs.call_args.args[1]
self.assertEqual(
[specs for specs, _ in outputs],
[[PageSpec(1, 90)], [PageSpec(3), PageSpec(2, 180)]],
)
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s")
def test_edit_pdf_with_user_override(self, mock_consume_file, mock_group) -> None:
@@ -1570,10 +1513,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.build_pdfs")
@mock.patch("pikepdf.new")
@mock.patch("pikepdf.open")
def test_edit_pdf_explicit_selection_uses_root_source_when_root_selected(
self,
mock_build_pdfs,
mock_open,
mock_new,
mock_consume_delay,
mock_magic,
) -> None:
@@ -1582,7 +1527,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
title="B version 1",
root_document=self.doc2,
)
mock_build_pdfs.return_value = [Path("edited.pdf")]
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock()]
mock_open.return_value.__enter__.return_value = fake_pdf
output_pdf = mock.MagicMock()
output_pdf.pages = []
mock_new.return_value = output_pdf
result = bulk_edit.edit_pdf(
[self.doc2.id],
@@ -1592,7 +1542,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
self.assertEqual(mock_build_pdfs.call_args.args[0], self.doc2.source_path)
mock_open.assert_called_once_with(self.doc2.source_path)
mock_consume_delay.assert_called_once()
@mock.patch("documents.bulk_edit.group")
@@ -1618,6 +1568,31 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK")
mock_group.return_value.delay.assert_called_once()
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s")
def test_edit_pdf_open_failure(
self,
mock_consume_file: mock.Mock,
mock_group: mock.Mock,
) -> None:
"""
GIVEN:
- Existing document
WHEN:
- edit_pdf fails to open PDF
THEN:
- Task group is not called
"""
doc_ids = [self.doc2.id]
operations = [
{"page": 9999}, # invalid page, forces error during PDF load
]
with self.assertLogs("paperless.bulk_edit", level="ERROR"):
with self.assertRaises(Exception):
bulk_edit.edit_pdf(doc_ids, operations)
mock_group.assert_not_called()
mock_consume_file.assert_not_called()
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s")
def test_edit_pdf_multiple_outputs_with_update_flag_errors(
@@ -1644,15 +1619,23 @@ class TestPDFActions(DirectoriesMixin, TestCase):
mock_group.assert_not_called()
mock_consume_file.assert_not_called()
@mock.patch("pikepdf.open")
def test_edit_pdf_rejects_invalid_operations(self, mock_open) -> None:
for operations in ([], [{"page": 1, "doc": 2**32}]):
with self.subTest(operations=operations):
with self.assertLogs("paperless.bulk_edit", level="ERROR"):
with self.assertRaisesRegex(ValueError, "index is out of bounds"):
bulk_edit.edit_pdf([self.doc2.id], operations)
mock_open.assert_not_called()
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file.delay")
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
@mock.patch("pikepdf.open")
def test_remove_password_update_document(
self,
mock_needs_decrypt,
mock_decrypt,
mock_open,
mock_mkdtemp,
mock_consume_delay,
mock_update_document,
@@ -1661,7 +1644,16 @@ class TestPDFActions(DirectoriesMixin, TestCase):
temp_dir = self.dirs.scratch_dir / "remove-password-update"
temp_dir.mkdir(parents=True, exist_ok=True)
mock_mkdtemp.return_value = str(temp_dir)
mock_decrypt.side_effect = self.fake_decrypt
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock(), mock.Mock()]
fake_pdf.is_encrypted = True
def save_side_effect(target_path):
Path(target_path).write_bytes(b"new pdf content")
fake_pdf.save.side_effect = save_side_effect
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.remove_password(
[doc.id],
@@ -1670,8 +1662,14 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
mock_needs_decrypt.assert_called_once_with(doc.source_path)
mock_decrypt.assert_called_once_with(doc.source_path, mock.ANY, "secret")
self.assertEqual(
mock_open.call_args_list,
[
mock.call(doc.source_path),
mock.call(doc.source_path, password="secret"),
],
)
fake_pdf.remove_unreferenced_resources.assert_called_once()
mock_update_document.assert_not_called()
mock_consume_delay.assert_called_once()
task_kwargs = mock_consume_delay.call_args.kwargs["kwargs"]
@@ -1684,15 +1682,40 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(task_kwargs["input_doc"].root_document_id, doc.id)
self.assertIsNotNone(task_kwargs["overrides"])
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("pikepdf.open")
def test_remove_password_update_document_skips_unencrypted_pdf(
self,
mock_open,
mock_mkdtemp,
mock_consume_delay,
) -> None:
doc = self.doc1
fake_pdf = mock.MagicMock()
fake_pdf.is_encrypted = False
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.remove_password(
[doc.id],
password="secret",
update_document=True,
)
self.assertEqual(result, "OK")
mock_open.assert_called_once_with(doc.source_path)
fake_pdf.remove_unreferenced_resources.assert_not_called()
fake_pdf.save.assert_not_called()
mock_mkdtemp.assert_not_called()
mock_consume_delay.assert_not_called()
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file.delay")
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
@mock.patch("pikepdf.open")
def test_remove_password_update_document_uses_source_paths(
self,
mock_needs_decrypt,
mock_decrypt,
mock_open,
mock_mkdtemp,
mock_consume_delay,
mock_update_document,
@@ -1703,7 +1726,14 @@ class TestPDFActions(DirectoriesMixin, TestCase):
temp_dir = self.dirs.scratch_dir / "remove-password-source-file"
temp_dir.mkdir(parents=True, exist_ok=True)
mock_mkdtemp.return_value = str(temp_dir)
mock_decrypt.side_effect = self.fake_decrypt
fake_pdf = mock.MagicMock()
self.mock_password_required_pdf(mock_open, fake_pdf)
def save_side_effect(target_path):
Path(target_path).write_bytes(b"new pdf content")
fake_pdf.save.side_effect = save_side_effect
result = bulk_edit.remove_password(
[doc.id],
@@ -1713,19 +1743,22 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
mock_needs_decrypt.assert_called_once_with(source_file)
mock_decrypt.assert_called_once_with(source_file, mock.ANY, "secret")
self.assertEqual(
mock_open.call_args_list,
[
mock.call(source_file),
mock.call(source_file, password="secret"),
],
)
mock_update_document.assert_not_called()
mock_consume_delay.assert_called_once()
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
@mock.patch("pikepdf.open")
def test_remove_password_explicit_selection_uses_root_source_when_root_selected(
self,
mock_needs_decrypt,
mock_decrypt,
mock_open,
mock_consume_delay,
mock_magic,
) -> None:
@@ -1734,7 +1767,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
title="A version 1",
root_document=self.doc1,
)
mock_decrypt.return_value = Path("unprotected.pdf")
fake_pdf = mock.MagicMock()
self.mock_password_required_pdf(mock_open, fake_pdf)
result = bulk_edit.remove_password(
[self.doc1.id],
@@ -1744,11 +1778,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
mock_needs_decrypt.assert_called_once_with(self.doc1.source_path)
mock_decrypt.assert_called_once_with(
self.doc1.source_path,
mock.ANY,
"secret",
self.assertEqual(
mock_open.call_args_list,
[
mock.call(self.doc1.source_path),
mock.call(self.doc1.source_path, password="secret"),
],
)
mock_consume_delay.assert_called_once()
@@ -1756,12 +1791,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
@mock.patch("pikepdf.open")
def test_remove_password_creates_consumable_document(
self,
mock_needs_decrypt: mock.Mock,
mock_decrypt: mock.Mock,
mock_open: mock.Mock,
mock_mkdtemp: mock.Mock,
mock_consume_file: mock.Mock,
mock_group: mock.Mock,
@@ -1771,7 +1804,15 @@ class TestPDFActions(DirectoriesMixin, TestCase):
temp_dir = self.dirs.scratch_dir / "remove-password"
temp_dir.mkdir(parents=True, exist_ok=True)
mock_mkdtemp.return_value = str(temp_dir)
mock_decrypt.side_effect = self.fake_decrypt
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock()]
self.mock_password_required_pdf(mock_open, fake_pdf)
def save_side_effect(target_path: Path) -> None:
target_path.write_bytes(b"password removed")
fake_pdf.save.side_effect = save_side_effect
mock_group.return_value.delay.return_value = None
user = User.objects.create(username="owner")
@@ -1786,7 +1827,13 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
mock_decrypt.assert_called_once_with(doc.source_path, mock.ANY, "secret")
self.assertEqual(
mock_open.call_args_list,
[
mock.call(doc.source_path),
mock.call(doc.source_path, password="secret"),
],
)
mock_consume_file.assert_called_once()
call_kwargs = mock_consume_file.call_args.kwargs
consumable_document = call_kwargs["input_doc"]
@@ -1808,18 +1855,21 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.bulk_edit.chord")
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=False)
@mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("pikepdf.open")
def test_remove_password_skips_unencrypted_pdf_without_queueing(
self,
mock_needs_decrypt: mock.Mock,
mock_decrypt: mock.Mock,
mock_open: mock.Mock,
mock_mkdtemp: mock.Mock,
mock_consume_file: mock.Mock,
mock_group: mock.Mock,
mock_chord: mock.Mock,
mock_delete: mock.Mock,
) -> None:
doc = self.doc2
fake_pdf = mock.MagicMock()
fake_pdf.is_encrypted = False
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.remove_password(
[doc.id],
@@ -1829,8 +1879,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
mock_needs_decrypt.assert_called_once_with(doc.source_path)
mock_decrypt.assert_not_called()
mock_open.assert_called_once_with(doc.source_path)
fake_pdf.remove_unreferenced_resources.assert_not_called()
fake_pdf.save.assert_not_called()
mock_mkdtemp.assert_not_called()
mock_consume_file.assert_not_called()
mock_group.assert_not_called()
mock_chord.assert_not_called()
@@ -1841,12 +1893,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
@mock.patch("pikepdf.open")
def test_remove_password_deletes_original(
self,
mock_needs_decrypt: mock.Mock,
mock_decrypt: mock.Mock,
mock_open: mock.Mock,
mock_mkdtemp: mock.Mock,
mock_consume_file: mock.Mock,
mock_group: mock.Mock,
@@ -1857,7 +1907,15 @@ class TestPDFActions(DirectoriesMixin, TestCase):
temp_dir = self.dirs.scratch_dir / "remove-password-delete"
temp_dir.mkdir(parents=True, exist_ok=True)
mock_mkdtemp.return_value = str(temp_dir)
mock_decrypt.side_effect = self.fake_decrypt
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock()]
self.mock_password_required_pdf(mock_open, fake_pdf)
def save_side_effect(target_path: Path) -> None:
target_path.write_bytes(b"password removed")
fake_pdf.save.side_effect = save_side_effect
mock_chord.return_value.delay.return_value = None
result = bulk_edit.remove_password(
@@ -1869,23 +1927,23 @@ class TestPDFActions(DirectoriesMixin, TestCase):
)
self.assertEqual(result, "OK")
mock_decrypt.assert_called_once_with(doc.source_path, mock.ANY, "secret")
self.assertEqual(
mock_open.call_args_list,
[
mock.call(doc.source_path),
mock.call(doc.source_path, password="secret"),
],
)
mock_consume_file.assert_called_once()
mock_group.assert_not_called()
mock_chord.assert_called_once()
mock_chord.return_value.delay.assert_called_once()
mock_delete.si.assert_called_once_with([doc.id])
@mock.patch(
"documents.pdf_ops.decrypt_pdf",
side_effect=RuntimeError("wrong password"),
)
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
def test_remove_password_failure_raises_value_error(
self,
mock_needs_decrypt: mock.Mock,
mock_decrypt: mock.Mock,
) -> None:
@mock.patch("pikepdf.open")
def test_remove_password_open_failure(self, mock_open: mock.Mock) -> None:
mock_open.side_effect = RuntimeError("wrong password")
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
with self.assertRaises(ValueError) as exc:
bulk_edit.remove_password([self.doc1.id], password="secret")
+9 -28
View File
@@ -39,6 +39,9 @@ from paperless_testing.factories import TagFactory
from paperless_testing.factories import UserFactory
from paperless_testing.fakes.progress import FakeProgressManager
from paperless_testing.permissions import grant_object
from paperless_testing.samples import MULTI_PAGE_DIGITAL_PDF
from paperless_testing.samples import MULTI_PAGE_IMAGES_PDF
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
class _BaseNewStyleParser:
@@ -115,9 +118,7 @@ class _BaseNewStyleParser:
class DummyParser(_BaseNewStyleParser):
_ARCHIVE_SRC = (
Path(__file__).parent / "samples" / "documents" / "archive" / "0000001.pdf"
)
_ARCHIVE_SRC = MULTI_PAGE_IMAGES_PDF
def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:
self._text = "The Text"
@@ -203,33 +204,19 @@ class TestConsumer(
self.addCleanup(patcher.stop)
def get_test_file(self):
src = (
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf"
)
src = SIMPLE_DIGITAL_PDF
dst = self.dirs.scratch_dir / "sample.pdf"
shutil.copy(src, dst)
return dst
def get_test_file2(self):
src = (
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000002.pdf"
)
src = MULTI_PAGE_DIGITAL_PDF
dst = self.dirs.scratch_dir / "sample2.pdf"
shutil.copy(src, dst)
return dst
def get_test_archive_file(self):
src = (
Path(__file__).parent / "samples" / "documents" / "archive" / "0000001.pdf"
)
src = MULTI_PAGE_IMAGES_PDF
dst = self.dirs.scratch_dir / "sample_archive.pdf"
shutil.copy(src, dst)
return dst
@@ -1124,7 +1111,7 @@ class TestConsumer(
@mock.patch("documents.consumer.get_parser_registry")
def test_similar_filenames(self, m) -> None:
shutil.copy(
Path(__file__).parent / "samples" / "simple.pdf",
SIMPLE_DIGITAL_PDF,
settings.CONSUMPTION_DIR / "simple.pdf",
)
shutil.copy(
@@ -1665,13 +1652,7 @@ class TestConsumerRemoteOCR(
self.addCleanup(patcher.stop)
def _consume(self, *, overrides: DocumentMetadataOverrides | None = None) -> bool:
src = (
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf"
)
src = SIMPLE_DIGITAL_PDF
dst = self.dirs.scratch_dir / "sample.pdf"
shutil.copy(src, dst)
+5 -2
View File
@@ -37,12 +37,15 @@ class TestDoubleSided(
self.double_sided_dir.mkdir()
self.staging_file = self.dirs.scratch_dir / STAGING_FILE_NAME
def _sample(self, name: str) -> Path:
return self.SIMPLE_PDF if name == "simple.pdf" else self.SAMPLE_DIR / name
def consume_file(self, srcname, dstname: str | Path = "foo.pdf"):
"""
Starts the consume process and also ensures the
destination file does not exist afterwards
"""
src = self.SAMPLE_DIR / srcname
src = self._sample(srcname)
dst = self.double_sided_dir / dstname
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(src, dst)
@@ -57,7 +60,7 @@ class TestDoubleSided(
return msg
def create_staging_file(self, src="double-sided-odd.pdf", datetime=None) -> None:
shutil.copy(self.SAMPLE_DIR / src, self.staging_file)
shutil.copy(self._sample(src), self.staging_file)
if datetime is None:
datetime = dt.datetime.now()
os.utime(str(self.staging_file), (datetime.timestamp(),) * 2)
+2 -1
View File
@@ -23,8 +23,9 @@ from documents.tasks import update_document_content_maybe_archive_file
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
sample_file: Path = SIMPLE_DIGITAL_PDF
@pytest.mark.management
+21 -80
View File
@@ -59,6 +59,7 @@ from paperless_testing.dirs import paperless_environment
from paperless_testing.factories import DocumentBarcodeFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.permissions import grant_object
from paperless_testing.samples import install_document_samples
@pytest.mark.management
@@ -204,10 +205,7 @@ class TestExportImport(
def test_exporter(self, *, use_filename_format=False) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
num_permission_objects = Permission.objects.count()
@@ -311,10 +309,7 @@ class TestExportImport(
def test_exporter_with_filename_format(self) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
with override_settings(
FILENAME_FORMAT="{created_year}/{correspondent}/{title}",
@@ -323,10 +318,7 @@ class TestExportImport(
def test_exporter_includes_share_links_and_bundles(self) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
share_link = ShareLink.objects.create(
slug="share-link-slug",
@@ -425,10 +417,7 @@ class TestExportImport(
def test_update_export_changed_time(self) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
self._do_export()
self.assertIsFile(self.target / "manifest.json")
@@ -464,10 +453,7 @@ class TestExportImport(
def test_update_export_changed_checksum(self) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
self._do_export()
@@ -494,10 +480,7 @@ class TestExportImport(
def test_update_export_deleted_document(self) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
manifest = self._do_export()
@@ -529,10 +512,7 @@ class TestExportImport(
@override_settings(FILENAME_FORMAT="{title}/{correspondent}")
def test_update_export_changed_location(self) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
self._do_export(use_filename_format=True)
self.assertIsFile(self.target / "wow1" / "c.pdf")
@@ -574,10 +554,7 @@ class TestExportImport(
- Zipfile contains exported files
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
args = ["document_exporter", self.target, "--zip"]
@@ -606,10 +583,7 @@ class TestExportImport(
- Zipfile contains exported files
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
args = ["document_exporter", self.target, "--zip", "--use-filename-format"]
@@ -645,10 +619,7 @@ class TestExportImport(
- The existing file and directory in target are removed
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
# Create stuff in target directory
existing_file = self.target / "test.txt"
@@ -744,10 +715,7 @@ class TestExportImport(
- Documents can be imported again
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
manifest = self._do_export()
has_archive = False
@@ -790,10 +758,7 @@ class TestExportImport(
- Documents can be imported again
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
manifest = self._do_export()
has_thumbnail = False
@@ -838,10 +803,7 @@ class TestExportImport(
- Documents can be imported again
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
manifest = self._do_export(split_manifest=True)
has_document = False
@@ -866,10 +828,7 @@ class TestExportImport(
def _export_import_barcodes(self, *, split_manifest: bool) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
DocumentBarcodeFactory(document=self.d1, value="https://example.com")
DocumentBarcodeFactory(document=self.d2, page=2, value="DE8937")
@@ -905,10 +864,7 @@ class TestExportImport(
- Documents can be imported again
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
self._do_export(use_folder_prefix=True)
@@ -935,10 +891,7 @@ class TestExportImport(
- Documents can be imported again
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
self._do_export(use_folder_prefix=True, split_manifest=True)
@@ -965,10 +918,7 @@ class TestExportImport(
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
num_content_type_objects = ContentType.objects.count()
num_permission_objects = Permission.objects.count()
@@ -1005,10 +955,7 @@ class TestExportImport(
def test_exporter_with_auditlog_disabled(self) -> None:
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
with override_settings(
AUDIT_LOG_ENABLED=False,
@@ -1028,10 +975,7 @@ class TestExportImport(
survive the round-trip with deleted_at preserved
"""
shutil.rmtree(Path(self.dirs.media_dir) / "documents")
shutil.copytree(
Path(__file__).parent / "samples" / "documents",
Path(self.dirs.media_dir) / "documents",
)
install_document_samples(Path(self.dirs.media_dir) / "documents")
# d1 has self.note and self.cfi1 attached via setUp
self.d1.delete()
@@ -1075,10 +1019,7 @@ class TestExportImport(
"""
shutil.rmtree(self.dirs.media_dir / "documents")
shutil.copytree(
self.SAMPLE_DIR / "documents",
self.dirs.media_dir / "documents",
)
install_document_samples(self.dirs.media_dir / "documents")
_ = self._do_export(data_only=True)
@@ -11,6 +11,7 @@ from documents.models import Document
from documents.parsers import get_default_thumbnail
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
@pytest.mark.management
@@ -24,7 +25,7 @@ class TestMakeThumbnails(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
filename="test.pdf",
)
shutil.copy(
Path(__file__).parent / "samples" / "simple.pdf",
SIMPLE_DIGITAL_PDF,
self.d1.source_path,
)
@@ -36,7 +37,7 @@ class TestMakeThumbnails(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
filename="test2.pdf",
)
shutil.copy(
Path(__file__).parent / "samples" / "simple.pdf",
SIMPLE_DIGITAL_PDF,
self.d2.source_path,
)
-520
View File
@@ -1,520 +0,0 @@
"""
Tests for documents.pdf_ops.
These use real PDFs from the sample directories. No database, Celery or mocks.
Pages are compared by a hash of their content stream, so page identity and order
are easy to assert.
"""
import hashlib
from collections.abc import Callable
from pathlib import Path
import pikepdf
import pytest
from documents import pdf_ops
from documents.pdf_ops import PageSpec
SRC_ROOT = Path(__file__).parents[2]
SAMPLES = Path(__file__).parent / "samples"
THREE_PAGES = SAMPLES / "documents" / "originals" / "0000002.pdf"
TWELVE_PAGES = SAMPLES / "barcodes" / "split-by-asn-2.pdf"
ENCRYPTED = SAMPLES / "password-is-test.pdf"
SIGNED = SRC_ROOT / "paperless" / "tests" / "samples" / "tesseract" / "signed.pdf"
def _page_fingerprint(page: pikepdf.Page) -> str:
contents = page.obj.get("/Contents")
assert contents is not None, "sample page has no /Contents"
streams = list(contents) if isinstance(contents, pikepdf.Array) else [contents]
return hashlib.sha256(b"".join(s.read_bytes() for s in streams)).hexdigest()
def fingerprints(path: Path) -> list[str]:
with pikepdf.open(path) as pdf:
return [_page_fingerprint(page) for page in pdf.pages]
def rotations(path: Path) -> list[int]:
with pikepdf.open(path) as pdf:
return [int(page.obj.get("/Rotate", 0)) for page in pdf.pages]
def docinfo_keys(path: Path) -> set[str]:
with pikepdf.open(path) as pdf:
return set(pdf.docinfo.keys())
def constant(path: Path) -> Callable[[], Path]:
return lambda: path
@pytest.fixture
def source_fingerprints() -> list[str]:
fps = fingerprints(THREE_PAGES)
assert len(set(fps)) == 3, "sample must have three distinct pages"
return fps
class TestRotatePdf:
def test_rotation_is_relative_and_applies_to_every_page(
self,
tmp_path: Path,
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- It is rotated by 90 degrees, then the result is rotated by 90 again
THEN:
- Every page is rotated relative to its current rotation
- The page content itself is unchanged
"""
once = tmp_path / "once.pdf"
twice = tmp_path / "twice.pdf"
pdf_ops.rotate_pdf(THREE_PAGES, once, 90)
pdf_ops.rotate_pdf(once, twice, 90)
assert rotations(once) == [90, 90, 90]
assert rotations(twice) == [180, 180, 180]
assert fingerprints(twice) == fingerprints(THREE_PAGES)
def test_keeps_document_info(self, tmp_path: Path) -> None:
"""
GIVEN:
- A PDF with document info
WHEN:
- It is rotated
THEN:
- The document info is still present in the output
"""
dst = tmp_path / "out.pdf"
pdf_ops.rotate_pdf(THREE_PAGES, dst, 90)
assert "/Creator" in docinfo_keys(dst)
class TestRemovePages:
@pytest.mark.parametrize(
("pages", "kept"),
[
pytest.param([2], [0, 2], id="single"),
pytest.param([3, 1], [1], id="unordered"),
pytest.param([2, 2], [0, 2], id="duplicates-remove-once"),
pytest.param([], [0, 1, 2], id="empty-keeps-everything"),
pytest.param([1, 2, 3], [], id="every-page"),
],
)
def test_removes_only_the_requested_pages(
self,
tmp_path: Path,
source_fingerprints: list[str],
pages: list[int],
kept: list[int],
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- Pages are removed, in any order and possibly repeated
THEN:
- Exactly the other pages remain, in their original order
"""
dst = tmp_path / "out.pdf"
pdf_ops.remove_pages(THREE_PAGES, dst, pages)
assert fingerprints(dst) == [source_fingerprints[i] for i in kept]
def test_keeps_document_info(self, tmp_path: Path) -> None:
"""
GIVEN:
- A PDF with document info
WHEN:
- A page is removed
THEN:
- The document info is still present in the output
"""
dst = tmp_path / "out.pdf"
pdf_ops.remove_pages(THREE_PAGES, dst, [1])
assert "/Creator" in docinfo_keys(dst)
@pytest.mark.parametrize(
"bad_page",
[
pytest.param(0, id="zero"),
pytest.param(-1, id="negative"),
],
)
def test_rejects_pages_below_one(self, tmp_path: Path, bad_page: int) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- Pages are removed and one of them is below 1
THEN:
- ValueError is raised
- No output file is written
"""
dst = tmp_path / "out.pdf"
with pytest.raises(ValueError, match="start at 1"):
pdf_ops.remove_pages(THREE_PAGES, dst, [1, bad_page])
assert not dst.exists()
class TestBuildPdfs:
def test_selects_and_orders_pages(
self,
tmp_path: Path,
source_fingerprints: list[str],
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- One output is built from pages 3 then 1
THEN:
- The output has those pages in that order
- Its path is returned
"""
dst = tmp_path / "out.pdf"
written = pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(3), PageSpec(1)], constant(dst))],
)
assert written == [dst]
assert fingerprints(dst) == [source_fingerprints[2], source_fingerprints[0]]
def test_rotates_only_the_requested_pages(self, tmp_path: Path) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- One output is built with a different rotation on each page
THEN:
- Each output page has exactly the rotation requested for it
"""
dst = tmp_path / "out.pdf"
pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(1), PageSpec(2, 90), PageSpec(3, 180)], constant(dst))],
)
assert rotations(dst) == [0, 90, 180]
def test_writes_one_file_per_output_in_order(self, tmp_path: Path) -> None:
"""
GIVEN:
- A twelve page PDF
WHEN:
- Two outputs are built from different page ranges
THEN:
- Two files are written and returned in output order
- Each holds exactly its own pages
"""
first = tmp_path / "first.pdf"
second = tmp_path / "second.pdf"
source = fingerprints(TWELVE_PAGES)
written = pdf_ops.build_pdfs(
TWELVE_PAGES,
[
([PageSpec(p) for p in (1, 2, 3)], constant(first)),
([PageSpec(p) for p in range(4, 13)], constant(second)),
],
)
assert written == [first, second]
assert fingerprints(first) == source[:3]
assert fingerprints(second) == source[3:]
def test_empty_page_list_writes_a_zero_page_file(self, tmp_path: Path) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- An output with no pages is built
THEN:
- A PDF with zero pages is written
"""
dst = tmp_path / "out.pdf"
pdf_ops.build_pdfs(THREE_PAGES, [([], constant(dst))])
assert fingerprints(dst) == []
def test_destination_is_not_requested_when_a_page_is_out_of_range(
self,
tmp_path: Path,
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- Several outputs are built
- A later output refers to a page past the end
THEN:
- IndexError is raised
- No destination was requested for any output, including earlier valid ones
"""
requested: list[Path] = []
def make_dst() -> Path:
requested.append(tmp_path / "out.pdf")
return requested[-1]
with pytest.raises(IndexError):
pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(1)], make_dst), ([PageSpec(99)], make_dst)],
)
assert requested == []
@pytest.mark.parametrize(
"bad_page",
[
pytest.param(0, id="zero"),
pytest.param(-1, id="negative"),
],
)
def test_rejects_pages_below_one_before_opening_anything(
self,
tmp_path: Path,
bad_page: int,
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- Several outputs are built and a later one has a page below 1
THEN:
- ValueError is raised
- No destination was requested for any output
"""
requested: list[Path] = []
def make_dst() -> Path:
requested.append(tmp_path / "out.pdf")
return requested[-1]
with pytest.raises(ValueError, match="start at 1"):
pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(1)], make_dst), ([PageSpec(bad_page)], make_dst)],
)
assert requested == []
class TestValidatePageOperations:
def test_returns_the_output_count(self) -> None:
"""
GIVEN:
- Operations that all target the default output
WHEN:
- They are validated
THEN:
- One output document is reported
"""
operations = [{"page": 1}, {"page": 2}, {"page": 3}]
assert pdf_ops.validate_page_operations(operations, single_output=True) == 1
def test_gap_in_output_indices_counts_up_to_the_highest(self) -> None:
"""
GIVEN:
- Operations that target outputs 0 and 2 but never 1
WHEN:
- They are validated
THEN:
- Three output documents are reported
"""
operations = [
{"page": 1, "doc": 0},
{"page": 2, "doc": 2},
{"page": 3, "doc": 0},
]
count = pdf_ops.validate_page_operations(operations, single_output=False)
assert count == 3
def test_empty_operations_are_rejected(self) -> None:
"""
GIVEN:
- No operations
WHEN:
- They are validated
THEN:
- ValueError is raised
"""
with pytest.raises(ValueError, match="index is out of bounds"):
pdf_ops.validate_page_operations([], single_output=False)
def test_multiple_outputs_rejected_when_single_output_required(self) -> None:
"""
GIVEN:
- Operations that target two outputs
WHEN:
- They are validated with a single output required
THEN:
- ValueError is raised
"""
operations = [{"page": 1, "doc": 0}, {"page": 2, "doc": 1}]
with pytest.raises(ValueError, match="Multiple output documents"):
pdf_ops.validate_page_operations(operations, single_output=True)
@pytest.mark.parametrize(
"doc",
[
pytest.param(-1, id="negative"),
pytest.param(2, id="equal-to-operation-count"),
pytest.param(2**32, id="huge"),
],
)
def test_output_index_out_of_bounds(self, doc: int) -> None:
"""
GIVEN:
- Two operations, one with an output index that is out of bounds
WHEN:
- They are validated
THEN:
- ValueError is raised
"""
operations = [{"page": 1, "doc": 0}, {"page": 2, "doc": doc}]
with pytest.raises(ValueError, match="index is out of bounds"):
pdf_ops.validate_page_operations(operations, single_output=False)
class TestDecrypt:
@pytest.mark.parametrize(
("path", "expected"),
[
pytest.param(ENCRYPTED, True, id="password-required"),
pytest.param(SIGNED, True, id="opens-without-password-but-encrypted"),
pytest.param(THREE_PAGES, False, id="not-encrypted"),
],
)
def test_needs_decrypt(self, path: Path, *, expected: bool) -> None:
"""
GIVEN:
- A PDF that is encrypted, or encrypted but openable, or plain
WHEN:
- needs_decrypt is asked about it
THEN:
- Only the unencrypted PDF reports False
"""
assert pdf_ops.needs_decrypt(path) is expected
def test_decrypt_writes_an_unencrypted_copy(self, tmp_path: Path) -> None:
"""
GIVEN:
- A password protected PDF
WHEN:
- It is decrypted with the correct password
THEN:
- The path from make_dst is returned
- The written copy no longer needs decrypting
"""
dst = tmp_path / "out.pdf"
result = pdf_ops.decrypt_pdf(ENCRYPTED, constant(dst), "test")
assert result == dst
assert pdf_ops.needs_decrypt(dst) is False
def test_wrong_password_raises_and_never_requests_a_destination(
self,
tmp_path: Path,
) -> None:
"""
GIVEN:
- A password protected PDF
WHEN:
- It is decrypted with the wrong password
THEN:
- PasswordError is raised
- No destination was requested
"""
requested: list[Path] = []
def make_dst() -> Path:
requested.append(tmp_path / "out.pdf")
return requested[-1]
with pytest.raises(pikepdf.PasswordError):
pdf_ops.decrypt_pdf(ENCRYPTED, make_dst, "wrong")
assert requested == []
class TestPdfMerger:
def test_pages_are_appended_in_the_order_added(
self,
tmp_path: Path,
source_fingerprints: list[str],
) -> None:
"""
GIVEN:
- A reordered PDF and the original three page PDF
WHEN:
- Both are added to a merger in that order and saved
THEN:
- The output holds all pages in the order they were added
"""
reordered = tmp_path / "reordered.pdf"
merged = tmp_path / "merged.pdf"
pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(3), PageSpec(1)], constant(reordered))],
)
with pdf_ops.PdfMerger() as merger:
merger.add(reordered)
merger.add(THREE_PAGES)
merger.save(merged)
assert fingerprints(merged) == [
source_fingerprints[2],
source_fingerprints[0],
*source_fingerprints,
]
def test_output_version_is_at_least_the_highest_source_version(
self,
tmp_path: Path,
) -> None:
"""
GIVEN:
- Two PDFs with different PDF versions
WHEN:
- Both are added to a merger and saved
THEN:
- The output version is at least the highest source version
"""
merged = tmp_path / "merged.pdf"
with pikepdf.open(TWELVE_PAGES) as pdf:
source_versions = [pdf.pdf_version]
with pikepdf.open(THREE_PAGES) as pdf:
source_versions.append(pdf.pdf_version)
with pdf_ops.PdfMerger() as merger:
merger.add(TWELVE_PAGES)
merger.add(THREE_PAGES)
merger.save(merged)
with pikepdf.open(merged) as pdf:
assert pdf.pdf_version >= max(source_versions)
+5 -23
View File
@@ -21,6 +21,7 @@ from documents.tests.helpers import dummy_preprocess
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
@pytest.mark.django_db
@@ -229,20 +230,12 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
"""
sample1 = self.dirs.scratch_dir / "sample.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf",
SIMPLE_DIGITAL_PDF,
sample1,
)
sample1_archive = self.dirs.archive_dir / "sample_archive.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf",
SIMPLE_DIGITAL_PDF,
sample1_archive,
)
doc = Document.objects.create(
@@ -270,11 +263,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
"""
sample1 = self.dirs.scratch_dir / "sample.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf",
SIMPLE_DIGITAL_PDF,
sample1,
)
doc = Document.objects.create(
@@ -290,14 +279,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
def _create_root_with_version(self) -> tuple[Document, Document]:
sample1 = self.dirs.scratch_dir / "sample.pdf"
shutil.copy(
Path(__file__).parent
/ "samples"
/ "documents"
/ "originals"
/ "0000001.pdf",
sample1,
)
shutil.copy(SIMPLE_DIGITAL_PDF, sample1)
root = DocumentFactory(content="root content", mime_type="application/pdf")
version = DocumentFactory(
content="my document",
+27 -27
View File
@@ -178,7 +178,7 @@ class TestWorkflows(
self.assertEqual(action.__str__(), "WorkflowAction 1")
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -290,7 +290,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -409,7 +409,7 @@ class TestWorkflows(
w2.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -478,7 +478,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -531,7 +531,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -608,7 +608,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -687,7 +687,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -765,7 +765,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -876,7 +876,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -928,7 +928,7 @@ class TestWorkflows(
generated = generate_unique_filename(doc)
destination = (settings.ORIGINALS_DIR / generated).resolve()
create_source_path_directory(destination)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", destination)
shutil.copy(self.SIMPLE_PDF, destination)
Document.objects.filter(pk=doc.pk).update(filename=generated.as_posix())
doc.refresh_from_db()
@@ -2023,7 +2023,7 @@ class TestWorkflows(
superuser = UserFactory(username="superuser", superuser=True)
self.client.force_authenticate(user=superuser)
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
tasks.consume_file(
@@ -3009,7 +3009,7 @@ class TestWorkflows(
generated = generate_unique_filename(doc)
destination = (settings.ORIGINALS_DIR / generated).resolve()
create_source_path_directory(destination)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", destination)
shutil.copy(self.SIMPLE_PDF, destination)
Document.objects.filter(pk=doc.pk).update(filename=generated.as_posix())
doc.refresh_from_db()
doc.tags.set([self.t1, self.t2])
@@ -3073,7 +3073,7 @@ class TestWorkflows(
generated = generate_unique_filename(doc)
destination = (settings.ORIGINALS_DIR / generated).resolve()
create_source_path_directory(destination)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", destination)
shutil.copy(self.SIMPLE_PDF, destination)
Document.objects.filter(pk=doc.pk).update(filename=generated.as_posix())
doc.refresh_from_db()
doc.tags.set([self.t1])
@@ -3222,7 +3222,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -3346,7 +3346,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -3496,7 +3496,7 @@ class TestWorkflows(
workflow.actions.set([assignment_action, email_action])
temp_working_copy = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "working-copy.pdf",
)
@@ -3596,7 +3596,7 @@ class TestWorkflows(
# move the file
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -3704,7 +3704,7 @@ class TestWorkflows(
generated = generate_unique_filename(doc)
destination = (settings.ORIGINALS_DIR / generated).resolve()
create_source_path_directory(destination)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", destination)
shutil.copy(self.SIMPLE_PDF, destination)
Document.objects.filter(pk=doc.pk).update(filename=generated.as_posix())
run_workflows(WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED, doc)
@@ -3915,7 +3915,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -4036,7 +4036,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -4387,7 +4387,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -4835,7 +4835,7 @@ class TestWorkflows(
w.save()
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
@@ -5027,7 +5027,7 @@ class TestWorkflows(
# Create a test file to be consumed
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
test_file_path = Path(test_file)
@@ -5091,7 +5091,7 @@ class TestWorkflows(
# Create a test file to be consumed
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple2.pdf",
)
test_file_path = Path(test_file)
@@ -5529,7 +5529,7 @@ class TestDateWorkflowLocalization(
)
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
tmp_path / "simple.pdf",
)
@@ -5596,7 +5596,7 @@ class TestRemoteOCRWorkflowAction(DirectoriesMixin, SampleDirMixin, APITestCase)
self._make_workflow(WorkflowTrigger.WorkflowTriggerType.CONSUMPTION)
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
overrides = DocumentMetadataOverrides()
@@ -5814,7 +5814,7 @@ class TestApplyAISuggestionsWorkflowAction(
)
test_file = shutil.copy(
self.SAMPLE_DIR / "simple.pdf",
self.SIMPLE_PDF,
self.dirs.scratch_dir / "simple.pdf",
)
+3
View File
@@ -11,6 +11,7 @@ from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from paperless_testing.fakes.progress import FakeProgressManager
from paperless_testing.samples import SIMPLE_DIGITAL_PDF
class ConsumeTaskMixin:
@@ -53,6 +54,8 @@ class SampleDirMixin:
BARCODE_SAMPLE_DIR = SAMPLE_DIR / "barcodes"
SIMPLE_PDF = SIMPLE_DIGITAL_PDF
class GetConsumerMixin:
@contextmanager
+10 -10
View File
@@ -2,7 +2,7 @@ msgid ""
msgstr ""
"Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2026-10-22 19:29+0000\n"
"POT-Creation-Date: 2026-10-22 12:54+0000\n"
"PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n"
"Language-Team: English\n"
@@ -1653,7 +1653,7 @@ msgid "workflow runs"
msgstr ""
#: documents/serialisers.py:515 documents/serialisers.py:872
#: documents/serialisers.py:2904 documents/views.py:364 documents/views.py:2787
#: documents/serialisers.py:2902 documents/views.py:364 documents/views.py:2787
#: paperless_mail/serialisers.py:156
msgid "Insufficient permissions."
msgstr ""
@@ -1662,39 +1662,39 @@ msgstr ""
msgid "Invalid color."
msgstr ""
#: documents/serialisers.py:2371
#: documents/serialisers.py:2369
#, python-format
msgid "File type %(type)s not supported"
msgstr ""
#: documents/serialisers.py:2415
#: documents/serialisers.py:2413
#, python-format
msgid "Custom field id must be an integer: %(id)s"
msgstr ""
#: documents/serialisers.py:2422
#: documents/serialisers.py:2420
#, python-format
msgid "Custom field with id %(id)s does not exist"
msgstr ""
#: documents/serialisers.py:2439 documents/serialisers.py:2449
#: documents/serialisers.py:2437 documents/serialisers.py:2447
msgid ""
"Custom fields must be a list of integers or an object mapping ids to values."
msgstr ""
#: documents/serialisers.py:2444
#: documents/serialisers.py:2442
msgid "Some custom fields don't exist or were specified twice."
msgstr ""
#: documents/serialisers.py:2591
#: documents/serialisers.py:2589
msgid "Invalid variable detected."
msgstr ""
#: documents/serialisers.py:2960
#: documents/serialisers.py:2958
msgid "Duplicate document identifiers are not allowed."
msgstr ""
#: documents/serialisers.py:2990 documents/views.py:4837
#: documents/serialisers.py:2988 documents/views.py:4837
#, python-format
msgid "Documents not found: %(ids)s"
msgstr ""
+12 -24
View File
@@ -432,30 +432,6 @@ def tesseract_samples_dir(parser_samples_dir: Path) -> Path:
return parser_samples_dir / "tesseract"
@pytest.fixture(scope="session")
def multi_page_images_pdf_file(tesseract_samples_dir: Path) -> Path:
"""Path to a multi-page PDF with images.
Returns
-------
Path
Absolute path to ``tesseract/multi-page-images.pdf``.
"""
return tesseract_samples_dir / "multi-page-images.pdf"
@pytest.fixture(scope="session")
def simple_digital_pdf_file(tesseract_samples_dir: Path) -> Path:
"""Path to a simple digital PDF sample file.
Returns
-------
Path
Absolute path to ``tesseract/simple-digital.pdf``.
"""
return tesseract_samples_dir / "simple-digital.pdf"
@pytest.fixture(scope="session")
def simple_no_dpi_png_file(tesseract_samples_dir: Path) -> Path:
"""Path to a simple PNG without DPI information.
@@ -480,6 +456,18 @@ def simple_png_file(tesseract_samples_dir: Path) -> Path:
return tesseract_samples_dir / "simple.png"
@pytest.fixture(scope="session")
def rotated_pdf_file(tesseract_samples_dir: Path) -> Path:
"""Path to a landscape (rotated) PDF sample file.
Returns
-------
Path
Absolute path to ``tesseract/rotated.pdf``.
"""
return tesseract_samples_dir / "rotated.pdf"
# ------------------------------------------------------------------
# Tesseract parser instance and settings helpers
# ------------------------------------------------------------------
@@ -161,11 +161,11 @@ class TestGetPageCount:
def test_single_page_pdf(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
simple_digital_pdf_file: Path,
) -> None:
assert (
tesseract_parser.get_page_count(
tesseract_samples_dir / "simple-digital.pdf",
simple_digital_pdf_file,
"application/pdf",
)
== 1
@@ -188,7 +188,7 @@ class TestGetPageCount:
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
simple_digital_pdf_file: Path,
caplog,
) -> None:
"""
@@ -204,7 +204,7 @@ class TestGetPageCount:
with caplog.at_level(logging.WARNING):
page_count = tesseract_parser.get_page_count(
tesseract_samples_dir / "simple-digital.pdf",
simple_digital_pdf_file,
"application/pdf",
)
assert page_count is None
@@ -273,26 +273,26 @@ class TestGetThumbnail:
def test_thumbnail_is_file(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
simple_digital_pdf_file: Path,
) -> None:
thumb = tesseract_parser.get_thumbnail(
tesseract_samples_dir / "simple-digital.pdf",
simple_digital_pdf_file,
"application/pdf",
)
assert thumb.is_file()
@pytest.mark.parametrize(
("filename", "expected_height"),
("sample_fixture", "expected_height"),
[
pytest.param("simple-digital.pdf", 647, id="portrait-letter"),
pytest.param("rotated.pdf", 386, id="landscape"),
pytest.param("simple_digital_pdf_file", 647, id="portrait-letter"),
pytest.param("rotated_pdf_file", 386, id="landscape"),
],
)
def test_thumbnail_is_correct_format_and_size(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
filename: str,
request: pytest.FixtureRequest,
sample_fixture: str,
expected_height: int,
) -> None:
"""
@@ -304,7 +304,7 @@ class TestGetThumbnail:
- A 500px wide WebP keeping the page's aspect ratio
"""
thumb = tesseract_parser.get_thumbnail(
tesseract_samples_dir / filename,
request.getfixturevalue(sample_fixture),
"application/pdf",
)
with Image.open(thumb) as im:
@@ -316,7 +316,7 @@ class TestGetThumbnail:
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
simple_digital_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -326,7 +326,7 @@ class TestGetThumbnail:
THEN:
- The PDF is repaired with qpdf and a real thumbnail is rendered
"""
original = tesseract_samples_dir / "simple-digital.pdf"
original = simple_digital_pdf_file
def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None:
if in_path == original:
@@ -368,11 +368,11 @@ class TestExtractText:
def test_extract_text_from_digital_pdf(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
simple_digital_pdf_file: Path,
) -> None:
text = tesseract_parser.extract_text(
None,
tesseract_samples_dir / "simple-digital.pdf",
simple_digital_pdf_file,
)
assert text is not None
assert "This is a test document." in text.strip()
@@ -387,7 +387,7 @@ class TestParsePdf:
def test_simple_digital_creates_archive(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
multi_page_digital_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -401,7 +401,7 @@ class TestParsePdf:
- Text is extracted
"""
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-digital.pdf",
multi_page_digital_pdf_file,
"application/pdf",
)
assert tesseract_parser.archive_path is not None
@@ -416,10 +416,10 @@ class TestParsePdf:
def test_with_form_default(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
with_form_pdf_file: Path,
) -> None:
tesseract_parser.parse(
tesseract_samples_dir / "with-form.pdf",
with_form_pdf_file,
"application/pdf",
)
assert tesseract_parser.archive_path is not None
@@ -432,11 +432,11 @@ class TestParsePdf:
def test_with_form_redo_no_archive_when_not_requested(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
with_form_pdf_file: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.REDO
tesseract_parser.parse(
tesseract_samples_dir / "with-form.pdf",
with_form_pdf_file,
"application/pdf",
produce_archive=False,
)
@@ -449,11 +449,11 @@ class TestParsePdf:
def test_with_form_force(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
with_form_pdf_file: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.FORCE
tesseract_parser.parse(
tesseract_samples_dir / "with-form.pdf",
with_form_pdf_file,
"application/pdf",
)
assert_ordered_substrings(
@@ -494,7 +494,7 @@ class TestParsePdf:
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
simple_digital_pdf_file: Path,
) -> None:
mocker.patch(
"ocrmypdf.ocr",
@@ -502,7 +502,7 @@ class TestParsePdf:
)
with pytest.raises(ParseError):
tesseract_parser.parse(
tesseract_samples_dir / "simple-digital.pdf",
simple_digital_pdf_file,
"application/pdf",
)
@@ -578,10 +578,10 @@ class TestParseMultiPage:
def test_multi_page_digital(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
multi_page_digital_pdf_file: Path,
) -> None:
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-digital.pdf",
multi_page_digital_pdf_file,
"application/pdf",
)
assert tesseract_parser.archive_path is not None
@@ -605,12 +605,12 @@ class TestParseMultiPage:
self,
mode: str,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
multi_page_digital_pdf_file: Path,
) -> None:
tesseract_parser.settings.pages = 2
tesseract_parser.settings.mode = mode
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-digital.pdf",
multi_page_digital_pdf_file,
"application/pdf",
)
assert tesseract_parser.archive_path is not None
@@ -624,11 +624,11 @@ class TestParseMultiPage:
def test_multi_page_images_skip(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
multi_page_images_pdf_file: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
multi_page_images_pdf_file,
"application/pdf",
)
assert tesseract_parser.archive_path is not None
@@ -642,7 +642,7 @@ class TestParseMultiPage:
def test_multi_page_images_redo_pages_2(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
multi_page_images_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -657,7 +657,7 @@ class TestParseMultiPage:
tesseract_parser.settings.pages = 2
tesseract_parser.settings.mode = ModeChoices.REDO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
multi_page_images_pdf_file,
"application/pdf",
)
assert tesseract_parser.archive_path is not None
@@ -670,7 +670,7 @@ class TestParseMultiPage:
def test_multi_page_images_force_page_1(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
multi_page_images_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -685,7 +685,7 @@ class TestParseMultiPage:
tesseract_parser.settings.pages = 1
tesseract_parser.settings.mode = ModeChoices.FORCE
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
multi_page_images_pdf_file,
"application/pdf",
)
assert tesseract_parser.archive_path is not None
@@ -781,7 +781,7 @@ class TestSkipArchive:
def test_skip_noarchive_with_text_layer(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
multi_page_digital_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -795,7 +795,7 @@ class TestSkipArchive:
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-digital.pdf",
multi_page_digital_pdf_file,
"application/pdf",
produce_archive=False,
)
@@ -810,7 +810,7 @@ class TestSkipArchive:
def test_skip_noarchive_image_only_creates_archive(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
multi_page_images_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -823,7 +823,7 @@ class TestSkipArchive:
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
multi_page_images_pdf_file,
"application/pdf",
)
assert tesseract_parser.archive_path is not None
@@ -835,29 +835,29 @@ class TestSkipArchive:
)
@pytest.mark.parametrize(
("produce_archive", "filename", "expect_archive"),
("produce_archive", "sample_fixture", "expect_archive"),
[
pytest.param(
True,
"multi-page-digital.pdf",
"multi_page_digital_pdf_file",
True,
id="produce-archive-with-text",
),
pytest.param(
True,
"multi-page-images.pdf",
"multi_page_images_pdf_file",
True,
id="produce-archive-no-text",
),
pytest.param(
False,
"multi-page-digital.pdf",
"multi_page_digital_pdf_file",
False,
id="no-archive-with-text-layer",
),
pytest.param(
False,
"multi-page-images.pdf",
"multi_page_images_pdf_file",
False,
id="no-archive-no-text-layer",
),
@@ -866,10 +866,10 @@ class TestSkipArchive:
def test_produce_archive_flag(
self,
produce_archive: bool, # noqa: FBT001
filename: str,
sample_fixture: str,
expect_archive: bool, # noqa: FBT001
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
request: pytest.FixtureRequest,
) -> None:
"""
GIVEN:
@@ -882,8 +882,9 @@ class TestSkipArchive:
- Text is always extracted
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
sample = request.getfixturevalue(sample_fixture)
tesseract_parser.parse(
tesseract_samples_dir / filename,
sample,
"application/pdf",
produce_archive=produce_archive,
)
@@ -900,7 +901,7 @@ class TestSkipArchive:
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
simple_digital_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -916,7 +917,7 @@ class TestSkipArchive:
tesseract_parser.settings.mode = ModeChoices.AUTO
mock_ocr = mocker.patch("ocrmypdf.ocr")
tesseract_parser.parse(
tesseract_samples_dir / "simple-digital.pdf",
simple_digital_pdf_file,
"application/pdf",
produce_archive=False,
)
@@ -955,7 +956,7 @@ class TestSkipArchive:
def test_tagged_pdf_produces_pdfa_archive_without_ocr(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
simple_digital_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -970,7 +971,7 @@ class TestSkipArchive:
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "simple-digital.pdf",
simple_digital_pdf_file,
"application/pdf",
produce_archive=True,
)
@@ -1,19 +0,0 @@
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
</head>
<body>
<p>Some Text</p>
<p>
<img src="cid:part1.pNdUSz0s.D3NqVtPg@example.de" alt="Has to be rewritten to work..">
<img src="http://localhost:8080/assets/logo_full_white.svg" alt="This image should not be shown.">
</p>
<p>and an embedded image.<br>
</p>
<p id="changeme">Paragraph unchanged.</p>
<scRipt>
document.getElementById("changeme").innerHTML = "Paragraph changed via Java Script.";
</script>
</body>
</html>
Binary file not shown.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 32 KiB

+11 -11
View File
@@ -18,8 +18,6 @@ from paperless.parsers.utils import read_file_handle_unicode_errors
if TYPE_CHECKING:
from pytest_mock import MockerFixture
SAMPLES = Path(__file__).parent / "samples" / "tesseract"
class TestReadFileHandleUnicodeErrors:
def test_plain_utf8(self, tmp_path: Path) -> None:
@@ -57,11 +55,11 @@ class TestReadFileHandleUnicodeErrors:
class TestIsTaggedPdf:
def test_tagged_pdf_returns_true(self) -> None:
assert is_tagged_pdf(SAMPLES / "simple-digital.pdf") is True
def test_tagged_pdf_returns_true(self, simple_digital_pdf_file: Path) -> None:
assert is_tagged_pdf(simple_digital_pdf_file) is True
def test_untagged_pdf_returns_false(self) -> None:
assert is_tagged_pdf(SAMPLES / "multi-page-images.pdf") is False
def test_untagged_pdf_returns_false(self, multi_page_images_pdf_file: Path) -> None:
assert is_tagged_pdf(multi_page_images_pdf_file) is False
def test_nonexistent_path_returns_false(self) -> None:
assert is_tagged_pdf(Path("/nonexistent/file.pdf")) is False
@@ -95,7 +93,7 @@ class TestGetPdfFirstPageSizePoints:
pdf.save(path)
return path
def test_letter_sample(self) -> None:
def test_letter_sample(self, simple_digital_pdf_file: Path) -> None:
"""
GIVEN:
- A US Letter sample PDF with no CropBox and no rotation
@@ -104,7 +102,7 @@ class TestGetPdfFirstPageSizePoints:
THEN:
- The MediaBox size in points is returned
"""
assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") == (
assert get_pdf_first_page_size_points(simple_digital_pdf_file) == (
612.0,
792.0,
)
@@ -199,6 +197,7 @@ class TestGetPdfFirstPageSizePoints:
def test_pdfinfo_not_installed_returns_none(
self,
mocker: MockerFixture,
simple_digital_pdf_file: Path,
) -> None:
"""
GIVEN:
@@ -209,7 +208,7 @@ class TestGetPdfFirstPageSizePoints:
- None is returned and nothing is raised
"""
mocker.patch("subprocess.run", side_effect=FileNotFoundError("pdfinfo"))
assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") is None
assert get_pdf_first_page_size_points(simple_digital_pdf_file) is None
def test_degenerate_box_returns_none(self, tmp_path: Path) -> None:
"""
@@ -251,7 +250,7 @@ class TestGetPdfFirstPageSizePoints:
bad.write_bytes(b"not a pdf")
assert get_pdf_first_page_size_points(bad) is None
def test_encrypted_pdf_returns_none(self) -> None:
def test_encrypted_pdf_returns_none(self, parser_samples_dir: Path) -> None:
"""
GIVEN:
- A password protected PDF
@@ -260,7 +259,8 @@ class TestGetPdfFirstPageSizePoints:
THEN:
- None is returned and nothing is raised
"""
assert get_pdf_first_page_size_points(SAMPLES / "encrypted.pdf") is None
encrypted = parser_samples_dir / "tesseract" / "encrypted.pdf"
assert get_pdf_first_page_size_points(encrypted) is None
class TestPostProcessText:
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import Final
__version__: Final[tuple[int, int, int]] = (3, 3, 4)
__version__: Final[tuple[int, int, int]] = (3, 3, 3)
# Version string like X.Y.Z
__full_version_str__: Final[str] = ".".join(map(str, __version__))
# Version string like X.Y
+48
View File
@@ -0,0 +1,48 @@
"""Test sample files shared across tests.
A sample used by a single app stays in that app's ``tests/samples`` tree.
Samples needed by more than one app, or stored under more than one name, live
here exactly once.
"""
import shutil
from pathlib import Path
SHARED_SAMPLES_DIR = (Path(__file__).parent / "sample_files").resolve()
SIMPLE_DIGITAL_PDF = SHARED_SAMPLES_DIR / "simple-digital.pdf"
MULTI_PAGE_DIGITAL_PDF = SHARED_SAMPLES_DIR / "multi-page-digital.pdf"
WITH_FORM_PDF = SHARED_SAMPLES_DIR / "with-form.pdf"
MULTI_PAGE_IMAGES_PDF = SHARED_SAMPLES_DIR / "multi-page-images.pdf"
THUMBNAIL_WEBP = SHARED_SAMPLES_DIR / "thumbnail.webp"
# The fake media tree under documents/tests/samples/documents keeps only the
# files unique to it. The files that duplicate a shared sample are laid back
# down under their historical opaque names (DB rows and exporter manifests
# refer to them) by install_document_samples().
_DOCUMENT_SAMPLES_DIR = (
Path(__file__).parent.parent / "documents" / "tests" / "samples" / "documents"
).resolve()
_SHARED_AS_MEDIA = (
("originals/0000001.pdf", SIMPLE_DIGITAL_PDF),
("originals/0000002.pdf", MULTI_PAGE_DIGITAL_PDF),
("originals/0000003.pdf", WITH_FORM_PDF),
("archive/0000001.pdf", MULTI_PAGE_IMAGES_PDF),
("thumbnails/0000001.webp", THUMBNAIL_WEBP),
("thumbnails/0000002.webp", THUMBNAIL_WEBP),
("thumbnails/0000003.webp", THUMBNAIL_WEBP),
("thumbnails/0000004.webp", THUMBNAIL_WEBP),
)
def install_document_samples(dest: Path) -> None:
"""Populate ``dest`` with the full fake media tree.
Replaces ``shutil.copytree(<samples>/documents, dest)`` in tests.
"""
shutil.copytree(_DOCUMENT_SAMPLES_DIR, dest, dirs_exist_ok=True)
for relpath, source in _SHARED_AS_MEDIA:
target = dest / relpath
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, target)
Generated
+8 -8
View File
@@ -1,5 +1,5 @@
version = 1
revision = 5
revision = 3
requires-python = ">=3.11"
resolution-markers = [
"python_full_version >= '3.15' and sys_platform == 'darwin'",
@@ -9,7 +9,7 @@ resolution-markers = [
"python_full_version >= '3.12' and python_full_version < '3.15' and sys_platform == 'darwin'",
"python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'",
"python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'",
"(python_full_version == '3.12.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.14.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform == 'linux')",
"(python_full_version >= '3.12' and python_full_version < '3.15' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux')",
"python_full_version < '3.12' and sys_platform == 'darwin'",
"python_full_version < '3.12' and sys_platform == 'linux'",
]
@@ -3040,7 +3040,7 @@ wheels = [
[[package]]
name = "paperless-ngx"
version = "3.3.4"
version = "3.3.3"
source = { virtual = "." }
dependencies = [
{ name = "azure-ai-documentintelligence" },
@@ -3114,7 +3114,7 @@ mariadb = [
]
postgres = [
{ name = "psycopg", extra = ["c", "pool"] },
{ name = "psycopg-c", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or python_full_version == '3.13.*' or python_full_version >= '3.15' or (platform_machine != 'aarch64' and platform_machine != 'x86_64') or sys_platform != 'linux'" },
{ name = "psycopg-c", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and platform_machine == 'aarch64') or (python_full_version == '3.13.*' and platform_machine == 'aarch64') or (python_full_version >= '3.15' and platform_machine == 'aarch64') or (python_full_version < '3.12' and platform_machine == 'x86_64') or (python_full_version == '3.13.*' and platform_machine == 'x86_64') or (python_full_version >= '3.15' and platform_machine == 'x86_64') or (platform_machine != 'aarch64' and platform_machine != 'x86_64') or sys_platform != 'linux'" },
{ name = "psycopg-c", version = "3.3.4", source = { url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp312-cp312-linux_aarch64.whl" }, marker = "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "psycopg-c", version = "3.3.4", source = { url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp312-cp312-linux_x86_64.whl" }, marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "psycopg-c", version = "3.3.4", source = { url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp314-cp314-linux_aarch64.whl" }, marker = "python_full_version == '3.14.*' and platform_machine == 'aarch64' and sys_platform == 'linux'" },
@@ -3243,7 +3243,7 @@ requires-dist = [
{ name = "psycopg-c", marker = "python_full_version == '3.14.*' and platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'postgres'", url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp314-cp314-linux_aarch64.whl" },
{ name = "psycopg-c", marker = "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'postgres'", url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp312-cp312-linux_x86_64.whl" },
{ name = "psycopg-c", marker = "python_full_version == '3.14.*' and platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'postgres'", url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp314-cp314-linux_x86_64.whl" },
{ name = "psycopg-c", marker = "(python_full_version < '3.12' and extra == 'postgres') or (python_full_version == '3.13.*' and extra == 'postgres') or (python_full_version >= '3.15' and extra == 'postgres') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'postgres') or (sys_platform != 'linux' and extra == 'postgres')", specifier = "==3.3.4" },
{ name = "psycopg-c", marker = "(python_full_version < '3.12' and platform_machine == 'aarch64' and extra == 'postgres') or (python_full_version == '3.13.*' and platform_machine == 'aarch64' and extra == 'postgres') or (python_full_version >= '3.15' and platform_machine == 'aarch64' and extra == 'postgres') or (python_full_version < '3.12' and platform_machine == 'x86_64' and extra == 'postgres') or (python_full_version == '3.13.*' and platform_machine == 'x86_64' and extra == 'postgres') or (python_full_version >= '3.15' and platform_machine == 'x86_64' and extra == 'postgres') or (platform_machine != 'aarch64' and platform_machine != 'x86_64' and extra == 'postgres') or (sys_platform != 'linux' and extra == 'postgres')", specifier = "==3.3.4" },
{ name = "psycopg-pool", marker = "extra == 'postgres'", specifier = "==3.3.3" },
{ name = "python-dateutil", specifier = "~=2.9.0" },
{ name = "python-dotenv", specifier = "~=1.2.1" },
@@ -3700,7 +3700,7 @@ wheels = [
[package.optional-dependencies]
c = [
{ name = "psycopg-c", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and implementation_name != 'pypy') or (python_full_version == '3.13.*' and implementation_name != 'pypy') or (python_full_version >= '3.15' and implementation_name != 'pypy') or (implementation_name != 'pypy' and platform_machine != 'aarch64' and platform_machine != 'x86_64') or (implementation_name != 'pypy' and sys_platform != 'linux')" },
{ name = "psycopg-c", version = "3.3.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and implementation_name != 'pypy' and platform_machine == 'aarch64') or (python_full_version == '3.13.*' and implementation_name != 'pypy' and platform_machine == 'aarch64') or (python_full_version >= '3.15' and implementation_name != 'pypy' and platform_machine == 'aarch64') or (python_full_version < '3.12' and implementation_name != 'pypy' and platform_machine == 'x86_64') or (python_full_version == '3.13.*' and implementation_name != 'pypy' and platform_machine == 'x86_64') or (python_full_version >= '3.15' and implementation_name != 'pypy' and platform_machine == 'x86_64') or (implementation_name != 'pypy' and platform_machine != 'aarch64' and platform_machine != 'x86_64') or (implementation_name != 'pypy' and sys_platform != 'linux')" },
{ name = "psycopg-c", version = "3.3.4", source = { url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp312-cp312-linux_aarch64.whl" }, marker = "python_full_version == '3.12.*' and implementation_name != 'pypy' and platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "psycopg-c", version = "3.3.4", source = { url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp312-cp312-linux_x86_64.whl" }, marker = "python_full_version == '3.12.*' and implementation_name != 'pypy' and platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "psycopg-c", version = "3.3.4", source = { url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp314-cp314-linux_aarch64.whl" }, marker = "python_full_version == '3.14.*' and implementation_name != 'pypy' and platform_machine == 'aarch64' and sys_platform == 'linux'" },
@@ -3718,7 +3718,7 @@ resolution-markers = [
"python_full_version >= '3.15' and sys_platform == 'darwin'",
"python_full_version >= '3.15' and sys_platform == 'linux'",
"python_full_version >= '3.12' and python_full_version < '3.15' and sys_platform == 'darwin'",
"(python_full_version == '3.12.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.14.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform == 'linux')",
"(python_full_version >= '3.12' and python_full_version < '3.15' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux')",
"python_full_version < '3.12' and sys_platform == 'darwin'",
"python_full_version < '3.12' and sys_platform == 'linux'",
]
@@ -5260,7 +5260,7 @@ resolution-markers = [
"python_full_version == '3.14.*' and platform_machine == 'aarch64' and sys_platform == 'linux'",
"python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'",
"python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'",
"(python_full_version == '3.12.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.14.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform == 'linux')",
"(python_full_version >= '3.12' and python_full_version < '3.15' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_machine == 'x86_64' and sys_platform == 'linux')",
"python_full_version < '3.12' and sys_platform == 'linux'",
]
dependencies = [