Compare commits

...
Author SHA1 Message Date
stumpylog 4d98be3b47 This is redundant now 2026-10-13 10:04:35 -07:00
stumpylog 6322a0bad3 Adds the given/when/then and some additional cleanups of pytest style 2026-10-13 10:04:35 -07:00
stumpylog 70eab32503 Validates pages ahead of time in build_pdfs rather than during the building 2026-10-13 10:04:35 -07:00
stumpylog 4f91865826 Now that I'm sure it works, re-work these tests to mock pdf_ops, rather than pikepdf 2026-10-13 10:04:35 -07:00
stumpylog a5c3e74b18 Drops tests we don't relly need + adds the typing annotation 2026-10-13 10:04:35 -07:00
Trenton H c06827acdc Refactor: add pure pdf_ops module with real-PDF tests 2026-10-13 10:04:35 -07:00
Trenton H 6e717817c6 Fix: make exporter derived file names unique when originals differ only by extension (#14429) 2026-10-12 15:14:08 -07:00
shamoon 3e0fa73771 Chore: adjust issue auto-triage 2026-10-12 14:16:45 -07:00
github-actions[bot]andshamoon 2b00cf6a95 Documentation: Add v3.3.1 changelog (#14427)
* Changelog v3.3.1 - GHA

* Update changelog for paperless-ngx 3.3.1

Removed MARIADB_HOST documentation from MariaDB compose files.

---------

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
2026-10-12 13:22:46 -07:00
13 changed files with 1122 additions and 345 deletions

No files matched your search

+11 -1
View File
@@ -31,9 +31,19 @@ body:
value: | value: |
#### 🤖 Use of AI tools #### 🤖 Use of AI tools
If this report was generated, in whole or in part, by an AI tool or agent you **must** clearly say so in the description below. Failing to clearly disclose this is a violation of our [Code of Conduct](https://github.com/paperless-ngx/paperless-ngx/blob/main/CODE_OF_CONDUCT.md). If this report was generated, in whole or in part, by an AI tool or agent you **must** say so using the field below. Failing to clearly disclose this is a violation of our [Code of Conduct](https://github.com/paperless-ngx/paperless-ngx/blob/main/CODE_OF_CONDUCT.md).
Reports written by an AI agent must describe the _observed behavior only_: what happened, how to reproduce it, logs etc. Do **not** include code analysis, speculation about the root cause, or suggest code fixes. Reports written by an AI agent must describe the _observed behavior only_: what happened, how to reproduce it, logs etc. Do **not** include code analysis, speculation about the root cause, or suggest code fixes.
- type: dropdown
id: ai-usage
attributes:
label: AI usage
description: Was any part of this report written by an AI tool or agent?
options:
- No AI tools were used
- Yes, AI tools helped write this report
validations:
required: true
- type: textarea - type: textarea
id: description id: description
attributes: attributes:
+20 -4
View File
@@ -43,6 +43,12 @@ jobs:
const haystack = `${issue.title}\n${issue.body ?? ''}`; const haystack = `${issue.title}\n${issue.body ?? ''}`;
const honeypot = haystack.toUpperCase().includes('ASLOP-PR-VERIFY'); const honeypot = haystack.toUpperCase().includes('ASLOP-PR-VERIFY');
// The form's required "AI usage" dropdown renders as a `### AI usage` section. An agent
// that follows the template trips the honeypot even when it discloses, so only treat
// the honeypot as undisclosed AI use when the reporter did not answer "Yes" here.
const aiUsage = (issue.body ?? '').match(/^###\s+AI usage\s*\n+\s*(.+?)\s*$/m)?.[1] ?? '';
const disclosedAI = aiUsage.toLowerCase().startsWith('yes');
// Issues opened through the form always get the template's default labels. GitHub // Issues opened through the form always get the template's default labels. GitHub
// applies them a second or two *after* creation, so re-read them instead of trusting // applies them a second or two *after* creation, so re-read them instead of trusting
// the webhook payload, and retry before concluding that there are none. // the webhook payload, and retry before concluding that there are none.
@@ -55,19 +61,29 @@ jobs:
if (labels.some((label) => templateLabels.includes(label))) break; if (labels.some((label) => templateLabels.includes(label))) break;
} }
const bypassedTemplate = !labels.some((label) => templateLabels.includes(label)); const bypassedTemplate = !labels.some((label) => templateLabels.includes(label));
core.info(`Labels: [${labels.join(', ')}], honeypot: ${honeypot}, bypassed template: ${bypassedTemplate}`); core.info(
`Labels: [${labels.join(', ')}], honeypot: ${honeypot}, AI usage: "${aiUsage}", ` +
`bypassed template: ${bypassedTemplate}`
);
if (!honeypot && !bypassedTemplate) { if (!bypassedTemplate && (disclosedAI || !honeypot)) {
if (disclosedAI) {
core.info('AI use disclosed, labeling only');
await github.rest.issues.addLabels({ ...common, labels: ['ai'] });
} else {
core.info('No slop signals found'); core.info('No slop signals found');
}
return; return;
} }
// The honeypot is only ever tripped deliberately, so that message can name the cause. // Reaching here with the honeypot means AI usage was not disclosed in the form, so that
// message can name the cause.
// A missing template label only tells us the issue did not come from the form, which // A missing template label only tells us the issue did not come from the form, which
// is reason enough to close it, but not proof of how it was written. // is reason enough to close it, but not proof of how it was written.
const body = honeypot const body = honeypot
? 'This issue was automatically closed because it contains a marker that is only visible to ' + ? 'This issue was automatically closed because it contains a marker that is only visible to ' +
'automated tools, which indicates it was generated by an AI agent without being disclosed as such.\n\n' + 'automated tools, which indicates it was generated by an AI agent, but the "AI usage" field does not ' +
'disclose this.\n\n' +
`Please see our [contributing guidelines](${contributing}) and [Code of Conduct](${codeOfConduct}). ` + `Please see our [contributing guidelines](${contributing}) and [Code of Conduct](${codeOfConduct}). ` +
'You are welcome to open a new issue that describes the problem you observed in your own words.' 'You are welcome to open a new issue that describes the problem you observed in your own words.'
: 'This issue was automatically closed because it was not opened using our bug report form. ' + : 'This issue was automatically closed because it was not opened using our bug report form. ' +
-1
View File
@@ -38,7 +38,6 @@ src/documents/bulk_edit.py:0: error: Incompatible types in assignment (expressio
src/documents/bulk_edit.py:0: error: Invalid index type "str" for "dict[FieldDataType, str]"; expected type "FieldDataType" [index] src/documents/bulk_edit.py:0: error: Invalid index type "str" for "dict[FieldDataType, str]"; expected type "FieldDataType" [index]
src/documents/bulk_edit.py:0: error: List comprehension has incompatible type List[tuple[int, Any]]; expected List[int] [misc] src/documents/bulk_edit.py:0: error: List comprehension has incompatible type List[tuple[int, Any]]; expected List[int] [misc]
src/documents/bulk_edit.py:0: error: List comprehension has incompatible type List[tuple[int, None]]; expected List[int] [misc] src/documents/bulk_edit.py:0: error: List comprehension has incompatible type List[tuple[int, None]]; expected List[int] [misc]
src/documents/bulk_edit.py:0: error: Missing named argument "p" for "remove" of "PageList" [call-arg]
src/documents/bulk_edit.py:0: error: Missing type arguments for generic type "dict" [type-arg] src/documents/bulk_edit.py:0: error: Missing type arguments for generic type "dict" [type-arg]
src/documents/bulk_edit.py:0: error: Missing type arguments for generic type "dict" [type-arg] src/documents/bulk_edit.py:0: error: Missing type arguments for generic type "dict" [type-arg]
src/documents/bulk_edit.py:0: error: Need type annotation for "to_create" (hint: "to_create: list[<type>] = ...") [var-annotated] src/documents/bulk_edit.py:0: error: Need type annotation for "to_create" (hint: "to_create: list[<type>] = ...") [var-annotated]
-7
View File
@@ -91,13 +91,6 @@
"concise_description": "Argument `list[int]` is not assignable to parameter `args` with type `tuple[Any, ...] | None` in function `celery.app.task.Task.apply_async`", "concise_description": "Argument `list[int]` is not assignable to parameter `args` with type `tuple[Any, ...] | None` in function `celery.app.task.Task.apply_async`",
"severity": "error" "severity": "error"
}, },
{
"column": 33,
"path": "src/documents/bulk_edit.py",
"name": "missing-argument",
"concise_description": "Missing argument `p` in function `pikepdf._core.PageList.remove`",
"severity": "error"
},
{ {
"column": 25, "column": 25,
"path": "src/documents/caching.py", "path": "src/documents/caching.py",
+49
View File
@@ -1,5 +1,54 @@
# Changelog # Changelog
## paperless-ngx 3.3.1
### Bug Fixes
- Fix: use a root document's effective content for the LLM index and pr… [@stumpylog](https://github.com/stumpylog) ([#14423](https://github.com/paperless-ngx/paperless-ngx/pull/14423))
- Fix: index document versions as their root document [@stumpylog](https://github.com/stumpylog) ([#14422](https://github.com/paperless-ngx/paperless-ngx/pull/14422))
- Fix: authorize document versions by their root in the single-object permission check [@stumpylog](https://github.com/stumpylog) ([#14421](https://github.com/paperless-ngx/paperless-ngx/pull/14421))
- Fix: Provide better error messages to users for invalid JSON [@stumpylog](https://github.com/stumpylog) ([#14419](https://github.com/paperless-ngx/paperless-ngx/pull/14419))
- Fix: authorize document versions by their root and speed up permission id sets [@stumpylog](https://github.com/stumpylog) ([#14396](https://github.com/paperless-ngx/paperless-ngx/pull/14396))
- Chore: update the db options warning [@shamoon](https://github.com/shamoon) ([#14398](https://github.com/paperless-ngx/paperless-ngx/pull/14398))
- Fix: bulk reprocess latest version for root documents [@shamoon](https://github.com/shamoon) ([#14385](https://github.com/paperless-ngx/paperless-ngx/pull/14385))
- Fix: consistently use root doc for version action permissions [@shamoon](https://github.com/shamoon) ([#14384](https://github.com/paperless-ngx/paperless-ngx/pull/14384))
- Fix: Don't attempt to index created dates which are not representable in the FTS index [@stumpylog](https://github.com/stumpylog) ([#14390](https://github.com/paperless-ngx/paperless-ngx/pull/14390))
- Fix: retry a search index rebuild that was interrupted [@stumpylog](https://github.com/stumpylog) ([#14379](https://github.com/paperless-ngx/paperless-ngx/pull/14379))
### Documentation
- Chore: silence gotenberg startup warning [@slankes](https://github.com/slankes) ([#14405](https://github.com/paperless-ngx/paperless-ngx/pull/14405))
- Documentation: remove MARIADB\_HOST from MariaDB compose files [@shamoon](https://github.com/shamoon) ([#14378](https://github.com/paperless-ngx/paperless-ngx/pull/14378))
- Documentation: Updates the troubleshooting with 2 common database collation issues [@stumpylog](https://github.com/stumpylog) ([#14370](https://github.com/paperless-ngx/paperless-ngx/pull/14370))
### Maintenance
- Chore: Upgrade runners to Ubuntu 26.04 [@stumpylog](https://github.com/stumpylog) ([#14347](https://github.com/paperless-ngx/paperless-ngx/pull/14347))
### Dependencies
- Chore(deps): Bump the npm\_and\_yarn group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#14417](https://github.com/paperless-ngx/paperless-ngx/pull/14417))
### All App Changes
<details>
<summary>12 changes</summary>
- Chore: share the authorizing owner rule between the id sets and the trash filter [@stumpylog](https://github.com/stumpylog) ([#14425](https://github.com/paperless-ngx/paperless-ngx/pull/14425))
- Fix: use a root document's effective content for the LLM index and pr… [@stumpylog](https://github.com/stumpylog) ([#14423](https://github.com/paperless-ngx/paperless-ngx/pull/14423))
- Fix: index document versions as their root document [@stumpylog](https://github.com/stumpylog) ([#14422](https://github.com/paperless-ngx/paperless-ngx/pull/14422))
- Fix: authorize document versions by their root in the single-object permission check [@stumpylog](https://github.com/stumpylog) ([#14421](https://github.com/paperless-ngx/paperless-ngx/pull/14421))
- Fix: Provide better error messages to users for invalid JSON [@stumpylog](https://github.com/stumpylog) ([#14419](https://github.com/paperless-ngx/paperless-ngx/pull/14419))
- Chore(deps): Bump the npm\_and\_yarn group across 1 directory with 2 updates @[dependabot[bot]](https://github.com/apps/dependabot) ([#14417](https://github.com/paperless-ngx/paperless-ngx/pull/14417))
- Fix: authorize document versions by their root and speed up permission id sets [@stumpylog](https://github.com/stumpylog) ([#14396](https://github.com/paperless-ngx/paperless-ngx/pull/14396))
- Chore: update the db options warning [@shamoon](https://github.com/shamoon) ([#14398](https://github.com/paperless-ngx/paperless-ngx/pull/14398))
- Fix: bulk reprocess latest version for root documents [@shamoon](https://github.com/shamoon) ([#14385](https://github.com/paperless-ngx/paperless-ngx/pull/14385))
- Fix: consistently use root doc for version action permissions [@shamoon](https://github.com/shamoon) ([#14384](https://github.com/paperless-ngx/paperless-ngx/pull/14384))
- Fix: Don't attempt to index created dates which are not representable in the FTS index [@stumpylog](https://github.com/stumpylog) ([#14390](https://github.com/paperless-ngx/paperless-ngx/pull/14390))
- Fix: retry a search index rebuild that was interrupted [@stumpylog](https://github.com/stumpylog) ([#14379](https://github.com/paperless-ngx/paperless-ngx/pull/14379))
</details>
## paperless-ngx 3.3.0 ## paperless-ngx 3.3.0
### Features / Enhancements ### Features / Enhancements
+64 -99
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import logging import logging
import tempfile import tempfile
import uuid import uuid
from functools import partial
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from typing import Literal from typing import Literal
@@ -23,6 +24,7 @@ from django.db.models import When
from django.db.models.functions import Coalesce from django.db.models.functions import Coalesce
from django.utils import timezone from django.utils import timezone
from documents import pdf_ops
from documents.data_models import ConsumableDocument from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource from documents.data_models import DocumentSource
@@ -123,6 +125,11 @@ def _resolve_root_and_source_doc(
) )
def _scratch_path(name: str) -> Path:
"""A path inside a fresh directory under SCRATCH_DIR."""
return Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) / name
def set_correspondent( def set_correspondent(
doc_ids: list[int], doc_ids: list[int],
correspondent: Correspondent, correspondent: Correspondent,
@@ -499,8 +506,6 @@ def rotate(
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode) pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
docs_by_root_id.setdefault(pair.root_doc.id, pair) docs_by_root_id.setdefault(pair.root_doc.id, pair)
import pikepdf
for pair in docs_by_root_id.values(): for pair in docs_by_root_id.values():
if pair.source_doc.mime_type != "application/pdf": if pair.source_doc.mime_type != "application/pdf":
logger.warning( logger.warning(
@@ -513,11 +518,7 @@ def rotate(
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
/ f"{pair.root_doc.id}_rotated.pdf" / f"{pair.root_doc.id}_rotated.pdf"
) )
with pikepdf.open(pair.source_doc.source_path) as pdf: pdf_ops.rotate_pdf(pair.source_doc.source_path, filepath, degrees)
for page in pdf.pages:
page.rotate(degrees, relative=True)
pdf.remove_unreferenced_resources()
pdf.save(filepath)
# Preserve metadata/permissions via overrides; mark as new version # Preserve metadata/permissions via overrides; mark as new version
overrides = DocumentMetadataOverrides().from_document(pair.root_doc) overrides = DocumentMetadataOverrides().from_document(pair.root_doc)
@@ -560,11 +561,8 @@ def merge(
qs = Document.objects.select_related("root_document").filter(id__in=doc_ids) qs = Document.objects.select_related("root_document").filter(id__in=doc_ids)
docs_by_id = {doc.id: doc for doc in qs} docs_by_id = {doc.id: doc for doc in qs}
affected_docs: list[int] = [] affected_docs: list[int] = []
import pikepdf
merged_pdf = pikepdf.new()
version: str = merged_pdf.pdf_version
handoff_asn: int | None = None handoff_asn: int | None = None
with pdf_ops.PdfMerger() as merger:
# use doc_ids to preserve order # use doc_ids to preserve order
for doc_id in doc_ids: for doc_id in doc_ids:
doc = docs_by_id.get(doc_id) doc = docs_by_id.get(doc_id)
@@ -572,16 +570,18 @@ def merge(
continue continue
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode) pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
try: try:
doc_path = ( # archive_path is None when there is no archive version
archive_path = (
pair.source_doc.archive_path pair.source_doc.archive_path
if archive_fallback if archive_fallback
and pair.source_doc.mime_type != "application/pdf" and pair.source_doc.mime_type != "application/pdf"
and pair.source_doc.has_archive_version else None
else pair.source_doc.source_path )
merger.add(
archive_path
if archive_path is not None
else pair.source_doc.source_path,
) )
with pikepdf.open(str(doc_path)) as pdf:
version = max(version, pdf.pdf_version)
merged_pdf.pages.extend(pdf.pages)
affected_docs.append(doc.id) affected_docs.append(doc.id)
if handoff_asn is None and doc.archive_serial_number is not None: if handoff_asn is None and doc.archive_serial_number is not None:
handoff_asn = doc.archive_serial_number handoff_asn = doc.archive_serial_number
@@ -599,9 +599,7 @@ def merge(
) )
/ f"{'_'.join([str(doc_id) for doc_id in affected_docs])[:100]}_merged.pdf" / f"{'_'.join([str(doc_id) for doc_id in affected_docs])[:100]}_merged.pdf"
) )
merged_pdf.remove_unreferenced_resources() merger.save(filepath)
merged_pdf.save(filepath, min_version=version)
merged_pdf.close()
if metadata_document_id: if metadata_document_id:
metadata_document = qs.get(id=metadata_document_id) metadata_document = qs.get(id=metadata_document_id)
@@ -777,26 +775,22 @@ def split(
) )
doc = Document.objects.select_related("root_document").get(id=doc_ids[0]) doc = Document.objects.select_related("root_document").get(id=doc_ids[0])
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode) pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
import pikepdf
consume_tasks = [] consume_tasks = []
try: try:
with pikepdf.open(pair.source_doc.source_path) as pdf: outputs = [
for idx, split_doc in enumerate(pages): (
dst: pikepdf.Pdf = pikepdf.new() [pdf_ops.PageSpec(page) for page in split_doc],
for page in split_doc: partial(_scratch_path, f"{doc.id}_{split_doc[0]}-{split_doc[-1]}.pdf"),
dst.pages.append(pdf.pages[page - 1])
filepath: Path = (
Path(
tempfile.mkdtemp(dir=settings.SCRATCH_DIR),
) )
/ f"{doc.id}_{split_doc[0]}-{split_doc[-1]}.pdf" for split_doc in pages
) ]
dst.remove_unreferenced_resources() filepaths = pdf_ops.build_pdfs(pair.source_doc.source_path, outputs)
dst.save(filepath)
dst.close()
for idx, (split_doc, filepath) in enumerate(
zip(pages, filepaths, strict=True),
):
overrides: DocumentMetadataOverrides = ( overrides: DocumentMetadataOverrides = (
DocumentMetadataOverrides().from_document(doc) DocumentMetadataOverrides().from_document(doc)
) )
@@ -855,8 +849,6 @@ def delete_pages(
) )
doc = Document.objects.select_related("root_document").get(id=doc_ids[0]) doc = Document.objects.select_related("root_document").get(id=doc_ids[0])
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode) pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
pages = sorted(pages) # sort pages to avoid index issues
import pikepdf
try: try:
# Produce edited PDF to a temp file and create a new version # Produce edited PDF to a temp file and create a new version
@@ -864,13 +856,7 @@ def delete_pages(
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
/ f"{pair.root_doc.id}_pages_deleted.pdf" / f"{pair.root_doc.id}_pages_deleted.pdf"
) )
with pikepdf.open(pair.source_doc.source_path) as pdf: pdf_ops.remove_pages(pair.source_doc.source_path, filepath, pages)
offset = 1 # pages are 1-indexed
for page_num in pages:
pdf.pages.remove(pdf.pages[page_num - offset])
offset += 1 # remove() changes the index of the pages
pdf.remove_unreferenced_resources()
pdf.save(filepath)
overrides = DocumentMetadataOverrides().from_document(pair.root_doc) overrides = DocumentMetadataOverrides().from_document(pair.root_doc)
if user is not None: if user is not None:
@@ -919,47 +905,28 @@ def edit_pdf(
) )
doc = Document.objects.select_related("root_document").get(id=doc_ids[0]) doc = Document.objects.select_related("root_document").get(id=doc_ids[0])
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode) pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
import pikepdf
pdf_docs: list[pikepdf.Pdf] = []
try: try:
if not operations: output_count = pdf_ops.validate_page_operations(
raise ValueError("Output document index is out of bounds") operations,
single_output=update_document,
max_idx = max(op.get("doc", 0) for op in operations)
if update_document and max_idx > 0:
logger.error(
"Update requested but multiple output documents specified",
) )
raise ValueError("Multiple output documents specified") page_specs: list[list[pdf_ops.PageSpec]] = [[] for _ in range(output_count)]
if any(
op.get("doc", 0) < 0 or op.get("doc", 0) >= len(operations)
for op in operations
):
raise ValueError("Output document index is out of bounds")
with pikepdf.open(pair.source_doc.source_path) as src:
# prepare output documents
pdf_docs = [pikepdf.new() for _ in range(max_idx + 1)]
for op in operations: for op in operations:
dst = pdf_docs[op.get("doc", 0)] page_specs[op.get("doc", 0)].append(
page = src.pages[op["page"] - 1] pdf_ops.PageSpec(op["page"], op.get("rotate", 0)),
dst.pages.append(page) )
if op.get("rotate"):
dst.pages[-1].rotate(op["rotate"], relative=True)
if update_document: if update_document:
# Create a new version from the edited PDF rather than replacing in-place # Create a new version from the edited PDF rather than replacing in-place
pdf = pdf_docs[0] (filepath,) = pdf_ops.build_pdfs(
pdf.remove_unreferenced_resources() pair.source_doc.source_path,
filepath: Path = ( [
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) (
/ f"{pair.root_doc.id}_edited.pdf" page_specs[0],
partial(_scratch_path, f"{pair.root_doc.id}_edited.pdf"),
),
],
) )
pdf.save(filepath)
overrides = ( overrides = (
DocumentMetadataOverrides().from_document(pair.root_doc) DocumentMetadataOverrides().from_document(pair.root_doc)
if include_metadata if include_metadata
@@ -980,6 +947,19 @@ def edit_pdf(
headers={"trigger_source": trigger_source}, headers={"trigger_source": trigger_source},
) )
else: else:
version_filepaths = pdf_ops.build_pdfs(
pair.source_doc.source_path,
[
(
specs,
partial(
_scratch_path,
f"{pair.root_doc.id}_edit_{idx}.pdf",
),
)
for idx, specs in enumerate(page_specs, start=1)
],
)
consume_tasks = [] consume_tasks = []
overrides = ( overrides = (
DocumentMetadataOverrides().from_document(pair.root_doc) DocumentMetadataOverrides().from_document(pair.root_doc)
@@ -991,15 +971,9 @@ def edit_pdf(
overrides.actor_id = user.id overrides.actor_id = user.id
if not delete_original: if not delete_original:
overrides.skip_asn_if_exists = True overrides.skip_asn_if_exists = True
if delete_original and len(pdf_docs) == 1: if delete_original and output_count == 1:
overrides.asn = pair.root_doc.archive_serial_number overrides.asn = pair.root_doc.archive_serial_number
for idx, pdf in enumerate(pdf_docs, start=1): for version_filepath in version_filepaths:
version_filepath: Path = (
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR))
/ f"{pair.root_doc.id}_edit_{idx}.pdf"
)
pdf.remove_unreferenced_resources()
pdf.save(version_filepath)
consume_tasks.append( consume_tasks.append(
consume_file.s( consume_file.s(
input_doc=ConsumableDocument( input_doc=ConsumableDocument(
@@ -1049,8 +1023,6 @@ def remove_password(
""" """
Remove password protection from PDF documents. Remove password protection from PDF documents.
""" """
import pikepdf
for doc_id in doc_ids: for doc_id in doc_ids:
doc = Document.objects.select_related("root_document").get(id=doc_id) doc = Document.objects.select_related("root_document").get(id=doc_id)
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode) pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
@@ -1064,26 +1036,19 @@ def remove_password(
doc.id, doc.id,
pair.source_doc.source_path, pair.source_doc.source_path,
) )
try: if not pdf_ops.needs_decrypt(source_path):
with pikepdf.open(source_path) as pdf:
if not pdf.is_encrypted:
logger.info( logger.info(
"Skipping password removal for document %s because the " "Skipping password removal for document %s because the "
"source PDF is not encrypted", "source PDF is not encrypted",
pair.root_doc.id, pair.root_doc.id,
) )
continue continue
except pikepdf.PasswordError:
# Password-protected PDFs need the supplied password below.
pass
with pikepdf.open(source_path, password=password) as pdf: filepath = pdf_ops.decrypt_pdf(
filepath: Path = ( source_path,
Path(tempfile.mkdtemp(dir=settings.SCRATCH_DIR)) partial(_scratch_path, f"{pair.root_doc.id}_unprotected.pdf"),
/ f"{pair.root_doc.id}_unprotected.pdf" password,
) )
pdf.remove_unreferenced_resources()
pdf.save(filepath)
if update_document: if update_document:
# Create a new version rather than modifying the root/original in place. # Create a new version rather than modifying the root/original in place.
@@ -514,8 +514,11 @@ class Command(CryptMixin, PaperlessCommand):
else: else:
base_name = document.get_public_filename(counter=filename_counter) base_name = document.get_public_filename(counter=filename_counter)
if base_name not in self.exported_files: # The thumbnail and archive names are derived from the stem, so
self.exported_files.add(base_name) # uniqueness must hold without the original's extension
unique_key = str(Path(base_name).with_suffix(""))
if unique_key not in self.exported_files:
self.exported_files.add(unique_key)
break break
else: else:
filename_counter += 1 filename_counter += 1
+194
View File
@@ -0,0 +1,194 @@
"""
Pure PDF page operations used by documents.bulk_edit.
This module deliberately knows nothing about Django, Celery or the documents
app: callers resolve documents, choose output paths and queue work. Every
function that writes a PDF removes unreferenced resources before saving.
pikepdf is always called as ``pikepdf.open(...)`` / ``pikepdf.new()`` (never
``from pikepdf import open``) so tests can patch those module attributes.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import NamedTuple
import pikepdf
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Iterable
from collections.abc import Mapping
from collections.abc import Sequence
from pathlib import Path
from types import TracebackType
class PageSpec(NamedTuple):
"""One page of an output PDF: a 1-indexed source page, optionally rotated."""
page: int
rotate: int = 0 # relative degrees, 0 leaves the page alone
def _require_positive(pages: Iterable[int]) -> None:
for page in pages:
if page < 1:
raise ValueError(f"Page numbers start at 1, got {page}")
def rotate_pdf(src: Path, dst: Path, degrees: int) -> None:
"""
Rotate every page relatively on the opened document, not a rebuild, so Info,
XMP and outlines are kept. ``src`` is not modified.
"""
with pikepdf.open(src) as pdf:
for page in pdf.pages:
page.rotate(degrees, relative=True)
pdf.remove_unreferenced_resources()
pdf.save(dst)
def remove_pages(src: Path, dst: Path, pages: Iterable[int]) -> None:
"""
Remove 1-indexed pages from the opened document, not a rebuild, so Info, XMP
and outlines are kept. ``src`` is not modified.
Duplicates are ignored. Pages are removed highest first so earlier removals
never shift the index of later ones.
"""
unique = sorted(set(pages))
_require_positive(unique)
with pikepdf.open(src) as pdf:
for page_num in reversed(unique):
del pdf.pages[page_num - 1]
pdf.remove_unreferenced_resources()
pdf.save(dst)
def build_pdfs(
src: Path,
outputs: Sequence[tuple[Sequence[PageSpec], Callable[[], Path]]],
) -> list[Path]:
"""
Build one new PDF per output from pages of ``src``, opening ``src`` once.
Each output is ``(page_specs, make_dst)``. Every page number is checked against
``src`` before any output is built, and ``make_dst`` is called after that
output's pages are copied and immediately before it is saved, so a bad page
number in any output never leaves a destination behind. Document-level data
(Info, XMP, outlines) is not carried over. Returns the written paths in output
order.
"""
for specs, _ in outputs:
_require_positive(spec.page for spec in specs)
written: list[Path] = []
with pikepdf.open(src) as source:
page_count = len(source.pages)
for specs, _ in outputs:
for spec in specs:
if spec.page > page_count:
raise IndexError(
f"Page {spec.page} is out of range, the PDF has "
f"{page_count} pages",
)
for specs, make_dst in outputs:
dst = pikepdf.new()
for spec in specs:
dst.pages.append(source.pages[spec.page - 1])
if spec.rotate:
dst.pages[-1].rotate(spec.rotate, relative=True)
dst.remove_unreferenced_resources()
path = make_dst()
dst.save(path)
dst.close()
written.append(path)
return written
def validate_page_operations(
operations: Sequence[Mapping[str, int]],
*,
single_output: bool,
) -> int:
"""
Validate ``edit_pdf`` style operations and return the output document count.
Each operation has ``page`` and optionally ``rotate`` and ``doc`` (the output
document index, default 0). The bounds rule is kept as it was: a ``doc`` index
must be below the number of operations.
"""
if not operations:
raise ValueError("Output document index is out of bounds")
max_idx = max(op.get("doc", 0) for op in operations)
if single_output and max_idx > 0:
raise ValueError("Multiple output documents specified")
if any(
op.get("doc", 0) < 0 or op.get("doc", 0) >= len(operations) for op in operations
):
raise ValueError("Output document index is out of bounds")
return max_idx + 1
def needs_decrypt(src: Path) -> bool:
"""
True if ``src`` is encrypted. A PDF that needs a password to open at all
counts as encrypted.
"""
try:
with pikepdf.open(src) as pdf:
return bool(pdf.is_encrypted)
except pikepdf.PasswordError:
return True
def decrypt_pdf(src: Path, make_dst: Callable[[], Path], password: str) -> Path:
"""
Write an unencrypted copy of ``src`` and return its path.
``make_dst`` is only called once the password has been accepted, so a wrong
password never leaves a destination behind.
"""
with pikepdf.open(src, password=password) as pdf:
pdf.remove_unreferenced_resources()
dst = make_dst()
pdf.save(dst)
return dst
class PdfMerger:
"""
Accumulates the pages of several PDFs into one new PDF.
``add`` raises if a source cannot be read; deciding whether to skip it is the
caller's policy. Use as a context manager so the merged PDF is closed.
"""
def __init__(self) -> None:
self._pdf = pikepdf.new()
self._version: str = self._pdf.pdf_version
def __enter__(self) -> PdfMerger:
return self
def __exit__(
self,
exc_type: type[BaseException] | None,
exc: BaseException | None,
tb: TracebackType | None,
) -> None:
self._pdf.close()
def add(self, path: Path) -> None:
with pikepdf.open(str(path)) as pdf:
self._version = max(self._version, pdf.pdf_version)
self._pdf.pages.extend(pdf.pages)
def save(self, dst: Path) -> None:
self._pdf.remove_unreferenced_resources()
self._pdf.save(dst, min_version=self._version)
+2
View File
@@ -2144,6 +2144,8 @@ class BulkEditSerializer(
raise serializers.ValidationError("pages must be a list") raise serializers.ValidationError("pages must be a list")
if not all(isinstance(i, int) for i in parameters["pages"]): if not all(isinstance(i, int) for i in parameters["pages"]):
raise serializers.ValidationError("pages must be a list of integers") raise serializers.ValidationError("pages must be a list of integers")
if any(i < 1 for i in parameters["pages"]):
raise serializers.ValidationError("pages must be positive integers")
def _validate_parameters_merge(self, parameters) -> None: def _validate_parameters_merge(self, parameters) -> None:
if "delete_originals" in parameters: if "delete_originals" in parameters:
+30
View File
@@ -1843,6 +1843,36 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
m.assert_called_once() m.assert_called_once()
self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]]) self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]])
@mock.patch("documents.serialisers.bulk_edit.delete_pages")
def test_bulk_edit_delete_pages_rejects_pages_below_one(self, m) -> None:
"""
GIVEN:
- A legacy delete_pages bulk edit
WHEN:
- API to bulk edit is called with a page number below 1
THEN:
- API returns HTTP 400
- delete_pages is not called
"""
self.setup_mock(m, "delete_pages")
for pages in ([0], [-1], [1, 0]):
with self.subTest(pages=pages):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "delete_pages",
"parameters": {"pages": pages},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"pages must be positive integers", response.content)
m.assert_not_called()
@mock.patch("documents.views.bulk_edit.rotate") @mock.patch("documents.views.bulk_edit.rotate")
def test_rotate_insufficient_permissions(self, m) -> None: def test_rotate_insufficient_permissions(self, m) -> None:
self.doc1.owner = User.objects.get(username="temp_admin") self.doc1.owner = User.objects.get(username="temp_admin")
+152 -231
View File
@@ -1,9 +1,9 @@
import shutil import shutil
from collections.abc import Callable
from datetime import date from datetime import date
from pathlib import Path from pathlib import Path
from unittest import mock from unittest import mock
import pikepdf
import pytest import pytest
from django.contrib.auth.models import Group from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission from django.contrib.auth.models import Permission
@@ -23,6 +23,7 @@ from documents.models import Document
from documents.models import DocumentType from documents.models import DocumentType
from documents.models import StoragePath from documents.models import StoragePath
from documents.models import Tag from documents.models import Tag
from documents.pdf_ops import PageSpec
from documents.permissions import set_permissions_for_objects from documents.permissions import set_permissions_for_objects
from paperless_testing.dirs import DirectoriesMixin from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory from paperless_testing.factories import DocumentFactory
@@ -796,16 +797,14 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.img_doc.save() self.img_doc.save()
@staticmethod @staticmethod
def mock_password_required_pdf( def fake_decrypt(
mock_open: mock.Mock, src: Path,
fake_pdf: mock.Mock, make_dst: Callable[[], Path],
) -> None: password: str,
password_context = mock.MagicMock() ) -> Path:
password_context.__enter__.return_value = fake_pdf dst = make_dst()
mock_open.side_effect = [ dst.write_bytes(b"password removed")
pikepdf.PasswordError("password required"), return dst
password_context,
]
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
def test_merge(self, mock_consume_file) -> None: def test_merge(self, mock_consume_file) -> None:
@@ -850,12 +849,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.PdfMerger")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
def test_merge_uses_latest_version_source_for_root_selection( def test_merge_uses_latest_version_source_for_root_selection(
self, self,
mock_consume_file, mock_consume_file,
mock_open_pdf, mock_merger,
) -> None: ) -> None:
version_file = self.dirs.scratch_dir / "sample2_version_merge.pdf" version_file = self.dirs.scratch_dir / "sample2_version_merge.pdf"
shutil.copy(self.doc2.source_path, version_file) shutil.copy(self.doc2.source_path, version_file)
@@ -866,16 +865,14 @@ class TestPDFActions(DirectoriesMixin, TestCase):
filename=version_file, filename=version_file,
mime_type="application/pdf", mime_type="application/pdf",
) )
fake_pdf = mock.MagicMock() merger = mock_merger.return_value.__enter__.return_value
fake_pdf.pdf_version = "1.7" merger.save.side_effect = lambda dst: shutil.copy(version.source_path, dst)
fake_pdf.pages = [mock.Mock()]
mock_open_pdf.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.merge([self.doc2.id]) result = bulk_edit.merge([self.doc2.id])
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_open_pdf.assert_called_once_with(str(version.source_path)) merger.add.assert_called_once_with(version.source_path)
mock_consume_file.assert_not_called() mock_consume_file.assert_called_once()
@mock.patch("documents.bulk_edit.delete.si") @mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
@@ -1037,18 +1034,18 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
@mock.patch("documents.tasks.consume_file.delay") @mock.patch("documents.tasks.consume_file.delay")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.PdfMerger.add")
def test_merge_with_errors(self, mock_open_pdf, mock_consume_file) -> None: def test_merge_with_errors(self, mock_add, mock_consume_file) -> None:
""" """
GIVEN: GIVEN:
- Existing documents - Existing documents
WHEN: WHEN:
- Merge action is called with 2 documents - Merge action is called with 2 documents
- Error occurs when opening both files - Error occurs when adding both files
THEN: THEN:
- Consume file should not be called - Consume file should not be called
""" """
mock_open_pdf.side_effect = Exception("Error opening PDF") mock_add.side_effect = Exception("Error opening PDF")
doc_ids = [self.doc2.id, self.doc3.id] doc_ids = [self.doc2.id, self.doc3.id]
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm: with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
@@ -1085,12 +1082,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
@mock.patch("documents.bulk_edit.group") @mock.patch("documents.bulk_edit.group")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.build_pdfs")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
def test_split_uses_latest_version_source_for_root_selection( def test_split_uses_latest_version_source_for_root_selection(
self, self,
mock_consume_file, mock_consume_file,
mock_open_pdf, mock_build_pdfs,
mock_group, mock_group,
) -> None: ) -> None:
version_file = self.dirs.scratch_dir / "sample2_version_split.pdf" version_file = self.dirs.scratch_dir / "sample2_version_split.pdf"
@@ -1102,17 +1099,15 @@ class TestPDFActions(DirectoriesMixin, TestCase):
filename=version_file, filename=version_file,
mime_type="application/pdf", mime_type="application/pdf",
) )
fake_pdf = mock.MagicMock() mock_build_pdfs.return_value = [version.source_path, version.source_path]
fake_pdf.pages = [mock.Mock(), mock.Mock()]
mock_open_pdf.return_value.__enter__.return_value = fake_pdf
mock_group.return_value.delay.return_value = None mock_group.return_value.delay.return_value = None
result = bulk_edit.split([self.doc2.id], [[1], [2]]) result = bulk_edit.split([self.doc2.id], [[1], [2]])
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_open_pdf.assert_called_once_with(version.source_path) self.assertEqual(mock_build_pdfs.call_args.args[0], version.source_path)
mock_consume_file.assert_not_called() self.assertEqual(mock_consume_file.call_count, 2)
mock_group.return_value.delay.assert_not_called() mock_group.return_value.delay.assert_called_once()
@mock.patch("documents.bulk_edit.delete.si") @mock.patch("documents.bulk_edit.delete.si")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
@@ -1200,18 +1195,18 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(self.doc2.archive_serial_number, 222) self.assertEqual(self.doc2.archive_serial_number, 222)
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("pikepdf.Pdf.save") @mock.patch("documents.pdf_ops.build_pdfs")
def test_split_with_errors(self, mock_save_pdf, mock_consume_file) -> None: def test_split_with_errors(self, mock_build_pdfs, mock_consume_file) -> None:
""" """
GIVEN: GIVEN:
- Existing documents - Existing documents
WHEN: WHEN:
- Split action is called with 1 document and 2 page groups - Split action is called with 1 document and 2 page groups
- Error occurs when saving the files - Error occurs when building the files
THEN: THEN:
- Consume file should not be called - Consume file should not be called
""" """
mock_save_pdf.side_effect = Exception("Error saving PDF") mock_build_pdfs.side_effect = Exception("Error building PDFs")
doc_ids = [self.doc2.id] doc_ids = [self.doc2.id]
pages = [[1, 2], [3]] pages = [[1, 2], [3]]
@@ -1246,10 +1241,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("pikepdf.Pdf.save") @mock.patch("documents.pdf_ops.rotate_pdf")
def test_rotate_with_error( def test_rotate_with_error(
self, self,
mock_pdf_save, mock_rotate_pdf,
mock_consume_delay, mock_consume_delay,
) -> None: ) -> None:
""" """
@@ -1257,11 +1252,11 @@ class TestPDFActions(DirectoriesMixin, TestCase):
- Existing documents - Existing documents
WHEN: WHEN:
- Rotate action is called with 2 documents - Rotate action is called with 2 documents
- PikePDF raises an error - Rotating the PDF raises an error
THEN: THEN:
- Rotate action should be called 0 times - Rotate action should be called 0 times
""" """
mock_pdf_save.side_effect = Exception("Error saving PDF") mock_rotate_pdf.side_effect = Exception("Error rotating PDF")
doc_ids = [self.doc2.id, self.doc3.id] doc_ids = [self.doc2.id, self.doc3.id]
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm: with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
@@ -1296,10 +1291,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf") @mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.rotate_pdf")
def test_rotate_explicit_selection_uses_root_source_when_root_selected( def test_rotate_explicit_selection_uses_root_source_when_root_selected(
self, self,
mock_open, mock_rotate_pdf,
mock_consume_delay, mock_consume_delay,
mock_magic, mock_magic,
) -> None: ) -> None:
@@ -1308,9 +1303,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
title="B version 1", title="B version 1",
root_document=self.doc2, root_document=self.doc2,
) )
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock()]
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.rotate( result = bulk_edit.rotate(
[self.doc2.id], [self.doc2.id],
@@ -1319,26 +1311,35 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_open.assert_called_once_with(self.doc2.source_path) self.assertEqual(mock_rotate_pdf.call_args.args[0], self.doc2.source_path)
mock_consume_delay.assert_called_once() mock_consume_delay.assert_called_once()
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("pikepdf.Pdf.save") @mock.patch("documents.pdf_ops.remove_pages")
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf") @mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
def test_delete_pages(self, mock_magic, mock_pdf_save, mock_consume_delay) -> None: def test_delete_pages(
self,
mock_magic,
mock_remove_pages,
mock_consume_delay,
) -> None:
""" """
GIVEN: GIVEN:
- Existing documents - Existing documents
WHEN: WHEN:
- Delete pages action is called with 1 document and 2 pages - Delete pages action is called with 1 document and 2 pages
THEN: THEN:
- Save should be called once - The pages are removed from the document's source PDF
- A new version should be enqueued via consume_file - A new version should be enqueued via consume_file
""" """
doc_ids = [self.doc2.id] doc_ids = [self.doc2.id]
pages = [1, 3] pages = [1, 3]
result = bulk_edit.delete_pages(doc_ids, pages) result = bulk_edit.delete_pages(doc_ids, pages)
mock_pdf_save.assert_called_once() mock_remove_pages.assert_called_once_with(
self.doc2.source_path,
mock.ANY,
[1, 3],
)
mock_consume_delay.assert_called_once() mock_consume_delay.assert_called_once()
task_kwargs = mock_consume_delay.call_args.kwargs["kwargs"] task_kwargs = mock_consume_delay.call_args.kwargs["kwargs"]
self.assertEqual(task_kwargs["input_doc"].root_document_id, self.doc2.id) self.assertEqual(task_kwargs["input_doc"].root_document_id, self.doc2.id)
@@ -1350,10 +1351,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf") @mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.remove_pages")
def test_delete_pages_explicit_selection_uses_root_source_when_root_selected( def test_delete_pages_explicit_selection_uses_root_source_when_root_selected(
self, self,
mock_open, mock_remove_pages,
mock_consume_delay, mock_consume_delay,
mock_magic, mock_magic,
) -> None: ) -> None:
@@ -1362,9 +1363,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
title="B version 1", title="B version 1",
root_document=self.doc2, root_document=self.doc2,
) )
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock()]
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.delete_pages( result = bulk_edit.delete_pages(
[self.doc2.id], [self.doc2.id],
@@ -1373,23 +1371,26 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_open.assert_called_once_with(self.doc2.source_path) self.assertEqual(mock_remove_pages.call_args.args[0], self.doc2.source_path)
mock_consume_delay.assert_called_once() mock_consume_delay.assert_called_once()
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("pikepdf.Pdf.save") @mock.patch("documents.pdf_ops.remove_pages")
def test_delete_pages_with_error(self, mock_pdf_save, mock_consume_delay) -> None: def test_delete_pages_with_error(
self,
mock_remove_pages,
mock_consume_delay,
) -> None:
""" """
GIVEN: GIVEN:
- Existing documents - Existing documents
WHEN: WHEN:
- Delete pages action is called with 1 document and 2 pages - Delete pages action is called with 1 document and 2 pages
- PikePDF raises an error - Removing the pages raises an error
THEN: THEN:
- Save should be called once
- No new version should be enqueued - No new version should be enqueued
""" """
mock_pdf_save.side_effect = Exception("Error saving PDF") mock_remove_pages.side_effect = Exception("Error removing pages")
doc_ids = [self.doc2.id] doc_ids = [self.doc2.id]
pages = [1, 3] pages = [1, 3]
@@ -1419,6 +1420,41 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_group.return_value.delay.assert_called_once() mock_group.return_value.delay.assert_called_once()
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.pdf_ops.build_pdfs")
@mock.patch("documents.tasks.consume_file.s")
def test_edit_pdf_maps_operations_to_outputs(
self,
mock_consume_file: mock.Mock,
mock_build_pdfs: mock.Mock,
mock_group: mock.Mock,
) -> None:
"""
GIVEN:
- Existing document
WHEN:
- edit_pdf is called with operations interleaved across two outputs,
some of them rotated
THEN:
- Each output is built from its own operations, in operation order,
with the requested rotation
"""
mock_build_pdfs.return_value = [self.doc2.source_path, self.doc2.source_path]
mock_group.return_value.delay.return_value = None
operations = [
{"page": 3, "doc": 1},
{"page": 1, "doc": 0, "rotate": 90},
{"page": 2, "doc": 1, "rotate": 180},
]
bulk_edit.edit_pdf([self.doc2.id], operations)
outputs = mock_build_pdfs.call_args.args[1]
self.assertEqual(
[specs for specs, _ in outputs],
[[PageSpec(1, 90)], [PageSpec(3), PageSpec(2, 180)]],
)
@mock.patch("documents.bulk_edit.group") @mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
def test_edit_pdf_with_user_override(self, mock_consume_file, mock_group) -> None: def test_edit_pdf_with_user_override(self, mock_consume_file, mock_group) -> None:
@@ -1534,12 +1570,10 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf") @mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("pikepdf.new") @mock.patch("documents.pdf_ops.build_pdfs")
@mock.patch("pikepdf.open")
def test_edit_pdf_explicit_selection_uses_root_source_when_root_selected( def test_edit_pdf_explicit_selection_uses_root_source_when_root_selected(
self, self,
mock_open, mock_build_pdfs,
mock_new,
mock_consume_delay, mock_consume_delay,
mock_magic, mock_magic,
) -> None: ) -> None:
@@ -1548,12 +1582,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
title="B version 1", title="B version 1",
root_document=self.doc2, root_document=self.doc2,
) )
fake_pdf = mock.MagicMock() mock_build_pdfs.return_value = [Path("edited.pdf")]
fake_pdf.pages = [mock.Mock()]
mock_open.return_value.__enter__.return_value = fake_pdf
output_pdf = mock.MagicMock()
output_pdf.pages = []
mock_new.return_value = output_pdf
result = bulk_edit.edit_pdf( result = bulk_edit.edit_pdf(
[self.doc2.id], [self.doc2.id],
@@ -1563,7 +1592,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_open.assert_called_once_with(self.doc2.source_path) self.assertEqual(mock_build_pdfs.call_args.args[0], self.doc2.source_path)
mock_consume_delay.assert_called_once() mock_consume_delay.assert_called_once()
@mock.patch("documents.bulk_edit.group") @mock.patch("documents.bulk_edit.group")
@@ -1589,31 +1618,6 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_group.return_value.delay.assert_called_once() mock_group.return_value.delay.assert_called_once()
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s")
def test_edit_pdf_open_failure(
self,
mock_consume_file: mock.Mock,
mock_group: mock.Mock,
) -> None:
"""
GIVEN:
- Existing document
WHEN:
- edit_pdf fails to open PDF
THEN:
- Task group is not called
"""
doc_ids = [self.doc2.id]
operations = [
{"page": 9999}, # invalid page, forces error during PDF load
]
with self.assertLogs("paperless.bulk_edit", level="ERROR"):
with self.assertRaises(Exception):
bulk_edit.edit_pdf(doc_ids, operations)
mock_group.assert_not_called()
mock_consume_file.assert_not_called()
@mock.patch("documents.bulk_edit.group") @mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
def test_edit_pdf_multiple_outputs_with_update_flag_errors( def test_edit_pdf_multiple_outputs_with_update_flag_errors(
@@ -1640,23 +1644,15 @@ class TestPDFActions(DirectoriesMixin, TestCase):
mock_group.assert_not_called() mock_group.assert_not_called()
mock_consume_file.assert_not_called() mock_consume_file.assert_not_called()
@mock.patch("pikepdf.open")
def test_edit_pdf_rejects_invalid_operations(self, mock_open) -> None:
for operations in ([], [{"page": 1, "doc": 2**32}]):
with self.subTest(operations=operations):
with self.assertLogs("paperless.bulk_edit", level="ERROR"):
with self.assertRaisesRegex(ValueError, "index is out of bounds"):
bulk_edit.edit_pdf([self.doc2.id], operations)
mock_open.assert_not_called()
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file.delay") @mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file.delay")
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp") @mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
def test_remove_password_update_document( def test_remove_password_update_document(
self, self,
mock_open, mock_needs_decrypt,
mock_decrypt,
mock_mkdtemp, mock_mkdtemp,
mock_consume_delay, mock_consume_delay,
mock_update_document, mock_update_document,
@@ -1665,16 +1661,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
temp_dir = self.dirs.scratch_dir / "remove-password-update" temp_dir = self.dirs.scratch_dir / "remove-password-update"
temp_dir.mkdir(parents=True, exist_ok=True) temp_dir.mkdir(parents=True, exist_ok=True)
mock_mkdtemp.return_value = str(temp_dir) mock_mkdtemp.return_value = str(temp_dir)
mock_decrypt.side_effect = self.fake_decrypt
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock(), mock.Mock()]
fake_pdf.is_encrypted = True
def save_side_effect(target_path):
Path(target_path).write_bytes(b"new pdf content")
fake_pdf.save.side_effect = save_side_effect
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.remove_password( result = bulk_edit.remove_password(
[doc.id], [doc.id],
@@ -1683,14 +1670,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
self.assertEqual( mock_needs_decrypt.assert_called_once_with(doc.source_path)
mock_open.call_args_list, mock_decrypt.assert_called_once_with(doc.source_path, mock.ANY, "secret")
[
mock.call(doc.source_path),
mock.call(doc.source_path, password="secret"),
],
)
fake_pdf.remove_unreferenced_resources.assert_called_once()
mock_update_document.assert_not_called() mock_update_document.assert_not_called()
mock_consume_delay.assert_called_once() mock_consume_delay.assert_called_once()
task_kwargs = mock_consume_delay.call_args.kwargs["kwargs"] task_kwargs = mock_consume_delay.call_args.kwargs["kwargs"]
@@ -1703,40 +1684,15 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertEqual(task_kwargs["input_doc"].root_document_id, doc.id) self.assertEqual(task_kwargs["input_doc"].root_document_id, doc.id)
self.assertIsNotNone(task_kwargs["overrides"]) self.assertIsNotNone(task_kwargs["overrides"])
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("pikepdf.open")
def test_remove_password_update_document_skips_unencrypted_pdf(
self,
mock_open,
mock_mkdtemp,
mock_consume_delay,
) -> None:
doc = self.doc1
fake_pdf = mock.MagicMock()
fake_pdf.is_encrypted = False
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.remove_password(
[doc.id],
password="secret",
update_document=True,
)
self.assertEqual(result, "OK")
mock_open.assert_called_once_with(doc.source_path)
fake_pdf.remove_unreferenced_resources.assert_not_called()
fake_pdf.save.assert_not_called()
mock_mkdtemp.assert_not_called()
mock_consume_delay.assert_not_called()
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file.delay") @mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file.delay")
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp") @mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
def test_remove_password_update_document_uses_source_paths( def test_remove_password_update_document_uses_source_paths(
self, self,
mock_open, mock_needs_decrypt,
mock_decrypt,
mock_mkdtemp, mock_mkdtemp,
mock_consume_delay, mock_consume_delay,
mock_update_document, mock_update_document,
@@ -1747,14 +1703,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
temp_dir = self.dirs.scratch_dir / "remove-password-source-file" temp_dir = self.dirs.scratch_dir / "remove-password-source-file"
temp_dir.mkdir(parents=True, exist_ok=True) temp_dir.mkdir(parents=True, exist_ok=True)
mock_mkdtemp.return_value = str(temp_dir) mock_mkdtemp.return_value = str(temp_dir)
mock_decrypt.side_effect = self.fake_decrypt
fake_pdf = mock.MagicMock()
self.mock_password_required_pdf(mock_open, fake_pdf)
def save_side_effect(target_path):
Path(target_path).write_bytes(b"new pdf content")
fake_pdf.save.side_effect = save_side_effect
result = bulk_edit.remove_password( result = bulk_edit.remove_password(
[doc.id], [doc.id],
@@ -1764,22 +1713,19 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
self.assertEqual( mock_needs_decrypt.assert_called_once_with(source_file)
mock_open.call_args_list, mock_decrypt.assert_called_once_with(source_file, mock.ANY, "secret")
[
mock.call(source_file),
mock.call(source_file, password="secret"),
],
)
mock_update_document.assert_not_called() mock_update_document.assert_not_called()
mock_consume_delay.assert_called_once() mock_consume_delay.assert_called_once()
@mock.patch("documents.data_models.magic.from_file", return_value="application/pdf") @mock.patch("documents.data_models.magic.from_file", return_value="application/pdf")
@mock.patch("documents.tasks.consume_file.apply_async") @mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
def test_remove_password_explicit_selection_uses_root_source_when_root_selected( def test_remove_password_explicit_selection_uses_root_source_when_root_selected(
self, self,
mock_open, mock_needs_decrypt,
mock_decrypt,
mock_consume_delay, mock_consume_delay,
mock_magic, mock_magic,
) -> None: ) -> None:
@@ -1788,8 +1734,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
title="A version 1", title="A version 1",
root_document=self.doc1, root_document=self.doc1,
) )
fake_pdf = mock.MagicMock() mock_decrypt.return_value = Path("unprotected.pdf")
self.mock_password_required_pdf(mock_open, fake_pdf)
result = bulk_edit.remove_password( result = bulk_edit.remove_password(
[self.doc1.id], [self.doc1.id],
@@ -1799,12 +1744,11 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
self.assertEqual( mock_needs_decrypt.assert_called_once_with(self.doc1.source_path)
mock_open.call_args_list, mock_decrypt.assert_called_once_with(
[ self.doc1.source_path,
mock.call(self.doc1.source_path), mock.ANY,
mock.call(self.doc1.source_path, password="secret"), "secret",
],
) )
mock_consume_delay.assert_called_once() mock_consume_delay.assert_called_once()
@@ -1812,10 +1756,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.bulk_edit.group") @mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp") @mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
def test_remove_password_creates_consumable_document( def test_remove_password_creates_consumable_document(
self, self,
mock_open: mock.Mock, mock_needs_decrypt: mock.Mock,
mock_decrypt: mock.Mock,
mock_mkdtemp: mock.Mock, mock_mkdtemp: mock.Mock,
mock_consume_file: mock.Mock, mock_consume_file: mock.Mock,
mock_group: mock.Mock, mock_group: mock.Mock,
@@ -1825,15 +1771,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
temp_dir = self.dirs.scratch_dir / "remove-password" temp_dir = self.dirs.scratch_dir / "remove-password"
temp_dir.mkdir(parents=True, exist_ok=True) temp_dir.mkdir(parents=True, exist_ok=True)
mock_mkdtemp.return_value = str(temp_dir) mock_mkdtemp.return_value = str(temp_dir)
mock_decrypt.side_effect = self.fake_decrypt
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock()]
self.mock_password_required_pdf(mock_open, fake_pdf)
def save_side_effect(target_path: Path) -> None:
target_path.write_bytes(b"password removed")
fake_pdf.save.side_effect = save_side_effect
mock_group.return_value.delay.return_value = None mock_group.return_value.delay.return_value = None
user = User.objects.create(username="owner") user = User.objects.create(username="owner")
@@ -1848,13 +1786,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
self.assertEqual( mock_decrypt.assert_called_once_with(doc.source_path, mock.ANY, "secret")
mock_open.call_args_list,
[
mock.call(doc.source_path),
mock.call(doc.source_path, password="secret"),
],
)
mock_consume_file.assert_called_once() mock_consume_file.assert_called_once()
call_kwargs = mock_consume_file.call_args.kwargs call_kwargs = mock_consume_file.call_args.kwargs
consumable_document = call_kwargs["input_doc"] consumable_document = call_kwargs["input_doc"]
@@ -1876,21 +1808,18 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.bulk_edit.chord") @mock.patch("documents.bulk_edit.chord")
@mock.patch("documents.bulk_edit.group") @mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp") @mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.needs_decrypt", return_value=False)
def test_remove_password_skips_unencrypted_pdf_without_queueing( def test_remove_password_skips_unencrypted_pdf_without_queueing(
self, self,
mock_open: mock.Mock, mock_needs_decrypt: mock.Mock,
mock_mkdtemp: mock.Mock, mock_decrypt: mock.Mock,
mock_consume_file: mock.Mock, mock_consume_file: mock.Mock,
mock_group: mock.Mock, mock_group: mock.Mock,
mock_chord: mock.Mock, mock_chord: mock.Mock,
mock_delete: mock.Mock, mock_delete: mock.Mock,
) -> None: ) -> None:
doc = self.doc2 doc = self.doc2
fake_pdf = mock.MagicMock()
fake_pdf.is_encrypted = False
mock_open.return_value.__enter__.return_value = fake_pdf
result = bulk_edit.remove_password( result = bulk_edit.remove_password(
[doc.id], [doc.id],
@@ -1900,10 +1829,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
mock_open.assert_called_once_with(doc.source_path) mock_needs_decrypt.assert_called_once_with(doc.source_path)
fake_pdf.remove_unreferenced_resources.assert_not_called() mock_decrypt.assert_not_called()
fake_pdf.save.assert_not_called()
mock_mkdtemp.assert_not_called()
mock_consume_file.assert_not_called() mock_consume_file.assert_not_called()
mock_group.assert_not_called() mock_group.assert_not_called()
mock_chord.assert_not_called() mock_chord.assert_not_called()
@@ -1914,10 +1841,12 @@ class TestPDFActions(DirectoriesMixin, TestCase):
@mock.patch("documents.bulk_edit.group") @mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s") @mock.patch("documents.tasks.consume_file.s")
@mock.patch("documents.bulk_edit.tempfile.mkdtemp") @mock.patch("documents.bulk_edit.tempfile.mkdtemp")
@mock.patch("pikepdf.open") @mock.patch("documents.pdf_ops.decrypt_pdf")
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
def test_remove_password_deletes_original( def test_remove_password_deletes_original(
self, self,
mock_open: mock.Mock, mock_needs_decrypt: mock.Mock,
mock_decrypt: mock.Mock,
mock_mkdtemp: mock.Mock, mock_mkdtemp: mock.Mock,
mock_consume_file: mock.Mock, mock_consume_file: mock.Mock,
mock_group: mock.Mock, mock_group: mock.Mock,
@@ -1928,15 +1857,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
temp_dir = self.dirs.scratch_dir / "remove-password-delete" temp_dir = self.dirs.scratch_dir / "remove-password-delete"
temp_dir.mkdir(parents=True, exist_ok=True) temp_dir.mkdir(parents=True, exist_ok=True)
mock_mkdtemp.return_value = str(temp_dir) mock_mkdtemp.return_value = str(temp_dir)
mock_decrypt.side_effect = self.fake_decrypt
fake_pdf = mock.MagicMock()
fake_pdf.pages = [mock.Mock(), mock.Mock()]
self.mock_password_required_pdf(mock_open, fake_pdf)
def save_side_effect(target_path: Path) -> None:
target_path.write_bytes(b"password removed")
fake_pdf.save.side_effect = save_side_effect
mock_chord.return_value.delay.return_value = None mock_chord.return_value.delay.return_value = None
result = bulk_edit.remove_password( result = bulk_edit.remove_password(
@@ -1948,23 +1869,23 @@ class TestPDFActions(DirectoriesMixin, TestCase):
) )
self.assertEqual(result, "OK") self.assertEqual(result, "OK")
self.assertEqual( mock_decrypt.assert_called_once_with(doc.source_path, mock.ANY, "secret")
mock_open.call_args_list,
[
mock.call(doc.source_path),
mock.call(doc.source_path, password="secret"),
],
)
mock_consume_file.assert_called_once() mock_consume_file.assert_called_once()
mock_group.assert_not_called() mock_group.assert_not_called()
mock_chord.assert_called_once() mock_chord.assert_called_once()
mock_chord.return_value.delay.assert_called_once() mock_chord.return_value.delay.assert_called_once()
mock_delete.si.assert_called_once_with([doc.id]) mock_delete.si.assert_called_once_with([doc.id])
@mock.patch("pikepdf.open") @mock.patch(
def test_remove_password_open_failure(self, mock_open: mock.Mock) -> None: "documents.pdf_ops.decrypt_pdf",
mock_open.side_effect = RuntimeError("wrong password") side_effect=RuntimeError("wrong password"),
)
@mock.patch("documents.pdf_ops.needs_decrypt", return_value=True)
def test_remove_password_failure_raises_value_error(
self,
mock_needs_decrypt: mock.Mock,
mock_decrypt: mock.Mock,
) -> None:
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm: with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
with self.assertRaises(ValueError) as exc: with self.assertRaises(ValueError) as exc:
bulk_edit.remove_password([self.doc1.id], password="secret") bulk_edit.remove_password([self.doc1.id], password="secret")
@@ -2,6 +2,7 @@ import hashlib
import json import json
import shutil import shutil
import tempfile import tempfile
from datetime import date
from datetime import timedelta from datetime import timedelta
from io import StringIO from io import StringIO
from pathlib import Path from pathlib import Path
@@ -44,14 +45,18 @@ from documents.models import Workflow
from documents.models import WorkflowAction from documents.models import WorkflowAction
from documents.models import WorkflowTrigger from documents.models import WorkflowTrigger
from documents.sanity_checker import check_sanity from documents.sanity_checker import check_sanity
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_FILE_NAME
from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME
from documents.settings import EXPORTER_THUMBNAIL_NAME
from documents.tests.utils import SampleDirMixin from documents.tests.utils import SampleDirMixin
from paperless_mail.models import MailAccount from paperless_mail.models import MailAccount
from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.dirs import PaperlessDirs
from paperless_testing.dirs import paperless_environment from paperless_testing.dirs import paperless_environment
from paperless_testing.factories import DocumentBarcodeFactory from paperless_testing.factories import DocumentBarcodeFactory
from paperless_testing.factories import DocumentFactory
from paperless_testing.permissions import grant_object from paperless_testing.permissions import grant_object
@@ -1462,3 +1467,73 @@ class TestCryptExportImport(
("No passphrase was given, sensitive fields will be in plaintext"), ("No passphrase was given, sensitive fields will be in plaintext"),
stdout.read(), stdout.read(),
) )
@pytest.mark.management
@pytest.mark.django_db
class TestExportDerivedFileNameCollisions:
def test_same_stem_different_extension_get_distinct_derived_files(
self,
paperless_dirs: PaperlessDirs,
tmp_path: Path,
) -> None:
"""
GIVEN:
- Two documents with the same title and created date
- One original is a PDF, the other a PNG, both with archive and thumbnail
WHEN:
- The exporter runs
THEN:
- Each document is exported under a distinct original, thumbnail and archive name
- Each exported file holds its own document's content
"""
for pk, (filename, mime_type) in enumerate(
[("collide.pdf", "application/pdf"), ("collide.png", "image/png")],
start=1,
):
(paperless_dirs.originals_dir / filename).write_bytes(
f"original {pk}".encode(),
)
(paperless_dirs.archive_dir / f"{pk:07}.pdf").write_bytes(
f"archive {pk}".encode(),
)
(paperless_dirs.thumbnail_dir / f"{pk:07}.webp").write_bytes(
f"thumbnail {pk}".encode(),
)
DocumentFactory(
pk=pk,
title="collide",
created=date(2026, 10, 7),
filename=filename,
archive_filename=f"{pk:07}.pdf",
mime_type=mime_type,
)
target = tmp_path / "export"
target.mkdir()
call_command(
"document_exporter",
str(target),
"--no-progress-bar",
skip_checks=True,
)
manifest = json.loads((target / "manifest.json").read_text())
records = {r["pk"]: r for r in manifest if r["model"] == "documents.document"}
for key in (
EXPORTER_FILE_NAME,
EXPORTER_THUMBNAIL_NAME,
EXPORTER_ARCHIVE_NAME,
):
assert records[1][key] != records[2][key]
for pk in (1, 2):
record = records[pk]
assert (target / record[EXPORTER_FILE_NAME]).read_text() == f"original {pk}"
assert (
target / record[EXPORTER_THUMBNAIL_NAME]
).read_text() == f"thumbnail {pk}"
assert (
target / record[EXPORTER_ARCHIVE_NAME]
).read_text() == f"archive {pk}"
+520
View File
@@ -0,0 +1,520 @@
"""
Tests for documents.pdf_ops.
These use real PDFs from the sample directories. No database, Celery or mocks.
Pages are compared by a hash of their content stream, so page identity and order
are easy to assert.
"""
import hashlib
from collections.abc import Callable
from pathlib import Path
import pikepdf
import pytest
from documents import pdf_ops
from documents.pdf_ops import PageSpec
SRC_ROOT = Path(__file__).parents[2]
SAMPLES = Path(__file__).parent / "samples"
THREE_PAGES = SAMPLES / "documents" / "originals" / "0000002.pdf"
TWELVE_PAGES = SAMPLES / "barcodes" / "split-by-asn-2.pdf"
ENCRYPTED = SAMPLES / "password-is-test.pdf"
SIGNED = SRC_ROOT / "paperless" / "tests" / "samples" / "tesseract" / "signed.pdf"
def _page_fingerprint(page: pikepdf.Page) -> str:
contents = page.obj.get("/Contents")
assert contents is not None, "sample page has no /Contents"
streams = list(contents) if isinstance(contents, pikepdf.Array) else [contents]
return hashlib.sha256(b"".join(s.read_bytes() for s in streams)).hexdigest()
def fingerprints(path: Path) -> list[str]:
with pikepdf.open(path) as pdf:
return [_page_fingerprint(page) for page in pdf.pages]
def rotations(path: Path) -> list[int]:
with pikepdf.open(path) as pdf:
return [int(page.obj.get("/Rotate", 0)) for page in pdf.pages]
def docinfo_keys(path: Path) -> set[str]:
with pikepdf.open(path) as pdf:
return set(pdf.docinfo.keys())
def constant(path: Path) -> Callable[[], Path]:
return lambda: path
@pytest.fixture
def source_fingerprints() -> list[str]:
fps = fingerprints(THREE_PAGES)
assert len(set(fps)) == 3, "sample must have three distinct pages"
return fps
class TestRotatePdf:
def test_rotation_is_relative_and_applies_to_every_page(
self,
tmp_path: Path,
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- It is rotated by 90 degrees, then the result is rotated by 90 again
THEN:
- Every page is rotated relative to its current rotation
- The page content itself is unchanged
"""
once = tmp_path / "once.pdf"
twice = tmp_path / "twice.pdf"
pdf_ops.rotate_pdf(THREE_PAGES, once, 90)
pdf_ops.rotate_pdf(once, twice, 90)
assert rotations(once) == [90, 90, 90]
assert rotations(twice) == [180, 180, 180]
assert fingerprints(twice) == fingerprints(THREE_PAGES)
def test_keeps_document_info(self, tmp_path: Path) -> None:
"""
GIVEN:
- A PDF with document info
WHEN:
- It is rotated
THEN:
- The document info is still present in the output
"""
dst = tmp_path / "out.pdf"
pdf_ops.rotate_pdf(THREE_PAGES, dst, 90)
assert "/Creator" in docinfo_keys(dst)
class TestRemovePages:
@pytest.mark.parametrize(
("pages", "kept"),
[
pytest.param([2], [0, 2], id="single"),
pytest.param([3, 1], [1], id="unordered"),
pytest.param([2, 2], [0, 2], id="duplicates-remove-once"),
pytest.param([], [0, 1, 2], id="empty-keeps-everything"),
pytest.param([1, 2, 3], [], id="every-page"),
],
)
def test_removes_only_the_requested_pages(
self,
tmp_path: Path,
source_fingerprints: list[str],
pages: list[int],
kept: list[int],
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- Pages are removed, in any order and possibly repeated
THEN:
- Exactly the other pages remain, in their original order
"""
dst = tmp_path / "out.pdf"
pdf_ops.remove_pages(THREE_PAGES, dst, pages)
assert fingerprints(dst) == [source_fingerprints[i] for i in kept]
def test_keeps_document_info(self, tmp_path: Path) -> None:
"""
GIVEN:
- A PDF with document info
WHEN:
- A page is removed
THEN:
- The document info is still present in the output
"""
dst = tmp_path / "out.pdf"
pdf_ops.remove_pages(THREE_PAGES, dst, [1])
assert "/Creator" in docinfo_keys(dst)
@pytest.mark.parametrize(
"bad_page",
[
pytest.param(0, id="zero"),
pytest.param(-1, id="negative"),
],
)
def test_rejects_pages_below_one(self, tmp_path: Path, bad_page: int) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- Pages are removed and one of them is below 1
THEN:
- ValueError is raised
- No output file is written
"""
dst = tmp_path / "out.pdf"
with pytest.raises(ValueError, match="start at 1"):
pdf_ops.remove_pages(THREE_PAGES, dst, [1, bad_page])
assert not dst.exists()
class TestBuildPdfs:
def test_selects_and_orders_pages(
self,
tmp_path: Path,
source_fingerprints: list[str],
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- One output is built from pages 3 then 1
THEN:
- The output has those pages in that order
- Its path is returned
"""
dst = tmp_path / "out.pdf"
written = pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(3), PageSpec(1)], constant(dst))],
)
assert written == [dst]
assert fingerprints(dst) == [source_fingerprints[2], source_fingerprints[0]]
def test_rotates_only_the_requested_pages(self, tmp_path: Path) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- One output is built with a different rotation on each page
THEN:
- Each output page has exactly the rotation requested for it
"""
dst = tmp_path / "out.pdf"
pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(1), PageSpec(2, 90), PageSpec(3, 180)], constant(dst))],
)
assert rotations(dst) == [0, 90, 180]
def test_writes_one_file_per_output_in_order(self, tmp_path: Path) -> None:
"""
GIVEN:
- A twelve page PDF
WHEN:
- Two outputs are built from different page ranges
THEN:
- Two files are written and returned in output order
- Each holds exactly its own pages
"""
first = tmp_path / "first.pdf"
second = tmp_path / "second.pdf"
source = fingerprints(TWELVE_PAGES)
written = pdf_ops.build_pdfs(
TWELVE_PAGES,
[
([PageSpec(p) for p in (1, 2, 3)], constant(first)),
([PageSpec(p) for p in range(4, 13)], constant(second)),
],
)
assert written == [first, second]
assert fingerprints(first) == source[:3]
assert fingerprints(second) == source[3:]
def test_empty_page_list_writes_a_zero_page_file(self, tmp_path: Path) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- An output with no pages is built
THEN:
- A PDF with zero pages is written
"""
dst = tmp_path / "out.pdf"
pdf_ops.build_pdfs(THREE_PAGES, [([], constant(dst))])
assert fingerprints(dst) == []
def test_destination_is_not_requested_when_a_page_is_out_of_range(
self,
tmp_path: Path,
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- Several outputs are built
- A later output refers to a page past the end
THEN:
- IndexError is raised
- No destination was requested for any output, including earlier valid ones
"""
requested: list[Path] = []
def make_dst() -> Path:
requested.append(tmp_path / "out.pdf")
return requested[-1]
with pytest.raises(IndexError):
pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(1)], make_dst), ([PageSpec(99)], make_dst)],
)
assert requested == []
@pytest.mark.parametrize(
"bad_page",
[
pytest.param(0, id="zero"),
pytest.param(-1, id="negative"),
],
)
def test_rejects_pages_below_one_before_opening_anything(
self,
tmp_path: Path,
bad_page: int,
) -> None:
"""
GIVEN:
- A three page PDF
WHEN:
- Several outputs are built and a later one has a page below 1
THEN:
- ValueError is raised
- No destination was requested for any output
"""
requested: list[Path] = []
def make_dst() -> Path:
requested.append(tmp_path / "out.pdf")
return requested[-1]
with pytest.raises(ValueError, match="start at 1"):
pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(1)], make_dst), ([PageSpec(bad_page)], make_dst)],
)
assert requested == []
class TestValidatePageOperations:
def test_returns_the_output_count(self) -> None:
"""
GIVEN:
- Operations that all target the default output
WHEN:
- They are validated
THEN:
- One output document is reported
"""
operations = [{"page": 1}, {"page": 2}, {"page": 3}]
assert pdf_ops.validate_page_operations(operations, single_output=True) == 1
def test_gap_in_output_indices_counts_up_to_the_highest(self) -> None:
"""
GIVEN:
- Operations that target outputs 0 and 2 but never 1
WHEN:
- They are validated
THEN:
- Three output documents are reported
"""
operations = [
{"page": 1, "doc": 0},
{"page": 2, "doc": 2},
{"page": 3, "doc": 0},
]
count = pdf_ops.validate_page_operations(operations, single_output=False)
assert count == 3
def test_empty_operations_are_rejected(self) -> None:
"""
GIVEN:
- No operations
WHEN:
- They are validated
THEN:
- ValueError is raised
"""
with pytest.raises(ValueError, match="index is out of bounds"):
pdf_ops.validate_page_operations([], single_output=False)
def test_multiple_outputs_rejected_when_single_output_required(self) -> None:
"""
GIVEN:
- Operations that target two outputs
WHEN:
- They are validated with a single output required
THEN:
- ValueError is raised
"""
operations = [{"page": 1, "doc": 0}, {"page": 2, "doc": 1}]
with pytest.raises(ValueError, match="Multiple output documents"):
pdf_ops.validate_page_operations(operations, single_output=True)
@pytest.mark.parametrize(
"doc",
[
pytest.param(-1, id="negative"),
pytest.param(2, id="equal-to-operation-count"),
pytest.param(2**32, id="huge"),
],
)
def test_output_index_out_of_bounds(self, doc: int) -> None:
"""
GIVEN:
- Two operations, one with an output index that is out of bounds
WHEN:
- They are validated
THEN:
- ValueError is raised
"""
operations = [{"page": 1, "doc": 0}, {"page": 2, "doc": doc}]
with pytest.raises(ValueError, match="index is out of bounds"):
pdf_ops.validate_page_operations(operations, single_output=False)
class TestDecrypt:
@pytest.mark.parametrize(
("path", "expected"),
[
pytest.param(ENCRYPTED, True, id="password-required"),
pytest.param(SIGNED, True, id="opens-without-password-but-encrypted"),
pytest.param(THREE_PAGES, False, id="not-encrypted"),
],
)
def test_needs_decrypt(self, path: Path, *, expected: bool) -> None:
"""
GIVEN:
- A PDF that is encrypted, or encrypted but openable, or plain
WHEN:
- needs_decrypt is asked about it
THEN:
- Only the unencrypted PDF reports False
"""
assert pdf_ops.needs_decrypt(path) is expected
def test_decrypt_writes_an_unencrypted_copy(self, tmp_path: Path) -> None:
"""
GIVEN:
- A password protected PDF
WHEN:
- It is decrypted with the correct password
THEN:
- The path from make_dst is returned
- The written copy no longer needs decrypting
"""
dst = tmp_path / "out.pdf"
result = pdf_ops.decrypt_pdf(ENCRYPTED, constant(dst), "test")
assert result == dst
assert pdf_ops.needs_decrypt(dst) is False
def test_wrong_password_raises_and_never_requests_a_destination(
self,
tmp_path: Path,
) -> None:
"""
GIVEN:
- A password protected PDF
WHEN:
- It is decrypted with the wrong password
THEN:
- PasswordError is raised
- No destination was requested
"""
requested: list[Path] = []
def make_dst() -> Path:
requested.append(tmp_path / "out.pdf")
return requested[-1]
with pytest.raises(pikepdf.PasswordError):
pdf_ops.decrypt_pdf(ENCRYPTED, make_dst, "wrong")
assert requested == []
class TestPdfMerger:
def test_pages_are_appended_in_the_order_added(
self,
tmp_path: Path,
source_fingerprints: list[str],
) -> None:
"""
GIVEN:
- A reordered PDF and the original three page PDF
WHEN:
- Both are added to a merger in that order and saved
THEN:
- The output holds all pages in the order they were added
"""
reordered = tmp_path / "reordered.pdf"
merged = tmp_path / "merged.pdf"
pdf_ops.build_pdfs(
THREE_PAGES,
[([PageSpec(3), PageSpec(1)], constant(reordered))],
)
with pdf_ops.PdfMerger() as merger:
merger.add(reordered)
merger.add(THREE_PAGES)
merger.save(merged)
assert fingerprints(merged) == [
source_fingerprints[2],
source_fingerprints[0],
*source_fingerprints,
]
def test_output_version_is_at_least_the_highest_source_version(
self,
tmp_path: Path,
) -> None:
"""
GIVEN:
- Two PDFs with different PDF versions
WHEN:
- Both are added to a merger and saved
THEN:
- The output version is at least the highest source version
"""
merged = tmp_path / "merged.pdf"
with pikepdf.open(TWELVE_PAGES) as pdf:
source_versions = [pdf.pdf_version]
with pikepdf.open(THREE_PAGES) as pdf:
source_versions.append(pdf.pdf_version)
with pdf_ops.PdfMerger() as merger:
merger.add(TWELVE_PAGES)
merger.add(THREE_PAGES)
merger.save(merged)
with pikepdf.open(merged) as pdf:
assert pdf.pdf_version >= max(source_versions)