mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-14 03:57:13 +00:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4c5927a45f | ||
|
|
884ddc401c | ||
|
|
e606dbffcd | ||
|
|
138160cbda | ||
|
|
6d960c11d5 | ||
|
|
a35bd6e238 | ||
|
|
474c630aa4 |
No files matched your search
@@ -111,7 +111,7 @@ jobs:
|
||||
timeout-minutes: 12
|
||||
uses: $/.github/actions/apt-install
|
||||
with:
|
||||
packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils
|
||||
packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils qpdf
|
||||
- name: Configure ImageMagick
|
||||
run: |
|
||||
sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml
|
||||
|
||||
+6
-18
@@ -1315,29 +1315,17 @@ valid crontab(5) expression describing when to run.
|
||||
|
||||
#### [`PAPERLESS_CONVERT_MEMORY_LIMIT=<num>`](#PAPERLESS_CONVERT_MEMORY_LIMIT) {#PAPERLESS_CONVERT_MEMORY_LIMIT}
|
||||
|
||||
: On smaller systems, or even in the case of Very Large Documents, the
|
||||
consumer may explode, complaining about how it's "unable to extend
|
||||
pixel cache". In such cases, try setting this to a reasonably low
|
||||
value, like 32. The default is to use whatever is necessary to do
|
||||
everything without writing to disk, and units are in megabytes.
|
||||
!!! warning
|
||||
|
||||
For more information on how to use this value, you should search the
|
||||
web for "MAGICK_MEMORY_LIMIT".
|
||||
|
||||
Defaults to 0, which disables the limit.
|
||||
Deprecated and has no effect, since PDF thumbnails no longer use
|
||||
ImageMagick. It will be removed in a future release.
|
||||
|
||||
#### [`PAPERLESS_CONVERT_TMPDIR=<path>`](#PAPERLESS_CONVERT_TMPDIR) {#PAPERLESS_CONVERT_TMPDIR}
|
||||
|
||||
: Similar to the memory limit, if you've got a small system and your
|
||||
OS mounts /tmp as tmpfs, you should set this to a path that's on a
|
||||
physical disk, like /home/your_user/tmp or something. ImageMagick
|
||||
will use this as scratch space when crunching through very large
|
||||
documents.
|
||||
!!! warning
|
||||
|
||||
For more information on how to use this value, you should search the
|
||||
web for "MAGICK_TMPDIR".
|
||||
|
||||
Default is none, which disables the temporary directory.
|
||||
Deprecated and has no effect, since PDF thumbnails no longer use
|
||||
ImageMagick. It will be removed in a future release.
|
||||
|
||||
#### [`PAPERLESS_APPS=<string>`](#PAPERLESS_APPS) {#PAPERLESS_APPS}
|
||||
|
||||
|
||||
+4
-7
@@ -177,12 +177,12 @@ to a positive number to enable polling and disable native filesystem notificatio
|
||||
- `pkg-config` for mysqlclient (python dependency)
|
||||
- `fonts-liberation` for generating thumbnails for plain text
|
||||
files
|
||||
- `imagemagick` >= 6 for PDF conversion
|
||||
- `imagemagick` >= 6 for image alpha handling
|
||||
- `gnupg` for decrypting GPG-encrypted email
|
||||
- `libpq-dev` for PostgreSQL
|
||||
- `libmagic-dev` for mime type detection
|
||||
- `mariadb-client` for MariaDB compile time
|
||||
- `poppler-utils` for barcode detection
|
||||
- `poppler-utils` for thumbnail generation and barcode detection
|
||||
|
||||
Use this list for your preferred package management:
|
||||
|
||||
@@ -416,11 +416,8 @@ to a positive number to enable polling and disable native filesystem notificatio
|
||||
You may need to change the path in the files. Example:
|
||||
`ExecStart=/opt/paperless/.local/bin/celery --app paperless worker --loglevel INFO`
|
||||
|
||||
12. Configure ImageMagick to allow processing of PDF documents and disable
|
||||
formats that Paperless-ngx does not use. Most distributions disable PDF
|
||||
processing by default, since PDF documents can contain malware. If you
|
||||
don't enable it, Paperless-ngx will fall back to Ghostscript for certain
|
||||
steps such as thumbnail generation.
|
||||
12. Harden ImageMagick by disabling formats that Paperless-ngx does not use.
|
||||
PDF processing is not needed and should stay disabled.
|
||||
|
||||
Configure the active ImageMagick policy file (commonly
|
||||
`/etc/ImageMagick-6/policy.xml` or `/etc/ImageMagick-7/policy.xml`) and
|
||||
|
||||
@@ -50,8 +50,6 @@ PAPERLESS_SECRET_KEY=change-me
|
||||
#PAPERLESS_OCR_ROTATE_PAGES=true
|
||||
#PAPERLESS_OCR_ROTATE_PAGES_THRESHOLD=12.0
|
||||
#PAPERLESS_OCR_USER_ARGS={}
|
||||
#PAPERLESS_CONVERT_MEMORY_LIMIT=0
|
||||
#PAPERLESS_CONVERT_TMPDIR=/var/tmp/paperless
|
||||
|
||||
# Software tweaks
|
||||
|
||||
|
||||
@@ -29,13 +29,8 @@
|
||||
</div>
|
||||
<div class="search-container flex-grow-1 py-2 pb-3 pb-sm-2 me-sm-auto order-3 order-sm-1"
|
||||
[class.mobile-hidden]="mobileSearchHidden()">
|
||||
<div class="col-12 header-search mx-auto d-flex align-items-stretch gap-3">
|
||||
<pngx-global-search class="flex-grow-1"></pngx-global-search>
|
||||
<button *pngxIfPermissions="{ action: PermissionAction.Add, type: PermissionType.Document }"
|
||||
class="btn btn-sm header-upload border-0 d-flex align-items-center justify-content-center flex-shrink-0 py-0 px-2" type="button"
|
||||
(click)="fileUpload.click()" aria-label="Upload documents" i18n-aria-label>
|
||||
<i-bs class="d-flex" width="1.1em" height="1.1em" name="file-earmark-arrow-up"></i-bs><span class="ms-2 d-sm-none d-md-inline" i18n>Upload</span>
|
||||
</button>
|
||||
<div class="col-12 header-search mx-auto">
|
||||
<pngx-global-search></pngx-global-search>
|
||||
</div>
|
||||
</div>
|
||||
<ul ngbNav class="order-sm-3">
|
||||
@@ -75,9 +70,6 @@
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
<input type="file" class="visually-hidden" id="navbar-file-upload" (change)="onFileSelected($event)" multiple #fileUpload tabindex="-1" aria-hidden="true">
|
||||
<label for="navbar-file-upload" class="visually-hidden" i18n>Upload Files</label>
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<nav id="sidebarMenu" class="d-md-block bg-light sidebar collapse"
|
||||
|
||||
@@ -532,23 +532,7 @@ main {
|
||||
|
||||
.header-search {
|
||||
width: 100%;
|
||||
max-width: 50rem;
|
||||
}
|
||||
|
||||
.header-search pngx-global-search {
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
// light w shadow to stand out against search
|
||||
.header-upload {
|
||||
--bs-btn-bg: rgba(255, 255, 255, .15);
|
||||
--bs-btn-hover-bg: rgba(255, 255, 255, .22);
|
||||
--bs-btn-active-bg: rgba(255, 255, 255, .3);
|
||||
color: var(--pngx-primary-text-contrast) !important; // .navbar .dropdown > .btn in styles.scss
|
||||
|
||||
&:not(:focus) {
|
||||
box-shadow: 0 1px 3px rgba(0, 0, 0, .35), 0 1px 1px rgba(0, 0, 0, .2); // leave room for the .btn:focus ring
|
||||
}
|
||||
max-width: 55rem;
|
||||
}
|
||||
|
||||
@media screen and (max-width: 575.98px) {
|
||||
@@ -678,7 +662,3 @@ main {
|
||||
::ng-deep .navItemDrag .position-absolute i-bs {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.smaller {
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
@@ -33,7 +33,6 @@ import { SearchService } from 'src/app/services/rest/search.service'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
import { TasksService } from 'src/app/services/tasks.service'
|
||||
import { ToastService } from 'src/app/services/toast.service'
|
||||
import { UploadDocumentsService } from 'src/app/services/upload-documents.service'
|
||||
import { environment } from 'src/environments/environment'
|
||||
import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
|
||||
import { DocumentDetailComponent } from '../document-detail/document-detail.component'
|
||||
@@ -463,29 +462,6 @@ describe('AppFrameComponent', () => {
|
||||
expect(routerSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should upload files selected from the navbar upload button', () => {
|
||||
const uploadDocumentsService = TestBed.inject(UploadDocumentsService)
|
||||
const uploadSpy = jest
|
||||
.spyOn(uploadDocumentsService, 'uploadFiles')
|
||||
.mockImplementation(() => {})
|
||||
const toastSpy = jest.spyOn(toastService, 'showInfo')
|
||||
const file = new File(['testing'], 'file.pdf')
|
||||
const input = window.document.createElement('input')
|
||||
jest.spyOn(input, 'files', 'get').mockReturnValue([file] as any)
|
||||
|
||||
component.onFileSelected({ target: input } as unknown as Event)
|
||||
expect(toastSpy).toHaveBeenCalled()
|
||||
expect(uploadSpy).toHaveBeenCalledWith([file])
|
||||
})
|
||||
|
||||
it('should not upload when no files are selected from the navbar', () => {
|
||||
const uploadDocumentsService = TestBed.inject(UploadDocumentsService)
|
||||
const uploadSpy = jest.spyOn(uploadDocumentsService, 'uploadFiles')
|
||||
const input = window.document.createElement('input')
|
||||
component.onFileSelected({ target: input } as unknown as Event)
|
||||
expect(uploadSpy).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should close all documents on logout', () => {
|
||||
const closeAllSpy = jest.spyOn(openDocumentsService, 'closeAll')
|
||||
component.onLogout()
|
||||
|
||||
@@ -48,13 +48,12 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service'
|
||||
import { SettingsService } from 'src/app/services/settings.service'
|
||||
import { TasksService } from 'src/app/services/tasks.service'
|
||||
import { ToastService } from 'src/app/services/toast.service'
|
||||
import { UploadDocumentsService } from 'src/app/services/upload-documents.service'
|
||||
import { environment } from 'src/environments/environment'
|
||||
import { ChatComponent } from '../chat/chat/chat.component'
|
||||
import { SwitchComponent } from '../common/input/switch/switch.component'
|
||||
import { BrandMarkComponent } from '../common/logo/brand-mark/brand-mark.component'
|
||||
import { LogoComponent } from '../common/logo/logo.component'
|
||||
import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
|
||||
import { SwitchComponent } from '../common/input/switch/switch.component'
|
||||
import { DocumentDetailComponent } from '../document-detail/document-detail.component'
|
||||
import { ComponentWithPermissions } from '../with-permissions/with-permissions.component'
|
||||
import { GlobalSearchComponent } from './global-search/global-search.component'
|
||||
@@ -102,7 +101,6 @@ export class AppFrameComponent
|
||||
private modalService = inject(NgbModal)
|
||||
permissionsService = inject(PermissionsService)
|
||||
private djangoMessagesService = inject(DjangoMessagesService)
|
||||
private uploadDocumentsService = inject(UploadDocumentsService)
|
||||
|
||||
readonly appRemoteVersion = signal<AppRemoteVersion>(null)
|
||||
readonly isMenuCollapsed = signal(true)
|
||||
@@ -513,15 +511,6 @@ export class AppFrameComponent
|
||||
}
|
||||
}
|
||||
|
||||
onFileSelected(event: Event) {
|
||||
const input = event.target as HTMLInputElement
|
||||
const files = Array.from(input.files ?? [])
|
||||
input.value = ''
|
||||
if (!files.length) return
|
||||
this.toastService.showInfo($localize`Initiating upload...`, 3000)
|
||||
this.uploadDocumentsService.uploadFiles(files)
|
||||
}
|
||||
|
||||
onLogout() {
|
||||
this.openDocumentsService.closeAll()
|
||||
}
|
||||
|
||||
+8
-4
@@ -85,11 +85,15 @@ describe('UploadFileWidgetComponent', () => {
|
||||
'file.pdf'
|
||||
)
|
||||
const fileInput = fixture.debugElement.query(By.css('input'))
|
||||
jest
|
||||
.spyOn(fileInput.nativeElement, 'files', 'get')
|
||||
.mockReturnValue([file] as any)
|
||||
jest.spyOn(fileInput.nativeElement, 'files', 'get').mockReturnValue({
|
||||
item: () => file,
|
||||
length: 1,
|
||||
[Symbol.iterator]: () => ({
|
||||
next: () => ({ done: false, value: file }),
|
||||
}),
|
||||
} as any)
|
||||
fileInput.nativeElement.dispatchEvent(new Event('change'))
|
||||
expect(uploadSpy).toHaveBeenCalledWith(file)
|
||||
expect(uploadSpy).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should generate stats summary', () => {
|
||||
|
||||
+5
-3
@@ -130,9 +130,11 @@ export class UploadFileWidgetComponent extends ComponentWithPermissions {
|
||||
}
|
||||
|
||||
public onFileSelected(event: Event) {
|
||||
const input = event.target as HTMLInputElement
|
||||
this.uploadDocumentsService.uploadFiles(Array.from(input.files ?? []))
|
||||
input.value = ''
|
||||
const files = (event.target as HTMLInputElement).files
|
||||
for (let i = 0; i < files?.length; i++) {
|
||||
const file = files.item(i)
|
||||
file && this.uploadDocumentsService.uploadFile(file)
|
||||
}
|
||||
}
|
||||
|
||||
get slimSidebarEnabled(): boolean {
|
||||
|
||||
@@ -135,7 +135,7 @@ export class FileDropComponent {
|
||||
.then((results) => {
|
||||
files.push(...[].concat(...results))
|
||||
this.toastService.showInfo($localize`Initiating upload...`, 3000)
|
||||
this.uploadDocumentsService.uploadFiles(files)
|
||||
files.forEach((file) => this.uploadDocumentsService.uploadFile(file))
|
||||
})
|
||||
.catch((e) => {
|
||||
this.toastService.showError(
|
||||
@@ -144,7 +144,7 @@ export class FileDropComponent {
|
||||
})
|
||||
} else if (files.length) {
|
||||
this.toastService.showInfo($localize`Initiating upload...`, 3000)
|
||||
this.uploadDocumentsService.uploadFiles(files)
|
||||
files.forEach((file) => this.uploadDocumentsService.uploadFile(file))
|
||||
}
|
||||
|
||||
this.onDragLeave(event, true)
|
||||
|
||||
@@ -54,18 +54,6 @@ describe('UploadDocumentsService', () => {
|
||||
req[0].flush('123-456')
|
||||
})
|
||||
|
||||
it('uploads each file when given multiple files', () => {
|
||||
const uploadSpy = jest.spyOn(uploadDocumentsService, 'uploadFile')
|
||||
const file1 = new File(['one'], 'one.pdf')
|
||||
const file2 = new File(['two'], 'two.pdf')
|
||||
uploadDocumentsService.uploadFiles([file1, file2])
|
||||
expect(uploadSpy).toHaveBeenCalledWith(file1)
|
||||
expect(uploadSpy).toHaveBeenCalledWith(file2)
|
||||
httpTestingController
|
||||
.match(`${environment.apiBaseUrl}documents/post_document/`)
|
||||
.forEach((req) => req.flush('123-456'))
|
||||
})
|
||||
|
||||
it('updates progress during upload and failure', () => {
|
||||
const file = new File(
|
||||
[new Blob(['testing'], { type: 'application/pdf' })],
|
||||
|
||||
@@ -16,12 +16,6 @@ export class UploadDocumentsService {
|
||||
|
||||
private uploadSubscriptions: Array<Subscription> = []
|
||||
|
||||
public uploadFiles(files: Iterable<File>) {
|
||||
for (const file of files) {
|
||||
this.uploadFile(file)
|
||||
}
|
||||
}
|
||||
|
||||
public uploadFile(file: File) {
|
||||
let formData = new FormData()
|
||||
formData.append('document', file, file.name)
|
||||
|
||||
@@ -90,7 +90,6 @@ import {
|
||||
exclamationTriangleFill,
|
||||
eye,
|
||||
fileEarmark,
|
||||
fileEarmarkArrowUp,
|
||||
fileEarmarkCheck,
|
||||
fileEarmarkDiff,
|
||||
fileEarmarkFill,
|
||||
@@ -342,7 +341,6 @@ const icons = {
|
||||
exclamationTriangleFill,
|
||||
eye,
|
||||
fileEarmark,
|
||||
fileEarmarkArrowUp,
|
||||
fileEarmarkCheck,
|
||||
fileEarmarkDiff,
|
||||
fileEarmarkFill,
|
||||
|
||||
@@ -13,8 +13,14 @@ from celery import group
|
||||
from celery import shared_task
|
||||
from django.conf import settings
|
||||
from django.db import transaction
|
||||
from django.db.models import Case
|
||||
from django.db.models import F
|
||||
from django.db.models import Max
|
||||
from django.db.models import OuterRef
|
||||
from django.db.models import Q
|
||||
from django.db.models import Subquery
|
||||
from django.db.models import When
|
||||
from django.db.models.functions import Coalesce
|
||||
from django.utils import timezone
|
||||
|
||||
from documents.data_models import ConsumableDocument
|
||||
@@ -36,6 +42,7 @@ from documents.tasks import remove_document_from_index
|
||||
from documents.tasks import update_document_content_maybe_archive_file
|
||||
from documents.versioning import get_latest_version_for_root
|
||||
from documents.versioning import get_root_document
|
||||
from documents.versioning import versions_newest_first
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Mapping
|
||||
@@ -408,10 +415,28 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
|
||||
|
||||
Consumption workflows do not run here, so ``remote_ocr`` is how the user
|
||||
asks for the remote engine when it is not configured to handle everything.
|
||||
|
||||
A root document with versions reprocesses its latest version, which is the
|
||||
file whose content, archive and thumbnail are shown for it.
|
||||
"""
|
||||
for document_id in doc_ids:
|
||||
latest_version = versions_newest_first(
|
||||
Document.objects.filter(root_document=OuterRef("pk")),
|
||||
).values("id")[:1]
|
||||
source_ids = (
|
||||
Document.objects.filter(id__in=doc_ids)
|
||||
.annotate(
|
||||
source_id=Case(
|
||||
When(root_document__isnull=False, then=F("id")),
|
||||
default=Coalesce(Subquery(latest_version), F("id")),
|
||||
),
|
||||
)
|
||||
.order_by()
|
||||
.values_list("source_id", flat=True)
|
||||
.distinct()
|
||||
)
|
||||
for source_id in source_ids:
|
||||
update_document_content_maybe_archive_file.apply_async(
|
||||
kwargs={"document_id": document_id, "remote_ocr": remote_ocr},
|
||||
kwargs={"document_id": source_id, "remote_ocr": remote_ocr},
|
||||
headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
|
||||
)
|
||||
|
||||
|
||||
+161
-98
@@ -1,8 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -68,58 +68,6 @@ def get_supported_file_extensions() -> set[str]:
|
||||
return extensions
|
||||
|
||||
|
||||
def run_convert(
|
||||
input_file,
|
||||
output_file,
|
||||
*,
|
||||
density=None,
|
||||
scale=None,
|
||||
alpha=None,
|
||||
strip=False,
|
||||
trim=False,
|
||||
type=None,
|
||||
depth=None,
|
||||
auto_orient=False,
|
||||
use_cropbox=False,
|
||||
extra=None,
|
||||
logging_group=None,
|
||||
) -> None:
|
||||
environment = os.environ.copy()
|
||||
if settings.CONVERT_MEMORY_LIMIT:
|
||||
# MAGICK_MEMORY_LIMIT sets the maximum amount of RAM the pixel cache can use.
|
||||
# MAGICK_MAP_LIMIT sets the maximum amount of memory-mapped I/O allowed.
|
||||
#
|
||||
# For large-format documents ImageMagick will hit the RAM limit and
|
||||
# immediately try to "map" the remaining data. If MAGICK_MAP_LIMIT isn't
|
||||
# also set, the process may trigger an OOM kill because the default
|
||||
# system/policy map limit is often too restrictive for these massive bitmaps.
|
||||
environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
|
||||
environment["MAGICK_MAP_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
|
||||
if settings.CONVERT_TMPDIR:
|
||||
environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR
|
||||
|
||||
args = [settings.CONVERT_BINARY]
|
||||
args += ["-density", str(density)] if density else []
|
||||
args += ["-scale", str(scale)] if scale else []
|
||||
args += ["-alpha", str(alpha)] if alpha else []
|
||||
args += ["-strip"] if strip else []
|
||||
args += ["-trim"] if trim else []
|
||||
args += ["-type", str(type)] if type else []
|
||||
args += ["-depth", str(depth)] if depth else []
|
||||
args += ["-auto-orient"] if auto_orient else []
|
||||
args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else []
|
||||
args += [str(input_file), str(output_file)]
|
||||
|
||||
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
|
||||
|
||||
try:
|
||||
run_subprocess(args, environment, logger)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise ParseError(f"Convert failed at {args}") from e
|
||||
except Exception as e: # pragma: no cover
|
||||
raise ParseError("Unknown error running convert") from e
|
||||
|
||||
|
||||
def get_default_thumbnail() -> Path:
|
||||
"""
|
||||
Returns the path to a generic thumbnail
|
||||
@@ -127,46 +75,168 @@ def get_default_thumbnail() -> Path:
|
||||
return (Path(__file__).parent / "resources" / "document.webp").resolve()
|
||||
|
||||
|
||||
def make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group=None) -> Path:
|
||||
out_path: Path = Path(temp_dir) / "convert_gs.webp"
|
||||
_THUMBNAIL_MAX_WIDTH = 500
|
||||
_THUMBNAIL_MAX_HEIGHT = 5000
|
||||
# Used only when the page geometry cannot be read
|
||||
_THUMBNAIL_FALLBACK_DPI = 150
|
||||
# Applied before supersampling, so tiny pages are not enlarged
|
||||
_THUMBNAIL_MAX_DPI = 300
|
||||
# Rendering at a multiple and downsampling keeps text crisper
|
||||
_THUMBNAIL_SUPERSAMPLE = 2
|
||||
|
||||
# if convert fails, fall back to extracting
|
||||
# the first PDF page as a PNG using Ghostscript
|
||||
logger.warning(
|
||||
"Thumbnail generation with ImageMagick failed, falling back "
|
||||
"to ghostscript. Check your /etc/ImageMagick-x/policy.xml!",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
# Ghostscript doesn't handle WebP outputs
|
||||
gs_out_path: Path = Path(temp_dir) / "gs_out.png"
|
||||
cmd = [settings.GS_BINARY, "-q", "-sDEVICE=pngalpha", "-o", gs_out_path, in_path]
|
||||
|
||||
def rasterize_pdf_page_to_png(
|
||||
in_path: Path,
|
||||
out_path: Path,
|
||||
*,
|
||||
dpi: int,
|
||||
logging_group=None,
|
||||
) -> None:
|
||||
"""
|
||||
Rasterizes the first page of a PDF to a PNG with pdftoppm.
|
||||
"""
|
||||
# -singlefile drops the page number and -png appends ".png", so pass the
|
||||
# path without its suffix
|
||||
args = [
|
||||
"pdftoppm",
|
||||
"-f",
|
||||
"1",
|
||||
"-l",
|
||||
"1",
|
||||
"-r",
|
||||
str(dpi),
|
||||
"-png",
|
||||
"-singlefile",
|
||||
"-cropbox",
|
||||
str(in_path),
|
||||
str(out_path.with_suffix("")),
|
||||
]
|
||||
|
||||
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
|
||||
|
||||
try:
|
||||
try:
|
||||
run_subprocess(cmd, logger=logger)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise ParseError(f"Thumbnail (gs) failed at {cmd}") from e
|
||||
# then run convert on the output from gs to make WebP
|
||||
run_convert(
|
||||
density=300,
|
||||
scale="500x5000>",
|
||||
alpha="remove",
|
||||
strip=True,
|
||||
trim=False,
|
||||
auto_orient=True,
|
||||
input_file=gs_out_path,
|
||||
output_file=out_path,
|
||||
logging_group=logging_group,
|
||||
)
|
||||
run_subprocess(args, logger=logger)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise ParseError(f"pdftoppm failed at {args}") from e
|
||||
except Exception as e: # pragma: no cover
|
||||
raise ParseError("Unknown error running pdftoppm") from e
|
||||
|
||||
|
||||
def encode_thumbnail_webp(
|
||||
png_path: Path,
|
||||
out_path: Path,
|
||||
*,
|
||||
supersample: int = 1,
|
||||
) -> None:
|
||||
"""
|
||||
Flattens alpha onto white, undoes supersampling, shrinks to fit and saves as WebP.
|
||||
"""
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
with Image.open(png_path) as im:
|
||||
if im.mode in ("RGBA", "LA"):
|
||||
flattened = Image.new("RGB", im.size, (255, 255, 255))
|
||||
flattened.paste(im, mask=im.split()[-1])
|
||||
else:
|
||||
flattened = im.convert("RGB")
|
||||
|
||||
if supersample > 1:
|
||||
flattened = flattened.resize(
|
||||
(
|
||||
max(1, round(flattened.width / supersample)),
|
||||
max(1, round(flattened.height / supersample)),
|
||||
),
|
||||
Image.Resampling.LANCZOS,
|
||||
)
|
||||
|
||||
flattened.thumbnail((_THUMBNAIL_MAX_WIDTH, _THUMBNAIL_MAX_HEIGHT))
|
||||
flattened.save(out_path, format="WEBP")
|
||||
except (OSError, Image.DecompressionBombError) as e:
|
||||
raise ParseError(f"Unable to encode thumbnail from {png_path}") from e
|
||||
|
||||
|
||||
def _compute_thumbnail_dpi(in_path: Path, logging_group=None) -> tuple[int, int]:
|
||||
"""
|
||||
Returns (dpi, supersample). Unknown geometry is not supersampled, since
|
||||
the render size cannot be bounded.
|
||||
"""
|
||||
from paperless.parsers.utils import get_pdf_first_page_size_points
|
||||
|
||||
size = get_pdf_first_page_size_points(in_path)
|
||||
if size is None:
|
||||
logger.debug(
|
||||
"Could not read PDF page size, using fallback DPI",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
return _THUMBNAIL_FALLBACK_DPI, 1
|
||||
|
||||
width_pts, height_pts = size
|
||||
dpi_for_width = _THUMBNAIL_MAX_WIDTH * 72 / width_pts
|
||||
dpi_for_height = _THUMBNAIL_MAX_HEIGHT * 72 / height_pts
|
||||
# Round up so the downsampled render is never a few pixels short of the
|
||||
# target; the shrink-only clamp in encode_thumbnail_webp trims the excess.
|
||||
dpi = max(
|
||||
1,
|
||||
math.ceil(min(_THUMBNAIL_MAX_DPI, dpi_for_width, dpi_for_height)),
|
||||
)
|
||||
# At the 1 DPI floor the page is already oversized, so do not supersample
|
||||
return dpi, 1 if dpi == 1 else _THUMBNAIL_SUPERSAMPLE
|
||||
|
||||
|
||||
def _render_pdf_thumbnail(
|
||||
in_path: Path,
|
||||
png_path: Path,
|
||||
out_path: Path,
|
||||
logging_group=None,
|
||||
) -> None:
|
||||
dpi, supersample = _compute_thumbnail_dpi(in_path, logging_group=logging_group)
|
||||
rasterize_pdf_page_to_png(
|
||||
in_path,
|
||||
png_path,
|
||||
dpi=dpi * supersample,
|
||||
logging_group=logging_group,
|
||||
)
|
||||
encode_thumbnail_webp(png_path, out_path, supersample=supersample)
|
||||
|
||||
|
||||
def _repair_pdf_with_qpdf(in_path: Path, out_path: Path) -> None:
|
||||
# qpdf exits 3 after a repair; --warning-exit-0 keeps that from failing
|
||||
try:
|
||||
shutil.copy(in_path, out_path)
|
||||
run_subprocess(
|
||||
["qpdf", "--warning-exit-0", "--replace-input", str(out_path)],
|
||||
logger=logger,
|
||||
)
|
||||
except (subprocess.CalledProcessError, OSError) as e:
|
||||
raise ParseError(f"qpdf repair failed for {in_path}") from e
|
||||
|
||||
|
||||
def make_thumbnail_from_pdf_qpdf_fallback(
|
||||
in_path: Path,
|
||||
temp_dir: Path,
|
||||
logging_group=None,
|
||||
) -> Path:
|
||||
png_path = temp_dir / "page1_repaired.png"
|
||||
out_path = temp_dir / "convert_qpdf.webp"
|
||||
repaired_path = temp_dir / "repaired.pdf"
|
||||
|
||||
logger.warning(
|
||||
"Thumbnail generation with pdftoppm failed, attempting qpdf repair and retry.",
|
||||
extra={"group": logging_group},
|
||||
)
|
||||
|
||||
try:
|
||||
_repair_pdf_with_qpdf(in_path, repaired_path)
|
||||
_render_pdf_thumbnail(repaired_path, png_path, out_path, logging_group)
|
||||
return out_path
|
||||
|
||||
except ParseError as e:
|
||||
logger.error(f"Unable to make thumbnail with Ghostscript: {e}")
|
||||
logger.error(f"Unable to make thumbnail after qpdf repair: {e}")
|
||||
# The caller might expect a generated thumbnail that can be moved,
|
||||
# so we need to copy it before it gets moved.
|
||||
# https://github.com/paperless-ngx/paperless-ngx/issues/3631
|
||||
default_thumbnail_path: Path = Path(temp_dir) / "document.webp"
|
||||
default_thumbnail_path = temp_dir / "document.webp"
|
||||
copy_file_with_basic_stats(get_default_thumbnail(), default_thumbnail_path)
|
||||
return default_thumbnail_path
|
||||
|
||||
@@ -175,25 +245,18 @@ def make_thumbnail_from_pdf(in_path: Path, temp_dir: Path, logging_group=None) -
|
||||
"""
|
||||
The thumbnail of a PDF is just a 500px wide image of the first page.
|
||||
"""
|
||||
png_path: Path = temp_dir / "page1.png"
|
||||
out_path: Path = temp_dir / "convert.webp"
|
||||
|
||||
# Run convert to get a decent thumbnail
|
||||
try:
|
||||
run_convert(
|
||||
density=300,
|
||||
scale="500x5000>",
|
||||
alpha="remove",
|
||||
strip=True,
|
||||
trim=False,
|
||||
auto_orient=True,
|
||||
use_cropbox=True,
|
||||
input_file=f"{in_path}[0]",
|
||||
output_file=str(out_path),
|
||||
logging_group=logging_group,
|
||||
)
|
||||
_render_pdf_thumbnail(in_path, png_path, out_path, logging_group)
|
||||
except ParseError as e:
|
||||
logger.error(f"Unable to make thumbnail with convert: {e}")
|
||||
out_path = make_thumbnail_from_pdf_gs_fallback(in_path, temp_dir, logging_group)
|
||||
logger.error(f"Unable to make thumbnail with pdftoppm: {e}")
|
||||
out_path = make_thumbnail_from_pdf_qpdf_fallback(
|
||||
in_path,
|
||||
temp_dir,
|
||||
logging_group,
|
||||
)
|
||||
|
||||
return out_path
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ from django.db.models import QuerySet
|
||||
from django.db.models import Value
|
||||
from django.db.models import When
|
||||
from django.db.models.functions import Cast
|
||||
from django.db.models.functions import Coalesce
|
||||
from guardian.core import ObjectPermissionChecker
|
||||
from guardian.models import GroupObjectPermission
|
||||
from guardian.models import UserObjectPermission
|
||||
@@ -474,6 +475,29 @@ def permitted_document_ids(
|
||||
return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
|
||||
|
||||
|
||||
def documents_without_permitted_root(
|
||||
documents: QuerySet[Document],
|
||||
user: User | None,
|
||||
*,
|
||||
perm: str = "view_document",
|
||||
include_deleted: bool = False,
|
||||
) -> QuerySet[Document]:
|
||||
"""
|
||||
The documents the user lacks ``perm`` on. Versions are authorized by their
|
||||
root document, so a version's own owner is ignored. A single query, without
|
||||
loading the documents or joining the root.
|
||||
"""
|
||||
return documents.annotate(
|
||||
root_id=Coalesce("root_document_id", "id"),
|
||||
).exclude(
|
||||
root_id__in=permitted_document_ids(
|
||||
user,
|
||||
perm=perm,
|
||||
include_deleted=include_deleted,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_document_count_filter_for_user(user, related_name: str = "documents"):
|
||||
"""
|
||||
Return the Q object used to filter document counts for the given user.
|
||||
|
||||
@@ -7,6 +7,7 @@ import threading
|
||||
import time
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from enum import StrEnum
|
||||
from itertools import islice
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -55,6 +56,19 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger("paperless.search")
|
||||
|
||||
# tantivy stores dates as signed 64-bit nanoseconds since the Unix epoch, which
|
||||
# covers 1677-09-21T00:12:43 to 2262-04-11T23:47:16 UTC
|
||||
_INDEX_DATE_NANOS_MIN: Final[int] = -(2**63)
|
||||
_INDEX_DATE_NANOS_MAX: Final[int] = 2**63 - 1
|
||||
_UNIX_EPOCH: Final[datetime] = datetime(1970, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def _is_indexable_date(value: datetime) -> bool:
|
||||
"""Whether value, at whole-second precision, fits tantivy's date range."""
|
||||
nanos = ((value - _UNIX_EPOCH) // timedelta(seconds=1)) * 1_000_000_000
|
||||
return _INDEX_DATE_NANOS_MIN <= nanos <= _INDEX_DATE_NANOS_MAX
|
||||
|
||||
|
||||
_LOCK_TIMEOUT_SECONDS: Final[float] = 10.0 # per-attempt acquire timeout
|
||||
_LOCK_RETRY_ATTEMPTS: Final[int] = 4 # total attempts (1 initial + 3 retries)
|
||||
_LOCK_BACKOFF_BASE: Final[float] = 1.0 # seconds
|
||||
@@ -628,7 +642,15 @@ class TantivyBackend:
|
||||
document.created.day,
|
||||
tzinfo=UTC,
|
||||
)
|
||||
doc.add_date("created", created_date)
|
||||
if _is_indexable_date(created_date):
|
||||
doc.add_date("created", created_date)
|
||||
else:
|
||||
logger.warning(
|
||||
"Document %s has a created date (%s) outside the range the search "
|
||||
"index can store; it will be indexed without a created date",
|
||||
document.pk,
|
||||
document.created,
|
||||
)
|
||||
doc.add_date("modified", document.modified)
|
||||
doc.add_date("added", document.added)
|
||||
|
||||
|
||||
@@ -90,6 +90,7 @@ from documents.templating.utils import convert_format_str_to_template_format
|
||||
from documents.templating.workflows import validate_workflow_template
|
||||
from documents.validators import uri_validator
|
||||
from documents.validators import url_validator
|
||||
from documents.versioning import get_root_document
|
||||
from documents.versioning import has_prefetched_effective_content
|
||||
from documents.versioning import sort_versions_newest_first
|
||||
|
||||
@@ -2894,7 +2895,7 @@ class ShareLinkSerializer(OwnedObjectSerializer):
|
||||
and has_perms_owner_aware(
|
||||
self.user,
|
||||
"view_document",
|
||||
document,
|
||||
get_root_document(document),
|
||||
)
|
||||
):
|
||||
return document
|
||||
|
||||
@@ -490,18 +490,23 @@ def update_document_content_maybe_archive_file(
|
||||
shutil.move(thumbnail, document.thumbnail_path)
|
||||
|
||||
document.refresh_from_db()
|
||||
root_document = (
|
||||
document.root_document if document.root_document_id else document
|
||||
)
|
||||
logger.info(
|
||||
f"Updating index for document {document_id} ({document.archive_checksum})",
|
||||
f"Updating index for document {root_document.pk} ({document.archive_checksum})",
|
||||
)
|
||||
from documents.search import get_backend
|
||||
|
||||
get_backend().add_or_update(document)
|
||||
get_backend().add_or_update(root_document)
|
||||
|
||||
ai_config = AIConfig()
|
||||
if ai_config.llm_index_enabled:
|
||||
llm_index_add_or_update_document(document)
|
||||
llm_index_add_or_update_document(root_document)
|
||||
|
||||
clear_document_caches(document.pk)
|
||||
if root_document.pk != document.pk:
|
||||
clear_document_caches(root_document.pk)
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import json
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -904,6 +906,79 @@ class TestRebuild:
|
||||
assert ids == [doc.pk]
|
||||
|
||||
|
||||
class TestCreatedDateOutOfRange:
|
||||
"""The index stores dates as nanosecond i64 values (1677-09-22 to 2262-04-11).
|
||||
|
||||
A document whose created date falls outside that window must not abort
|
||||
indexing: it is indexed without a created value and a warning names it.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("created", "expected_warnings"),
|
||||
[
|
||||
pytest.param(date(1677, 9, 22), 0, id="first-representable-day"),
|
||||
pytest.param(date(2262, 4, 11), 0, id="last-representable-day"),
|
||||
pytest.param(date(1677, 9, 21), 1, id="day-before-first"),
|
||||
pytest.param(date(2262, 4, 12), 1, id="day-after-last"),
|
||||
pytest.param(date(16, 8, 30), 1, id="two-digit-year-read-as-year-16"),
|
||||
pytest.param(date(9999, 12, 31), 1, id="max-python-date"),
|
||||
],
|
||||
)
|
||||
def test_add_or_update_indexes_document_and_warns_when_out_of_range(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
created: date,
|
||||
expected_warnings: int,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document with a created date at or beyond the index date limits
|
||||
WHEN:
|
||||
- The document is added to the index
|
||||
THEN:
|
||||
- The document is indexed and searchable either way
|
||||
- A warning naming the document is logged only for out-of-range dates
|
||||
"""
|
||||
doc = DocumentFactory(created=created, content="boundarycontent")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="paperless.search"):
|
||||
backend.add_or_update(doc)
|
||||
|
||||
assert backend.search_ids("boundarycontent", user=None) == [doc.pk]
|
||||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
||||
assert len(warnings) == expected_warnings
|
||||
if expected_warnings:
|
||||
assert f"Document {doc.pk}" in warnings[0].getMessage()
|
||||
|
||||
def test_rebuild_continues_past_out_of_range_document(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document with an unrepresentable created date among valid ones
|
||||
WHEN:
|
||||
- The index is rebuilt
|
||||
THEN:
|
||||
- Rebuild completes and every document is searchable
|
||||
- A warning names the offending document
|
||||
"""
|
||||
good = DocumentFactory(created=date(2016, 8, 30), content="rebuildcontent")
|
||||
bad = DocumentFactory(created=date(16, 8, 30), content="rebuildcontent")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="paperless.search"):
|
||||
backend.rebuild(Document.objects.all())
|
||||
|
||||
assert sorted(backend.search_ids("rebuildcontent", user=None)) == sorted(
|
||||
[good.pk, bad.pk],
|
||||
)
|
||||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
||||
assert len(warnings) == 1
|
||||
assert f"Document {bad.pk}" in warnings[0].getMessage()
|
||||
|
||||
|
||||
class TestAutocomplete:
|
||||
"""Test autocomplete functionality."""
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from auditlog.models import LogEntry # type: ignore[import-untyped]
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.test import TestCase as DjangoTestCase
|
||||
from django.test import override_settings
|
||||
from django.utils import timezone
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
@@ -16,13 +17,17 @@ from documents.data_models import DocumentSource
|
||||
from documents.filters import EffectiveContentFilter
|
||||
from documents.filters import TitleContentFilter
|
||||
from documents.models import Document
|
||||
from documents.models import Note
|
||||
from documents.models import ShareLink
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.views import DocumentSelectionMixin
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.http import read_streaming_response
|
||||
from paperless_testing.permissions import grant_all_global
|
||||
from paperless_testing.permissions import grant_global
|
||||
from paperless_testing.permissions import grant_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
@@ -1043,3 +1048,152 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(selected, [root.id])
|
||||
|
||||
|
||||
class TestVersionActionPermissions(DirectoriesMixin, APITestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.user = UserFactory()
|
||||
grant_all_global(self.user)
|
||||
self.client.force_authenticate(self.user)
|
||||
self.root = DocumentFactory(owner=UserFactory())
|
||||
self.version = DocumentFactory(root_document=self.root, owner=None)
|
||||
|
||||
@override_settings(AUDIT_LOG_ENABLED=True)
|
||||
def test_actions_reject_stale_version_ownership(self):
|
||||
note = Note.objects.create(document=self.version, note="Version note")
|
||||
for owner in (None, self.user):
|
||||
self.version.owner = owner
|
||||
self.version.save(update_fields=["owner"])
|
||||
for action in (
|
||||
"notes",
|
||||
"suggestions",
|
||||
"ai_suggestions",
|
||||
"history",
|
||||
"share_links",
|
||||
):
|
||||
with self.subTest(owner=owner, action=action):
|
||||
response = self.client.get(
|
||||
f"/api/documents/{self.version.pk}/{action}/",
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
response = self.client.post(
|
||||
f"/api/documents/{self.version.pk}/notes/",
|
||||
{"note": "New note"},
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
response = self.client.delete(
|
||||
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
response = self.client.post(
|
||||
"/api/share_links/",
|
||||
{"document": self.version.pk, "file_version": "original"},
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
response = self.client.post(
|
||||
"/api/share_link_bundles/",
|
||||
{"document_ids": [self.version.pk], "file_version": "original"},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, 400)
|
||||
response = self.client.post(
|
||||
"/api/documents/email/",
|
||||
{
|
||||
"documents": [self.version.pk],
|
||||
"addresses": "recipient@example.com",
|
||||
"subject": "Version",
|
||||
"message": "Version",
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
with (
|
||||
mock.patch("documents.views.AIConfig") as ai_config,
|
||||
mock.patch("documents.views.stream_chat_with_documents") as chat,
|
||||
):
|
||||
ai_config.return_value.ai_enabled = True
|
||||
response = self.client.post(
|
||||
"/api/documents/chat/",
|
||||
{"q": "Version?", "document_id": self.version.pk},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
chat.assert_not_called()
|
||||
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
|
||||
self.assertFalse(ShareLink.objects.exists())
|
||||
|
||||
@mock.patch("documents.views.build_share_link_bundle.apply_async")
|
||||
def test_root_permissions_allow_sharing_a_private_version(self, build_mock):
|
||||
self.version.owner = UserFactory()
|
||||
self.version.save(update_fields=["owner"])
|
||||
grant_object(self.user, self.root, "view_document", "change_document")
|
||||
note = Note.objects.create(document=self.version, note="Version note")
|
||||
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(response.data[0]["id"], note.pk)
|
||||
response = self.client.post(
|
||||
"/api/share_links/",
|
||||
{"document": self.version.pk, "file_version": "original"},
|
||||
)
|
||||
self.assertEqual(response.status_code, 201)
|
||||
self.assertEqual(ShareLink.objects.get().document_id, self.version.pk)
|
||||
response = self.client.get(f"/api/documents/{self.version.pk}/share_links/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(len(response.data), 1)
|
||||
response = self.client.post(
|
||||
"/api/share_link_bundles/",
|
||||
{"document_ids": [self.version.pk], "file_version": "original"},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, 201)
|
||||
build_mock.assert_called_once()
|
||||
|
||||
def test_root_view_permission_does_not_allow_note_changes(self):
|
||||
grant_object(self.user, self.root, "view_document")
|
||||
note = Note.objects.create(document=self.version, note="Version note")
|
||||
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
response = self.client.post(
|
||||
f"/api/documents/{self.version.pk}/notes/",
|
||||
{"note": "New note"},
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
response = self.client.delete(
|
||||
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
|
||||
|
||||
@override_settings(AUDIT_LOG_ENABLED=True)
|
||||
def test_history_uses_root_ownership(self):
|
||||
self.root.owner = self.user
|
||||
self.root.save(update_fields=["owner"])
|
||||
self.version.owner = UserFactory()
|
||||
self.version.save(update_fields=["owner"])
|
||||
response = self.client.get(f"/api/documents/{self.version.pk}/history/")
|
||||
self.assertEqual(response.status_code, 200)
|
||||
|
||||
def test_selection_data_rejects_stale_version_ownership(self):
|
||||
for owner in (None, self.user):
|
||||
self.version.owner = owner
|
||||
self.version.save(update_fields=["owner"])
|
||||
with self.subTest(owner=owner):
|
||||
response = self.client.post(
|
||||
"/api/documents/selection_data/",
|
||||
{"documents": [self.version.pk]},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
|
||||
def test_selection_data_allows_private_version_of_permitted_root(self):
|
||||
self.version.owner = UserFactory()
|
||||
self.version.save(update_fields=["owner"])
|
||||
grant_object(self.user, self.root, "view_document")
|
||||
other = DocumentFactory(owner=self.user)
|
||||
response = self.client.post(
|
||||
"/api/documents/selection_data/",
|
||||
{"documents": [self.version.pk, other.pk]},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
@@ -6,6 +6,7 @@ from rest_framework.test import APITestCase
|
||||
|
||||
from documents.models import Document
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.permissions import grant_all_global
|
||||
|
||||
@@ -279,3 +280,53 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
|
||||
Document.objects.filter(root_document=root).values_list("id", flat=True),
|
||||
[version.pk for version in versions],
|
||||
)
|
||||
|
||||
def test_api_trash_version_follows_root_owner(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A deleted version of user2's document, owned by nobody
|
||||
- A deleted version of the user's document, owned by user2
|
||||
WHEN:
|
||||
- The user lists the trash and tries to restore or empty the versions
|
||||
THEN:
|
||||
- Only the version of the user's own document is listed
|
||||
- The other version can't be restored or emptied
|
||||
- The version of the user's own document can be restored
|
||||
"""
|
||||
user2 = UserFactory(username="user2")
|
||||
other_version = DocumentFactory(
|
||||
root_document=DocumentFactory(owner=user2),
|
||||
version_index=1,
|
||||
)
|
||||
other_version.delete()
|
||||
own_version = DocumentFactory(
|
||||
owner=user2,
|
||||
root_document=DocumentFactory(owner=self.user),
|
||||
version_index=1,
|
||||
)
|
||||
own_version.delete()
|
||||
|
||||
resp = self.client.get("/api/trash/")
|
||||
self.assertEqual(resp.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(
|
||||
[doc["id"] for doc in resp.data["results"]],
|
||||
[own_version.pk],
|
||||
)
|
||||
|
||||
for action in ("restore", "empty"):
|
||||
with self.subTest(action=action):
|
||||
resp = self.client.post(
|
||||
"/api/trash/",
|
||||
{"action": action, "documents": [other_version.pk]},
|
||||
)
|
||||
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
|
||||
self.assertTrue(
|
||||
Document.deleted_objects.filter(pk=other_version.pk).exists(),
|
||||
)
|
||||
|
||||
resp = self.client.post(
|
||||
"/api/trash/",
|
||||
{"action": "restore", "documents": [own_version.pk]},
|
||||
)
|
||||
self.assertEqual(resp.status_code, status.HTTP_200_OK)
|
||||
self.assertTrue(Document.objects.filter(pk=own_version.pk).exists())
|
||||
@@ -4,6 +4,7 @@ from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pikepdf
|
||||
import pytest
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import Permission
|
||||
from django.contrib.auth.models import User
|
||||
@@ -12,6 +13,7 @@ from django.test import TestCase
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from guardian.shortcuts import get_groups_with_perms
|
||||
from guardian.shortcuts import get_users_with_perms
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from documents import bulk_edit
|
||||
from documents.models import Correspondent
|
||||
@@ -23,6 +25,7 @@ from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import set_permissions_for_objects
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.permissions import grant_object
|
||||
|
||||
|
||||
@@ -1970,18 +1973,22 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
self.assertIn("Error removing password from document", cm.output[0])
|
||||
|
||||
|
||||
class TestBulkEditReprocess(DirectoriesMixin, TestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
|
||||
self.doc = Document.objects.create(
|
||||
title="test",
|
||||
checksum="A",
|
||||
mime_type="application/pdf",
|
||||
@pytest.mark.django_db
|
||||
class TestBulkEditReprocess:
|
||||
@pytest.fixture
|
||||
def mock_task(self, mocker: MockerFixture) -> mock.MagicMock:
|
||||
return mocker.patch(
|
||||
"documents.bulk_edit.update_document_content_maybe_archive_file",
|
||||
)
|
||||
|
||||
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
|
||||
def test_reprocess_defaults_to_local(self, mock_task: mock.Mock) -> None:
|
||||
@staticmethod
|
||||
def _queued_ids(mock_task: mock.MagicMock) -> list[int]:
|
||||
return [
|
||||
call.kwargs["kwargs"]["document_id"]
|
||||
for call in mock_task.apply_async.call_args_list
|
||||
]
|
||||
|
||||
def test_reprocess_defaults_to_local(self, mock_task: mock.MagicMock) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A reprocess request that says nothing about remote OCR
|
||||
@@ -1990,18 +1997,17 @@ class TestBulkEditReprocess(DirectoriesMixin, TestCase):
|
||||
THEN:
|
||||
- The task is queued without asking for the remote engine
|
||||
"""
|
||||
result = bulk_edit.reprocess([self.doc.id])
|
||||
doc = DocumentFactory()
|
||||
|
||||
assert bulk_edit.reprocess([doc.id]) == "OK"
|
||||
|
||||
self.assertEqual(result, "OK")
|
||||
mock_task.apply_async.assert_called_once()
|
||||
_, kwargs = mock_task.apply_async.call_args
|
||||
self.assertEqual(
|
||||
kwargs["kwargs"],
|
||||
{"document_id": self.doc.id, "remote_ocr": False},
|
||||
)
|
||||
assert mock_task.apply_async.call_args.kwargs["kwargs"] == {
|
||||
"document_id": doc.id,
|
||||
"remote_ocr": False,
|
||||
}
|
||||
|
||||
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
|
||||
def test_reprocess_passes_remote_ocr(self, mock_task: mock.Mock) -> None:
|
||||
def test_reprocess_passes_remote_ocr(self, mock_task: mock.MagicMock) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A reprocess request that explicitly asks for remote OCR
|
||||
@@ -2010,14 +2016,72 @@ class TestBulkEditReprocess(DirectoriesMixin, TestCase):
|
||||
THEN:
|
||||
- The request is forwarded to the task for every document
|
||||
"""
|
||||
other = Document.objects.create(
|
||||
title="test2",
|
||||
checksum="B",
|
||||
mime_type="application/pdf",
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
|
||||
bulk_edit.reprocess([doc.id for doc in docs], remote_ocr=True)
|
||||
|
||||
assert mock_task.apply_async.call_count == 2
|
||||
for call in mock_task.apply_async.call_args_list:
|
||||
assert call.kwargs["kwargs"]["remote_ocr"]
|
||||
|
||||
def test_reprocess_root_uses_latest_version(
|
||||
self,
|
||||
mock_task: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with two versions
|
||||
WHEN:
|
||||
- reprocess is called with the root document
|
||||
THEN:
|
||||
- The latest version is reprocessed, not the root's original file
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
DocumentFactory(root_document=root, version_index=1)
|
||||
latest = DocumentFactory(root_document=root, version_index=2)
|
||||
|
||||
bulk_edit.reprocess([root.id])
|
||||
|
||||
assert self._queued_ids(mock_task) == [latest.id]
|
||||
|
||||
def test_reprocess_explicit_version(self, mock_task: mock.MagicMock) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with two versions
|
||||
WHEN:
|
||||
- reprocess is called with the older version
|
||||
THEN:
|
||||
- That version is reprocessed
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
older = DocumentFactory(root_document=root, version_index=1)
|
||||
DocumentFactory(root_document=root, version_index=2)
|
||||
|
||||
bulk_edit.reprocess([older.id])
|
||||
|
||||
assert self._queued_ids(mock_task) == [older.id]
|
||||
|
||||
def test_reprocess_root_and_latest_version_dispatches_once(
|
||||
self,
|
||||
mock_task: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with two versions, the latest created on a
|
||||
different date than the root
|
||||
WHEN:
|
||||
- reprocess is called with both the root and its latest version
|
||||
THEN:
|
||||
- The latest version is reprocessed only once
|
||||
"""
|
||||
root = DocumentFactory(created=date(2024, 1, 1))
|
||||
DocumentFactory(root_document=root, version_index=1)
|
||||
latest = DocumentFactory(
|
||||
root_document=root,
|
||||
version_index=2,
|
||||
created=date(2025, 1, 1),
|
||||
)
|
||||
|
||||
bulk_edit.reprocess([self.doc.id, other.id], remote_ocr=True)
|
||||
bulk_edit.reprocess([root.id, latest.id])
|
||||
|
||||
self.assertEqual(mock_task.apply_async.call_count, 2)
|
||||
for call in mock_task.apply_async.call_args_list:
|
||||
self.assertTrue(call.kwargs["kwargs"]["remote_ocr"])
|
||||
assert self._queued_ids(mock_task) == [latest.id]
|
||||
@@ -1,11 +1,22 @@
|
||||
import subprocess
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
import pikepdf
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from pytest_django.fixtures import Settings
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from documents.parsers import ParseError
|
||||
from documents.parsers import _compute_thumbnail_dpi
|
||||
from documents.parsers import encode_thumbnail_webp
|
||||
from documents.parsers import get_default_file_extension
|
||||
from documents.parsers import get_default_thumbnail
|
||||
from documents.parsers import get_supported_file_extensions
|
||||
from documents.parsers import is_file_ext_supported
|
||||
from documents.parsers import make_thumbnail_from_pdf
|
||||
from documents.parsers import rasterize_pdf_page_to_png
|
||||
from paperless.parsers.registry import get_parser_registry
|
||||
from paperless.parsers.registry import reset_parser_registry
|
||||
from paperless.parsers.tesseract import RasterisedDocumentParser
|
||||
@@ -125,3 +136,374 @@ class TestParserAvailability:
|
||||
assert is_file_ext_supported(".pdf")
|
||||
assert not is_file_ext_supported(".hsdfh")
|
||||
assert not is_file_ext_supported("")
|
||||
|
||||
|
||||
class TestComputeThumbnailDpi:
|
||||
@pytest.mark.parametrize(
|
||||
("size", "expected"),
|
||||
[
|
||||
pytest.param((612.0, 792.0), (59, 2), id="letter-width-bound"),
|
||||
pytest.param((792.0, 612.0), (46, 2), id="landscape-rounded-up"),
|
||||
pytest.param((612.0, 100000.0), (4, 2), id="tall-strip-height-bound"),
|
||||
pytest.param((200.0, 300.0), (180, 2), id="small-page-width-bound"),
|
||||
pytest.param((72.0, 72.0), (300, 2), id="tiny-page-capped-at-300"),
|
||||
pytest.param(
|
||||
(1000000.0, 1000000.0),
|
||||
(1, 1),
|
||||
id="huge-page-minimum-one-unsupersampled",
|
||||
),
|
||||
pytest.param(None, (150, 1), id="unreadable-geometry-fallback"),
|
||||
],
|
||||
)
|
||||
def test_dpi_from_page_size(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tmp_path: Path,
|
||||
size: tuple[float, float] | None,
|
||||
expected: tuple[int, int],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A first page of the given size, or unreadable geometry
|
||||
WHEN:
|
||||
- The thumbnail DPI is computed
|
||||
THEN:
|
||||
- The expected DPI and supersample factor are returned
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.parsers.utils.get_pdf_first_page_size_points",
|
||||
return_value=size,
|
||||
)
|
||||
assert _compute_thumbnail_dpi(tmp_path / "doc.pdf") == expected
|
||||
|
||||
|
||||
class TestMakeThumbnailFromPdf:
|
||||
@pytest.fixture
|
||||
def work_dir(self, tmp_path: Path) -> Path:
|
||||
path = tmp_path / "work"
|
||||
path.mkdir()
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _write_blank_pdf(path: Path, page_size: tuple[int, int] = (612, 792)) -> Path:
|
||||
pdf = pikepdf.new()
|
||||
pdf.add_blank_page(page_size=page_size)
|
||||
pdf.save(path, object_stream_mode=pikepdf.ObjectStreamMode.disable)
|
||||
return path
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("size", "expected_dpi", "expected_supersample"),
|
||||
[
|
||||
pytest.param((612.0, 792.0), 118, 2, id="known-geometry-2x"),
|
||||
pytest.param(None, 150, 1, id="unreadable-geometry-plain-fallback"),
|
||||
],
|
||||
)
|
||||
def test_render_dpi_requested(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tmp_path: Path,
|
||||
work_dir: Path,
|
||||
size: tuple[float, float] | None,
|
||||
expected_dpi: int,
|
||||
expected_supersample: int,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Readable or unreadable page geometry
|
||||
WHEN:
|
||||
- A thumbnail is made
|
||||
THEN:
|
||||
- Rasterize and encode get the matching DPI and supersample factor
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.parsers.utils.get_pdf_first_page_size_points",
|
||||
return_value=size,
|
||||
)
|
||||
rasterize = mocker.patch("documents.parsers.rasterize_pdf_page_to_png")
|
||||
encode = mocker.patch("documents.parsers.encode_thumbnail_webp")
|
||||
|
||||
make_thumbnail_from_pdf(tmp_path / "in.pdf", work_dir)
|
||||
|
||||
assert rasterize.call_args.kwargs["dpi"] == expected_dpi
|
||||
assert encode.call_args.kwargs["supersample"] == expected_supersample
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_size", "expected_width"),
|
||||
[
|
||||
pytest.param((612, 792), 500, id="letter"),
|
||||
pytest.param((792, 612), 500, id="landscape-letter"),
|
||||
pytest.param((595, 842), 500, id="a4"),
|
||||
pytest.param((200, 300), 500, id="small-page"),
|
||||
pytest.param((72, 72), 300, id="tiny-page-capped"),
|
||||
],
|
||||
)
|
||||
def test_thumbnail_width(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
work_dir: Path,
|
||||
page_size: tuple[int, int],
|
||||
expected_width: int,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF whose first page has the given size in points
|
||||
WHEN:
|
||||
- A thumbnail is made from it
|
||||
THEN:
|
||||
- The thumbnail has the expected width
|
||||
"""
|
||||
pdf_path = self._write_blank_pdf(tmp_path / "in.pdf", page_size)
|
||||
|
||||
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
|
||||
|
||||
assert thumb == work_dir / "convert.webp"
|
||||
with Image.open(thumb) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.width == expected_width
|
||||
|
||||
@classmethod
|
||||
def _write_pdf_without_xref(cls, path: Path) -> Path:
|
||||
"""
|
||||
Cuts off the xref and trailer, which pdftoppm cannot recover from but qpdf can.
|
||||
"""
|
||||
cls._write_blank_pdf(path)
|
||||
data = path.read_bytes()
|
||||
path.write_bytes(data[: data.rindex(b"\nxref")])
|
||||
return path
|
||||
|
||||
def test_qpdf_repair_produces_real_thumbnail(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
work_dir: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF with its xref table and trailer cut off
|
||||
WHEN:
|
||||
- A thumbnail is made from it
|
||||
THEN:
|
||||
- The thumbnail is rendered from a qpdf repaired copy
|
||||
- The original file is unchanged
|
||||
"""
|
||||
pdf_path = self._write_pdf_without_xref(tmp_path / "broken.pdf")
|
||||
original_bytes = pdf_path.read_bytes()
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
rasterize_pdf_page_to_png(pdf_path, work_dir / "probe.png", dpi=50)
|
||||
|
||||
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
|
||||
|
||||
assert thumb == work_dir / "convert_qpdf.webp"
|
||||
with Image.open(thumb) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.width == 500
|
||||
assert pdf_path.read_bytes() == original_bytes
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"qpdf_error",
|
||||
[
|
||||
pytest.param(subprocess.CalledProcessError(2, "qpdf"), id="qpdf-fails"),
|
||||
pytest.param(None, id="repaired-still-unrenderable"),
|
||||
],
|
||||
)
|
||||
def test_double_failure_uses_default_thumbnail(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tmp_path: Path,
|
||||
work_dir: Path,
|
||||
qpdf_error: subprocess.CalledProcessError | None,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF that cannot be rendered, even after qpdf repair
|
||||
WHEN:
|
||||
- A thumbnail is made from it
|
||||
THEN:
|
||||
- A copy of the default thumbnail is returned
|
||||
"""
|
||||
mocker.patch(
|
||||
"documents.parsers.rasterize_pdf_page_to_png",
|
||||
side_effect=ParseError("Does not compute."),
|
||||
)
|
||||
if qpdf_error is not None:
|
||||
mocker.patch("documents.parsers.run_subprocess", side_effect=qpdf_error)
|
||||
pdf_path = self._write_blank_pdf(tmp_path / "in.pdf")
|
||||
|
||||
thumb = make_thumbnail_from_pdf(pdf_path, work_dir)
|
||||
|
||||
assert thumb == work_dir / "document.webp"
|
||||
assert thumb.read_bytes() == get_default_thumbnail().read_bytes()
|
||||
|
||||
|
||||
class TestRasterizePdfPageToPng:
|
||||
@staticmethod
|
||||
def _write_pdf(
|
||||
path: Path,
|
||||
*,
|
||||
crop_box: tuple[float, float, float, float] | None = None,
|
||||
rotate: int | None = None,
|
||||
) -> Path:
|
||||
pdf = pikepdf.new()
|
||||
pdf.add_blank_page(page_size=(144, 72))
|
||||
pdf.add_blank_page(page_size=(300, 300))
|
||||
page = pdf.pages[0]
|
||||
if crop_box is not None:
|
||||
page.obj.CropBox = pikepdf.Array(crop_box)
|
||||
if rotate is not None:
|
||||
page.obj.Rotate = rotate
|
||||
pdf.save(path)
|
||||
return path
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("crop_box", "rotate", "expected_size"),
|
||||
[
|
||||
pytest.param(None, None, (144, 72), id="plain"),
|
||||
pytest.param(None, 90, (72, 144), id="rotated-90"),
|
||||
pytest.param((0, 0, 72, 36), None, (72, 36), id="crop-box"),
|
||||
],
|
||||
)
|
||||
def test_renders_first_page_to_exact_path(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
crop_box: tuple[float, float, float, float] | None,
|
||||
rotate: int | None,
|
||||
expected_size: tuple[int, int],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A two page PDF, first page optionally cropped or rotated
|
||||
WHEN:
|
||||
- The first page is rasterized at 72 DPI
|
||||
THEN:
|
||||
- Only out_path is written, sized to the first page's crop and rotation
|
||||
"""
|
||||
pdf_path = self._write_pdf(
|
||||
tmp_path / "in.pdf",
|
||||
crop_box=crop_box,
|
||||
rotate=rotate,
|
||||
)
|
||||
out_dir = tmp_path / "out"
|
||||
out_dir.mkdir()
|
||||
out_path = out_dir / "page1.png"
|
||||
|
||||
rasterize_pdf_page_to_png(pdf_path, out_path, dpi=72)
|
||||
|
||||
assert list(out_dir.iterdir()) == [out_path]
|
||||
with Image.open(out_path) as im:
|
||||
assert im.format == "PNG"
|
||||
assert im.size == expected_size
|
||||
|
||||
def test_failure_raises_parse_error(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A file that is not a PDF
|
||||
WHEN:
|
||||
- Rasterization is attempted
|
||||
THEN:
|
||||
- A ParseError is raised
|
||||
"""
|
||||
bad = tmp_path / "bad.pdf"
|
||||
bad.write_bytes(b"not a pdf")
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
rasterize_pdf_page_to_png(bad, tmp_path / "page1.png", dpi=72)
|
||||
|
||||
|
||||
class TestEncodeThumbnailWebp:
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "color"),
|
||||
[
|
||||
pytest.param("RGBA", (0, 0, 0, 0), id="rgba"),
|
||||
pytest.param("LA", (0, 0), id="la"),
|
||||
],
|
||||
)
|
||||
def test_alpha_flattened_onto_white(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
mode: str,
|
||||
color: tuple[int, ...],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A fully transparent PNG with an alpha channel
|
||||
WHEN:
|
||||
- It is encoded as a thumbnail
|
||||
THEN:
|
||||
- The WebP output is RGB with the transparency flattened to white
|
||||
"""
|
||||
png_path = tmp_path / "in.png"
|
||||
Image.new(mode, (20, 10), color).save(png_path)
|
||||
out_path = tmp_path / "out.webp"
|
||||
|
||||
encode_thumbnail_webp(png_path, out_path)
|
||||
|
||||
with Image.open(out_path) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.mode == "RGB"
|
||||
assert im.size == (20, 10)
|
||||
red, green, blue = im.getpixel((10, 5))
|
||||
assert min(red, green, blue) >= 250
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("in_size", "supersample", "expected_size"),
|
||||
[
|
||||
pytest.param((1000, 2000), 1, (500, 1000), id="too-wide-shrunk"),
|
||||
pytest.param((100, 10000), 1, (50, 5000), id="too-tall-shrunk"),
|
||||
pytest.param((100, 200), 1, (100, 200), id="small-not-enlarged"),
|
||||
pytest.param((1000, 1400), 2, (500, 700), id="2x-halved"),
|
||||
pytest.param((1001, 1401), 2, (500, 700), id="2x-odd-rounded"),
|
||||
pytest.param((600, 800), 2, (300, 400), id="2x-small-not-enlarged"),
|
||||
pytest.param((900, 1200), 2, (450, 600), id="2x-below-clamp"),
|
||||
],
|
||||
)
|
||||
def test_size_clamped_and_downsampled(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
in_size: tuple[int, int],
|
||||
supersample: int,
|
||||
expected_size: tuple[int, int],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A rendered image and its supersample factor
|
||||
WHEN:
|
||||
- It is encoded as a thumbnail with that factor
|
||||
THEN:
|
||||
- It is downsampled, fit within 500x5000 and never enlarged
|
||||
"""
|
||||
png_path = tmp_path / "in.png"
|
||||
Image.new("RGB", in_size, (255, 255, 255)).save(png_path)
|
||||
out_path = tmp_path / "out.webp"
|
||||
|
||||
encode_thumbnail_webp(png_path, out_path, supersample=supersample)
|
||||
|
||||
with Image.open(out_path) as im:
|
||||
assert im.size == expected_size
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
pytest.param(OSError("broken image"), id="os-error"),
|
||||
pytest.param(Image.DecompressionBombError("too large"), id="bomb"),
|
||||
],
|
||||
)
|
||||
def test_decode_failure_raises_parse_error(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
mocker: MockerFixture,
|
||||
error: Exception,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Opening the rendered image fails
|
||||
WHEN:
|
||||
- It is encoded as a thumbnail
|
||||
THEN:
|
||||
- A ParseError is raised
|
||||
"""
|
||||
png_path = tmp_path / "in.png"
|
||||
Image.new("RGB", (10, 10)).save(png_path)
|
||||
mocker.patch("PIL.Image.open", side_effect=error)
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
encode_thumbnail_webp(png_path, tmp_path / "out.webp")
|
||||
@@ -90,10 +90,13 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn("document_ids", response.data)
|
||||
|
||||
@mock.patch("documents.views.permitted_document_ids", return_value=set())
|
||||
def test_create_bundle_rejects_insufficient_permissions(self, perms_mock) -> None:
|
||||
def test_create_bundle_rejects_insufficient_permissions(self) -> None:
|
||||
requester = UserFactory(username="bundle_creator")
|
||||
grant_global(requester, "add_sharelinkbundle", "view_document")
|
||||
self.client.force_authenticate(requester)
|
||||
document = DocumentFactory(owner=UserFactory(username="document_owner"))
|
||||
payload = {
|
||||
"document_ids": [self.document.pk],
|
||||
"document_ids": [self.document.pk, document.pk],
|
||||
"file_version": ShareLink.FileVersion.ARCHIVE,
|
||||
"expiration_days": 7,
|
||||
}
|
||||
@@ -101,8 +104,8 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
|
||||
response = self.client.post(self.ENDPOINT, payload, format="json")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn("document_ids", response.data)
|
||||
perms_mock.assert_called()
|
||||
self.assertIn(str(document.pk), str(response.data["document_ids"]))
|
||||
self.assertFalse(ShareLinkBundle.objects.exists())
|
||||
|
||||
@mock.patch("documents.views.build_share_link_bundle.apply_async")
|
||||
def test_rebuild_bundle_resets_state(self, delay_mock) -> None:
|
||||
|
||||
@@ -20,6 +20,7 @@ from documents.sanity_checker import SanityCheckMessages
|
||||
from documents.tests.helpers import dummy_preprocess
|
||||
from paperless_testing.assertions import FileSystemAssertsMixin
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -287,6 +288,82 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
||||
tasks.update_document_content_maybe_archive_file(doc.pk)
|
||||
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
|
||||
|
||||
def _create_root_with_version(self) -> tuple[Document, Document]:
|
||||
sample1 = self.dirs.scratch_dir / "sample.pdf"
|
||||
shutil.copy(
|
||||
Path(__file__).parent
|
||||
/ "samples"
|
||||
/ "documents"
|
||||
/ "originals"
|
||||
/ "0000001.pdf",
|
||||
sample1,
|
||||
)
|
||||
root = DocumentFactory(content="root content", mime_type="application/pdf")
|
||||
version = DocumentFactory(
|
||||
content="my document",
|
||||
filename=sample1,
|
||||
mime_type="application/pdf",
|
||||
root_document=root,
|
||||
version_index=1,
|
||||
)
|
||||
return root, version
|
||||
|
||||
@mock.patch("documents.tasks.clear_document_caches")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
def test_update_content_version_indexes_root(
|
||||
self,
|
||||
mock_get_backend: mock.Mock,
|
||||
mock_clear_caches: mock.Mock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- Update content task is called for the version
|
||||
THEN:
|
||||
- The version's content is updated
|
||||
- The root document is indexed rather than the version
|
||||
- Caches are cleared for both
|
||||
"""
|
||||
root, version = self._create_root_with_version()
|
||||
|
||||
tasks.update_document_content_maybe_archive_file(version.pk)
|
||||
|
||||
self.assertNotEqual(
|
||||
Document.objects.get(pk=version.pk).content,
|
||||
"my document",
|
||||
)
|
||||
self.assertEqual(Document.objects.get(pk=root.pk).content, "root content")
|
||||
indexed = mock_get_backend.return_value.add_or_update.call_args.args[0]
|
||||
self.assertEqual(indexed.pk, root.pk)
|
||||
mock_clear_caches.assert_has_calls(
|
||||
[mock.call(version.pk), mock.call(root.pk)],
|
||||
)
|
||||
|
||||
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
|
||||
@mock.patch("documents.tasks.llm_index_add_or_update_document")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
def test_update_content_version_updates_llm_index_for_root(
|
||||
self,
|
||||
mock_get_backend: mock.Mock,
|
||||
mock_llm_index: mock.Mock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
- The LLM index is enabled
|
||||
WHEN:
|
||||
- Update content task is called for the version
|
||||
THEN:
|
||||
- The LLM index is updated for the root document, not the version
|
||||
"""
|
||||
root, version = self._create_root_with_version()
|
||||
|
||||
tasks.update_document_content_maybe_archive_file(version.pk)
|
||||
|
||||
mock_llm_index.assert_called_once()
|
||||
self.assertEqual(mock_llm_index.call_args.args[0].pk, root.pk)
|
||||
|
||||
|
||||
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
||||
"""
|
||||
|
||||
+81
-41
@@ -174,6 +174,7 @@ from documents.permissions import TrashPermissions
|
||||
from documents.permissions import ViewDocumentsPermissions
|
||||
from documents.permissions import annotate_document_count_by_ids
|
||||
from documents.permissions import annotate_document_count_for_related_queryset
|
||||
from documents.permissions import documents_without_permitted_root
|
||||
from documents.permissions import get_document_count_filter_for_user
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
from documents.permissions import has_global_statistics_permission
|
||||
@@ -1550,13 +1551,16 @@ class DocumentViewSet(
|
||||
)
|
||||
def suggestions(self, request, pk=None):
|
||||
doc = get_object_or_404(
|
||||
Document.objects.select_related("owner").prefetch_related("versions"),
|
||||
Document.objects.select_related(
|
||||
"owner",
|
||||
"root_document__owner",
|
||||
).prefetch_related("versions"),
|
||||
pk=pk,
|
||||
)
|
||||
if request.user is not None and not has_perms_owner_aware(
|
||||
request.user,
|
||||
"change_document",
|
||||
doc,
|
||||
get_root_document(doc),
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
@@ -1610,13 +1614,16 @@ class DocumentViewSet(
|
||||
@method_decorator(cache_control(no_cache=True))
|
||||
def ai_suggestions(self, request, pk=None):
|
||||
doc = get_object_or_404(
|
||||
Document.objects.select_related("owner").prefetch_related("versions"),
|
||||
Document.objects.select_related(
|
||||
"owner",
|
||||
"root_document__owner",
|
||||
).prefetch_related("versions"),
|
||||
pk=pk,
|
||||
)
|
||||
if request.user is not None and not has_perms_owner_aware(
|
||||
request.user,
|
||||
"change_document",
|
||||
doc,
|
||||
get_root_document(doc),
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
@@ -1856,15 +1863,20 @@ class DocumentViewSet(
|
||||
currentUser = request.user
|
||||
try:
|
||||
doc = (
|
||||
Document.objects.select_related("owner")
|
||||
Document.objects.select_related("owner", "root_document__owner")
|
||||
.prefetch_related("notes")
|
||||
.only("pk", "owner__id")
|
||||
.only(
|
||||
"pk",
|
||||
"owner__id",
|
||||
"root_document__id",
|
||||
"root_document__owner__id",
|
||||
)
|
||||
.get(pk=pk)
|
||||
)
|
||||
if currentUser is not None and not has_perms_owner_aware(
|
||||
currentUser,
|
||||
"view_document",
|
||||
doc,
|
||||
get_root_document(doc),
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions to view notes")
|
||||
except Document.DoesNotExist:
|
||||
@@ -1886,7 +1898,7 @@ class DocumentViewSet(
|
||||
if currentUser is not None and not has_perms_owner_aware(
|
||||
currentUser,
|
||||
"change_document",
|
||||
doc,
|
||||
get_root_document(doc),
|
||||
):
|
||||
return HttpResponseForbidden(
|
||||
"Insufficient permissions to create notes",
|
||||
@@ -1929,7 +1941,7 @@ class DocumentViewSet(
|
||||
if currentUser is not None and not has_perms_owner_aware(
|
||||
currentUser,
|
||||
"change_document",
|
||||
doc,
|
||||
get_root_document(doc),
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions to delete notes")
|
||||
|
||||
@@ -1973,11 +1985,13 @@ class DocumentViewSet(
|
||||
def share_links(self, request, pk=None):
|
||||
currentUser = request.user
|
||||
try:
|
||||
doc = Document.objects.select_related("owner").get(pk=pk)
|
||||
doc = Document.objects.select_related("owner", "root_document__owner").get(
|
||||
pk=pk,
|
||||
)
|
||||
if currentUser is not None and not has_perms_owner_aware(
|
||||
currentUser,
|
||||
"change_document",
|
||||
doc,
|
||||
get_root_document(doc),
|
||||
):
|
||||
return HttpResponseForbidden(
|
||||
"Insufficient permissions to add share link",
|
||||
@@ -2008,10 +2022,11 @@ class DocumentViewSet(
|
||||
if not settings.AUDIT_LOG_ENABLED:
|
||||
return HttpResponseBadRequest("Audit log is disabled")
|
||||
try:
|
||||
doc = Document.objects.get(pk=pk)
|
||||
doc = Document.objects.select_related("root_document__owner").get(pk=pk)
|
||||
root_doc = get_root_document(doc)
|
||||
if not request.user.has_perm("auditlog.view_logentry") or (
|
||||
doc.owner is not None
|
||||
and doc.owner != request.user
|
||||
root_doc.owner is not None
|
||||
and root_doc.owner != request.user
|
||||
and not request.user.is_superuser
|
||||
):
|
||||
return HttpResponseForbidden(
|
||||
@@ -2102,9 +2117,7 @@ class DocumentViewSet(
|
||||
documents = Document.objects.filter(pk__in=document_ids)
|
||||
if (
|
||||
request.user is not None
|
||||
and documents.exclude(
|
||||
pk__in=permitted_document_ids(request.user),
|
||||
).exists()
|
||||
and documents_without_permitted_root(documents, request.user).exists()
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
@@ -2430,11 +2443,17 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
|
||||
if doc_id:
|
||||
try:
|
||||
document = Document.objects.get(id=doc_id)
|
||||
document = Document.objects.select_related(
|
||||
"root_document__owner",
|
||||
).get(id=doc_id)
|
||||
except Document.DoesNotExist:
|
||||
return HttpResponseBadRequest("Document not found")
|
||||
|
||||
if not has_perms_owner_aware(request.user, "view_document", document):
|
||||
if not has_perms_owner_aware(
|
||||
request.user,
|
||||
"view_document",
|
||||
get_root_document(document),
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
documents = Document.objects.filter(pk=document.pk)
|
||||
@@ -3605,10 +3624,11 @@ class SelectionDataView(DocumentSelectionMixin, GenericAPIView[Any]):
|
||||
user=request.user,
|
||||
validated_data=serializer.validated_data,
|
||||
)
|
||||
permitted_documents = Document.objects.filter(
|
||||
id__in=permitted_document_ids(request.user),
|
||||
)
|
||||
if permitted_documents.filter(pk__in=ids).count() != len(ids):
|
||||
documents = Document.objects.filter(pk__in=ids)
|
||||
if (
|
||||
documents.count() != len(ids)
|
||||
or documents_without_permitted_root(documents, request.user).exists()
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
correspondents = Correspondent.objects.annotate(
|
||||
@@ -4790,19 +4810,23 @@ class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
|
||||
},
|
||||
)
|
||||
|
||||
documents = list(documents_qs)
|
||||
permitted_ids = set(permitted_document_ids(request.user))
|
||||
for document in documents:
|
||||
if document.pk not in permitted_ids:
|
||||
raise ValidationError(
|
||||
{
|
||||
"document_ids": _(
|
||||
"Insufficient permissions to share document %(id)s.",
|
||||
)
|
||||
% {"id": document.pk},
|
||||
},
|
||||
)
|
||||
denied_id = (
|
||||
documents_without_permitted_root(documents_qs, request.user)
|
||||
.order_by("pk")
|
||||
.values_list("pk", flat=True)
|
||||
.first()
|
||||
)
|
||||
if denied_id is not None:
|
||||
raise ValidationError(
|
||||
{
|
||||
"document_ids": _(
|
||||
"Insufficient permissions to share document %(id)s.",
|
||||
)
|
||||
% {"id": denied_id},
|
||||
},
|
||||
)
|
||||
|
||||
documents = list(documents_qs)
|
||||
document_map = {document.pk: document for document in documents}
|
||||
ordered_documents = [document_map[doc_id] for doc_id in document_ids]
|
||||
|
||||
@@ -5587,6 +5611,23 @@ class TrashView(ListModelMixin, PassUserMixin):
|
||||
class _TrashPermittedObjectsFilter(PermittedObjectsFilter):
|
||||
include_granted = False
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
if request.user.is_superuser or not request.user.is_active:
|
||||
return super().filter_queryset(request, queryset, view)
|
||||
|
||||
# A version belongs to whoever owns its root
|
||||
def owned_or_unowned(prefix: str) -> Q:
|
||||
return Q(**{f"{prefix}owner": request.user}) | Q(
|
||||
**{f"{prefix}owner__isnull": True},
|
||||
)
|
||||
|
||||
return queryset.filter(
|
||||
(Q(root_document__isnull=True) & owned_or_unowned(""))
|
||||
| (
|
||||
Q(root_document__isnull=False) & owned_or_unowned("root_document__")
|
||||
),
|
||||
)
|
||||
|
||||
filter_backends = (_TrashPermittedObjectsFilter,)
|
||||
pagination_class = StandardPagination
|
||||
|
||||
@@ -5616,12 +5657,11 @@ class TrashView(ListModelMixin, PassUserMixin):
|
||||
if doc_ids is not None
|
||||
else self.filter_queryset(self.get_queryset()).all()
|
||||
)
|
||||
if docs.exclude(
|
||||
pk__in=permitted_document_ids(
|
||||
request.user,
|
||||
perm="delete_document",
|
||||
include_deleted=True,
|
||||
),
|
||||
if documents_without_permitted_root(
|
||||
docs,
|
||||
request.user,
|
||||
perm="delete_document",
|
||||
include_deleted=True,
|
||||
).exists():
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
action = serializer.validated_data.get("action")
|
||||
|
||||
@@ -2,7 +2,7 @@ msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: paperless-ngx\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2026-10-06 15:12+0000\n"
|
||||
"POT-Creation-Date: 2026-10-08 18:39+0000\n"
|
||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||
"Last-Translator: \n"
|
||||
"Language-Team: English\n"
|
||||
@@ -1652,49 +1652,49 @@ msgstr ""
|
||||
msgid "workflow runs"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:515 documents/serialisers.py:872
|
||||
#: documents/serialisers.py:2902 documents/views.py:343 documents/views.py:2732
|
||||
#: documents/serialisers.py:516 documents/serialisers.py:873
|
||||
#: documents/serialisers.py:2903 documents/views.py:344 documents/views.py:2751
|
||||
#: paperless_mail/serialisers.py:156
|
||||
msgid "Insufficient permissions."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:708
|
||||
#: documents/serialisers.py:709
|
||||
msgid "Invalid color."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2369
|
||||
#: documents/serialisers.py:2370
|
||||
#, python-format
|
||||
msgid "File type %(type)s not supported"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2413
|
||||
#: documents/serialisers.py:2414
|
||||
#, python-format
|
||||
msgid "Custom field id must be an integer: %(id)s"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2420
|
||||
#: documents/serialisers.py:2421
|
||||
#, python-format
|
||||
msgid "Custom field with id %(id)s does not exist"
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2437 documents/serialisers.py:2447
|
||||
#: documents/serialisers.py:2438 documents/serialisers.py:2448
|
||||
msgid ""
|
||||
"Custom fields must be a list of integers or an object mapping ids to values."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2442
|
||||
#: documents/serialisers.py:2443
|
||||
msgid "Some custom fields don't exist or were specified twice."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2589
|
||||
#: documents/serialisers.py:2590
|
||||
msgid "Invalid variable detected."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2958
|
||||
#: documents/serialisers.py:2959
|
||||
msgid "Duplicate document identifiers are not allowed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/serialisers.py:2988 documents/views.py:4787
|
||||
#: documents/serialisers.py:2989 documents/views.py:4807
|
||||
#, python-format
|
||||
msgid "Documents not found: %(ids)s"
|
||||
msgstr ""
|
||||
@@ -1945,40 +1945,40 @@ msgstr ""
|
||||
msgid ", "
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:336 documents/views.py:2729
|
||||
#: documents/views.py:337 documents/views.py:2748
|
||||
msgid "Invalid more_like_id"
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1676
|
||||
#: documents/views.py:1683
|
||||
msgid "Invalid AI configuration."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1687
|
||||
#: documents/views.py:1694
|
||||
msgid "AI backend request timed out."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:1699
|
||||
#: documents/views.py:1706
|
||||
msgid "AI backend rejected the request. Check logs for details."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:2554 documents/views.py:2870
|
||||
#: documents/views.py:2573 documents/views.py:2889
|
||||
msgid "Specify only one of text, title_search, query, or more_like_id."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4800
|
||||
#: documents/views.py:4823
|
||||
#, python-format
|
||||
msgid "Insufficient permissions to share document %(id)s."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4846
|
||||
#: documents/views.py:4870
|
||||
msgid "Bundle is already being processed."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4910
|
||||
#: documents/views.py:4934
|
||||
msgid "The share link bundle is still being prepared. Please try again later."
|
||||
msgstr ""
|
||||
|
||||
#: documents/views.py:4924
|
||||
#: documents/views.py:4948
|
||||
msgid "The share link bundle is unavailable."
|
||||
msgstr ""
|
||||
|
||||
|
||||
+10
-11
@@ -272,27 +272,26 @@ def check_deprecated_db_settings(
|
||||
Detects legacy advanced options that should be migrated to
|
||||
PAPERLESS_DB_OPTIONS. Returns one Warning per deprecated variable found.
|
||||
"""
|
||||
deprecated_vars: dict[str, str] = {
|
||||
"PAPERLESS_DB_TIMEOUT": "timeout",
|
||||
"PAPERLESS_DB_POOLSIZE": "pool.min_size / pool.max_size",
|
||||
"PAPERLESS_DBSSLMODE": "sslmode",
|
||||
"PAPERLESS_DBSSLROOTCERT": "sslrootcert",
|
||||
"PAPERLESS_DBSSLCERT": "sslcert",
|
||||
"PAPERLESS_DBSSLKEY": "sslkey",
|
||||
}
|
||||
deprecated_vars = (
|
||||
"PAPERLESS_DB_TIMEOUT",
|
||||
"PAPERLESS_DB_POOLSIZE",
|
||||
"PAPERLESS_DBSSLMODE",
|
||||
"PAPERLESS_DBSSLROOTCERT",
|
||||
"PAPERLESS_DBSSLCERT",
|
||||
"PAPERLESS_DBSSLKEY",
|
||||
)
|
||||
|
||||
warnings: list[Warning] = []
|
||||
|
||||
for var_name, db_option_key in deprecated_vars.items():
|
||||
for var_name in deprecated_vars:
|
||||
if not os.getenv(var_name):
|
||||
continue
|
||||
warnings.append(
|
||||
Warning(
|
||||
f"Deprecated environment variable: {var_name}",
|
||||
hint=(
|
||||
f"{var_name} is no longer supported and will be removed in v3.2. "
|
||||
f"{var_name} is deprecated. "
|
||||
f"Set the equivalent option via PAPERLESS_DB_OPTIONS instead. "
|
||||
f'Example: PAPERLESS_DB_OPTIONS=\'{{"{db_option_key}": "<value>"}}\'. '
|
||||
"See https://docs.paperless-ngx.com/migration-v3/ for the full reference."
|
||||
),
|
||||
id="paperless.W001",
|
||||
|
||||
@@ -265,6 +265,52 @@ def get_page_count_for_pdf(
|
||||
return None
|
||||
|
||||
|
||||
def get_pdf_first_page_size_points(
|
||||
path: Path,
|
||||
log: logging.Logger | None = None,
|
||||
) -> tuple[float, float] | None:
|
||||
"""Return the first page's (width, height) in PDF points, post-rotation.
|
||||
|
||||
Uses the CropBox (MediaBox if absent), which must match pdftoppm's
|
||||
``-cropbox`` or the computed DPI targets the wrong box.
|
||||
|
||||
Swaps width and height for 90/270 rotation. ``page.rotation`` resolves
|
||||
inherited and negative ``/Rotate`` values, a raw lookup does not.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
Absolute path to the PDF file.
|
||||
log:
|
||||
Logger for warnings. Falls back to the module-level logger when omitted.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tuple[float, float] | None
|
||||
``(width_points, height_points)``, or ``None`` if the file cannot be
|
||||
opened, has no pages, or the page box is degenerate.
|
||||
"""
|
||||
import pikepdf
|
||||
|
||||
_log = log or logger
|
||||
try:
|
||||
with pikepdf.Pdf.open(path) as pdf:
|
||||
if len(pdf.pages) == 0:
|
||||
return None
|
||||
page = pdf.pages[0]
|
||||
llx, lly, urx, ury = (float(v) for v in page.cropbox)
|
||||
width = abs(urx - llx)
|
||||
height = abs(ury - lly)
|
||||
if width <= 0 or height <= 0:
|
||||
return None
|
||||
if page.rotation in (90, 270):
|
||||
width, height = height, width
|
||||
return width, height
|
||||
except Exception as e:
|
||||
_log.warning("Could not determine PDF page size for %s: %s", path, e)
|
||||
return None
|
||||
|
||||
|
||||
def extract_pdf_metadata(
|
||||
document_path: Path,
|
||||
log: logging.Logger | None = None,
|
||||
|
||||
@@ -986,8 +986,6 @@ GNUPG_HOME = os.getenv("HOME", "/tmp")
|
||||
|
||||
# Convert is part of the ImageMagick package
|
||||
CONVERT_BINARY = os.getenv("PAPERLESS_CONVERT_BINARY", "convert")
|
||||
CONVERT_TMPDIR = os.getenv("PAPERLESS_CONVERT_TMPDIR")
|
||||
CONVERT_MEMORY_LIMIT = os.getenv("PAPERLESS_CONVERT_MEMORY_LIMIT")
|
||||
|
||||
GS_BINARY = os.getenv("PAPERLESS_GS_BINARY", "gs")
|
||||
|
||||
|
||||
@@ -15,9 +15,10 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from ocrmypdf import SubprocessOutputError
|
||||
from PIL import Image
|
||||
|
||||
from documents.parsers import ParseError
|
||||
from documents.parsers import run_convert
|
||||
from documents.parsers import rasterize_pdf_page_to_png
|
||||
from paperless.models import ModeChoices
|
||||
from paperless.parsers import ParserProtocol
|
||||
from paperless.parsers.tesseract import RasterisedDocumentParser
|
||||
@@ -280,24 +281,71 @@ class TestGetThumbnail:
|
||||
)
|
||||
assert thumb.is_file()
|
||||
|
||||
def test_thumbnail_fallback_on_convert_error(
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "expected_height"),
|
||||
[
|
||||
pytest.param("simple-digital.pdf", 647, id="portrait-letter"),
|
||||
pytest.param("rotated.pdf", 386, id="landscape"),
|
||||
],
|
||||
)
|
||||
def test_thumbnail_is_correct_format_and_size(
|
||||
self,
|
||||
tesseract_parser: RasterisedDocumentParser,
|
||||
tesseract_samples_dir: Path,
|
||||
filename: str,
|
||||
expected_height: int,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A portrait or landscape PDF
|
||||
WHEN:
|
||||
- A thumbnail is generated
|
||||
THEN:
|
||||
- A 500px wide WebP keeping the page's aspect ratio
|
||||
"""
|
||||
thumb = tesseract_parser.get_thumbnail(
|
||||
tesseract_samples_dir / filename,
|
||||
"application/pdf",
|
||||
)
|
||||
with Image.open(thumb) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.width == 500
|
||||
assert im.height == pytest.approx(expected_height, abs=2)
|
||||
|
||||
def test_thumbnail_fallback_on_pdftoppm_error(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tesseract_parser: RasterisedDocumentParser,
|
||||
tesseract_samples_dir: Path,
|
||||
) -> None:
|
||||
def _raise_on_pdf(input_file, output_file, **kwargs) -> None:
|
||||
if ".pdf" in str(input_file):
|
||||
"""
|
||||
GIVEN:
|
||||
- Rasterizing the original PDF fails
|
||||
WHEN:
|
||||
- A thumbnail is generated
|
||||
THEN:
|
||||
- The PDF is repaired with qpdf and a real thumbnail is rendered
|
||||
"""
|
||||
original = tesseract_samples_dir / "simple-digital.pdf"
|
||||
|
||||
def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None:
|
||||
if in_path == original:
|
||||
raise ParseError("Does not compute.")
|
||||
run_convert(input_file=input_file, output_file=output_file, **kwargs)
|
||||
rasterize_pdf_page_to_png(in_path, out_path, **kwargs)
|
||||
|
||||
mocker.patch("documents.parsers.run_convert", side_effect=_raise_on_pdf)
|
||||
|
||||
thumb = tesseract_parser.get_thumbnail(
|
||||
tesseract_samples_dir / "simple-digital.pdf",
|
||||
"application/pdf",
|
||||
rasterize = mocker.patch(
|
||||
"documents.parsers.rasterize_pdf_page_to_png",
|
||||
side_effect=_fail_on_original,
|
||||
)
|
||||
|
||||
thumb = tesseract_parser.get_thumbnail(original, "application/pdf")
|
||||
|
||||
assert rasterize.call_count == 2
|
||||
assert thumb.is_file()
|
||||
assert thumb.name == "convert_qpdf.webp"
|
||||
with Image.open(thumb) as im:
|
||||
assert im.format == "WEBP"
|
||||
assert im.width == 500
|
||||
|
||||
def test_thumbnail_encrypted_pdf(
|
||||
self,
|
||||
|
||||
@@ -211,14 +211,14 @@ class TestAuditLogChecks:
|
||||
assert "auditlog table was found but audit log is disabled." in msgs[0].msg
|
||||
|
||||
|
||||
DEPRECATED_VARS: dict[str, str] = {
|
||||
"PAPERLESS_DB_TIMEOUT": "timeout",
|
||||
"PAPERLESS_DB_POOLSIZE": "pool.min_size / pool.max_size",
|
||||
"PAPERLESS_DBSSLMODE": "sslmode",
|
||||
"PAPERLESS_DBSSLROOTCERT": "sslrootcert",
|
||||
"PAPERLESS_DBSSLCERT": "sslcert",
|
||||
"PAPERLESS_DBSSLKEY": "sslkey",
|
||||
}
|
||||
DEPRECATED_VARS = (
|
||||
"PAPERLESS_DB_TIMEOUT",
|
||||
"PAPERLESS_DB_POOLSIZE",
|
||||
"PAPERLESS_DBSSLMODE",
|
||||
"PAPERLESS_DBSSLROOTCERT",
|
||||
"PAPERLESS_DBSSLCERT",
|
||||
"PAPERLESS_DBSSLKEY",
|
||||
)
|
||||
|
||||
|
||||
class TestDeprecatedDbSettings:
|
||||
@@ -234,26 +234,11 @@ class TestDeprecatedDbSettings:
|
||||
result = check_deprecated_db_settings(None)
|
||||
assert result == []
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env_var", "db_option_key"),
|
||||
[
|
||||
pytest.param("PAPERLESS_DB_TIMEOUT", "timeout", id="db-timeout"),
|
||||
pytest.param(
|
||||
"PAPERLESS_DB_POOLSIZE",
|
||||
"pool.min_size / pool.max_size",
|
||||
id="db-poolsize",
|
||||
),
|
||||
pytest.param("PAPERLESS_DBSSLMODE", "sslmode", id="ssl-mode"),
|
||||
pytest.param("PAPERLESS_DBSSLROOTCERT", "sslrootcert", id="ssl-rootcert"),
|
||||
pytest.param("PAPERLESS_DBSSLCERT", "sslcert", id="ssl-cert"),
|
||||
pytest.param("PAPERLESS_DBSSLKEY", "sslkey", id="ssl-key"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("env_var", DEPRECATED_VARS)
|
||||
def test_single_deprecated_var_produces_one_warning(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
env_var: str,
|
||||
db_option_key: str,
|
||||
) -> None:
|
||||
"""Each deprecated var in isolation produces exactly one warning."""
|
||||
mocker.patch.dict(os.environ, {env_var: "some_value"}, clear=True)
|
||||
@@ -264,7 +249,8 @@ class TestDeprecatedDbSettings:
|
||||
assert isinstance(warning, Warning)
|
||||
assert warning.id == "paperless.W001"
|
||||
assert env_var in warning.hint
|
||||
assert db_option_key in warning.hint
|
||||
assert "PAPERLESS_DB_OPTIONS" in warning.hint
|
||||
assert "https://docs.paperless-ngx.com/migration-v3/" in warning.hint
|
||||
|
||||
def test_multiple_deprecated_vars_produce_one_warning_each(
|
||||
self,
|
||||
|
||||
@@ -6,8 +6,10 @@ import codecs
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pikepdf
|
||||
import pytest
|
||||
|
||||
from paperless.parsers.utils import get_pdf_first_page_size_points
|
||||
from paperless.parsers.utils import is_tagged_pdf
|
||||
from paperless.parsers.utils import pdf_born_digital_text
|
||||
from paperless.parsers.utils import post_process_text
|
||||
@@ -70,6 +72,149 @@ class TestIsTaggedPdf:
|
||||
assert is_tagged_pdf(bad) is False
|
||||
|
||||
|
||||
class TestGetPdfFirstPageSizePoints:
|
||||
@staticmethod
|
||||
def _write_pdf(
|
||||
path: Path,
|
||||
*,
|
||||
media_box: tuple[float, float, float, float] = (0, 0, 600, 800),
|
||||
crop_box: tuple[float, float, float, float] | None = None,
|
||||
page_rotate: int | None = None,
|
||||
inherited_rotate: int | None = None,
|
||||
) -> Path:
|
||||
pdf = pikepdf.new()
|
||||
pdf.add_blank_page(page_size=(media_box[2], media_box[3]))
|
||||
page = pdf.pages[0]
|
||||
page.obj.MediaBox = pikepdf.Array(media_box)
|
||||
if crop_box is not None:
|
||||
page.obj.CropBox = pikepdf.Array(crop_box)
|
||||
if page_rotate is not None:
|
||||
page.obj.Rotate = page_rotate
|
||||
if inherited_rotate is not None:
|
||||
pdf.Root.Pages.Rotate = inherited_rotate
|
||||
pdf.save(path)
|
||||
return path
|
||||
|
||||
def test_letter_sample(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A US Letter sample PDF with no CropBox and no rotation
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- The MediaBox size in points is returned
|
||||
"""
|
||||
assert get_pdf_first_page_size_points(SAMPLES / "simple-digital.pdf") == (
|
||||
612.0,
|
||||
792.0,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rotate", "expected"),
|
||||
[
|
||||
pytest.param(0, (600.0, 800.0), id="rotate-0"),
|
||||
pytest.param(90, (800.0, 600.0), id="rotate-90"),
|
||||
pytest.param(180, (600.0, 800.0), id="rotate-180"),
|
||||
pytest.param(270, (800.0, 600.0), id="rotate-270"),
|
||||
pytest.param(-90, (800.0, 600.0), id="rotate-negative-90"),
|
||||
],
|
||||
)
|
||||
def test_page_rotation_swaps_dimensions(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
rotate: int,
|
||||
expected: tuple[float, float],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A portrait PDF page with /Rotate set directly on the page
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- Width and height are swapped for quarter-turn rotations only
|
||||
"""
|
||||
pdf_path = self._write_pdf(tmp_path / "rotated.pdf", page_rotate=rotate)
|
||||
assert get_pdf_first_page_size_points(pdf_path) == expected
|
||||
|
||||
def test_inherited_rotation_swaps_dimensions(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A page inheriting /Rotate 90 from the /Pages node
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- Width and height are swapped
|
||||
"""
|
||||
pdf_path = self._write_pdf(tmp_path / "inherited.pdf", inherited_rotate=90)
|
||||
assert get_pdf_first_page_size_points(pdf_path) == (800.0, 600.0)
|
||||
|
||||
def test_crop_box_preferred_over_media_box(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF page with a CropBox smaller than its MediaBox
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- The CropBox size is returned
|
||||
"""
|
||||
pdf_path = self._write_pdf(
|
||||
tmp_path / "cropped.pdf",
|
||||
crop_box=(50, 100, 350, 500),
|
||||
)
|
||||
assert get_pdf_first_page_size_points(pdf_path) == (300.0, 400.0)
|
||||
|
||||
def test_degenerate_box_returns_none(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A PDF page whose box has zero width
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- None is returned
|
||||
"""
|
||||
pdf_path = self._write_pdf(
|
||||
tmp_path / "degenerate.pdf",
|
||||
media_box=(0, 0, 600, 800),
|
||||
crop_box=(100, 0, 100, 800),
|
||||
)
|
||||
assert get_pdf_first_page_size_points(pdf_path) is None
|
||||
|
||||
def test_nonexistent_path_returns_none(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A path that does not exist
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- None is returned and nothing is raised
|
||||
"""
|
||||
assert get_pdf_first_page_size_points(Path("/nonexistent/file.pdf")) is None
|
||||
|
||||
def test_corrupt_pdf_returns_none(self, tmp_path: Path) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A file that is not a PDF
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- None is returned and nothing is raised
|
||||
"""
|
||||
bad = tmp_path / "bad.pdf"
|
||||
bad.write_bytes(b"not a pdf")
|
||||
assert get_pdf_first_page_size_points(bad) is None
|
||||
|
||||
def test_encrypted_pdf_returns_none(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A password protected PDF
|
||||
WHEN:
|
||||
- The first page size is requested
|
||||
THEN:
|
||||
- None is returned and nothing is raised
|
||||
"""
|
||||
assert get_pdf_first_page_size_points(SAMPLES / "encrypted.pdf") is None
|
||||
|
||||
|
||||
class TestPostProcessText:
|
||||
@pytest.mark.parametrize(
|
||||
("source", "expected"),
|
||||
|
||||
Reference in new issue
Block a user