mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-14 03:57:13 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b420fc1954 | ||
|
|
05321b10d3 |
No files matched your search
@@ -29,8 +29,13 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="search-container flex-grow-1 py-2 pb-3 pb-sm-2 me-sm-auto order-3 order-sm-1"
|
<div class="search-container flex-grow-1 py-2 pb-3 pb-sm-2 me-sm-auto order-3 order-sm-1"
|
||||||
[class.mobile-hidden]="mobileSearchHidden()">
|
[class.mobile-hidden]="mobileSearchHidden()">
|
||||||
<div class="col-12 header-search mx-auto">
|
<div class="col-12 header-search mx-auto d-flex align-items-stretch gap-3">
|
||||||
<pngx-global-search></pngx-global-search>
|
<pngx-global-search class="flex-grow-1"></pngx-global-search>
|
||||||
|
<button *pngxIfPermissions="{ action: PermissionAction.Add, type: PermissionType.Document }"
|
||||||
|
class="btn btn-sm header-upload border-0 d-flex align-items-center justify-content-center flex-shrink-0 py-0 px-2" type="button"
|
||||||
|
(click)="fileUpload.click()" aria-label="Upload documents" i18n-aria-label>
|
||||||
|
<i-bs class="d-flex" width="1.1em" height="1.1em" name="file-earmark-arrow-up"></i-bs><span class="ms-2 d-sm-none d-md-inline" i18n>Upload</span>
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<ul ngbNav class="order-sm-3">
|
<ul ngbNav class="order-sm-3">
|
||||||
@@ -70,6 +75,9 @@
|
|||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
|
<input type="file" class="visually-hidden" id="navbar-file-upload" (change)="onFileSelected($event)" multiple #fileUpload tabindex="-1" aria-hidden="true">
|
||||||
|
<label for="navbar-file-upload" class="visually-hidden" i18n>Upload Files</label>
|
||||||
|
|
||||||
<div class="container-fluid">
|
<div class="container-fluid">
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<nav id="sidebarMenu" class="d-md-block bg-light sidebar collapse"
|
<nav id="sidebarMenu" class="d-md-block bg-light sidebar collapse"
|
||||||
|
|||||||
@@ -532,7 +532,23 @@ main {
|
|||||||
|
|
||||||
.header-search {
|
.header-search {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
max-width: 55rem;
|
max-width: 50rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.header-search pngx-global-search {
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// light w shadow to stand out against search
|
||||||
|
.header-upload {
|
||||||
|
--bs-btn-bg: rgba(255, 255, 255, .15);
|
||||||
|
--bs-btn-hover-bg: rgba(255, 255, 255, .22);
|
||||||
|
--bs-btn-active-bg: rgba(255, 255, 255, .3);
|
||||||
|
color: var(--pngx-primary-text-contrast) !important; // .navbar .dropdown > .btn in styles.scss
|
||||||
|
|
||||||
|
&:not(:focus) {
|
||||||
|
box-shadow: 0 1px 3px rgba(0, 0, 0, .35), 0 1px 1px rgba(0, 0, 0, .2); // leave room for the .btn:focus ring
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@media screen and (max-width: 575.98px) {
|
@media screen and (max-width: 575.98px) {
|
||||||
@@ -662,3 +678,7 @@ main {
|
|||||||
::ng-deep .navItemDrag .position-absolute i-bs {
|
::ng-deep .navItemDrag .position-absolute i-bs {
|
||||||
display: none;
|
display: none;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.smaller {
|
||||||
|
font-size: 0.75rem;
|
||||||
|
}
|
||||||
@@ -33,6 +33,7 @@ import { SearchService } from 'src/app/services/rest/search.service'
|
|||||||
import { SettingsService } from 'src/app/services/settings.service'
|
import { SettingsService } from 'src/app/services/settings.service'
|
||||||
import { TasksService } from 'src/app/services/tasks.service'
|
import { TasksService } from 'src/app/services/tasks.service'
|
||||||
import { ToastService } from 'src/app/services/toast.service'
|
import { ToastService } from 'src/app/services/toast.service'
|
||||||
|
import { UploadDocumentsService } from 'src/app/services/upload-documents.service'
|
||||||
import { environment } from 'src/environments/environment'
|
import { environment } from 'src/environments/environment'
|
||||||
import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
|
import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
|
||||||
import { DocumentDetailComponent } from '../document-detail/document-detail.component'
|
import { DocumentDetailComponent } from '../document-detail/document-detail.component'
|
||||||
@@ -462,6 +463,29 @@ describe('AppFrameComponent', () => {
|
|||||||
expect(routerSpy).toHaveBeenCalled()
|
expect(routerSpy).toHaveBeenCalled()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should upload files selected from the navbar upload button', () => {
|
||||||
|
const uploadDocumentsService = TestBed.inject(UploadDocumentsService)
|
||||||
|
const uploadSpy = jest
|
||||||
|
.spyOn(uploadDocumentsService, 'uploadFiles')
|
||||||
|
.mockImplementation(() => {})
|
||||||
|
const toastSpy = jest.spyOn(toastService, 'showInfo')
|
||||||
|
const file = new File(['testing'], 'file.pdf')
|
||||||
|
const input = window.document.createElement('input')
|
||||||
|
jest.spyOn(input, 'files', 'get').mockReturnValue([file] as any)
|
||||||
|
|
||||||
|
component.onFileSelected({ target: input } as unknown as Event)
|
||||||
|
expect(toastSpy).toHaveBeenCalled()
|
||||||
|
expect(uploadSpy).toHaveBeenCalledWith([file])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not upload when no files are selected from the navbar', () => {
|
||||||
|
const uploadDocumentsService = TestBed.inject(UploadDocumentsService)
|
||||||
|
const uploadSpy = jest.spyOn(uploadDocumentsService, 'uploadFiles')
|
||||||
|
const input = window.document.createElement('input')
|
||||||
|
component.onFileSelected({ target: input } as unknown as Event)
|
||||||
|
expect(uploadSpy).not.toHaveBeenCalled()
|
||||||
|
})
|
||||||
|
|
||||||
it('should close all documents on logout', () => {
|
it('should close all documents on logout', () => {
|
||||||
const closeAllSpy = jest.spyOn(openDocumentsService, 'closeAll')
|
const closeAllSpy = jest.spyOn(openDocumentsService, 'closeAll')
|
||||||
component.onLogout()
|
component.onLogout()
|
||||||
|
|||||||
@@ -48,12 +48,13 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service'
|
|||||||
import { SettingsService } from 'src/app/services/settings.service'
|
import { SettingsService } from 'src/app/services/settings.service'
|
||||||
import { TasksService } from 'src/app/services/tasks.service'
|
import { TasksService } from 'src/app/services/tasks.service'
|
||||||
import { ToastService } from 'src/app/services/toast.service'
|
import { ToastService } from 'src/app/services/toast.service'
|
||||||
|
import { UploadDocumentsService } from 'src/app/services/upload-documents.service'
|
||||||
import { environment } from 'src/environments/environment'
|
import { environment } from 'src/environments/environment'
|
||||||
import { ChatComponent } from '../chat/chat/chat.component'
|
import { ChatComponent } from '../chat/chat/chat.component'
|
||||||
|
import { SwitchComponent } from '../common/input/switch/switch.component'
|
||||||
import { BrandMarkComponent } from '../common/logo/brand-mark/brand-mark.component'
|
import { BrandMarkComponent } from '../common/logo/brand-mark/brand-mark.component'
|
||||||
import { LogoComponent } from '../common/logo/logo.component'
|
import { LogoComponent } from '../common/logo/logo.component'
|
||||||
import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
|
import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
|
||||||
import { SwitchComponent } from '../common/input/switch/switch.component'
|
|
||||||
import { DocumentDetailComponent } from '../document-detail/document-detail.component'
|
import { DocumentDetailComponent } from '../document-detail/document-detail.component'
|
||||||
import { ComponentWithPermissions } from '../with-permissions/with-permissions.component'
|
import { ComponentWithPermissions } from '../with-permissions/with-permissions.component'
|
||||||
import { GlobalSearchComponent } from './global-search/global-search.component'
|
import { GlobalSearchComponent } from './global-search/global-search.component'
|
||||||
@@ -101,6 +102,7 @@ export class AppFrameComponent
|
|||||||
private modalService = inject(NgbModal)
|
private modalService = inject(NgbModal)
|
||||||
permissionsService = inject(PermissionsService)
|
permissionsService = inject(PermissionsService)
|
||||||
private djangoMessagesService = inject(DjangoMessagesService)
|
private djangoMessagesService = inject(DjangoMessagesService)
|
||||||
|
private uploadDocumentsService = inject(UploadDocumentsService)
|
||||||
|
|
||||||
readonly appRemoteVersion = signal<AppRemoteVersion>(null)
|
readonly appRemoteVersion = signal<AppRemoteVersion>(null)
|
||||||
readonly isMenuCollapsed = signal(true)
|
readonly isMenuCollapsed = signal(true)
|
||||||
@@ -511,6 +513,15 @@ export class AppFrameComponent
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
onFileSelected(event: Event) {
|
||||||
|
const input = event.target as HTMLInputElement
|
||||||
|
const files = Array.from(input.files ?? [])
|
||||||
|
input.value = ''
|
||||||
|
if (!files.length) return
|
||||||
|
this.toastService.showInfo($localize`Initiating upload...`, 3000)
|
||||||
|
this.uploadDocumentsService.uploadFiles(files)
|
||||||
|
}
|
||||||
|
|
||||||
onLogout() {
|
onLogout() {
|
||||||
this.openDocumentsService.closeAll()
|
this.openDocumentsService.closeAll()
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-8
@@ -85,15 +85,11 @@ describe('UploadFileWidgetComponent', () => {
|
|||||||
'file.pdf'
|
'file.pdf'
|
||||||
)
|
)
|
||||||
const fileInput = fixture.debugElement.query(By.css('input'))
|
const fileInput = fixture.debugElement.query(By.css('input'))
|
||||||
jest.spyOn(fileInput.nativeElement, 'files', 'get').mockReturnValue({
|
jest
|
||||||
item: () => file,
|
.spyOn(fileInput.nativeElement, 'files', 'get')
|
||||||
length: 1,
|
.mockReturnValue([file] as any)
|
||||||
[Symbol.iterator]: () => ({
|
|
||||||
next: () => ({ done: false, value: file }),
|
|
||||||
}),
|
|
||||||
} as any)
|
|
||||||
fileInput.nativeElement.dispatchEvent(new Event('change'))
|
fileInput.nativeElement.dispatchEvent(new Event('change'))
|
||||||
expect(uploadSpy).toHaveBeenCalled()
|
expect(uploadSpy).toHaveBeenCalledWith(file)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should generate stats summary', () => {
|
it('should generate stats summary', () => {
|
||||||
|
|||||||
+3
-5
@@ -130,11 +130,9 @@ export class UploadFileWidgetComponent extends ComponentWithPermissions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public onFileSelected(event: Event) {
|
public onFileSelected(event: Event) {
|
||||||
const files = (event.target as HTMLInputElement).files
|
const input = event.target as HTMLInputElement
|
||||||
for (let i = 0; i < files?.length; i++) {
|
this.uploadDocumentsService.uploadFiles(Array.from(input.files ?? []))
|
||||||
const file = files.item(i)
|
input.value = ''
|
||||||
file && this.uploadDocumentsService.uploadFile(file)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get slimSidebarEnabled(): boolean {
|
get slimSidebarEnabled(): boolean {
|
||||||
|
|||||||
@@ -135,7 +135,7 @@ export class FileDropComponent {
|
|||||||
.then((results) => {
|
.then((results) => {
|
||||||
files.push(...[].concat(...results))
|
files.push(...[].concat(...results))
|
||||||
this.toastService.showInfo($localize`Initiating upload...`, 3000)
|
this.toastService.showInfo($localize`Initiating upload...`, 3000)
|
||||||
files.forEach((file) => this.uploadDocumentsService.uploadFile(file))
|
this.uploadDocumentsService.uploadFiles(files)
|
||||||
})
|
})
|
||||||
.catch((e) => {
|
.catch((e) => {
|
||||||
this.toastService.showError(
|
this.toastService.showError(
|
||||||
@@ -144,7 +144,7 @@ export class FileDropComponent {
|
|||||||
})
|
})
|
||||||
} else if (files.length) {
|
} else if (files.length) {
|
||||||
this.toastService.showInfo($localize`Initiating upload...`, 3000)
|
this.toastService.showInfo($localize`Initiating upload...`, 3000)
|
||||||
files.forEach((file) => this.uploadDocumentsService.uploadFile(file))
|
this.uploadDocumentsService.uploadFiles(files)
|
||||||
}
|
}
|
||||||
|
|
||||||
this.onDragLeave(event, true)
|
this.onDragLeave(event, true)
|
||||||
|
|||||||
@@ -54,6 +54,18 @@ describe('UploadDocumentsService', () => {
|
|||||||
req[0].flush('123-456')
|
req[0].flush('123-456')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('uploads each file when given multiple files', () => {
|
||||||
|
const uploadSpy = jest.spyOn(uploadDocumentsService, 'uploadFile')
|
||||||
|
const file1 = new File(['one'], 'one.pdf')
|
||||||
|
const file2 = new File(['two'], 'two.pdf')
|
||||||
|
uploadDocumentsService.uploadFiles([file1, file2])
|
||||||
|
expect(uploadSpy).toHaveBeenCalledWith(file1)
|
||||||
|
expect(uploadSpy).toHaveBeenCalledWith(file2)
|
||||||
|
httpTestingController
|
||||||
|
.match(`${environment.apiBaseUrl}documents/post_document/`)
|
||||||
|
.forEach((req) => req.flush('123-456'))
|
||||||
|
})
|
||||||
|
|
||||||
it('updates progress during upload and failure', () => {
|
it('updates progress during upload and failure', () => {
|
||||||
const file = new File(
|
const file = new File(
|
||||||
[new Blob(['testing'], { type: 'application/pdf' })],
|
[new Blob(['testing'], { type: 'application/pdf' })],
|
||||||
|
|||||||
@@ -16,6 +16,12 @@ export class UploadDocumentsService {
|
|||||||
|
|
||||||
private uploadSubscriptions: Array<Subscription> = []
|
private uploadSubscriptions: Array<Subscription> = []
|
||||||
|
|
||||||
|
public uploadFiles(files: Iterable<File>) {
|
||||||
|
for (const file of files) {
|
||||||
|
this.uploadFile(file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public uploadFile(file: File) {
|
public uploadFile(file: File) {
|
||||||
let formData = new FormData()
|
let formData = new FormData()
|
||||||
formData.append('document', file, file.name)
|
formData.append('document', file, file.name)
|
||||||
|
|||||||
@@ -90,6 +90,7 @@ import {
|
|||||||
exclamationTriangleFill,
|
exclamationTriangleFill,
|
||||||
eye,
|
eye,
|
||||||
fileEarmark,
|
fileEarmark,
|
||||||
|
fileEarmarkArrowUp,
|
||||||
fileEarmarkCheck,
|
fileEarmarkCheck,
|
||||||
fileEarmarkDiff,
|
fileEarmarkDiff,
|
||||||
fileEarmarkFill,
|
fileEarmarkFill,
|
||||||
@@ -341,6 +342,7 @@ const icons = {
|
|||||||
exclamationTriangleFill,
|
exclamationTriangleFill,
|
||||||
eye,
|
eye,
|
||||||
fileEarmark,
|
fileEarmark,
|
||||||
|
fileEarmarkArrowUp,
|
||||||
fileEarmarkCheck,
|
fileEarmarkCheck,
|
||||||
fileEarmarkDiff,
|
fileEarmarkDiff,
|
||||||
fileEarmarkFill,
|
fileEarmarkFill,
|
||||||
|
|||||||
@@ -13,14 +13,8 @@ from celery import group
|
|||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
from django.db.models import Case
|
|
||||||
from django.db.models import F
|
|
||||||
from django.db.models import Max
|
from django.db.models import Max
|
||||||
from django.db.models import OuterRef
|
|
||||||
from django.db.models import Q
|
from django.db.models import Q
|
||||||
from django.db.models import Subquery
|
|
||||||
from django.db.models import When
|
|
||||||
from django.db.models.functions import Coalesce
|
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from documents.data_models import ConsumableDocument
|
from documents.data_models import ConsumableDocument
|
||||||
@@ -42,7 +36,6 @@ from documents.tasks import remove_document_from_index
|
|||||||
from documents.tasks import update_document_content_maybe_archive_file
|
from documents.tasks import update_document_content_maybe_archive_file
|
||||||
from documents.versioning import get_latest_version_for_root
|
from documents.versioning import get_latest_version_for_root
|
||||||
from documents.versioning import get_root_document
|
from documents.versioning import get_root_document
|
||||||
from documents.versioning import versions_newest_first
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
@@ -415,28 +408,10 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
|
|||||||
|
|
||||||
Consumption workflows do not run here, so ``remote_ocr`` is how the user
|
Consumption workflows do not run here, so ``remote_ocr`` is how the user
|
||||||
asks for the remote engine when it is not configured to handle everything.
|
asks for the remote engine when it is not configured to handle everything.
|
||||||
|
|
||||||
A root document with versions reprocesses its latest version, which is the
|
|
||||||
file whose content, archive and thumbnail are shown for it.
|
|
||||||
"""
|
"""
|
||||||
latest_version = versions_newest_first(
|
for document_id in doc_ids:
|
||||||
Document.objects.filter(root_document=OuterRef("pk")),
|
|
||||||
).values("id")[:1]
|
|
||||||
source_ids = (
|
|
||||||
Document.objects.filter(id__in=doc_ids)
|
|
||||||
.annotate(
|
|
||||||
source_id=Case(
|
|
||||||
When(root_document__isnull=False, then=F("id")),
|
|
||||||
default=Coalesce(Subquery(latest_version), F("id")),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.order_by()
|
|
||||||
.values_list("source_id", flat=True)
|
|
||||||
.distinct()
|
|
||||||
)
|
|
||||||
for source_id in source_ids:
|
|
||||||
update_document_content_maybe_archive_file.apply_async(
|
update_document_content_maybe_archive_file.apply_async(
|
||||||
kwargs={"document_id": source_id, "remote_ocr": remote_ocr},
|
kwargs={"document_id": document_id, "remote_ocr": remote_ocr},
|
||||||
headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
|
headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -67,22 +67,18 @@ class Command(PaperlessCommand):
|
|||||||
if options.get("recreate"):
|
if options.get("recreate"):
|
||||||
wipe_index(settings.INDEX_DIR)
|
wipe_index(settings.INDEX_DIR)
|
||||||
|
|
||||||
documents = (
|
documents = Document.objects.select_related(
|
||||||
Document.objects.filter(root_document__isnull=True)
|
"correspondent",
|
||||||
.select_related(
|
"document_type",
|
||||||
"correspondent",
|
"storage_path",
|
||||||
"document_type",
|
"owner",
|
||||||
"storage_path",
|
).prefetch_related(
|
||||||
"owner",
|
"tags",
|
||||||
)
|
"notes__user",
|
||||||
.prefetch_related(
|
"custom_fields__field",
|
||||||
"tags",
|
"versions",
|
||||||
"notes__user",
|
"barcodes",
|
||||||
"custom_fields__field",
|
"versions__barcodes",
|
||||||
"versions",
|
|
||||||
"barcodes",
|
|
||||||
"versions__barcodes",
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
total = documents.count()
|
total = documents.count()
|
||||||
rebuild_kwargs = {}
|
rebuild_kwargs = {}
|
||||||
|
|||||||
@@ -6,19 +6,14 @@ from django.contrib.auth.models import Permission
|
|||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
from django.contrib.contenttypes.models import ContentType
|
from django.contrib.contenttypes.models import ContentType
|
||||||
from django.db.models import Case
|
from django.db.models import Case
|
||||||
from django.db.models import CharField
|
|
||||||
from django.db.models import Count
|
from django.db.models import Count
|
||||||
from django.db.models import Exists
|
|
||||||
from django.db.models import F
|
|
||||||
from django.db.models import IntegerField
|
from django.db.models import IntegerField
|
||||||
from django.db.models import Model
|
from django.db.models import Model
|
||||||
from django.db.models import OuterRef
|
|
||||||
from django.db.models import Q
|
from django.db.models import Q
|
||||||
from django.db.models import QuerySet
|
from django.db.models import QuerySet
|
||||||
from django.db.models import Value
|
from django.db.models import Value
|
||||||
from django.db.models import When
|
from django.db.models import When
|
||||||
from django.db.models.functions import Cast
|
from django.db.models.functions import Cast
|
||||||
from django.db.models.functions import Coalesce
|
|
||||||
from guardian.core import ObjectPermissionChecker
|
from guardian.core import ObjectPermissionChecker
|
||||||
from guardian.models import GroupObjectPermission
|
from guardian.models import GroupObjectPermission
|
||||||
from guardian.models import UserObjectPermission
|
from guardian.models import UserObjectPermission
|
||||||
@@ -30,7 +25,6 @@ from rest_framework.permissions import BasePermission
|
|||||||
from rest_framework.permissions import DjangoObjectPermissions
|
from rest_framework.permissions import DjangoObjectPermissions
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from documents.versioning import get_root_document
|
|
||||||
|
|
||||||
|
|
||||||
class PaperlessObjectPermissions(DjangoObjectPermissions):
|
class PaperlessObjectPermissions(DjangoObjectPermissions):
|
||||||
@@ -354,7 +348,6 @@ def permitted_object_ids(
|
|||||||
perm: str,
|
perm: str,
|
||||||
*,
|
*,
|
||||||
include_deleted: bool = False,
|
include_deleted: bool = False,
|
||||||
parent_field: str | None = None,
|
|
||||||
) -> QuerySet[int]:
|
) -> QuerySet[int]:
|
||||||
"""
|
"""
|
||||||
Generic version of ``permitted_document_ids`` for any model with an
|
Generic version of ``permitted_document_ids`` for any model with an
|
||||||
@@ -363,20 +356,6 @@ def permitted_object_ids(
|
|||||||
soft-delete pattern (currently only ``Document``); for every other model
|
soft-delete pattern (currently only ``Document``); for every other model
|
||||||
it is accepted but has no effect, since those models have no soft-delete
|
it is accepted but has no effect, since those models have no soft-delete
|
||||||
concept.
|
concept.
|
||||||
|
|
||||||
``parent_field`` names a self-referencing foreign key whose target
|
|
||||||
authorizes the row (``Document.root_document``). A row with a parent is
|
|
||||||
visible exactly when its parent is, judged by the parent's owner and
|
|
||||||
grants, so the row's own owner and grants are ignored.
|
|
||||||
|
|
||||||
Guardian stores ``object_pk`` as a string, so each grant is an ``EXISTS``
|
|
||||||
keyed on the row's id cast to a string, which can use guardian's
|
|
||||||
(user, permission, object_pk) unique index. Casting every ``object_pk`` to
|
|
||||||
an integer for an ``id IN (...)`` is not indexable, and MariaDB cannot
|
|
||||||
materialize it inside the owner ``OR``, so it re-scans the user's grants
|
|
||||||
for every row. The user's groups are matched with an ``IN`` subquery
|
|
||||||
rather than a join through the membership table, which SQLite plans badly
|
|
||||||
once the grant tables grow.
|
|
||||||
"""
|
"""
|
||||||
has_soft_delete = hasattr(model, "global_objects")
|
has_soft_delete = hasattr(model, "global_objects")
|
||||||
manager = (
|
manager = (
|
||||||
@@ -384,21 +363,8 @@ def permitted_object_ids(
|
|||||||
)
|
)
|
||||||
base_qs = manager.all().only("id", "owner")
|
base_qs = manager.all().only("id", "owner")
|
||||||
|
|
||||||
owner_field, key_field = "owner", "pk"
|
|
||||||
if parent_field is not None:
|
|
||||||
owner_field, key_field = "authorizing_owner", "authorizing_id"
|
|
||||||
base_qs = base_qs.annotate(
|
|
||||||
authorizing_id=Coalesce(f"{parent_field}_id", "id"),
|
|
||||||
authorizing_owner=Case(
|
|
||||||
When(**{f"{parent_field}_id__isnull": True}, then=F("owner_id")),
|
|
||||||
default=F(f"{parent_field}__owner_id"),
|
|
||||||
output_field=IntegerField(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
unowned = Q(**{f"{owner_field}__isnull": True})
|
|
||||||
|
|
||||||
if user is None or not getattr(user, "is_authenticated", False):
|
if user is None or not getattr(user, "is_authenticated", False):
|
||||||
return base_qs.filter(unowned).values_list("id", flat=True)
|
return base_qs.filter(owner__isnull=True).values_list("id", flat=True)
|
||||||
|
|
||||||
# Deactivated users get nothing, deactivated superusers included, so this
|
# Deactivated users get nothing, deactivated superusers included, so this
|
||||||
# has to come before the superuser shortcut. guardian's
|
# has to come before the superuser shortcut. guardian's
|
||||||
@@ -422,24 +388,20 @@ def permitted_object_ids(
|
|||||||
"permission__content_type": content_type,
|
"permission__content_type": content_type,
|
||||||
}
|
}
|
||||||
|
|
||||||
key_as_text = Cast(OuterRef(key_field), CharField(max_length=64))
|
user_perm_ids = (
|
||||||
granted_to_user = Exists(
|
UserObjectPermission.objects.filter(user=user, **perm_filter)
|
||||||
UserObjectPermission.objects.filter(
|
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
||||||
user=user,
|
.values_list("object_pk_int", flat=True)
|
||||||
object_pk=key_as_text,
|
|
||||||
**perm_filter,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
granted_to_group = Exists(
|
group_perm_ids = (
|
||||||
GroupObjectPermission.objects.filter(
|
GroupObjectPermission.objects.filter(group__user=user, **perm_filter)
|
||||||
group_id__in=user.groups.values("id"),
|
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
||||||
object_pk=key_as_text,
|
.values_list("object_pk_int", flat=True)
|
||||||
**perm_filter,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
permitted_ids = user_perm_ids.union(group_perm_ids)
|
||||||
|
|
||||||
return base_qs.filter(
|
return base_qs.filter(
|
||||||
Q(**{owner_field: user.pk}) | unowned | granted_to_user | granted_to_group,
|
Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_ids),
|
||||||
).values_list("id", flat=True)
|
).values_list("id", flat=True)
|
||||||
|
|
||||||
|
|
||||||
@@ -508,17 +470,8 @@ def permitted_document_ids(
|
|||||||
``include_deleted=True`` for callers that need to check permission on
|
``include_deleted=True`` for callers that need to check permission on
|
||||||
soft-deleted documents (e.g. trash restore). This intentionally avoids
|
soft-deleted documents (e.g. trash restore). This intentionally avoids
|
||||||
``get_objects_for_user`` to keep the subquery small and index-friendly.
|
``get_objects_for_user`` to keep the subquery small and index-friendly.
|
||||||
|
|
||||||
A version is authorized by its root document, so a version's own owner and
|
|
||||||
grants never matter.
|
|
||||||
"""
|
"""
|
||||||
return permitted_object_ids(
|
return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
|
||||||
user,
|
|
||||||
Document,
|
|
||||||
perm,
|
|
||||||
include_deleted=include_deleted,
|
|
||||||
parent_field="root_document",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def get_document_count_filter_for_user(user, related_name: str = "documents"):
|
def get_document_count_filter_for_user(user, related_name: str = "documents"):
|
||||||
@@ -677,14 +630,7 @@ def has_perms_owner_aware(user, perms, obj):
|
|||||||
single-object check still has many production callers. Several callers
|
single-object check still has many production callers. Several callers
|
||||||
remain across ``documents/``, ``paperless_mail/``, and ``paperless_ai/``
|
remain across ``documents/``, ``paperless_mail/``, and ``paperless_ai/``
|
||||||
-- grep for this function name before removing it.
|
-- grep for this function name before removing it.
|
||||||
|
|
||||||
A document version is authorized by its root document, like in
|
|
||||||
``permitted_document_ids``, so a version's own owner and grants never
|
|
||||||
matter. Fetch the root with ``select_related("root_document__owner")`` to
|
|
||||||
avoid extra queries.
|
|
||||||
"""
|
"""
|
||||||
if isinstance(obj, Document):
|
|
||||||
obj = get_root_document(obj)
|
|
||||||
checker = ObjectPermissionChecker(user)
|
checker = ObjectPermissionChecker(user)
|
||||||
return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj)
|
return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj)
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from datetime import UTC
|
from datetime import UTC
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from datetime import timedelta
|
|
||||||
from enum import StrEnum
|
from enum import StrEnum
|
||||||
from itertools import islice
|
from itertools import islice
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
@@ -56,19 +55,6 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
logger = logging.getLogger("paperless.search")
|
logger = logging.getLogger("paperless.search")
|
||||||
|
|
||||||
# tantivy stores dates as signed 64-bit nanoseconds since the Unix epoch, which
|
|
||||||
# covers 1677-09-21T00:12:43 to 2262-04-11T23:47:16 UTC
|
|
||||||
_INDEX_DATE_NANOS_MIN: Final[int] = -(2**63)
|
|
||||||
_INDEX_DATE_NANOS_MAX: Final[int] = 2**63 - 1
|
|
||||||
_UNIX_EPOCH: Final[datetime] = datetime(1970, 1, 1, tzinfo=UTC)
|
|
||||||
|
|
||||||
|
|
||||||
def _is_indexable_date(value: datetime) -> bool:
|
|
||||||
"""Whether value, at whole-second precision, fits tantivy's date range."""
|
|
||||||
nanos = ((value - _UNIX_EPOCH) // timedelta(seconds=1)) * 1_000_000_000
|
|
||||||
return _INDEX_DATE_NANOS_MIN <= nanos <= _INDEX_DATE_NANOS_MAX
|
|
||||||
|
|
||||||
|
|
||||||
_LOCK_TIMEOUT_SECONDS: Final[float] = 10.0 # per-attempt acquire timeout
|
_LOCK_TIMEOUT_SECONDS: Final[float] = 10.0 # per-attempt acquire timeout
|
||||||
_LOCK_RETRY_ATTEMPTS: Final[int] = 4 # total attempts (1 initial + 3 retries)
|
_LOCK_RETRY_ATTEMPTS: Final[int] = 4 # total attempts (1 initial + 3 retries)
|
||||||
_LOCK_BACKOFF_BASE: Final[float] = 1.0 # seconds
|
_LOCK_BACKOFF_BASE: Final[float] = 1.0 # seconds
|
||||||
@@ -284,14 +270,9 @@ class WriteBatch:
|
|||||||
and adding the new version. This ensures stale document data (e.g., after
|
and adding the new version. This ensures stale document data (e.g., after
|
||||||
permission changes) doesn't persist in the index.
|
permission changes) doesn't persist in the index.
|
||||||
|
|
||||||
Only root documents are indexed, with their effective content, so a
|
|
||||||
version is indexed as its root document.
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
document: Django Document instance to index
|
document: Django Document instance to index
|
||||||
"""
|
"""
|
||||||
if document.root_document_id is not None:
|
|
||||||
document = document.root_document
|
|
||||||
self.remove(document.pk)
|
self.remove(document.pk)
|
||||||
doc = self._backend._build_tantivy_doc(document)
|
doc = self._backend._build_tantivy_doc(document)
|
||||||
self._writer.add_document(doc)
|
self._writer.add_document(doc)
|
||||||
@@ -316,22 +297,20 @@ class WriteBatch:
|
|||||||
An id with no matching document (e.g. deleted between the caller
|
An id with no matching document (e.g. deleted between the caller
|
||||||
collecting ids and the batch running) is silently skipped, matching
|
collecting ids and the batch running) is silently skipped, matching
|
||||||
``add_or_update()``'s existing single-document deferred-task behavior
|
``add_or_update()``'s existing single-document deferred-task behavior
|
||||||
rather than erroring or leaving a stale index entry. The id of a
|
rather than erroring or leaving a stale index entry.
|
||||||
version stands for its root document.
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
ids: Primary keys of Document instances to index
|
ids: Primary keys of Document instances to index
|
||||||
"""
|
"""
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from documents.versioning import annotate_effective_content
|
from documents.versioning import annotate_effective_content
|
||||||
from documents.versioning import root_document_ids
|
|
||||||
|
|
||||||
ids = list(ids)
|
ids = list(ids)
|
||||||
if not ids:
|
if not ids:
|
||||||
return
|
return
|
||||||
|
|
||||||
queryset = annotate_effective_content(
|
queryset = annotate_effective_content(
|
||||||
Document.objects.filter(pk__in=root_document_ids(ids))
|
Document.objects.filter(pk__in=ids)
|
||||||
.select_related("correspondent", "document_type", "storage_path", "owner")
|
.select_related("correspondent", "document_type", "storage_path", "owner")
|
||||||
.prefetch_related(
|
.prefetch_related(
|
||||||
"tags",
|
"tags",
|
||||||
@@ -649,15 +628,7 @@ class TantivyBackend:
|
|||||||
document.created.day,
|
document.created.day,
|
||||||
tzinfo=UTC,
|
tzinfo=UTC,
|
||||||
)
|
)
|
||||||
if _is_indexable_date(created_date):
|
doc.add_date("created", created_date)
|
||||||
doc.add_date("created", created_date)
|
|
||||||
else:
|
|
||||||
logger.warning(
|
|
||||||
"Document %s has a created date (%s) outside the range the search "
|
|
||||||
"index can store; it will be indexed without a created date",
|
|
||||||
document.pk,
|
|
||||||
document.created,
|
|
||||||
)
|
|
||||||
doc.add_date("modified", document.modified)
|
doc.add_date("modified", document.modified)
|
||||||
doc.add_date("added", document.added)
|
doc.add_date("added", document.added)
|
||||||
|
|
||||||
|
|||||||
@@ -502,8 +502,6 @@ def update_document_content_maybe_archive_file(
|
|||||||
llm_index_add_or_update_document(document)
|
llm_index_add_or_update_document(document)
|
||||||
|
|
||||||
clear_document_caches(document.pk)
|
clear_document_caches(document.pk)
|
||||||
if document.root_document_id is not None:
|
|
||||||
clear_document_caches(document.root_document_id)
|
|
||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
import json
|
import json
|
||||||
import logging
|
|
||||||
from datetime import date
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -293,80 +291,6 @@ class TestAddOrUpdateIds:
|
|||||||
assert backend.search_ids("updated", user=None) == [doc.pk]
|
assert backend.search_ids("updated", user=None) == [doc.pk]
|
||||||
|
|
||||||
|
|
||||||
class TestVersionsAreIndexedAsTheirRoot:
|
|
||||||
"""Only root documents are indexed, with their effective content, so
|
|
||||||
every write path that is handed a version indexes its root instead."""
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _root_with_version() -> tuple[Document, Document]:
|
|
||||||
root = DocumentFactory(title="Statement", content="stale text")
|
|
||||||
version = DocumentFactory(
|
|
||||||
title="Statement",
|
|
||||||
content="latest text",
|
|
||||||
root_document=root,
|
|
||||||
version_index=1,
|
|
||||||
)
|
|
||||||
return root, version
|
|
||||||
|
|
||||||
def test_add_or_update_indexes_the_root_of_a_version(
|
|
||||||
self,
|
|
||||||
backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- The version is passed to add_or_update
|
|
||||||
THEN:
|
|
||||||
- The root is indexed with the version's text, and the version is not
|
|
||||||
"""
|
|
||||||
root, version = self._root_with_version()
|
|
||||||
|
|
||||||
backend.add_or_update(version)
|
|
||||||
|
|
||||||
assert backend.search_ids("latest", user=None) == [root.pk]
|
|
||||||
assert backend.search_ids("stale", user=None) == []
|
|
||||||
|
|
||||||
def test_add_or_update_ids_indexes_each_root_once(
|
|
||||||
self,
|
|
||||||
backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- Both ids are passed to add_or_update_ids
|
|
||||||
THEN:
|
|
||||||
- The root is indexed once and the version is not indexed
|
|
||||||
"""
|
|
||||||
root, version = self._root_with_version()
|
|
||||||
|
|
||||||
with backend.batch_update() as batch:
|
|
||||||
batch.add_or_update_ids([version.pk, root.pk])
|
|
||||||
|
|
||||||
assert backend.search_ids("Statement", user=None) == [root.pk]
|
|
||||||
assert backend.search_ids("latest", user=None) == [root.pk]
|
|
||||||
|
|
||||||
def test_add_or_update_ids_resolves_a_lone_version(
|
|
||||||
self,
|
|
||||||
backend: TantivyBackend,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- Only the version's id is passed to add_or_update_ids
|
|
||||||
THEN:
|
|
||||||
- The root is indexed
|
|
||||||
"""
|
|
||||||
root, version = self._root_with_version()
|
|
||||||
|
|
||||||
with backend.batch_update() as batch:
|
|
||||||
batch.add_or_update_ids([version.pk])
|
|
||||||
|
|
||||||
assert backend.search_ids("latest", user=None) == [root.pk]
|
|
||||||
|
|
||||||
|
|
||||||
class TestSearch:
|
class TestSearch:
|
||||||
"""Test search query parsing and matching via search_ids."""
|
"""Test search query parsing and matching via search_ids."""
|
||||||
|
|
||||||
@@ -980,79 +904,6 @@ class TestRebuild:
|
|||||||
assert ids == [doc.pk]
|
assert ids == [doc.pk]
|
||||||
|
|
||||||
|
|
||||||
class TestCreatedDateOutOfRange:
|
|
||||||
"""The index stores dates as nanosecond i64 values (1677-09-22 to 2262-04-11).
|
|
||||||
|
|
||||||
A document whose created date falls outside that window must not abort
|
|
||||||
indexing: it is indexed without a created value and a warning names it.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("created", "expected_warnings"),
|
|
||||||
[
|
|
||||||
pytest.param(date(1677, 9, 22), 0, id="first-representable-day"),
|
|
||||||
pytest.param(date(2262, 4, 11), 0, id="last-representable-day"),
|
|
||||||
pytest.param(date(1677, 9, 21), 1, id="day-before-first"),
|
|
||||||
pytest.param(date(2262, 4, 12), 1, id="day-after-last"),
|
|
||||||
pytest.param(date(16, 8, 30), 1, id="two-digit-year-read-as-year-16"),
|
|
||||||
pytest.param(date(9999, 12, 31), 1, id="max-python-date"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_add_or_update_indexes_document_and_warns_when_out_of_range(
|
|
||||||
self,
|
|
||||||
backend: TantivyBackend,
|
|
||||||
caplog: pytest.LogCaptureFixture,
|
|
||||||
created: date,
|
|
||||||
expected_warnings: int,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A document with a created date at or beyond the index date limits
|
|
||||||
WHEN:
|
|
||||||
- The document is added to the index
|
|
||||||
THEN:
|
|
||||||
- The document is indexed and searchable either way
|
|
||||||
- A warning naming the document is logged only for out-of-range dates
|
|
||||||
"""
|
|
||||||
doc = DocumentFactory(created=created, content="boundarycontent")
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING, logger="paperless.search"):
|
|
||||||
backend.add_or_update(doc)
|
|
||||||
|
|
||||||
assert backend.search_ids("boundarycontent", user=None) == [doc.pk]
|
|
||||||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
|
||||||
assert len(warnings) == expected_warnings
|
|
||||||
if expected_warnings:
|
|
||||||
assert f"Document {doc.pk}" in warnings[0].getMessage()
|
|
||||||
|
|
||||||
def test_rebuild_continues_past_out_of_range_document(
|
|
||||||
self,
|
|
||||||
backend: TantivyBackend,
|
|
||||||
caplog: pytest.LogCaptureFixture,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A document with an unrepresentable created date among valid ones
|
|
||||||
WHEN:
|
|
||||||
- The index is rebuilt
|
|
||||||
THEN:
|
|
||||||
- Rebuild completes and every document is searchable
|
|
||||||
- A warning names the offending document
|
|
||||||
"""
|
|
||||||
good = DocumentFactory(created=date(2016, 8, 30), content="rebuildcontent")
|
|
||||||
bad = DocumentFactory(created=date(16, 8, 30), content="rebuildcontent")
|
|
||||||
|
|
||||||
with caplog.at_level(logging.WARNING, logger="paperless.search"):
|
|
||||||
backend.rebuild(Document.objects.all())
|
|
||||||
|
|
||||||
assert sorted(backend.search_ids("rebuildcontent", user=None)) == sorted(
|
|
||||||
[good.pk, bad.pk],
|
|
||||||
)
|
|
||||||
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
|
||||||
assert len(warnings) == 1
|
|
||||||
assert f"Document {bad.pk}" in warnings[0].getMessage()
|
|
||||||
|
|
||||||
|
|
||||||
class TestAutocomplete:
|
class TestAutocomplete:
|
||||||
"""Test autocomplete functionality."""
|
"""Test autocomplete functionality."""
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ from auditlog.models import LogEntry # type: ignore[import-untyped]
|
|||||||
from django.contrib.contenttypes.models import ContentType
|
from django.contrib.contenttypes.models import ContentType
|
||||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||||
from django.test import TestCase as DjangoTestCase
|
from django.test import TestCase as DjangoTestCase
|
||||||
from django.test import override_settings
|
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from rest_framework import status
|
from rest_framework import status
|
||||||
from rest_framework.test import APITestCase
|
from rest_framework.test import APITestCase
|
||||||
@@ -17,17 +16,13 @@ from documents.data_models import DocumentSource
|
|||||||
from documents.filters import EffectiveContentFilter
|
from documents.filters import EffectiveContentFilter
|
||||||
from documents.filters import TitleContentFilter
|
from documents.filters import TitleContentFilter
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from documents.models import Note
|
|
||||||
from documents.models import ShareLink
|
|
||||||
from documents.versioning import annotate_effective_content
|
from documents.versioning import annotate_effective_content
|
||||||
from documents.views import DocumentSelectionMixin
|
from documents.views import DocumentSelectionMixin
|
||||||
from paperless_testing.dirs import DirectoriesMixin
|
from paperless_testing.dirs import DirectoriesMixin
|
||||||
from paperless_testing.factories import DocumentFactory
|
from paperless_testing.factories import DocumentFactory
|
||||||
from paperless_testing.factories import UserFactory
|
from paperless_testing.factories import UserFactory
|
||||||
from paperless_testing.http import read_streaming_response
|
from paperless_testing.http import read_streaming_response
|
||||||
from paperless_testing.permissions import grant_all_global
|
|
||||||
from paperless_testing.permissions import grant_global
|
from paperless_testing.permissions import grant_global
|
||||||
from paperless_testing.permissions import grant_object
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -1048,152 +1043,3 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self.assertEqual(selected, [root.id])
|
self.assertEqual(selected, [root.id])
|
||||||
|
|
||||||
|
|
||||||
class TestVersionActionPermissions(DirectoriesMixin, APITestCase):
|
|
||||||
def setUp(self):
|
|
||||||
super().setUp()
|
|
||||||
self.user = UserFactory()
|
|
||||||
grant_all_global(self.user)
|
|
||||||
self.client.force_authenticate(self.user)
|
|
||||||
self.root = DocumentFactory(owner=UserFactory())
|
|
||||||
self.version = DocumentFactory(root_document=self.root, owner=None)
|
|
||||||
|
|
||||||
@override_settings(AUDIT_LOG_ENABLED=True)
|
|
||||||
def test_actions_reject_stale_version_ownership(self):
|
|
||||||
note = Note.objects.create(document=self.version, note="Version note")
|
|
||||||
for owner in (None, self.user):
|
|
||||||
self.version.owner = owner
|
|
||||||
self.version.save(update_fields=["owner"])
|
|
||||||
for action in (
|
|
||||||
"notes",
|
|
||||||
"suggestions",
|
|
||||||
"ai_suggestions",
|
|
||||||
"history",
|
|
||||||
"share_links",
|
|
||||||
):
|
|
||||||
with self.subTest(owner=owner, action=action):
|
|
||||||
response = self.client.get(
|
|
||||||
f"/api/documents/{self.version.pk}/{action}/",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
response = self.client.post(
|
|
||||||
f"/api/documents/{self.version.pk}/notes/",
|
|
||||||
{"note": "New note"},
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
response = self.client.delete(
|
|
||||||
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
response = self.client.post(
|
|
||||||
"/api/share_links/",
|
|
||||||
{"document": self.version.pk, "file_version": "original"},
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
response = self.client.post(
|
|
||||||
"/api/share_link_bundles/",
|
|
||||||
{"document_ids": [self.version.pk], "file_version": "original"},
|
|
||||||
format="json",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 400)
|
|
||||||
response = self.client.post(
|
|
||||||
"/api/documents/email/",
|
|
||||||
{
|
|
||||||
"documents": [self.version.pk],
|
|
||||||
"addresses": "recipient@example.com",
|
|
||||||
"subject": "Version",
|
|
||||||
"message": "Version",
|
|
||||||
},
|
|
||||||
format="json",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
with (
|
|
||||||
mock.patch("documents.views.AIConfig") as ai_config,
|
|
||||||
mock.patch("documents.views.stream_chat_with_documents") as chat,
|
|
||||||
):
|
|
||||||
ai_config.return_value.ai_enabled = True
|
|
||||||
response = self.client.post(
|
|
||||||
"/api/documents/chat/",
|
|
||||||
{"q": "Version?", "document_id": self.version.pk},
|
|
||||||
format="json",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
chat.assert_not_called()
|
|
||||||
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
|
|
||||||
self.assertFalse(ShareLink.objects.exists())
|
|
||||||
|
|
||||||
@mock.patch("documents.views.build_share_link_bundle.apply_async")
|
|
||||||
def test_root_permissions_allow_sharing_a_private_version(self, build_mock):
|
|
||||||
self.version.owner = UserFactory()
|
|
||||||
self.version.save(update_fields=["owner"])
|
|
||||||
grant_object(self.user, self.root, "view_document", "change_document")
|
|
||||||
note = Note.objects.create(document=self.version, note="Version note")
|
|
||||||
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
|
|
||||||
self.assertEqual(response.status_code, 200)
|
|
||||||
self.assertEqual(response.data[0]["id"], note.pk)
|
|
||||||
response = self.client.post(
|
|
||||||
"/api/share_links/",
|
|
||||||
{"document": self.version.pk, "file_version": "original"},
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 201)
|
|
||||||
self.assertEqual(ShareLink.objects.get().document_id, self.version.pk)
|
|
||||||
response = self.client.get(f"/api/documents/{self.version.pk}/share_links/")
|
|
||||||
self.assertEqual(response.status_code, 200)
|
|
||||||
self.assertEqual(len(response.data), 1)
|
|
||||||
response = self.client.post(
|
|
||||||
"/api/share_link_bundles/",
|
|
||||||
{"document_ids": [self.version.pk], "file_version": "original"},
|
|
||||||
format="json",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 201)
|
|
||||||
build_mock.assert_called_once()
|
|
||||||
|
|
||||||
def test_root_view_permission_does_not_allow_note_changes(self):
|
|
||||||
grant_object(self.user, self.root, "view_document")
|
|
||||||
note = Note.objects.create(document=self.version, note="Version note")
|
|
||||||
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
|
|
||||||
self.assertEqual(response.status_code, 200)
|
|
||||||
response = self.client.post(
|
|
||||||
f"/api/documents/{self.version.pk}/notes/",
|
|
||||||
{"note": "New note"},
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
response = self.client.delete(
|
|
||||||
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
|
|
||||||
|
|
||||||
@override_settings(AUDIT_LOG_ENABLED=True)
|
|
||||||
def test_history_uses_root_ownership(self):
|
|
||||||
self.root.owner = self.user
|
|
||||||
self.root.save(update_fields=["owner"])
|
|
||||||
self.version.owner = UserFactory()
|
|
||||||
self.version.save(update_fields=["owner"])
|
|
||||||
response = self.client.get(f"/api/documents/{self.version.pk}/history/")
|
|
||||||
self.assertEqual(response.status_code, 200)
|
|
||||||
|
|
||||||
def test_selection_data_rejects_stale_version_ownership(self):
|
|
||||||
for owner in (None, self.user):
|
|
||||||
self.version.owner = owner
|
|
||||||
self.version.save(update_fields=["owner"])
|
|
||||||
with self.subTest(owner=owner):
|
|
||||||
response = self.client.post(
|
|
||||||
"/api/documents/selection_data/",
|
|
||||||
{"documents": [self.version.pk]},
|
|
||||||
format="json",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 403)
|
|
||||||
|
|
||||||
def test_selection_data_allows_private_version_of_permitted_root(self):
|
|
||||||
self.version.owner = UserFactory()
|
|
||||||
self.version.save(update_fields=["owner"])
|
|
||||||
grant_object(self.user, self.root, "view_document")
|
|
||||||
other = DocumentFactory(owner=self.user)
|
|
||||||
response = self.client.post(
|
|
||||||
"/api/documents/selection_data/",
|
|
||||||
{"documents": [self.version.pk, other.pk]},
|
|
||||||
format="json",
|
|
||||||
)
|
|
||||||
self.assertEqual(response.status_code, 200)
|
|
||||||
@@ -1196,59 +1196,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
|||||||
self.assertIn(d3.id, result_ids)
|
self.assertIn(d3.id, result_ids)
|
||||||
self.assertNotIn(d4.id, result_ids)
|
self.assertNotIn(d4.id, result_ids)
|
||||||
|
|
||||||
def test_search_more_like_version_uses_its_root(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A document similar in content to a root document, and one that is not
|
|
||||||
- A version of the root document, which is never indexed
|
|
||||||
WHEN:
|
|
||||||
- API request for more like the version
|
|
||||||
THEN:
|
|
||||||
- The documents similar to the root are returned, not the version
|
|
||||||
"""
|
|
||||||
indexed = {}
|
|
||||||
for name, title, content, day in (
|
|
||||||
("root", "bank statement 1", "things i paid for in august", (2019, 3, 4)),
|
|
||||||
(
|
|
||||||
"similar",
|
|
||||||
"bank statement 3",
|
|
||||||
"things i paid for in september",
|
|
||||||
(2020, 7, 9),
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"other",
|
|
||||||
"Quarterly Report",
|
|
||||||
"quarterly revenue profit margin",
|
|
||||||
(2021, 11, 30),
|
|
||||||
),
|
|
||||||
):
|
|
||||||
with time_machine.travel(
|
|
||||||
timezone.make_aware(datetime.datetime(*day)),
|
|
||||||
tick=False,
|
|
||||||
):
|
|
||||||
indexed[name] = DocumentFactory(
|
|
||||||
title=title,
|
|
||||||
content=content,
|
|
||||||
created=datetime.date(*day),
|
|
||||||
added=timezone.make_aware(datetime.datetime(*day)),
|
|
||||||
)
|
|
||||||
version = DocumentFactory(
|
|
||||||
root_document=indexed["root"],
|
|
||||||
version_index=1,
|
|
||||||
content="things i paid for in august",
|
|
||||||
)
|
|
||||||
backend = get_backend()
|
|
||||||
for document in indexed.values():
|
|
||||||
backend.add_or_update(document)
|
|
||||||
|
|
||||||
response = self.client.get(f"/api/documents/?more_like_id={version.id}")
|
|
||||||
|
|
||||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
|
||||||
result_ids = [r["id"] for r in response.data["results"]]
|
|
||||||
self.assertIn(indexed["similar"].id, result_ids)
|
|
||||||
self.assertNotIn(indexed["other"].id, result_ids)
|
|
||||||
self.assertNotIn(version.id, result_ids)
|
|
||||||
|
|
||||||
def test_more_like_requires_id_of_existing_document(self) -> None:
|
def test_more_like_requires_id_of_existing_document(self) -> None:
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ from rest_framework.test import APITestCase
|
|||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from paperless_testing.dirs import DirectoriesMixin
|
from paperless_testing.dirs import DirectoriesMixin
|
||||||
from paperless_testing.factories import DocumentFactory
|
|
||||||
from paperless_testing.factories import UserFactory
|
from paperless_testing.factories import UserFactory
|
||||||
from paperless_testing.permissions import grant_all_global
|
from paperless_testing.permissions import grant_all_global
|
||||||
|
|
||||||
@@ -280,53 +279,3 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
|
|||||||
Document.objects.filter(root_document=root).values_list("id", flat=True),
|
Document.objects.filter(root_document=root).values_list("id", flat=True),
|
||||||
[version.pk for version in versions],
|
[version.pk for version in versions],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_api_trash_version_follows_root_owner(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A deleted version of user2's document, owned by nobody
|
|
||||||
- A deleted version of the user's document, owned by user2
|
|
||||||
WHEN:
|
|
||||||
- The user lists the trash and tries to restore or empty the versions
|
|
||||||
THEN:
|
|
||||||
- Only the version of the user's own document is listed
|
|
||||||
- The other version can't be restored or emptied
|
|
||||||
- The version of the user's own document can be restored
|
|
||||||
"""
|
|
||||||
user2 = UserFactory(username="user2")
|
|
||||||
other_version = DocumentFactory(
|
|
||||||
root_document=DocumentFactory(owner=user2),
|
|
||||||
version_index=1,
|
|
||||||
)
|
|
||||||
other_version.delete()
|
|
||||||
own_version = DocumentFactory(
|
|
||||||
owner=user2,
|
|
||||||
root_document=DocumentFactory(owner=self.user),
|
|
||||||
version_index=1,
|
|
||||||
)
|
|
||||||
own_version.delete()
|
|
||||||
|
|
||||||
resp = self.client.get("/api/trash/")
|
|
||||||
self.assertEqual(resp.status_code, status.HTTP_200_OK)
|
|
||||||
self.assertEqual(
|
|
||||||
[doc["id"] for doc in resp.data["results"]],
|
|
||||||
[own_version.pk],
|
|
||||||
)
|
|
||||||
|
|
||||||
for action in ("restore", "empty"):
|
|
||||||
with self.subTest(action=action):
|
|
||||||
resp = self.client.post(
|
|
||||||
"/api/trash/",
|
|
||||||
{"action": action, "documents": [other_version.pk]},
|
|
||||||
)
|
|
||||||
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
|
|
||||||
self.assertTrue(
|
|
||||||
Document.deleted_objects.filter(pk=other_version.pk).exists(),
|
|
||||||
)
|
|
||||||
|
|
||||||
resp = self.client.post(
|
|
||||||
"/api/trash/",
|
|
||||||
{"action": "restore", "documents": [own_version.pk]},
|
|
||||||
)
|
|
||||||
self.assertEqual(resp.status_code, status.HTTP_200_OK)
|
|
||||||
self.assertTrue(Document.objects.filter(pk=own_version.pk).exists())
|
|
||||||
@@ -4,7 +4,6 @@ from pathlib import Path
|
|||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
import pikepdf
|
import pikepdf
|
||||||
import pytest
|
|
||||||
from django.contrib.auth.models import Group
|
from django.contrib.auth.models import Group
|
||||||
from django.contrib.auth.models import Permission
|
from django.contrib.auth.models import Permission
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
@@ -13,7 +12,6 @@ from django.test import TestCase
|
|||||||
from django.test.utils import CaptureQueriesContext
|
from django.test.utils import CaptureQueriesContext
|
||||||
from guardian.shortcuts import get_groups_with_perms
|
from guardian.shortcuts import get_groups_with_perms
|
||||||
from guardian.shortcuts import get_users_with_perms
|
from guardian.shortcuts import get_users_with_perms
|
||||||
from pytest_mock import MockerFixture
|
|
||||||
|
|
||||||
from documents import bulk_edit
|
from documents import bulk_edit
|
||||||
from documents.models import Correspondent
|
from documents.models import Correspondent
|
||||||
@@ -25,7 +23,6 @@ from documents.models import StoragePath
|
|||||||
from documents.models import Tag
|
from documents.models import Tag
|
||||||
from documents.permissions import set_permissions_for_objects
|
from documents.permissions import set_permissions_for_objects
|
||||||
from paperless_testing.dirs import DirectoriesMixin
|
from paperless_testing.dirs import DirectoriesMixin
|
||||||
from paperless_testing.factories import DocumentFactory
|
|
||||||
from paperless_testing.permissions import grant_object
|
from paperless_testing.permissions import grant_object
|
||||||
|
|
||||||
|
|
||||||
@@ -1973,22 +1970,18 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
|||||||
self.assertIn("Error removing password from document", cm.output[0])
|
self.assertIn("Error removing password from document", cm.output[0])
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
class TestBulkEditReprocess(DirectoriesMixin, TestCase):
|
||||||
class TestBulkEditReprocess:
|
def setUp(self) -> None:
|
||||||
@pytest.fixture
|
super().setUp()
|
||||||
def mock_task(self, mocker: MockerFixture) -> mock.MagicMock:
|
|
||||||
return mocker.patch(
|
self.doc = Document.objects.create(
|
||||||
"documents.bulk_edit.update_document_content_maybe_archive_file",
|
title="test",
|
||||||
|
checksum="A",
|
||||||
|
mime_type="application/pdf",
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
|
||||||
def _queued_ids(mock_task: mock.MagicMock) -> list[int]:
|
def test_reprocess_defaults_to_local(self, mock_task: mock.Mock) -> None:
|
||||||
return [
|
|
||||||
call.kwargs["kwargs"]["document_id"]
|
|
||||||
for call in mock_task.apply_async.call_args_list
|
|
||||||
]
|
|
||||||
|
|
||||||
def test_reprocess_defaults_to_local(self, mock_task: mock.MagicMock) -> None:
|
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
- A reprocess request that says nothing about remote OCR
|
- A reprocess request that says nothing about remote OCR
|
||||||
@@ -1997,17 +1990,18 @@ class TestBulkEditReprocess:
|
|||||||
THEN:
|
THEN:
|
||||||
- The task is queued without asking for the remote engine
|
- The task is queued without asking for the remote engine
|
||||||
"""
|
"""
|
||||||
doc = DocumentFactory()
|
result = bulk_edit.reprocess([self.doc.id])
|
||||||
|
|
||||||
assert bulk_edit.reprocess([doc.id]) == "OK"
|
|
||||||
|
|
||||||
|
self.assertEqual(result, "OK")
|
||||||
mock_task.apply_async.assert_called_once()
|
mock_task.apply_async.assert_called_once()
|
||||||
assert mock_task.apply_async.call_args.kwargs["kwargs"] == {
|
_, kwargs = mock_task.apply_async.call_args
|
||||||
"document_id": doc.id,
|
self.assertEqual(
|
||||||
"remote_ocr": False,
|
kwargs["kwargs"],
|
||||||
}
|
{"document_id": self.doc.id, "remote_ocr": False},
|
||||||
|
)
|
||||||
|
|
||||||
def test_reprocess_passes_remote_ocr(self, mock_task: mock.MagicMock) -> None:
|
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
|
||||||
|
def test_reprocess_passes_remote_ocr(self, mock_task: mock.Mock) -> None:
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
- A reprocess request that explicitly asks for remote OCR
|
- A reprocess request that explicitly asks for remote OCR
|
||||||
@@ -2016,72 +2010,14 @@ class TestBulkEditReprocess:
|
|||||||
THEN:
|
THEN:
|
||||||
- The request is forwarded to the task for every document
|
- The request is forwarded to the task for every document
|
||||||
"""
|
"""
|
||||||
docs = DocumentFactory.create_batch(2)
|
other = Document.objects.create(
|
||||||
|
title="test2",
|
||||||
bulk_edit.reprocess([doc.id for doc in docs], remote_ocr=True)
|
checksum="B",
|
||||||
|
mime_type="application/pdf",
|
||||||
assert mock_task.apply_async.call_count == 2
|
|
||||||
for call in mock_task.apply_async.call_args_list:
|
|
||||||
assert call.kwargs["kwargs"]["remote_ocr"]
|
|
||||||
|
|
||||||
def test_reprocess_root_uses_latest_version(
|
|
||||||
self,
|
|
||||||
mock_task: mock.MagicMock,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with two versions
|
|
||||||
WHEN:
|
|
||||||
- reprocess is called with the root document
|
|
||||||
THEN:
|
|
||||||
- The latest version is reprocessed, not the root's original file
|
|
||||||
"""
|
|
||||||
root = DocumentFactory()
|
|
||||||
DocumentFactory(root_document=root, version_index=1)
|
|
||||||
latest = DocumentFactory(root_document=root, version_index=2)
|
|
||||||
|
|
||||||
bulk_edit.reprocess([root.id])
|
|
||||||
|
|
||||||
assert self._queued_ids(mock_task) == [latest.id]
|
|
||||||
|
|
||||||
def test_reprocess_explicit_version(self, mock_task: mock.MagicMock) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with two versions
|
|
||||||
WHEN:
|
|
||||||
- reprocess is called with the older version
|
|
||||||
THEN:
|
|
||||||
- That version is reprocessed
|
|
||||||
"""
|
|
||||||
root = DocumentFactory()
|
|
||||||
older = DocumentFactory(root_document=root, version_index=1)
|
|
||||||
DocumentFactory(root_document=root, version_index=2)
|
|
||||||
|
|
||||||
bulk_edit.reprocess([older.id])
|
|
||||||
|
|
||||||
assert self._queued_ids(mock_task) == [older.id]
|
|
||||||
|
|
||||||
def test_reprocess_root_and_latest_version_dispatches_once(
|
|
||||||
self,
|
|
||||||
mock_task: mock.MagicMock,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with two versions, the latest created on a
|
|
||||||
different date than the root
|
|
||||||
WHEN:
|
|
||||||
- reprocess is called with both the root and its latest version
|
|
||||||
THEN:
|
|
||||||
- The latest version is reprocessed only once
|
|
||||||
"""
|
|
||||||
root = DocumentFactory(created=date(2024, 1, 1))
|
|
||||||
DocumentFactory(root_document=root, version_index=1)
|
|
||||||
latest = DocumentFactory(
|
|
||||||
root_document=root,
|
|
||||||
version_index=2,
|
|
||||||
created=date(2025, 1, 1),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
bulk_edit.reprocess([root.id, latest.id])
|
bulk_edit.reprocess([self.doc.id, other.id], remote_ocr=True)
|
||||||
|
|
||||||
assert self._queued_ids(mock_task) == [latest.id]
|
self.assertEqual(mock_task.apply_async.call_count, 2)
|
||||||
|
for call in mock_task.apply_async.call_args_list:
|
||||||
|
self.assertTrue(call.kwargs["kwargs"]["remote_ocr"])
|
||||||
@@ -22,7 +22,6 @@ from documents.models import Document
|
|||||||
from documents.tasks import update_document_content_maybe_archive_file
|
from documents.tasks import update_document_content_maybe_archive_file
|
||||||
from paperless_testing.assertions import FileSystemAssertsMixin
|
from paperless_testing.assertions import FileSystemAssertsMixin
|
||||||
from paperless_testing.dirs import DirectoriesMixin
|
from paperless_testing.dirs import DirectoriesMixin
|
||||||
from paperless_testing.factories import DocumentFactory
|
|
||||||
|
|
||||||
sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
|
sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
|
||||||
|
|
||||||
@@ -117,27 +116,6 @@ class TestMakeIndex:
|
|||||||
call_command("document_index", "reindex", skip_checks=True)
|
call_command("document_index", "reindex", skip_checks=True)
|
||||||
mock_get_backend.return_value.rebuild.assert_called_once()
|
mock_get_backend.return_value.rebuild.assert_called_once()
|
||||||
|
|
||||||
def test_reindex_skips_versions(self, mocker: MockerFixture) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- The reindex command runs
|
|
||||||
THEN:
|
|
||||||
- Only the root document is handed to the rebuild, since a version
|
|
||||||
is indexed as its root
|
|
||||||
"""
|
|
||||||
root = DocumentFactory()
|
|
||||||
DocumentFactory(root_document=root, version_index=1)
|
|
||||||
mock_get_backend = mocker.patch(
|
|
||||||
"documents.management.commands.document_index.get_backend",
|
|
||||||
)
|
|
||||||
|
|
||||||
call_command("document_index", "reindex", skip_checks=True)
|
|
||||||
|
|
||||||
documents = mock_get_backend.return_value.rebuild.call_args.args[0]
|
|
||||||
assert list(documents.values_list("pk", flat=True)) == [root.pk]
|
|
||||||
|
|
||||||
def test_optimize(self) -> None:
|
def test_optimize(self) -> None:
|
||||||
"""Optimize command must execute without error (Tantivy handles optimization automatically)."""
|
"""Optimize command must execute without error (Tantivy handles optimization automatically)."""
|
||||||
call_command("document_index", "optimize", skip_checks=True)
|
call_command("document_index", "optimize", skip_checks=True)
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ from documents.models import Correspondent
|
|||||||
from documents.models import DocumentType
|
from documents.models import DocumentType
|
||||||
from documents.models import StoragePath
|
from documents.models import StoragePath
|
||||||
from documents.models import Tag
|
from documents.models import Tag
|
||||||
from documents.permissions import has_perms_owner_aware
|
|
||||||
from documents.permissions import permitted_document_ids
|
from documents.permissions import permitted_document_ids
|
||||||
from documents.permissions import permitted_object_ids
|
from documents.permissions import permitted_object_ids
|
||||||
from documents.permissions import restrict_queryset_to_visible
|
from documents.permissions import restrict_queryset_to_visible
|
||||||
@@ -33,8 +32,6 @@ from paperless_testing.permissions import grant_global
|
|||||||
from paperless_testing.permissions import grant_object
|
from paperless_testing.permissions import grant_object
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from django.contrib.auth.models import User
|
|
||||||
|
|
||||||
from paperless_testing.dirs import PaperlessDirs
|
from paperless_testing.dirs import PaperlessDirs
|
||||||
|
|
||||||
|
|
||||||
@@ -181,382 +178,6 @@ class TestPermittedDocumentIdsIncludeDeleted:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
class TestPermittedDocumentIdsVersions:
|
|
||||||
"""
|
|
||||||
A version is authorized by its root document: the version's own owner and
|
|
||||||
grants never matter.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("root_owner", "version_owner", "expected_visible"),
|
|
||||||
[
|
|
||||||
pytest.param(
|
|
||||||
"other",
|
|
||||||
"nobody",
|
|
||||||
False,
|
|
||||||
id="unowned-version-of-private-root",
|
|
||||||
),
|
|
||||||
pytest.param("other", "user", False, id="own-version-of-private-root"),
|
|
||||||
pytest.param("user", "other", True, id="foreign-version-of-own-root"),
|
|
||||||
pytest.param("user", "nobody", True, id="unowned-version-of-own-root"),
|
|
||||||
pytest.param("nobody", "other", True, id="private-version-of-unowned-root"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_version_follows_root_owner(
|
|
||||||
self,
|
|
||||||
root_owner: str,
|
|
||||||
version_owner: str,
|
|
||||||
*,
|
|
||||||
expected_visible: bool,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document and a version with differing owners
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved for the user
|
|
||||||
THEN:
|
|
||||||
- The version is visible exactly when its root is
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
owners = {"user": user, "other": UserFactory(), "nobody": None}
|
|
||||||
root = DocumentFactory(owner=owners[root_owner])
|
|
||||||
version = DocumentFactory(root_document=root, owner=owners[version_owner])
|
|
||||||
|
|
||||||
visible = set(permitted_document_ids(user))
|
|
||||||
|
|
||||||
assert (version.pk in visible) is expected_visible
|
|
||||||
assert (root.pk in visible) is expected_visible
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def grantee(user: User, kind: str) -> User | Group:
|
|
||||||
"""The user itself, or a new group the user belongs to."""
|
|
||||||
if kind == "user":
|
|
||||||
return user
|
|
||||||
group = Group.objects.create(name="shared")
|
|
||||||
user.groups.add(group)
|
|
||||||
return group
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"grantee_kind",
|
|
||||||
[pytest.param("user", id="user"), pytest.param("group", id="group")],
|
|
||||||
)
|
|
||||||
def test_grant_on_root_applies_to_version(self, grantee_kind: str) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A private root document shared with a user or one of their groups
|
|
||||||
- A version of it owned by someone else
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved for the user
|
|
||||||
THEN:
|
|
||||||
- Both the root and the version are visible
|
|
||||||
- A user without the grant sees neither
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
stranger = UserFactory()
|
|
||||||
root = DocumentFactory(owner=UserFactory())
|
|
||||||
version = DocumentFactory(root_document=root, owner=UserFactory())
|
|
||||||
grant_object(self.grantee(user, grantee_kind), root, "view_document")
|
|
||||||
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(user),
|
|
||||||
expected_visible=[root.pk, version.pk],
|
|
||||||
expected_hidden=[],
|
|
||||||
)
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(stranger),
|
|
||||||
expected_visible=[],
|
|
||||||
expected_hidden=[root.pk, version.pk],
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"grantee_kind",
|
|
||||||
[pytest.param("user", id="user"), pytest.param("group", id="group")],
|
|
||||||
)
|
|
||||||
def test_grant_on_version_is_ignored(self, grantee_kind: str) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A private root document
|
|
||||||
- A version with an explicit grant for the user or one of their groups
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved for the user
|
|
||||||
THEN:
|
|
||||||
- Neither the root nor the version is visible
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
root = DocumentFactory(owner=UserFactory())
|
|
||||||
version = DocumentFactory(root_document=root, owner=UserFactory())
|
|
||||||
grant_object(self.grantee(user, grantee_kind), version, "view_document")
|
|
||||||
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(user),
|
|
||||||
expected_visible=[],
|
|
||||||
expected_hidden=[root.pk, version.pk],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_grant_on_one_root_does_not_reach_another_roots_version(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- Two private roots, each with a version
|
|
||||||
- The user may view only the first root
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved for the user
|
|
||||||
THEN:
|
|
||||||
- Only the first root and its version are visible
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
first = DocumentFactory(owner=UserFactory())
|
|
||||||
first_version = DocumentFactory(root_document=first, owner=UserFactory())
|
|
||||||
second = DocumentFactory(owner=UserFactory())
|
|
||||||
second_version = DocumentFactory(root_document=second, owner=user)
|
|
||||||
grant_object(user, first, "view_document")
|
|
||||||
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(user),
|
|
||||||
expected_visible=[first.pk, first_version.pk],
|
|
||||||
expected_hidden=[second.pk, second_version.pk],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_user_in_several_groups(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A user in two groups
|
|
||||||
- Two private roots shared with one group each, and a third shared with nobody
|
|
||||||
- A version of each root
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved for the user
|
|
||||||
THEN:
|
|
||||||
- The two shared roots and their versions are visible
|
|
||||||
- The third root and its version are not
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
groups = [Group.objects.create(name=f"group{i}") for i in range(2)]
|
|
||||||
user.groups.add(*groups)
|
|
||||||
shared = [DocumentFactory(owner=UserFactory()) for _ in groups]
|
|
||||||
for root, group in zip(shared, groups, strict=True):
|
|
||||||
grant_object(group, root, "view_document")
|
|
||||||
unshared = DocumentFactory(owner=UserFactory())
|
|
||||||
shared_versions = [
|
|
||||||
DocumentFactory(root_document=root, owner=UserFactory()) for root in shared
|
|
||||||
]
|
|
||||||
unshared_version = DocumentFactory(root_document=unshared, owner=None)
|
|
||||||
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(user),
|
|
||||||
expected_visible=[
|
|
||||||
*(root.pk for root in shared),
|
|
||||||
*(version.pk for version in shared_versions),
|
|
||||||
],
|
|
||||||
expected_hidden=[unshared.pk, unshared_version.pk],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_permission_is_resolved_through_the_root(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A private root document where the user may view and change
|
|
||||||
WHEN:
|
|
||||||
- The permitted ids are resolved for view, change and delete
|
|
||||||
THEN:
|
|
||||||
- The version is visible for view and change only
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
root = DocumentFactory(owner=UserFactory())
|
|
||||||
version = DocumentFactory(root_document=root, owner=UserFactory())
|
|
||||||
grant_object(user, root, "view_document", "change_document")
|
|
||||||
|
|
||||||
assert version.pk in set(permitted_document_ids(user))
|
|
||||||
assert version.pk in set(permitted_document_ids(user, perm="change_document"))
|
|
||||||
assert version.pk in set(
|
|
||||||
permitted_document_ids(user, perm="documents.change_document"),
|
|
||||||
)
|
|
||||||
assert version.pk not in set(
|
|
||||||
permitted_document_ids(user, perm="delete_document"),
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_anonymous_sees_versions_of_unowned_roots_only(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A version owned by nobody under a private root
|
|
||||||
- A version owned by someone under an unowned root
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved for an anonymous user
|
|
||||||
THEN:
|
|
||||||
- Only the version of the unowned root is visible
|
|
||||||
"""
|
|
||||||
private_root = DocumentFactory(owner=UserFactory())
|
|
||||||
private_version = DocumentFactory(root_document=private_root, owner=None)
|
|
||||||
open_root = DocumentFactory(owner=None)
|
|
||||||
open_version = DocumentFactory(root_document=open_root, owner=UserFactory())
|
|
||||||
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(AnonymousUser()),
|
|
||||||
expected_visible=[open_root.pk, open_version.pk],
|
|
||||||
expected_hidden=[private_root.pk, private_version.pk],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_deleted_versions_follow_their_deleted_root(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A soft-deleted root document and its version, which deleting the
|
|
||||||
root soft-deletes too; the version is owned by someone else
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved with and without deleted
|
|
||||||
documents
|
|
||||||
THEN:
|
|
||||||
- Nothing is visible by default
|
|
||||||
- With deleted documents included, the version is visible to the
|
|
||||||
root's owner and not to the version's own owner
|
|
||||||
"""
|
|
||||||
owner = UserFactory()
|
|
||||||
version_owner = UserFactory()
|
|
||||||
root = DocumentFactory(owner=owner)
|
|
||||||
version = DocumentFactory(root_document=root, owner=version_owner)
|
|
||||||
root.delete()
|
|
||||||
|
|
||||||
assert not {root.pk, version.pk} & set(permitted_document_ids(owner))
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(owner, include_deleted=True),
|
|
||||||
expected_visible=[root.pk, version.pk],
|
|
||||||
expected_hidden=[],
|
|
||||||
)
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(version_owner, include_deleted=True),
|
|
||||||
expected_visible=[],
|
|
||||||
expected_hidden=[root.pk, version.pk],
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
"is_superuser",
|
|
||||||
[
|
|
||||||
pytest.param(False, id="regular-user"),
|
|
||||||
pytest.param(True, id="superuser"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_inactive_user_sees_no_versions(self, *, is_superuser: bool) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- An inactive user, possibly a superuser, who owns a root and its version
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved for them
|
|
||||||
THEN:
|
|
||||||
- Nothing is visible
|
|
||||||
"""
|
|
||||||
user = UserFactory(is_active=False, is_superuser=is_superuser)
|
|
||||||
root = DocumentFactory(owner=user)
|
|
||||||
version = DocumentFactory(root_document=root, owner=user)
|
|
||||||
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(user),
|
|
||||||
expected_visible=[],
|
|
||||||
expected_hidden=[root.pk, version.pk],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_superuser_sees_all_versions(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A private root owned by someone else, with a version
|
|
||||||
WHEN:
|
|
||||||
- The permitted document ids are resolved for a superuser
|
|
||||||
THEN:
|
|
||||||
- Both the root and the version are visible
|
|
||||||
"""
|
|
||||||
superuser = UserFactory(superuser=True)
|
|
||||||
root = DocumentFactory(owner=UserFactory())
|
|
||||||
version = DocumentFactory(root_document=root, owner=UserFactory())
|
|
||||||
|
|
||||||
assert_visible_document_ids(
|
|
||||||
permitted_document_ids(superuser),
|
|
||||||
expected_visible=[root.pk, version.pk],
|
|
||||||
expected_hidden=[],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
class TestHasPermsOwnerAwareVersions:
|
|
||||||
"""
|
|
||||||
The single-object check agrees with permitted_document_ids: a version is
|
|
||||||
authorized by its root document.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
|
||||||
("root_owner", "version_owner", "expected"),
|
|
||||||
[
|
|
||||||
pytest.param(
|
|
||||||
"other",
|
|
||||||
"nobody",
|
|
||||||
False,
|
|
||||||
id="unowned-version-of-private-root",
|
|
||||||
),
|
|
||||||
pytest.param("other", "user", False, id="own-version-of-private-root"),
|
|
||||||
pytest.param("user", "other", True, id="foreign-version-of-own-root"),
|
|
||||||
pytest.param("nobody", "other", True, id="private-version-of-unowned-root"),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
def test_version_follows_root_owner(
|
|
||||||
self,
|
|
||||||
root_owner: str,
|
|
||||||
version_owner: str,
|
|
||||||
*,
|
|
||||||
expected: bool,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document and a version with differing owners
|
|
||||||
WHEN:
|
|
||||||
- The single-object check runs for the version
|
|
||||||
THEN:
|
|
||||||
- The version is allowed exactly when its root is
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
owners = {"user": user, "other": UserFactory(), "nobody": None}
|
|
||||||
root = DocumentFactory(owner=owners[root_owner])
|
|
||||||
version = DocumentFactory(root_document=root, owner=owners[version_owner])
|
|
||||||
|
|
||||||
assert has_perms_owner_aware(user, "view_document", version) is expected
|
|
||||||
assert has_perms_owner_aware(user, "view_document", root) is expected
|
|
||||||
|
|
||||||
def test_grant_on_root_applies_and_grant_on_version_does_not(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A private root with a version, and a second private root with a version
|
|
||||||
- The user may change only the first root, and was granted the second
|
|
||||||
root's version directly
|
|
||||||
WHEN:
|
|
||||||
- The single-object check runs for each version
|
|
||||||
THEN:
|
|
||||||
- Only the first root's version is allowed
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
shared_root = DocumentFactory(owner=UserFactory())
|
|
||||||
shared_version = DocumentFactory(root_document=shared_root, owner=UserFactory())
|
|
||||||
private_root = DocumentFactory(owner=UserFactory())
|
|
||||||
private_version = DocumentFactory(
|
|
||||||
root_document=private_root,
|
|
||||||
owner=UserFactory(),
|
|
||||||
)
|
|
||||||
grant_object(user, shared_root, "change_document")
|
|
||||||
grant_object(user, private_version, "change_document")
|
|
||||||
|
|
||||||
assert has_perms_owner_aware(user, "change_document", shared_version)
|
|
||||||
assert not has_perms_owner_aware(user, "change_document", private_version)
|
|
||||||
|
|
||||||
def test_other_models_use_their_own_owner(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A tag owned by someone else, and one owned by the user
|
|
||||||
WHEN:
|
|
||||||
- The single-object check runs for each
|
|
||||||
THEN:
|
|
||||||
- Only the user's own tag is allowed without a grant
|
|
||||||
"""
|
|
||||||
user = UserFactory()
|
|
||||||
mine = TagFactory(owner=user)
|
|
||||||
theirs = TagFactory(owner=UserFactory())
|
|
||||||
|
|
||||||
assert has_perms_owner_aware(user, "view_tag", mine)
|
|
||||||
assert not has_perms_owner_aware(user, "view_tag", theirs)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
class TestAiChatAllDocumentsPermissionBoundary:
|
class TestAiChatAllDocumentsPermissionBoundary:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -90,13 +90,10 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
|
|||||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||||
self.assertIn("document_ids", response.data)
|
self.assertIn("document_ids", response.data)
|
||||||
|
|
||||||
def test_create_bundle_rejects_insufficient_permissions(self) -> None:
|
@mock.patch("documents.views.permitted_document_ids", return_value=set())
|
||||||
requester = UserFactory(username="bundle_creator")
|
def test_create_bundle_rejects_insufficient_permissions(self, perms_mock) -> None:
|
||||||
grant_global(requester, "add_sharelinkbundle", "view_document")
|
|
||||||
self.client.force_authenticate(requester)
|
|
||||||
document = DocumentFactory(owner=UserFactory(username="document_owner"))
|
|
||||||
payload = {
|
payload = {
|
||||||
"document_ids": [self.document.pk, document.pk],
|
"document_ids": [self.document.pk],
|
||||||
"file_version": ShareLink.FileVersion.ARCHIVE,
|
"file_version": ShareLink.FileVersion.ARCHIVE,
|
||||||
"expiration_days": 7,
|
"expiration_days": 7,
|
||||||
}
|
}
|
||||||
@@ -104,8 +101,8 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
|
|||||||
response = self.client.post(self.ENDPOINT, payload, format="json")
|
response = self.client.post(self.ENDPOINT, payload, format="json")
|
||||||
|
|
||||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||||
self.assertIn(str(document.pk), str(response.data["document_ids"]))
|
self.assertIn("document_ids", response.data)
|
||||||
self.assertFalse(ShareLinkBundle.objects.exists())
|
perms_mock.assert_called()
|
||||||
|
|
||||||
@mock.patch("documents.views.build_share_link_bundle.apply_async")
|
@mock.patch("documents.views.build_share_link_bundle.apply_async")
|
||||||
def test_rebuild_bundle_resets_state(self, delay_mock) -> None:
|
def test_rebuild_bundle_resets_state(self, delay_mock) -> None:
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ from documents.sanity_checker import SanityCheckMessages
|
|||||||
from documents.tests.helpers import dummy_preprocess
|
from documents.tests.helpers import dummy_preprocess
|
||||||
from paperless_testing.assertions import FileSystemAssertsMixin
|
from paperless_testing.assertions import FileSystemAssertsMixin
|
||||||
from paperless_testing.dirs import DirectoriesMixin
|
from paperless_testing.dirs import DirectoriesMixin
|
||||||
from paperless_testing.factories import DocumentFactory
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@@ -288,80 +287,6 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
|||||||
tasks.update_document_content_maybe_archive_file(doc.pk)
|
tasks.update_document_content_maybe_archive_file(doc.pk)
|
||||||
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
|
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
|
||||||
|
|
||||||
def _create_root_with_version(self) -> tuple[Document, Document]:
|
|
||||||
sample1 = self.dirs.scratch_dir / "sample.pdf"
|
|
||||||
shutil.copy(
|
|
||||||
Path(__file__).parent
|
|
||||||
/ "samples"
|
|
||||||
/ "documents"
|
|
||||||
/ "originals"
|
|
||||||
/ "0000001.pdf",
|
|
||||||
sample1,
|
|
||||||
)
|
|
||||||
root = DocumentFactory(content="root content", mime_type="application/pdf")
|
|
||||||
version = DocumentFactory(
|
|
||||||
content="my document",
|
|
||||||
filename=sample1,
|
|
||||||
mime_type="application/pdf",
|
|
||||||
root_document=root,
|
|
||||||
version_index=1,
|
|
||||||
)
|
|
||||||
return root, version
|
|
||||||
|
|
||||||
@mock.patch("documents.tasks.clear_document_caches")
|
|
||||||
@mock.patch("documents.search.get_backend")
|
|
||||||
def test_update_content_version_clears_caches_for_root(
|
|
||||||
self,
|
|
||||||
mock_get_backend: mock.Mock,
|
|
||||||
mock_clear_caches: mock.Mock,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- Update content task is called for the version
|
|
||||||
THEN:
|
|
||||||
- The version's content is updated, not the root's
|
|
||||||
- The document is indexed
|
|
||||||
- Caches are cleared for both
|
|
||||||
"""
|
|
||||||
root, version = self._create_root_with_version()
|
|
||||||
|
|
||||||
tasks.update_document_content_maybe_archive_file(version.pk)
|
|
||||||
|
|
||||||
self.assertNotEqual(
|
|
||||||
Document.objects.get(pk=version.pk).content,
|
|
||||||
"my document",
|
|
||||||
)
|
|
||||||
self.assertEqual(Document.objects.get(pk=root.pk).content, "root content")
|
|
||||||
mock_get_backend.return_value.add_or_update.assert_called_once()
|
|
||||||
mock_clear_caches.assert_has_calls(
|
|
||||||
[mock.call(version.pk), mock.call(root.pk)],
|
|
||||||
)
|
|
||||||
|
|
||||||
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
@mock.patch("documents.tasks.llm_index_add_or_update_document")
|
|
||||||
@mock.patch("documents.search.get_backend")
|
|
||||||
def test_update_content_version_updates_llm_index(
|
|
||||||
self,
|
|
||||||
mock_get_backend: mock.Mock,
|
|
||||||
mock_llm_index: mock.Mock,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
- The LLM index is enabled
|
|
||||||
WHEN:
|
|
||||||
- Update content task is called for the version
|
|
||||||
THEN:
|
|
||||||
- The LLM index is updated
|
|
||||||
"""
|
|
||||||
_, version = self._create_root_with_version()
|
|
||||||
|
|
||||||
tasks.update_document_content_maybe_archive_file(version.pk)
|
|
||||||
|
|
||||||
mock_llm_index.assert_called_once()
|
|
||||||
|
|
||||||
|
|
||||||
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -17,26 +17,9 @@ from django.db.models.functions import RowNumber
|
|||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Iterable
|
|
||||||
|
|
||||||
from rest_framework.request import Request
|
from rest_framework.request import Request
|
||||||
|
|
||||||
|
|
||||||
def root_document_ids(ids: Iterable[int]) -> QuerySet[int]:
|
|
||||||
"""
|
|
||||||
The ids of the root documents of the given documents: a root stands for
|
|
||||||
itself and a version for its root. Only the indexes' bookkeeping needs
|
|
||||||
this, since they hold root documents only.
|
|
||||||
"""
|
|
||||||
return (
|
|
||||||
Document.objects.filter(pk__in=ids)
|
|
||||||
.annotate(root_id=Coalesce("root_document_id", "id"))
|
|
||||||
.order_by()
|
|
||||||
.values_list("root_id", flat=True)
|
|
||||||
.distinct()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
|
def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
|
||||||
"""
|
"""
|
||||||
Sorts versions so the newest one comes first using version_index and not on id,
|
Sorts versions so the newest one comes first using version_index and not on id,
|
||||||
|
|||||||
+51
-84
@@ -329,10 +329,9 @@ def _get_tantivy_query_and_mode(params):
|
|||||||
def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
|
def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
|
||||||
try:
|
try:
|
||||||
more_like_doc_id = int(query_params["more_like_id"])
|
more_like_doc_id = int(query_params["more_like_id"])
|
||||||
more_like_doc = Document.objects.select_related(
|
more_like_doc = Document.objects.select_related("owner").get(
|
||||||
"owner",
|
pk=more_like_doc_id,
|
||||||
"root_document__owner",
|
)
|
||||||
).get(pk=more_like_doc_id)
|
|
||||||
except (TypeError, ValueError, Document.DoesNotExist):
|
except (TypeError, ValueError, Document.DoesNotExist):
|
||||||
raise PermissionDenied(_("Invalid more_like_id"))
|
raise PermissionDenied(_("Invalid more_like_id"))
|
||||||
|
|
||||||
@@ -343,8 +342,7 @@ def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
|
|||||||
):
|
):
|
||||||
raise PermissionDenied(_("Insufficient permissions."))
|
raise PermissionDenied(_("Insufficient permissions."))
|
||||||
|
|
||||||
# Only root documents are indexed, a version stands for its root
|
return more_like_doc_id
|
||||||
return more_like_doc.root_document_id or more_like_doc.pk
|
|
||||||
|
|
||||||
|
|
||||||
class SearchParams(NamedTuple):
|
class SearchParams(NamedTuple):
|
||||||
@@ -1552,10 +1550,7 @@ class DocumentViewSet(
|
|||||||
)
|
)
|
||||||
def suggestions(self, request, pk=None):
|
def suggestions(self, request, pk=None):
|
||||||
doc = get_object_or_404(
|
doc = get_object_or_404(
|
||||||
Document.objects.select_related(
|
Document.objects.select_related("owner").prefetch_related("versions"),
|
||||||
"owner",
|
|
||||||
"root_document__owner",
|
|
||||||
).prefetch_related("versions"),
|
|
||||||
pk=pk,
|
pk=pk,
|
||||||
)
|
)
|
||||||
if request.user is not None and not has_perms_owner_aware(
|
if request.user is not None and not has_perms_owner_aware(
|
||||||
@@ -1615,10 +1610,7 @@ class DocumentViewSet(
|
|||||||
@method_decorator(cache_control(no_cache=True))
|
@method_decorator(cache_control(no_cache=True))
|
||||||
def ai_suggestions(self, request, pk=None):
|
def ai_suggestions(self, request, pk=None):
|
||||||
doc = get_object_or_404(
|
doc = get_object_or_404(
|
||||||
Document.objects.select_related(
|
Document.objects.select_related("owner").prefetch_related("versions"),
|
||||||
"owner",
|
|
||||||
"root_document__owner",
|
|
||||||
).prefetch_related("versions"),
|
|
||||||
pk=pk,
|
pk=pk,
|
||||||
)
|
)
|
||||||
if request.user is not None and not has_perms_owner_aware(
|
if request.user is not None and not has_perms_owner_aware(
|
||||||
@@ -1864,14 +1856,9 @@ class DocumentViewSet(
|
|||||||
currentUser = request.user
|
currentUser = request.user
|
||||||
try:
|
try:
|
||||||
doc = (
|
doc = (
|
||||||
Document.objects.select_related("owner", "root_document__owner")
|
Document.objects.select_related("owner")
|
||||||
.prefetch_related("notes")
|
.prefetch_related("notes")
|
||||||
.only(
|
.only("pk", "owner__id")
|
||||||
"pk",
|
|
||||||
"owner__id",
|
|
||||||
"root_document__id",
|
|
||||||
"root_document__owner__id",
|
|
||||||
)
|
|
||||||
.get(pk=pk)
|
.get(pk=pk)
|
||||||
)
|
)
|
||||||
if currentUser is not None and not has_perms_owner_aware(
|
if currentUser is not None and not has_perms_owner_aware(
|
||||||
@@ -1986,9 +1973,7 @@ class DocumentViewSet(
|
|||||||
def share_links(self, request, pk=None):
|
def share_links(self, request, pk=None):
|
||||||
currentUser = request.user
|
currentUser = request.user
|
||||||
try:
|
try:
|
||||||
doc = Document.objects.select_related("owner", "root_document__owner").get(
|
doc = Document.objects.select_related("owner").get(pk=pk)
|
||||||
pk=pk,
|
|
||||||
)
|
|
||||||
if currentUser is not None and not has_perms_owner_aware(
|
if currentUser is not None and not has_perms_owner_aware(
|
||||||
currentUser,
|
currentUser,
|
||||||
"change_document",
|
"change_document",
|
||||||
@@ -2023,11 +2008,10 @@ class DocumentViewSet(
|
|||||||
if not settings.AUDIT_LOG_ENABLED:
|
if not settings.AUDIT_LOG_ENABLED:
|
||||||
return HttpResponseBadRequest("Audit log is disabled")
|
return HttpResponseBadRequest("Audit log is disabled")
|
||||||
try:
|
try:
|
||||||
doc = Document.objects.select_related("root_document__owner").get(pk=pk)
|
doc = Document.objects.get(pk=pk)
|
||||||
root_doc = get_root_document(doc)
|
|
||||||
if not request.user.has_perm("auditlog.view_logentry") or (
|
if not request.user.has_perm("auditlog.view_logentry") or (
|
||||||
root_doc.owner is not None
|
doc.owner is not None
|
||||||
and root_doc.owner != request.user
|
and doc.owner != request.user
|
||||||
and not request.user.is_superuser
|
and not request.user.is_superuser
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden(
|
return HttpResponseForbidden(
|
||||||
@@ -2118,7 +2102,9 @@ class DocumentViewSet(
|
|||||||
documents = Document.objects.filter(pk__in=document_ids)
|
documents = Document.objects.filter(pk__in=document_ids)
|
||||||
if (
|
if (
|
||||||
request.user is not None
|
request.user is not None
|
||||||
and documents.exclude(id__in=permitted_document_ids(request.user)).exists()
|
and documents.exclude(
|
||||||
|
pk__in=permitted_document_ids(request.user),
|
||||||
|
).exists()
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
|
|
||||||
@@ -2444,17 +2430,11 @@ class ChatStreamingView(GenericAPIView[Any]):
|
|||||||
|
|
||||||
if doc_id:
|
if doc_id:
|
||||||
try:
|
try:
|
||||||
document = Document.objects.select_related(
|
document = Document.objects.get(id=doc_id)
|
||||||
"root_document__owner",
|
|
||||||
).get(id=doc_id)
|
|
||||||
except Document.DoesNotExist:
|
except Document.DoesNotExist:
|
||||||
return HttpResponseBadRequest("Document not found")
|
return HttpResponseBadRequest("Document not found")
|
||||||
|
|
||||||
if not has_perms_owner_aware(
|
if not has_perms_owner_aware(request.user, "view_document", document):
|
||||||
request.user,
|
|
||||||
"view_document",
|
|
||||||
document,
|
|
||||||
):
|
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
|
|
||||||
documents = Document.objects.filter(pk=document.pk)
|
documents = Document.objects.filter(pk=document.pk)
|
||||||
@@ -3010,8 +2990,12 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
|||||||
user.has_perm(
|
user.has_perm(
|
||||||
"documents.change_document",
|
"documents.change_document",
|
||||||
)
|
)
|
||||||
and not Document.global_objects.filter(pk__in=documents)
|
and not Document.global_objects.filter(
|
||||||
.exclude(pk__in=permitted_document_ids(user, perm="change_document"))
|
pk__in=[doc.pk for doc in root_docs],
|
||||||
|
)
|
||||||
|
.exclude(
|
||||||
|
pk__in=permitted_document_ids(user, perm="change_document"),
|
||||||
|
)
|
||||||
.exists()
|
.exists()
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -3621,11 +3605,10 @@ class SelectionDataView(DocumentSelectionMixin, GenericAPIView[Any]):
|
|||||||
user=request.user,
|
user=request.user,
|
||||||
validated_data=serializer.validated_data,
|
validated_data=serializer.validated_data,
|
||||||
)
|
)
|
||||||
documents = Document.objects.filter(pk__in=ids)
|
permitted_documents = Document.objects.filter(
|
||||||
if (
|
id__in=permitted_document_ids(request.user),
|
||||||
documents.count() != len(ids)
|
)
|
||||||
or documents.exclude(id__in=permitted_document_ids(request.user)).exists()
|
if permitted_documents.filter(pk__in=ids).count() != len(ids):
|
||||||
):
|
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
|
|
||||||
correspondents = Correspondent.objects.annotate(
|
correspondents = Correspondent.objects.annotate(
|
||||||
@@ -4121,16 +4104,21 @@ class BulkDownloadView(DocumentSelectionMixin, GenericAPIView[Any]):
|
|||||||
validated_data=serializer.validated_data,
|
validated_data=serializer.validated_data,
|
||||||
)
|
)
|
||||||
documents = Document.objects.filter(pk__in=ids)
|
documents = Document.objects.filter(pk__in=ids)
|
||||||
|
versioned_documents = []
|
||||||
compression = serializer.validated_data.get("compression")
|
compression = serializer.validated_data.get("compression")
|
||||||
content = serializer.validated_data.get("content")
|
content = serializer.validated_data.get("content")
|
||||||
follow_filename_format = serializer.validated_data.get("follow_formatting")
|
follow_filename_format = serializer.validated_data.get("follow_formatting")
|
||||||
|
|
||||||
if documents.exclude(id__in=permitted_document_ids(request.user)).exists():
|
permitted_ids = set(permitted_document_ids(request.user))
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
for document in documents:
|
||||||
versioned_documents = [
|
root_doc = get_root_document(document)
|
||||||
get_latest_version_for_root(get_root_document(document))
|
if root_doc.pk not in permitted_ids:
|
||||||
for document in documents
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
]
|
versioned_documents.append(
|
||||||
|
get_latest_version_for_root(
|
||||||
|
root_doc,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
if content == "both":
|
if content == "both":
|
||||||
strategy_class = OriginalAndArchiveStrategy
|
strategy_class = OriginalAndArchiveStrategy
|
||||||
@@ -4802,23 +4790,19 @@ class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
denied_id = (
|
|
||||||
documents_qs.exclude(id__in=permitted_document_ids(request.user))
|
|
||||||
.order_by("pk")
|
|
||||||
.values_list("pk", flat=True)
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
if denied_id is not None:
|
|
||||||
raise ValidationError(
|
|
||||||
{
|
|
||||||
"document_ids": _(
|
|
||||||
"Insufficient permissions to share document %(id)s.",
|
|
||||||
)
|
|
||||||
% {"id": denied_id},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
documents = list(documents_qs)
|
documents = list(documents_qs)
|
||||||
|
permitted_ids = set(permitted_document_ids(request.user))
|
||||||
|
for document in documents:
|
||||||
|
if document.pk not in permitted_ids:
|
||||||
|
raise ValidationError(
|
||||||
|
{
|
||||||
|
"document_ids": _(
|
||||||
|
"Insufficient permissions to share document %(id)s.",
|
||||||
|
)
|
||||||
|
% {"id": document.pk},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
document_map = {document.pk: document for document in documents}
|
document_map = {document.pk: document for document in documents}
|
||||||
ordered_documents = [document_map[doc_id] for doc_id in document_ids]
|
ordered_documents = [document_map[doc_id] for doc_id in document_ids]
|
||||||
|
|
||||||
@@ -5603,23 +5587,6 @@ class TrashView(ListModelMixin, PassUserMixin):
|
|||||||
class _TrashPermittedObjectsFilter(PermittedObjectsFilter):
|
class _TrashPermittedObjectsFilter(PermittedObjectsFilter):
|
||||||
include_granted = False
|
include_granted = False
|
||||||
|
|
||||||
def filter_queryset(self, request, queryset, view):
|
|
||||||
if request.user.is_superuser or not request.user.is_active:
|
|
||||||
return super().filter_queryset(request, queryset, view)
|
|
||||||
|
|
||||||
# A version belongs to whoever owns its root
|
|
||||||
def owned_or_unowned(prefix: str) -> Q:
|
|
||||||
return Q(**{f"{prefix}owner": request.user}) | Q(
|
|
||||||
**{f"{prefix}owner__isnull": True},
|
|
||||||
)
|
|
||||||
|
|
||||||
return queryset.filter(
|
|
||||||
(Q(root_document__isnull=True) & owned_or_unowned(""))
|
|
||||||
| (
|
|
||||||
Q(root_document__isnull=False) & owned_or_unowned("root_document__")
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
filter_backends = (_TrashPermittedObjectsFilter,)
|
filter_backends = (_TrashPermittedObjectsFilter,)
|
||||||
pagination_class = StandardPagination
|
pagination_class = StandardPagination
|
||||||
|
|
||||||
@@ -5650,7 +5617,7 @@ class TrashView(ListModelMixin, PassUserMixin):
|
|||||||
else self.filter_queryset(self.get_queryset()).all()
|
else self.filter_queryset(self.get_queryset()).all()
|
||||||
)
|
)
|
||||||
if docs.exclude(
|
if docs.exclude(
|
||||||
id__in=permitted_document_ids(
|
pk__in=permitted_document_ids(
|
||||||
request.user,
|
request.user,
|
||||||
perm="delete_document",
|
perm="delete_document",
|
||||||
include_deleted=True,
|
include_deleted=True,
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ msgid ""
|
|||||||
msgstr ""
|
msgstr ""
|
||||||
"Project-Id-Version: paperless-ngx\n"
|
"Project-Id-Version: paperless-ngx\n"
|
||||||
"Report-Msgid-Bugs-To: \n"
|
"Report-Msgid-Bugs-To: \n"
|
||||||
"POT-Creation-Date: 2026-10-08 18:39+0000\n"
|
"POT-Creation-Date: 2026-10-06 15:12+0000\n"
|
||||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||||
"Last-Translator: \n"
|
"Last-Translator: \n"
|
||||||
"Language-Team: English\n"
|
"Language-Team: English\n"
|
||||||
@@ -1652,49 +1652,49 @@ msgstr ""
|
|||||||
msgid "workflow runs"
|
msgid "workflow runs"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:516 documents/serialisers.py:873
|
#: documents/serialisers.py:515 documents/serialisers.py:872
|
||||||
#: documents/serialisers.py:2903 documents/views.py:344 documents/views.py:2751
|
#: documents/serialisers.py:2902 documents/views.py:343 documents/views.py:2732
|
||||||
#: paperless_mail/serialisers.py:156
|
#: paperless_mail/serialisers.py:156
|
||||||
msgid "Insufficient permissions."
|
msgid "Insufficient permissions."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:709
|
#: documents/serialisers.py:708
|
||||||
msgid "Invalid color."
|
msgid "Invalid color."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2370
|
#: documents/serialisers.py:2369
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "File type %(type)s not supported"
|
msgid "File type %(type)s not supported"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2414
|
#: documents/serialisers.py:2413
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "Custom field id must be an integer: %(id)s"
|
msgid "Custom field id must be an integer: %(id)s"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2421
|
#: documents/serialisers.py:2420
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "Custom field with id %(id)s does not exist"
|
msgid "Custom field with id %(id)s does not exist"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2438 documents/serialisers.py:2448
|
#: documents/serialisers.py:2437 documents/serialisers.py:2447
|
||||||
msgid ""
|
msgid ""
|
||||||
"Custom fields must be a list of integers or an object mapping ids to values."
|
"Custom fields must be a list of integers or an object mapping ids to values."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2443
|
#: documents/serialisers.py:2442
|
||||||
msgid "Some custom fields don't exist or were specified twice."
|
msgid "Some custom fields don't exist or were specified twice."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2590
|
#: documents/serialisers.py:2589
|
||||||
msgid "Invalid variable detected."
|
msgid "Invalid variable detected."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2959
|
#: documents/serialisers.py:2958
|
||||||
msgid "Duplicate document identifiers are not allowed."
|
msgid "Duplicate document identifiers are not allowed."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/serialisers.py:2989 documents/views.py:4807
|
#: documents/serialisers.py:2988 documents/views.py:4787
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "Documents not found: %(ids)s"
|
msgid "Documents not found: %(ids)s"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
@@ -1945,40 +1945,40 @@ msgstr ""
|
|||||||
msgid ", "
|
msgid ", "
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:337 documents/views.py:2748
|
#: documents/views.py:336 documents/views.py:2729
|
||||||
msgid "Invalid more_like_id"
|
msgid "Invalid more_like_id"
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:1683
|
#: documents/views.py:1676
|
||||||
msgid "Invalid AI configuration."
|
msgid "Invalid AI configuration."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:1694
|
#: documents/views.py:1687
|
||||||
msgid "AI backend request timed out."
|
msgid "AI backend request timed out."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:1706
|
#: documents/views.py:1699
|
||||||
msgid "AI backend rejected the request. Check logs for details."
|
msgid "AI backend rejected the request. Check logs for details."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:2573 documents/views.py:2889
|
#: documents/views.py:2554 documents/views.py:2870
|
||||||
msgid "Specify only one of text, title_search, query, or more_like_id."
|
msgid "Specify only one of text, title_search, query, or more_like_id."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:4823
|
#: documents/views.py:4800
|
||||||
#, python-format
|
#, python-format
|
||||||
msgid "Insufficient permissions to share document %(id)s."
|
msgid "Insufficient permissions to share document %(id)s."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:4870
|
#: documents/views.py:4846
|
||||||
msgid "Bundle is already being processed."
|
msgid "Bundle is already being processed."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:4934
|
#: documents/views.py:4910
|
||||||
msgid "The share link bundle is still being prepared. Please try again later."
|
msgid "The share link bundle is still being prepared. Please try again later."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
#: documents/views.py:4948
|
#: documents/views.py:4924
|
||||||
msgid "The share link bundle is unavailable."
|
msgid "The share link bundle is unavailable."
|
||||||
msgstr ""
|
msgstr ""
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ from documents.models import Document
|
|||||||
from documents.permissions import permitted_object_ids
|
from documents.permissions import permitted_object_ids
|
||||||
from documents.permissions import restrict_queryset_to_visible
|
from documents.permissions import restrict_queryset_to_visible
|
||||||
from documents.permissions import user_is_unrestricted
|
from documents.permissions import user_is_unrestricted
|
||||||
from documents.versioning import annotate_effective_content
|
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
from paperless_ai.base_model import ClassificationSuggestions
|
from paperless_ai.base_model import ClassificationSuggestions
|
||||||
from paperless_ai.base_model import TaxonomyChoiceDict
|
from paperless_ai.base_model import TaxonomyChoiceDict
|
||||||
@@ -114,7 +113,7 @@ def build_prompt_without_rag(
|
|||||||
) -> str:
|
) -> str:
|
||||||
filename = document.filename or ""
|
filename = document.filename or ""
|
||||||
content = truncate_content(
|
content = truncate_content(
|
||||||
(document.get_effective_content() or "")[:4000],
|
document.content[:4000] or "",
|
||||||
chunk_size=config.llm_embedding_chunk_size,
|
chunk_size=config.llm_embedding_chunk_size,
|
||||||
context_size=config.llm_context_size,
|
context_size=config.llm_context_size,
|
||||||
)
|
)
|
||||||
@@ -228,9 +227,7 @@ def get_taxonomy_context(
|
|||||||
|
|
||||||
# similar_documents is already ordered by descending weight; don't lose it.
|
# similar_documents is already ordered by descending weight; don't lose it.
|
||||||
similar_document_ids = [s["document_id"] for s in similar_documents]
|
similar_document_ids = [s["document_id"] for s in similar_documents]
|
||||||
similar_documents_by_id = annotate_effective_content(
|
similar_documents_by_id = Document.objects.in_bulk(similar_document_ids)
|
||||||
Document.objects.all(),
|
|
||||||
).in_bulk(similar_document_ids)
|
|
||||||
similar_docs = [
|
similar_docs = [
|
||||||
similar_documents_by_id[document_id]
|
similar_documents_by_id[document_id]
|
||||||
for document_id in similar_document_ids
|
for document_id in similar_document_ids
|
||||||
@@ -238,7 +235,7 @@ def get_taxonomy_context(
|
|||||||
][:max_docs]
|
][:max_docs]
|
||||||
context_blocks = []
|
context_blocks = []
|
||||||
for similar in similar_docs:
|
for similar in similar_docs:
|
||||||
text = (similar.get_effective_content() or "")[:1000]
|
text = similar.content[:1000] or ""
|
||||||
title = similar.title or similar.filename or "Untitled"
|
title = similar.title or similar.filename or "Untitled"
|
||||||
context_blocks.append(f"TITLE: {title}\n{text}")
|
context_blocks.append(f"TITLE: {title}\n{text}")
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -135,6 +135,6 @@ def build_llm_index_text(doc: Document) -> str:
|
|||||||
lines.append(f"Custom Field - {instance.field.name}: {instance}")
|
lines.append(f"Custom Field - {instance.field.name}: {instance}")
|
||||||
|
|
||||||
lines.append("\nContent:\n")
|
lines.append("\nContent:\n")
|
||||||
lines.append(doc.get_effective_content() or "")
|
lines.append(doc.content or "")
|
||||||
|
|
||||||
return _normalize_llm_index_text("\n".join(lines))
|
return _normalize_llm_index_text("\n".join(lines))
|
||||||
@@ -17,8 +17,6 @@ from documents.models import PaperlessTask
|
|||||||
from documents.utils import IterWrapper
|
from documents.utils import IterWrapper
|
||||||
from documents.utils import QuerySetStream
|
from documents.utils import QuerySetStream
|
||||||
from documents.utils import identity
|
from documents.utils import identity
|
||||||
from documents.versioning import annotate_effective_content
|
|
||||||
from documents.versioning import root_document_ids
|
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
from paperless_ai.db import db_connection_released
|
from paperless_ai.db import db_connection_released
|
||||||
from paperless_ai.embedding import build_llm_index_text
|
from paperless_ai.embedding import build_llm_index_text
|
||||||
@@ -445,11 +443,11 @@ def update_llm_index(
|
|||||||
"Skipping LLM index update: migration check deferred; "
|
"Skipping LLM index update: migration check deferred; "
|
||||||
"will retry next run."
|
"will retry next run."
|
||||||
)
|
)
|
||||||
documents = annotate_effective_content(
|
documents = Document.objects.select_related(
|
||||||
Document.objects.filter(root_document__isnull=True)
|
"correspondent",
|
||||||
.select_related("correspondent", "document_type", "storage_path")
|
"document_type",
|
||||||
.prefetch_related("tags", "notes", "custom_fields__field"),
|
"storage_path",
|
||||||
)
|
).prefetch_related("tags", "notes", "custom_fields__field")
|
||||||
no_documents = not documents.exists()
|
no_documents = not documents.exists()
|
||||||
|
|
||||||
# Fast exit before touching config: nothing to index and no existing index.
|
# Fast exit before touching config: nothing to index and no existing index.
|
||||||
@@ -485,7 +483,7 @@ def update_llm_index(
|
|||||||
msg = "LLM index rebuilt successfully."
|
msg = "LLM index rebuilt successfully."
|
||||||
else:
|
else:
|
||||||
scoped_documents = (
|
scoped_documents = (
|
||||||
documents.filter(id__in=root_document_ids(document_ids))
|
documents.filter(id__in=document_ids)
|
||||||
if document_ids is not None
|
if document_ids is not None
|
||||||
else documents
|
else documents
|
||||||
)
|
)
|
||||||
@@ -512,12 +510,7 @@ def update_llm_index(
|
|||||||
|
|
||||||
|
|
||||||
def llm_index_add_or_update_document(document: Document):
|
def llm_index_add_or_update_document(document: Document):
|
||||||
"""
|
"""Add or atomically replace a document's chunks in the index."""
|
||||||
Add or atomically replace a document's chunks in the index. Only root
|
|
||||||
documents are indexed, so a version is indexed as its root document.
|
|
||||||
"""
|
|
||||||
if document.root_document_id is not None:
|
|
||||||
document = document.root_document
|
|
||||||
config = AIConfig()
|
config = AIConfig()
|
||||||
new_nodes = build_document_node(
|
new_nodes = build_document_node(
|
||||||
document,
|
document,
|
||||||
@@ -695,7 +688,7 @@ def retrieve_similar_nodes(
|
|||||||
)
|
)
|
||||||
|
|
||||||
query_text = truncate_embedding_query(
|
query_text = truncate_embedding_query(
|
||||||
(document.title or "") + "\n" + (document.get_effective_content() or ""),
|
(document.title or "") + "\n" + (document.content or ""),
|
||||||
chunk_size=config.llm_embedding_chunk_size,
|
chunk_size=config.llm_embedding_chunk_size,
|
||||||
)
|
)
|
||||||
# Hold the shared read lock for the whole retrieval so the connection is
|
# Hold the shared read lock for the whole retrieval so the connection is
|
||||||
|
|||||||
@@ -55,7 +55,6 @@ def mock_document():
|
|||||||
doc.storage_path = None
|
doc.storage_path = None
|
||||||
doc.archive_serial_number = "12345"
|
doc.archive_serial_number = "12345"
|
||||||
doc.content = "This is the document content."
|
doc.content = "This is the document content."
|
||||||
doc.get_effective_content.return_value = "This is the document content."
|
|
||||||
|
|
||||||
cf1 = MagicMock(__str__=lambda x: "Value1")
|
cf1 = MagicMock(__str__=lambda x: "Value1")
|
||||||
cf1.field = MagicMock()
|
cf1.field = MagicMock()
|
||||||
@@ -435,55 +434,6 @@ def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
class TestClassifierEffectiveContent:
|
|
||||||
"""A root document's text for the LLM is its newest version's content."""
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _root_with_version() -> Document:
|
|
||||||
root = DocumentFactory(title="Statement", content="stale text")
|
|
||||||
DocumentFactory(root_document=root, version_index=1, content="latest text")
|
|
||||||
return root
|
|
||||||
|
|
||||||
def test_prompt_uses_the_newest_versions_content(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- The classification prompt is built for the root
|
|
||||||
THEN:
|
|
||||||
- It contains the newest version's content
|
|
||||||
"""
|
|
||||||
prompt = build_prompt_without_rag(self._root_with_version(), AIConfig())
|
|
||||||
|
|
||||||
assert "latest text" in prompt
|
|
||||||
assert "stale text" not in prompt
|
|
||||||
|
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
|
||||||
def test_similar_document_context_uses_the_newest_versions_content(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A similar root document with a version
|
|
||||||
WHEN:
|
|
||||||
- The similar-document context is built
|
|
||||||
THEN:
|
|
||||||
- It contains the newest version's content
|
|
||||||
"""
|
|
||||||
similar = self._root_with_version()
|
|
||||||
document = DocumentFactory(content="Some content")
|
|
||||||
fake_nodes = [
|
|
||||||
SimpleNamespace(metadata={"document_id": str(similar.pk)}, score=0.9),
|
|
||||||
]
|
|
||||||
|
|
||||||
with patch(
|
|
||||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
|
||||||
return_value=fake_nodes,
|
|
||||||
):
|
|
||||||
_candidates, context = get_taxonomy_context(document, user=None)
|
|
||||||
|
|
||||||
assert context == "TITLE: Statement\nlatest text"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
def test_get_taxonomy_context_no_similar_docs():
|
def test_get_taxonomy_context_no_similar_docs():
|
||||||
|
|||||||
@@ -388,108 +388,6 @@ def test_update_llm_index_partial_update(
|
|||||||
assert after[str(doc2.pk)] == before[str(doc2.pk)]
|
assert after[str(doc2.pk)] == before[str(doc2.pk)]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
class TestLlmIndexVersions:
|
|
||||||
"""The LLM index holds root documents only: a version is indexed as its root."""
|
|
||||||
|
|
||||||
def test_add_or_update_document_indexes_a_version_as_its_root(
|
|
||||||
self,
|
|
||||||
temp_llm_index_dir: Path,
|
|
||||||
mock_embed_model: FakeEmbedding,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- The version is passed to llm_index_add_or_update_document
|
|
||||||
THEN:
|
|
||||||
- Only the root document is in the index
|
|
||||||
"""
|
|
||||||
root = DocumentFactory(content="root content")
|
|
||||||
version = DocumentFactory(root_document=root, version_index=1)
|
|
||||||
|
|
||||||
indexing.llm_index_add_or_update_document(version)
|
|
||||||
|
|
||||||
with indexing.get_vector_store() as store:
|
|
||||||
indexed = store.get_modified_times()
|
|
||||||
|
|
||||||
assert set(indexed) == {str(root.pk)}
|
|
||||||
|
|
||||||
def test_rebuild_skips_versions(
|
|
||||||
self,
|
|
||||||
temp_llm_index_dir: Path,
|
|
||||||
mock_embed_model: FakeEmbedding,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- The LLM index is rebuilt
|
|
||||||
THEN:
|
|
||||||
- Only the root document is in the index
|
|
||||||
"""
|
|
||||||
root = DocumentFactory()
|
|
||||||
DocumentFactory(root_document=root, version_index=1)
|
|
||||||
|
|
||||||
indexing.update_llm_index(rebuild=True)
|
|
||||||
|
|
||||||
with indexing.get_vector_store() as store:
|
|
||||||
indexed = store.get_modified_times()
|
|
||||||
|
|
||||||
assert set(indexed) == {str(root.pk)}
|
|
||||||
|
|
||||||
def test_rebuild_indexes_the_newest_versions_content(
|
|
||||||
self,
|
|
||||||
temp_llm_index_dir: Path,
|
|
||||||
mock_embed_model: FakeEmbedding,
|
|
||||||
mocker: pytest_mock.MockerFixture,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with a version
|
|
||||||
WHEN:
|
|
||||||
- The LLM index is rebuilt
|
|
||||||
THEN:
|
|
||||||
- The root's text for the index is the version's content, answered
|
|
||||||
from the query rather than a query per document
|
|
||||||
"""
|
|
||||||
root = DocumentFactory(content="stale text")
|
|
||||||
DocumentFactory(root_document=root, version_index=1, content="latest text")
|
|
||||||
spy = mocker.spy(indexing, "build_document_node")
|
|
||||||
|
|
||||||
indexing.update_llm_index(rebuild=True)
|
|
||||||
|
|
||||||
indexed = spy.call_args.args[0]
|
|
||||||
assert indexed.pk == root.pk
|
|
||||||
assert indexed.effective_content == "latest text"
|
|
||||||
|
|
||||||
def test_incremental_update_by_version_id_refreshes_the_root(
|
|
||||||
self,
|
|
||||||
temp_llm_index_dir: Path,
|
|
||||||
mock_embed_model: FakeEmbedding,
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- An indexed root document with a version whose root was modified since
|
|
||||||
WHEN:
|
|
||||||
- An incremental update is scoped to the version's id
|
|
||||||
THEN:
|
|
||||||
- The root's entry is refreshed and no entry exists for the version
|
|
||||||
"""
|
|
||||||
root = DocumentFactory()
|
|
||||||
version = DocumentFactory(root_document=root, version_index=1)
|
|
||||||
indexing.update_llm_index(rebuild=True)
|
|
||||||
Document.objects.filter(pk=root.pk).update(modified=timezone.now())
|
|
||||||
root.refresh_from_db()
|
|
||||||
|
|
||||||
indexing.update_llm_index(document_ids=[version.pk])
|
|
||||||
|
|
||||||
with indexing.get_vector_store() as store:
|
|
||||||
indexed = store.get_modified_times()
|
|
||||||
|
|
||||||
assert indexed == {str(root.pk): root.modified.isoformat()}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
def test_add_or_update_document_updates_existing_entry(
|
def test_add_or_update_document_updates_existing_entry(
|
||||||
temp_llm_index_dir: Path,
|
temp_llm_index_dir: Path,
|
||||||
@@ -739,7 +637,6 @@ class TestLlmIndexAddOrUpdateDocumentEmptyContent:
|
|||||||
|
|
||||||
doc = MagicMock(spec=Document)
|
doc = MagicMock(spec=Document)
|
||||||
doc.id = 42
|
doc.id = 42
|
||||||
doc.root_document_id = None
|
|
||||||
# Must not raise
|
# Must not raise
|
||||||
indexing.llm_index_add_or_update_document(doc)
|
indexing.llm_index_add_or_update_document(doc)
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ from paperless_ai.embedding import _normalize_llm_index_text
|
|||||||
from paperless_ai.embedding import build_llm_index_text
|
from paperless_ai.embedding import build_llm_index_text
|
||||||
from paperless_ai.embedding import get_configured_model_name
|
from paperless_ai.embedding import get_configured_model_name
|
||||||
from paperless_ai.embedding import get_embedding_model
|
from paperless_ai.embedding import get_embedding_model
|
||||||
from paperless_testing.factories import DocumentFactory
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -47,7 +46,6 @@ def mock_document():
|
|||||||
doc.correspondent.name = "Test Correspondent"
|
doc.correspondent.name = "Test Correspondent"
|
||||||
doc.archive_serial_number = "12345"
|
doc.archive_serial_number = "12345"
|
||||||
doc.content = "This is the document content."
|
doc.content = "This is the document content."
|
||||||
doc.get_effective_content.return_value = "This is the document content."
|
|
||||||
|
|
||||||
cf1 = MagicMock(__str__=lambda x: "Value1")
|
cf1 = MagicMock(__str__=lambda x: "Value1")
|
||||||
cf1.field = MagicMock()
|
cf1.field = MagicMock()
|
||||||
@@ -282,7 +280,7 @@ def test_build_llm_index_text(mock_document):
|
|||||||
|
|
||||||
|
|
||||||
def test_build_llm_index_text_normalizes_ocr_punctuation_runs(mock_document):
|
def test_build_llm_index_text_normalizes_ocr_punctuation_runs(mock_document):
|
||||||
mock_document.get_effective_content.return_value = (
|
mock_document.content = (
|
||||||
"Introduction ................................................ 7\n"
|
"Introduction ................................................ 7\n"
|
||||||
"Hardware Limitation ________________________________________ 9\n"
|
"Hardware Limitation ________________________________________ 9\n"
|
||||||
"Keep short punctuation like INV-100 and ellipses..."
|
"Keep short punctuation like INV-100 and ellipses..."
|
||||||
@@ -296,43 +294,6 @@ def test_build_llm_index_text_normalizes_ocr_punctuation_runs(mock_document):
|
|||||||
assert "ellipses..." in result
|
assert "ellipses..." in result
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
class TestBuildLlmIndexTextVersions:
|
|
||||||
"""A root document is indexed with its effective content, like in the search index."""
|
|
||||||
|
|
||||||
def test_root_uses_the_newest_versions_content(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document with two versions
|
|
||||||
WHEN:
|
|
||||||
- The LLM index text is built for the root
|
|
||||||
THEN:
|
|
||||||
- It contains the newest version's content and not the others'
|
|
||||||
"""
|
|
||||||
root = DocumentFactory(content="stale text")
|
|
||||||
DocumentFactory(root_document=root, version_index=1, content="older text")
|
|
||||||
DocumentFactory(root_document=root, version_index=2, content="latest text")
|
|
||||||
|
|
||||||
text = build_llm_index_text(root)
|
|
||||||
|
|
||||||
assert "latest text" in text
|
|
||||||
assert "stale text" not in text
|
|
||||||
assert "older text" not in text
|
|
||||||
|
|
||||||
def test_root_without_versions_uses_its_own_content(self) -> None:
|
|
||||||
"""
|
|
||||||
GIVEN:
|
|
||||||
- A root document without versions
|
|
||||||
WHEN:
|
|
||||||
- The LLM index text is built for it
|
|
||||||
THEN:
|
|
||||||
- It contains the document's own content
|
|
||||||
"""
|
|
||||||
root = DocumentFactory(content="own text")
|
|
||||||
|
|
||||||
assert "own text" in build_llm_index_text(root)
|
|
||||||
|
|
||||||
|
|
||||||
def test_normalize_llm_index_text_collapses_ocr_leaders_without_joining_lines():
|
def test_normalize_llm_index_text_collapses_ocr_leaders_without_joining_lines():
|
||||||
assert _normalize_llm_index_text("A........B\nC____D----E") == "A B\nC D E"
|
assert _normalize_llm_index_text("A........B\nC____D----E") == "A B\nC D E"
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user