mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-10 11:48:00 +00:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0ef2c39c0b | ||
|
|
9e076e7cdf | ||
|
|
e1df61a192 | ||
|
|
41524a6230 | ||
|
|
e5c9b8facb | ||
|
|
cf3a080694 | ||
|
|
018de125ff |
@@ -1200,15 +1200,6 @@ still perform some basic text pre-processing before matching.
|
||||
|
||||
Defaults to true, enabling the feature.
|
||||
|
||||
#### [`PAPERLESS_CLASSIFIER_MATCH_THRESHOLD=<float>`](#PAPERLESS_CLASSIFIER_MATCH_THRESHOLD) {#PAPERLESS_CLASSIFIER_MATCH_THRESHOLD}
|
||||
|
||||
: Sets the minimum confidence score (0.0-1.0) required for the automatic
|
||||
classifier to assign a correspondent, document type, or storage path to a
|
||||
document. Predictions below this threshold are discarded and the field is
|
||||
left unassigned, preventing low-confidence guesses from being applied.
|
||||
|
||||
Defaults to 0.6.
|
||||
|
||||
#### [`PAPERLESS_DATE_PARSER_LANGUAGES=<lang>`](#PAPERLESS_DATE_PARSER_LANGUAGES) {#PAPERLESS_DATE_PARSER_LANGUAGES}
|
||||
|
||||
: Specifies which language Paperless should use when parsing dates from documents.
|
||||
|
||||
+203
-250
File diff suppressed because it is too large
Load Diff
@@ -112,22 +112,6 @@
|
||||
|
||||
<pngx-input-check i18n-title title="Use 'slim' sidebar (icons only)" formControlName="slimSidebarEnabled"></pngx-input-check>
|
||||
|
||||
<p class="mb-2 mt-3" i18n>Sidebar items to show:</p>
|
||||
@for (option of sidebarItemOptions; track option.id) {
|
||||
<div class="form-check">
|
||||
<input
|
||||
class="form-check-input"
|
||||
type="checkbox"
|
||||
[id]="'sidebar-item-setting-' + option.id"
|
||||
[checked]="isSidebarItemShown(option.id)"
|
||||
(change)="toggleSidebarItem(option.id, $event.target.checked)"
|
||||
/>
|
||||
<label class="form-check-label" [for]="'sidebar-item-setting-' + option.id">
|
||||
{{ option.label }}
|
||||
</label>
|
||||
</div>
|
||||
}
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ import {
|
||||
SystemStatus,
|
||||
SystemStatusItemStatus,
|
||||
} from 'src/app/data/system-status'
|
||||
import { HideableSidebarItemID, SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { IfOwnerDirective } from 'src/app/directives/if-owner.directive'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import { PermissionsGuard } from 'src/app/guards/permissions.guard'
|
||||
@@ -209,45 +209,6 @@ describe('SettingsComponent', () => {
|
||||
fixture.detectChanges()
|
||||
}
|
||||
|
||||
it('supports configuring sidebar items and canceling changes', () => {
|
||||
completeSetup()
|
||||
|
||||
component.toggleSidebarItem(HideableSidebarItemID.Workflows, false)
|
||||
fixture.detectChanges()
|
||||
|
||||
expect(component.settingsForm.value.sidebarHiddenItems).toContain(
|
||||
HideableSidebarItemID.Workflows
|
||||
)
|
||||
|
||||
settingsService.updateSidebarItemVisibility(
|
||||
HideableSidebarItemID.Mail,
|
||||
false
|
||||
)
|
||||
|
||||
expect(component.settingsForm.value.sidebarHiddenItems).toContain(
|
||||
HideableSidebarItemID.Mail
|
||||
)
|
||||
|
||||
component.reset()
|
||||
|
||||
expect(component.settingsForm.value.sidebarHiddenItems).not.toContain(
|
||||
HideableSidebarItemID.Workflows
|
||||
)
|
||||
expect(component.settingsForm.value.sidebarHiddenItems).not.toContain(
|
||||
HideableSidebarItemID.Mail
|
||||
)
|
||||
})
|
||||
|
||||
it('enables sidebar item controls on general settings until destroyed', () => {
|
||||
completeSetup()
|
||||
|
||||
expect(settingsService.organizingSidebarItems()).toBe(true)
|
||||
|
||||
component.ngOnDestroy()
|
||||
|
||||
expect(settingsService.organizingSidebarItems()).toBe(false)
|
||||
})
|
||||
|
||||
it('should support tabbed settings & change URL, prevent navigation if dirty confirmation rejected', async () => {
|
||||
completeSetup()
|
||||
const navigateSpy = jest.spyOn(router, 'navigate')
|
||||
@@ -288,7 +249,6 @@ describe('SettingsComponent', () => {
|
||||
|
||||
it('should support save local settings updating appearance settings and calling API, show error', () => {
|
||||
completeSetup()
|
||||
component.toggleSidebarItem(HideableSidebarItemID.Workflows, false)
|
||||
const toastErrorSpy = jest.spyOn(toastService, 'showError')
|
||||
const toastSpy = jest.spyOn(toastService, 'show')
|
||||
const storeSpy = jest.spyOn(settingsService, 'storeSettings')
|
||||
@@ -307,10 +267,7 @@ describe('SettingsComponent', () => {
|
||||
expect(toastErrorSpy).toHaveBeenCalled()
|
||||
expect(storeSpy).toHaveBeenCalled()
|
||||
expect(appearanceSettingsSpy).not.toHaveBeenCalled()
|
||||
expect(setSpy).toHaveBeenCalledTimes(34)
|
||||
expect(setSpy).toHaveBeenCalledWith(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [
|
||||
HideableSidebarItemID.Workflows,
|
||||
])
|
||||
expect(setSpy).toHaveBeenCalledTimes(33)
|
||||
|
||||
// succeed
|
||||
storeSpy.mockReturnValueOnce(of(true))
|
||||
|
||||
@@ -39,12 +39,7 @@ import {
|
||||
SystemStatus,
|
||||
SystemStatusItemStatus,
|
||||
} from 'src/app/data/system-status'
|
||||
import {
|
||||
GlobalSearchType,
|
||||
HIDEABLE_SIDEBAR_ITEM_IDS,
|
||||
HideableSidebarItemID,
|
||||
SETTINGS_KEYS,
|
||||
} from 'src/app/data/ui-settings'
|
||||
import { GlobalSearchType, SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { User } from 'src/app/data/user'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
|
||||
@@ -107,14 +102,6 @@ const documentDetailFieldOptions = [
|
||||
{ id: DocumentDetailFieldID.Tags, label: $localize`Tags` },
|
||||
]
|
||||
|
||||
const sidebarItemLabels: Record<HideableSidebarItemID, string> = {
|
||||
[HideableSidebarItemID.Dashboard]: $localize`Dashboard`,
|
||||
[HideableSidebarItemID.SavedViews]: $localize`Saved Views`,
|
||||
[HideableSidebarItemID.Workflows]: $localize`Workflows`,
|
||||
[HideableSidebarItemID.Mail]: $localize`Mail`,
|
||||
[HideableSidebarItemID.Documentation]: $localize`Documentation`,
|
||||
}
|
||||
|
||||
@Component({
|
||||
selector: 'pngx-settings',
|
||||
templateUrl: './settings.component.html',
|
||||
@@ -162,7 +149,6 @@ export class SettingsComponent
|
||||
bulkEditApplyOnClose: new FormControl(null),
|
||||
documentListItemPerPage: new FormControl(null),
|
||||
slimSidebarEnabled: new FormControl(null),
|
||||
sidebarHiddenItems: new FormControl<HideableSidebarItemID[]>([]),
|
||||
darkModeUseSystem: new FormControl(null),
|
||||
darkModeEnabled: new FormControl(null),
|
||||
darkModeInvertThumbs: new FormControl(null),
|
||||
@@ -200,7 +186,6 @@ export class SettingsComponent
|
||||
|
||||
store: BehaviorSubject<any>
|
||||
storeSub: Subscription
|
||||
sidebarItemsSub: Subscription
|
||||
isDirty$: Observable<boolean>
|
||||
isDirty: boolean = false
|
||||
unsubscribeNotifier: Subject<any> = new Subject()
|
||||
@@ -218,10 +203,6 @@ export class SettingsComponent
|
||||
public readonly PdfEditorEditMode = PdfEditorEditMode
|
||||
|
||||
public readonly documentDetailFieldOptions = documentDetailFieldOptions
|
||||
public readonly sidebarItemOptions = HIDEABLE_SIDEBAR_ITEM_IDS.map((id) => ({
|
||||
id,
|
||||
label: sidebarItemLabels[id],
|
||||
}))
|
||||
|
||||
get systemStatusHasErrors(): boolean {
|
||||
const status = this.systemStatus()
|
||||
@@ -249,10 +230,6 @@ export class SettingsComponent
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.sidebarItemsSub =
|
||||
this.settings.sidebarHiddenItemsEditingChanged.subscribe((hiddenItems) =>
|
||||
this.settingsForm.controls.sidebarHiddenItems.setValue(hiddenItems)
|
||||
)
|
||||
this.settings.settingsSaved.subscribe(() => {
|
||||
if (!this.savePending) this.initialize()
|
||||
this.savedViewsService.maybeRefreshDocumentCounts()
|
||||
@@ -302,21 +279,14 @@ export class SettingsComponent
|
||||
|
||||
this.activatedRoute.paramMap.subscribe((paramMap) => {
|
||||
const section = paramMap.get('section')
|
||||
let navID = SettingsNavIDs.General
|
||||
if (section) {
|
||||
const navIDKey: string = Object.keys(SettingsNavIDs).find(
|
||||
(navID) => navID.toLowerCase() == section
|
||||
)
|
||||
if (navIDKey) {
|
||||
navID = SettingsNavIDs[navIDKey]
|
||||
this.activeNavID.set(SettingsNavIDs[navIDKey])
|
||||
}
|
||||
}
|
||||
this.activeNavID.set(navID)
|
||||
this.settings.sidebarHiddenItemsEditing.set(
|
||||
navID === SettingsNavIDs.General
|
||||
? [...this.settingsForm.controls.sidebarHiddenItems.value]
|
||||
: null
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -340,7 +310,6 @@ export class SettingsComponent
|
||||
SETTINGS_KEYS.DOCUMENT_LIST_SIZE
|
||||
),
|
||||
slimSidebarEnabled: this.settings.get(SETTINGS_KEYS.SLIM_SIDEBAR),
|
||||
sidebarHiddenItems: this.settings.get(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS),
|
||||
darkModeUseSystem: this.settings.get(SETTINGS_KEYS.DARK_MODE_USE_SYSTEM),
|
||||
darkModeEnabled: this.settings.get(SETTINGS_KEYS.DARK_MODE_ENABLED),
|
||||
darkModeInvertThumbs: this.settings.get(
|
||||
@@ -467,12 +436,6 @@ export class SettingsComponent
|
||||
this.settingsForm.patchValue(currentFormValue)
|
||||
}
|
||||
|
||||
if (this.settings.organizingSidebarItems()) {
|
||||
this.settings.sidebarHiddenItemsEditing.set([
|
||||
...this.settingsForm.controls.sidebarHiddenItems.value,
|
||||
])
|
||||
}
|
||||
|
||||
if (this.canViewSystemStatus) {
|
||||
this.systemStatusService.get().subscribe((status) => {
|
||||
this.systemStatus.set(status)
|
||||
@@ -481,18 +444,8 @@ export class SettingsComponent
|
||||
}
|
||||
|
||||
ngOnDestroy() {
|
||||
this.settings.sidebarHiddenItemsEditing.set(null)
|
||||
if (this.isDirty) this.settings.updateAppearanceSettings() // in case user changed appearance but didn't save
|
||||
this.storeSub && this.storeSub.unsubscribe()
|
||||
this.sidebarItemsSub.unsubscribe()
|
||||
}
|
||||
|
||||
isSidebarItemShown(item: HideableSidebarItemID): boolean {
|
||||
return !(this.settingsForm.value.sidebarHiddenItems || []).includes(item)
|
||||
}
|
||||
|
||||
toggleSidebarItem(item: HideableSidebarItemID, checked: boolean): void {
|
||||
this.settings.updateSidebarItemVisibility(item, checked)
|
||||
}
|
||||
|
||||
public saveSettings() {
|
||||
@@ -520,10 +473,6 @@ export class SettingsComponent
|
||||
SETTINGS_KEYS.SLIM_SIDEBAR,
|
||||
this.settingsForm.value.slimSidebarEnabled
|
||||
)
|
||||
this.settings.set(
|
||||
SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS,
|
||||
this.settingsForm.value.sidebarHiddenItems
|
||||
)
|
||||
this.settings.set(
|
||||
SETTINGS_KEYS.DARK_MODE_USE_SYSTEM,
|
||||
this.settingsForm.value.darkModeUseSystem
|
||||
@@ -683,11 +632,6 @@ export class SettingsComponent
|
||||
|
||||
reset() {
|
||||
this.settingsForm.patchValue(this.store.getValue())
|
||||
if (this.settings.organizingSidebarItems()) {
|
||||
this.settings.sidebarHiddenItemsEditing.set([
|
||||
...this.settingsForm.controls.sidebarHiddenItems.value,
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
clearThemeColor() {
|
||||
|
||||
@@ -86,15 +86,12 @@
|
||||
}
|
||||
<div class="sidebar-sticky pt-3 pb-1 d-flex flex-column justify-space-around">
|
||||
<ul class="nav flex-column">
|
||||
<li class="nav-item app-link position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Dashboard) && !settingsService.organizingSidebarItems()">
|
||||
<a class="nav-link" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Dashboard)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()"
|
||||
<li class="nav-item app-link">
|
||||
<a class="nav-link" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()"
|
||||
ngbPopover="Dashboard" i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end"
|
||||
container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
|
||||
<i-bs class="me-2" name="house"></i-bs><span class="nav-link-label"><ng-container i18n>Dashboard</ng-container></span>
|
||||
</a>
|
||||
@if (settingsService.organizingSidebarItems()) {
|
||||
<pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Dashboard" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.Dashboard)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.Dashboard, $event)"></pngx-input-switch>
|
||||
}
|
||||
</li>
|
||||
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Document }">
|
||||
<a class="nav-link" routerLink="documents" routerLinkActive="active"
|
||||
@@ -240,38 +237,29 @@
|
||||
</div>
|
||||
</li>
|
||||
}
|
||||
<li class="nav-item app-link position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.SavedViews) && !settingsService.organizingSidebarItems()" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.SavedView }">
|
||||
<a class="nav-link" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.SavedViews)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()" routerLink="savedviews" routerLinkActive="active" (click)="closeMenu()"
|
||||
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.SavedView }">
|
||||
<a class="nav-link" routerLink="savedviews" routerLinkActive="active" (click)="closeMenu()"
|
||||
ngbPopover="Saved Views" i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end"
|
||||
container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
|
||||
<i-bs class="me-2" name="window-stack"></i-bs><span class="nav-link-label"><ng-container i18n>Saved Views</ng-container></span>
|
||||
</a>
|
||||
@if (settingsService.organizingSidebarItems()) {
|
||||
<pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Saved Views" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.SavedViews)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.SavedViews, $event)"></pngx-input-switch>
|
||||
}
|
||||
</li>
|
||||
<li class="nav-item app-link position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows) && !settingsService.organizingSidebarItems()"
|
||||
<li class="nav-item app-link"
|
||||
*pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Workflow }"
|
||||
tourAnchor="tour.workflows">
|
||||
<a class="nav-link" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()" routerLink="workflows" routerLinkActive="active" (click)="closeMenu()"
|
||||
<a class="nav-link" routerLink="workflows" routerLinkActive="active" (click)="closeMenu()"
|
||||
ngbPopover="Workflows" i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end"
|
||||
container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
|
||||
<i-bs class="me-2" name="boxes"></i-bs><span class="nav-link-label"><ng-container i18n>Workflows</ng-container></span>
|
||||
</a>
|
||||
@if (settingsService.organizingSidebarItems()) {
|
||||
<pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Workflows" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.Workflows, $event)"></pngx-input-switch>
|
||||
}
|
||||
</li>
|
||||
<li class="nav-item app-link position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Mail) && !settingsService.organizingSidebarItems()" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.MailAccount }"
|
||||
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.MailAccount }"
|
||||
tourAnchor="tour.mail">
|
||||
<a class="nav-link" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Mail)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()" routerLink="mail" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Mail"
|
||||
<a class="nav-link" routerLink="mail" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Mail"
|
||||
i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end" container="body"
|
||||
triggers="mouseenter:mouseleave" popoverClass="popover-slim">
|
||||
<i-bs class="me-2" name="envelope"></i-bs><span class="nav-link-label"><ng-container i18n>Mail</ng-container></span>
|
||||
</a>
|
||||
@if (settingsService.organizingSidebarItems()) {
|
||||
<pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Mail" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.Mail)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.Mail, $event)"></pngx-input-switch>
|
||||
}
|
||||
</li>
|
||||
<li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.Delete, type: PermissionType.Document }">
|
||||
<a class="nav-link" routerLink="trash" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Trash"
|
||||
@@ -334,16 +322,13 @@
|
||||
</a>
|
||||
</li>
|
||||
}
|
||||
<li class="nav-item mt-2 position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Documentation) && !settingsService.organizingSidebarItems()" tourAnchor="tour.outro">
|
||||
<a class="text-muted small d-flex align-items-center flex-wrap text-decoration-none nav-anchor" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Documentation)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()"
|
||||
<li class="nav-item mt-2" tourAnchor="tour.outro">
|
||||
<a class="text-muted small d-flex align-items-center flex-wrap text-decoration-none nav-anchor"
|
||||
target="_blank" rel="noopener noreferrer" href="https://docs.paperless-ngx.com" ngbPopover="Documentation"
|
||||
i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end" container="body"
|
||||
triggers="mouseenter:mouseleave" popoverClass="popover-slim">
|
||||
<i-bs class="d-flex me-2" name="question-circle"></i-bs><span><ng-container i18n>Documentation</ng-container></span>
|
||||
</a>
|
||||
@if (settingsService.organizingSidebarItems()) {
|
||||
<pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Documentation" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.Documentation)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.Documentation, $event)"></pngx-input-switch>
|
||||
}
|
||||
</li>
|
||||
<li class="nav-item" [class.visually-hidden]="slimSidebarEnabled">
|
||||
<div class="text-muted small d-flex align-items-center flex-wrap nav-label">
|
||||
|
||||
@@ -15,7 +15,7 @@ import { provideUiTour } from 'ngx-ui-tour-ng-bootstrap'
|
||||
import { of, throwError } from 'rxjs'
|
||||
import { routes } from 'src/app/app-routing.module'
|
||||
import { SavedView } from 'src/app/data/saved-view'
|
||||
import { HideableSidebarItemID, SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import { PermissionsGuard } from 'src/app/guards/permissions.guard'
|
||||
import {
|
||||
@@ -287,82 +287,6 @@ describe('AppFrameComponent', () => {
|
||||
jest.useRealTimers()
|
||||
})
|
||||
|
||||
it('should hide configured sidebar items', () => {
|
||||
settingsService.set(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [
|
||||
HideableSidebarItemID.Dashboard,
|
||||
HideableSidebarItemID.Workflows,
|
||||
])
|
||||
fixture.detectChanges()
|
||||
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('[routerLink="dashboard"]')
|
||||
.parentElement.classList
|
||||
).toContain('d-none')
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('[routerLink="workflows"]')
|
||||
.parentElement.classList
|
||||
).toContain('d-none')
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('[routerLink="mail"]').parentElement
|
||||
.classList
|
||||
).not.toContain('d-none')
|
||||
})
|
||||
|
||||
it('should show hidden items and visibility switches while customizing', () => {
|
||||
settingsService.set(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [
|
||||
HideableSidebarItemID.Dashboard,
|
||||
])
|
||||
settingsService.sidebarHiddenItemsEditing.set([
|
||||
HideableSidebarItemID.Dashboard,
|
||||
])
|
||||
fixture.detectChanges()
|
||||
|
||||
expect(
|
||||
fixture.nativeElement.querySelectorAll('pngx-input-switch').length
|
||||
).toBe(5)
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('[routerLink="dashboard"]')
|
||||
.parentElement.classList
|
||||
).not.toContain('d-none')
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('[routerLink="dashboard"]').classList
|
||||
).toContain('opacity-50')
|
||||
|
||||
settingsService.set(SETTINGS_KEYS.SLIM_SIDEBAR, true)
|
||||
fixture.detectChanges()
|
||||
|
||||
expect(
|
||||
Array.from(
|
||||
fixture.nativeElement.querySelectorAll('pngx-input-switch')
|
||||
).every((toggle: HTMLElement) => toggle.classList.contains('d-none'))
|
||||
).toBe(true)
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('[routerLink="dashboard"]').classList
|
||||
).not.toContain('pe-5')
|
||||
|
||||
settingsService.set(SETTINGS_KEYS.SLIM_SIDEBAR, false)
|
||||
component.slimSidebarAnimating.set(true)
|
||||
fixture.detectChanges()
|
||||
|
||||
expect(
|
||||
Array.from(
|
||||
fixture.nativeElement.querySelectorAll('pngx-input-switch')
|
||||
).every((toggle: HTMLElement) => toggle.classList.contains('d-none'))
|
||||
).toBe(true)
|
||||
|
||||
component.slimSidebarAnimating.set(false)
|
||||
fixture.detectChanges()
|
||||
|
||||
expect(
|
||||
Array.from(
|
||||
fixture.nativeElement.querySelectorAll('pngx-input-switch')
|
||||
).every((toggle: HTMLElement) => !toggle.classList.contains('d-none'))
|
||||
).toBe(true)
|
||||
expect(
|
||||
fixture.nativeElement.querySelector('[routerLink="dashboard"]').classList
|
||||
).toContain('pe-5')
|
||||
})
|
||||
|
||||
it('should show error on toggle slim sidebar if store settings fails', () => {
|
||||
jest.spyOn(console, 'warn').mockImplementation(() => {})
|
||||
const toastSpy = jest.spyOn(toastService, 'showError')
|
||||
|
||||
@@ -7,7 +7,6 @@ import {
|
||||
} from '@angular/cdk/drag-drop'
|
||||
import { NgClass } from '@angular/common'
|
||||
import { Component, HostListener, inject, OnInit, signal } from '@angular/core'
|
||||
import { FormsModule } from '@angular/forms'
|
||||
import { ActivatedRoute, Router, RouterModule } from '@angular/router'
|
||||
import {
|
||||
NgbCollapseModule,
|
||||
@@ -22,11 +21,7 @@ import { Observable } from 'rxjs'
|
||||
import { first } from 'rxjs/operators'
|
||||
import { Document } from 'src/app/data/document'
|
||||
import { SavedView } from 'src/app/data/saved-view'
|
||||
import {
|
||||
CollapsibleSection,
|
||||
HideableSidebarItemID,
|
||||
SETTINGS_KEYS,
|
||||
} from 'src/app/data/ui-settings'
|
||||
import { CollapsibleSection, SETTINGS_KEYS } from 'src/app/data/ui-settings'
|
||||
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
|
||||
import { ComponentCanDeactivate } from 'src/app/guards/dirty-doc.guard'
|
||||
import { DocumentTitlePipe } from 'src/app/pipes/document-title.pipe'
|
||||
@@ -53,7 +48,6 @@ import { ChatComponent } from '../chat/chat/chat.component'
|
||||
import { BrandMarkComponent } from '../common/logo/brand-mark/brand-mark.component'
|
||||
import { LogoComponent } from '../common/logo/logo.component'
|
||||
import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
|
||||
import { SwitchComponent } from '../common/input/switch/switch.component'
|
||||
import { DocumentDetailComponent } from '../document-detail/document-detail.component'
|
||||
import { ComponentWithPermissions } from '../with-permissions/with-permissions.component'
|
||||
import { GlobalSearchComponent } from './global-search/global-search.component'
|
||||
@@ -82,8 +76,6 @@ const SCROLL_THRESHOLD = 16
|
||||
NgxBootstrapIconsModule,
|
||||
DragDropModule,
|
||||
TourNgBootstrap,
|
||||
FormsModule,
|
||||
SwitchComponent,
|
||||
],
|
||||
})
|
||||
export class AppFrameComponent
|
||||
@@ -106,7 +98,6 @@ export class AppFrameComponent
|
||||
readonly isMenuCollapsed = signal(true)
|
||||
readonly slimSidebarAnimating = signal(false)
|
||||
readonly mobileSearchHidden = signal(false)
|
||||
readonly HideableSidebarItemID = HideableSidebarItemID
|
||||
private readonly versionSetting = this.settingsService.getSignal<string>(
|
||||
SETTINGS_KEYS.VERSION
|
||||
)
|
||||
@@ -204,10 +195,6 @@ export class AppFrameComponent
|
||||
}, 200) // slightly longer than css animation for slim sidebar
|
||||
}
|
||||
|
||||
toggleSidebarItem(item: HideableSidebarItemID, visible: boolean): void {
|
||||
this.settingsService.updateSidebarItemVisibility(item, visible)
|
||||
}
|
||||
|
||||
toggleAttributesSections(event?: Event): void {
|
||||
event?.preventDefault()
|
||||
event?.stopPropagation()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<div [class.mb-3]="!compact">
|
||||
<div [class.row]="!compact">
|
||||
@if (!horizontal && !compact) {
|
||||
<div class="mb-3">
|
||||
<div class="row">
|
||||
@if (!horizontal) {
|
||||
<div class="d-flex align-items-center position-relative hidden-button-container col-md-3">
|
||||
<label class="form-label" [for]="inputId" [ngbTooltip]="showUnsetNote && isUnset ? tipContent: null" placement="end">
|
||||
{{title}}
|
||||
@@ -17,8 +17,8 @@
|
||||
}
|
||||
<div [ngClass]="{'align-items-center': horizontal, 'd-flex': horizontal}">
|
||||
<div class="form-check form-switch">
|
||||
<input #inputField type="checkbox" class="form-check-input" [id]="inputId" [(ngModel)]="value" [ngModelOptions]="{standalone: true}" (change)="onChange(value)" (blur)="onTouched()" [disabled]="disabled" [attr.aria-label]="compact ? title : null">
|
||||
@if (horizontal && !compact) {
|
||||
<input #inputField type="checkbox" class="form-check-input" [id]="inputId" [(ngModel)]="value" [ngModelOptions]="{standalone: true}" (change)="onChange(value)" (blur)="onTouched()" [disabled]="disabled">
|
||||
@if (horizontal) {
|
||||
<label class="form-check-label" [class.text-muted]="showUnsetNote && isUnset" [for]="inputId" [ngbTooltip]="showUnsetNote && isUnset ? tipContent: null" placement="end">
|
||||
{{title}}
|
||||
@if (showUnsetNote && isUnset) {
|
||||
|
||||
@@ -48,14 +48,4 @@ describe('SwitchComponent', () => {
|
||||
component.value = undefined
|
||||
expect(component.isUnset).toBeTruthy()
|
||||
})
|
||||
|
||||
it('should support a compact layout', () => {
|
||||
component.compact = true
|
||||
component.title = 'Test switch'
|
||||
fixture.detectChanges()
|
||||
|
||||
expect(fixture.nativeElement.querySelector('.mb-3')).toBeNull()
|
||||
expect(fixture.nativeElement.querySelector('.row')).toBeNull()
|
||||
expect(input.getAttribute('aria-label')).toEqual('Test switch')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -25,9 +25,6 @@ export class SwitchComponent extends AbstractInputComponent<boolean> {
|
||||
@Input()
|
||||
showUnsetNote: boolean = false
|
||||
|
||||
@Input()
|
||||
compact: boolean = false
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
}
|
||||
|
||||
@@ -24,16 +24,6 @@ export enum CollapsibleSection {
|
||||
ATTRIBUTES = 'attributes',
|
||||
}
|
||||
|
||||
export enum HideableSidebarItemID {
|
||||
Dashboard = 'dashboard',
|
||||
SavedViews = 'saved_views',
|
||||
Workflows = 'workflows',
|
||||
Mail = 'mail',
|
||||
Documentation = 'documentation',
|
||||
}
|
||||
|
||||
export const HIDEABLE_SIDEBAR_ITEM_IDS = Object.values(HideableSidebarItemID)
|
||||
|
||||
export const PAPERLESS_GREEN_HEX = '#17541f'
|
||||
|
||||
export const SETTINGS_KEYS = {
|
||||
@@ -66,7 +56,6 @@ export const SETTINGS_KEYS = {
|
||||
NOTES_ENABLED: 'general-settings:notes-enabled',
|
||||
AUDITLOG_ENABLED: 'general-settings:auditlog-enabled',
|
||||
SLIM_SIDEBAR: 'general-settings:slim-sidebar',
|
||||
SIDEBAR_HIDDEN_ITEMS: 'general-settings:sidebar:hidden-items',
|
||||
ATTRIBUTES_SECTIONS_COLLAPSED:
|
||||
'general-settings:attributes-sections-collapsed',
|
||||
UPDATE_CHECKING_ENABLED: 'general-settings:update-checking:enabled',
|
||||
@@ -138,11 +127,6 @@ export const SETTINGS: UiSetting[] = [
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS,
|
||||
type: 'array',
|
||||
default: [],
|
||||
},
|
||||
{
|
||||
key: SETTINGS_KEYS.ATTRIBUTES_SECTIONS_COLLAPSED,
|
||||
type: 'array',
|
||||
|
||||
@@ -14,11 +14,7 @@ import { CustomFieldDataType } from '../data/custom-field'
|
||||
import { DEFAULT_DISPLAY_FIELDS, DisplayField } from '../data/document'
|
||||
import { SavedView } from '../data/saved-view'
|
||||
import { RemoteOCRModeConfig } from '../data/paperless-config'
|
||||
import {
|
||||
HideableSidebarItemID,
|
||||
SETTINGS_KEYS,
|
||||
UiSettings,
|
||||
} from '../data/ui-settings'
|
||||
import { SETTINGS_KEYS, UiSettings } from '../data/ui-settings'
|
||||
import { PermissionsService } from './permissions.service'
|
||||
import { CustomFieldsService } from './rest/custom-fields.service'
|
||||
import { SettingsService } from './settings.service'
|
||||
@@ -234,35 +230,6 @@ describe('SettingsService', () => {
|
||||
expect(notesEnabled()).toBeFalsy()
|
||||
})
|
||||
|
||||
it('updates sidebar item visibility', () => {
|
||||
httpTestingController
|
||||
.expectOne(`${environment.apiBaseUrl}ui_settings/`)
|
||||
.flush(ui_settings)
|
||||
|
||||
expect(
|
||||
settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)
|
||||
).toBe(false)
|
||||
|
||||
settingsService.updateSidebarItemVisibility(
|
||||
HideableSidebarItemID.Workflows,
|
||||
false
|
||||
)
|
||||
|
||||
expect(
|
||||
settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)
|
||||
).toBe(true)
|
||||
expect(settingsService.get(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS)).toEqual([])
|
||||
|
||||
settingsService.updateSidebarItemVisibility(
|
||||
HideableSidebarItemID.Workflows,
|
||||
true
|
||||
)
|
||||
|
||||
expect(
|
||||
settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('updates setting signals when settings are reinitialized', () => {
|
||||
let req = httpTestingController.expectOne(
|
||||
`${environment.apiBaseUrl}ui_settings/`
|
||||
|
||||
@@ -24,7 +24,6 @@ import { DEFAULT_DISPLAY_FIELDS, DisplayField } from '../data/document'
|
||||
import { RemoteOCRModeConfig } from '../data/paperless-config'
|
||||
import { SavedView } from '../data/saved-view'
|
||||
import {
|
||||
HideableSidebarItemID,
|
||||
PAPERLESS_GREEN_HEX,
|
||||
SETTINGS,
|
||||
SETTINGS_KEYS,
|
||||
@@ -314,18 +313,6 @@ export class SettingsService {
|
||||
readonly globalDropzoneEnabled = signal(true)
|
||||
readonly globalDropzoneActive = signal(false)
|
||||
readonly organizingSidebarSavedViews = signal(false)
|
||||
readonly sidebarHiddenItemsEditing = signal<HideableSidebarItemID[] | null>(
|
||||
null
|
||||
)
|
||||
readonly organizingSidebarItems = computed(
|
||||
() => this.sidebarHiddenItemsEditing() !== null
|
||||
)
|
||||
readonly sidebarHiddenItemsEditingChanged = new EventEmitter<
|
||||
HideableSidebarItemID[]
|
||||
>()
|
||||
readonly hiddenSidebarItems = this.getSignal<HideableSidebarItemID[]>(
|
||||
SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS
|
||||
)
|
||||
|
||||
readonly allDisplayFields = signal<Array<{ id: DisplayField; name: string }>>(
|
||||
DEFAULT_DISPLAY_FIELDS
|
||||
@@ -762,29 +749,6 @@ export class SettingsService {
|
||||
return this.storeSettings()
|
||||
}
|
||||
|
||||
sidebarItemIsHidden(item: HideableSidebarItemID): boolean {
|
||||
return (
|
||||
this.sidebarHiddenItemsEditing() ?? this.hiddenSidebarItems()
|
||||
).includes(item)
|
||||
}
|
||||
|
||||
updateSidebarItemVisibility(
|
||||
item: HideableSidebarItemID,
|
||||
visible: boolean
|
||||
): void {
|
||||
const hiddenItems = new Set(
|
||||
this.sidebarHiddenItemsEditing() ?? this.hiddenSidebarItems()
|
||||
)
|
||||
if (visible) {
|
||||
hiddenItems.delete(item)
|
||||
} else {
|
||||
hiddenItems.add(item)
|
||||
}
|
||||
const updatedHiddenItems = [...hiddenItems]
|
||||
this.sidebarHiddenItemsEditing.set(updatedHiddenItems)
|
||||
this.sidebarHiddenItemsEditingChanged.emit(updatedHiddenItems)
|
||||
}
|
||||
|
||||
updateSavedViewsVisibility(
|
||||
dashboardVisibleViewIds: number[],
|
||||
sidebarVisibleViewIds: number[]
|
||||
|
||||
+42
-44
@@ -28,7 +28,7 @@ from documents.models import DocumentType
|
||||
from documents.models import PaperlessTask
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import set_permissions_for_object
|
||||
from documents.permissions import set_permissions_for_objects
|
||||
from documents.plugins.helpers import DocumentsStatusManager
|
||||
from documents.tasks import bulk_update_documents
|
||||
from documents.tasks import consume_file
|
||||
@@ -299,55 +299,53 @@ def modify_custom_fields(
|
||||
) -> Literal["OK"]:
|
||||
qs = Document.objects.filter(id__in=doc_ids).only("pk")
|
||||
affected_docs = list(qs.values_list("pk", flat=True))
|
||||
# Ensure add_custom_fields is a list of (int, value) tuples, supports old API
|
||||
# Ensure add_custom_fields is a list of tuples, supports old API
|
||||
add_custom_fields = (
|
||||
[(int(field), value) for field, value in add_custom_fields.items()]
|
||||
add_custom_fields.items()
|
||||
if isinstance(add_custom_fields, dict)
|
||||
else [(int(field), None) for field in add_custom_fields]
|
||||
else [(field, None) for field in add_custom_fields]
|
||||
)
|
||||
|
||||
# Resolved once, instead of re-querying the same field for every document
|
||||
custom_fields_by_id: dict[int, CustomField] = CustomField.objects.in_bulk(
|
||||
[field_id for field_id, _ in add_custom_fields],
|
||||
)
|
||||
# Passed to update_or_create() below rather than a bare id, so the FK is
|
||||
# cached on the created instance and auditlog's post_save receiver does
|
||||
# not reload it per row. Only needed for additions. content is deferred:
|
||||
# the one field here that is both large and unused.
|
||||
docs_by_id: dict[int, Document] = (
|
||||
Document.objects.defer("content").in_bulk(affected_docs)
|
||||
if add_custom_fields
|
||||
else {}
|
||||
)
|
||||
custom_fields = CustomField.objects.filter(
|
||||
id__in=[int(field) for field, _ in add_custom_fields],
|
||||
).distinct()
|
||||
for field_id, value in add_custom_fields:
|
||||
custom_field = custom_fields_by_id[field_id]
|
||||
value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
custom_field.data_type
|
||||
]
|
||||
is_doclink = custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
|
||||
for doc_id in affected_docs:
|
||||
if is_doclink and value and doc_id in value:
|
||||
# Prevent self-linking
|
||||
continue
|
||||
defaults = {}
|
||||
custom_field = custom_fields.get(id=field_id)
|
||||
if custom_field:
|
||||
value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
|
||||
custom_field.data_type
|
||||
]
|
||||
defaults[value_field] = value
|
||||
if (
|
||||
custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
|
||||
and value
|
||||
and doc_id in value
|
||||
):
|
||||
# Prevent self-linking
|
||||
continue
|
||||
CustomFieldInstance.objects.update_or_create(
|
||||
document=docs_by_id[doc_id],
|
||||
field=custom_field,
|
||||
defaults={value_field: value},
|
||||
document_id=doc_id,
|
||||
field_id=field_id,
|
||||
defaults=defaults,
|
||||
)
|
||||
if is_doclink:
|
||||
reflect_doclinks(docs_by_id[doc_id], custom_field, value)
|
||||
if custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
|
||||
doc = Document.objects.get(id=doc_id)
|
||||
reflect_doclinks(doc, custom_field, value)
|
||||
|
||||
# For doc link fields that are being removed, remove symmetrical links.
|
||||
# select_related avoids a per-instance reload of the document and field.
|
||||
# For doc link fields that are being removed, remove symmetrical links
|
||||
for doclink_being_removed_instance in CustomFieldInstance.objects.filter(
|
||||
document_id__in=affected_docs,
|
||||
field__id__in=remove_custom_fields,
|
||||
field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
|
||||
value_document_ids__isnull=False,
|
||||
).select_related("field", "document"):
|
||||
):
|
||||
for target_doc_id in doclink_being_removed_instance.value:
|
||||
remove_doclink(
|
||||
document=doclink_being_removed_instance.document,
|
||||
document=Document.objects.get(
|
||||
id=doclink_being_removed_instance.document.id,
|
||||
),
|
||||
field=doclink_being_removed_instance.field,
|
||||
target_doc_id=target_doc_id,
|
||||
)
|
||||
@@ -433,10 +431,13 @@ def set_permissions(
|
||||
else:
|
||||
qs.update(owner=owner)
|
||||
|
||||
for doc in qs:
|
||||
set_permissions_for_object(permissions=set_permissions, object=doc, merge=merge)
|
||||
|
||||
affected_docs = list(qs.values_list("pk", flat=True))
|
||||
set_permissions_for_objects(
|
||||
permissions=set_permissions,
|
||||
model=Document,
|
||||
pks=affected_docs,
|
||||
merge=merge,
|
||||
)
|
||||
|
||||
bulk_update_documents.apply_async(
|
||||
kwargs={"document_ids": affected_docs},
|
||||
@@ -1180,13 +1181,10 @@ def remove_doclink(
|
||||
"""
|
||||
Removes a 'symmetrical' link to `document` from the target document's existing custom field instance
|
||||
"""
|
||||
# select_related: a signal receiver (auditlog) touches .document/.field on
|
||||
# the save() below, without this that is a per-call reload query
|
||||
target_doc_field_instance = (
|
||||
CustomFieldInstance.objects.filter(document_id=target_doc_id, field=field)
|
||||
.select_related("document", "field")
|
||||
.first()
|
||||
)
|
||||
target_doc_field_instance = CustomFieldInstance.objects.filter(
|
||||
document_id=target_doc_id,
|
||||
field=field,
|
||||
).first()
|
||||
if (
|
||||
target_doc_field_instance is not None
|
||||
and document.id in target_doc_field_instance.value
|
||||
|
||||
+28
-66
@@ -34,27 +34,6 @@ from paperless.signed_pickle import signed_pickle_loads
|
||||
|
||||
logger = logging.getLogger("paperless.classifier")
|
||||
|
||||
|
||||
def _predict_with_threshold(classifier, X, threshold: float) -> int | None:
|
||||
"""
|
||||
Return the predicted class id, or None if:
|
||||
- the prediction is -1 (no match), or
|
||||
- the winning class probability is below the configured threshold.
|
||||
|
||||
Using predict_proba() instead of predict() lets us apply a minimum-confidence
|
||||
cutoff so that uncertain predictions are discarded rather than assigned.
|
||||
"""
|
||||
probas = classifier.predict_proba(X)[0]
|
||||
best_idx = int(probas.argmax())
|
||||
best_class = int(classifier.classes_[best_idx])
|
||||
|
||||
if best_class == -1:
|
||||
return None
|
||||
if threshold > 0.0 and probas[best_idx] < threshold:
|
||||
return None
|
||||
return best_class
|
||||
|
||||
|
||||
ADVANCED_TEXT_PROCESSING_ENABLED = (
|
||||
settings.NLTK_LANGUAGE is not None and settings.NLTK_ENABLED
|
||||
)
|
||||
@@ -123,8 +102,7 @@ class DocumentClassifier:
|
||||
# v8 - Added storage path classifier
|
||||
# v9 - Changed from hashing to time/ids for re-train check
|
||||
# v10 - HMAC-signed model file
|
||||
# v11 - Use sample_weight for balanced training; predict_proba with threshold
|
||||
FORMAT_VERSION = 11
|
||||
FORMAT_VERSION = 10
|
||||
|
||||
HMAC_SIZE = 32 # SHA-256 digest length
|
||||
|
||||
@@ -346,13 +324,6 @@ class DocumentClassifier:
|
||||
from sklearn.preprocessing import LabelBinarizer
|
||||
from sklearn.preprocessing import MultiLabelBinarizer
|
||||
|
||||
# MLPClassifier does not support class_weight directly
|
||||
# (https://github.com/scikit-learn/scikit-learn/issues/9113), so we use
|
||||
# compute_sample_weight to balance classes during training and prevent
|
||||
# over-represented correspondents from dominating predictions.
|
||||
# https://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_sample_weight.html
|
||||
from sklearn.utils.class_weight import compute_sample_weight
|
||||
|
||||
# Step 2: vectorize data
|
||||
logger.debug("Vectorizing data...")
|
||||
notify("Vectorizing document content...")
|
||||
@@ -398,7 +369,7 @@ class DocumentClassifier:
|
||||
self.tags_binarizer = MultiLabelBinarizer()
|
||||
labels_tags_vectorized = self.tags_binarizer.fit_transform(labels_tags)
|
||||
|
||||
self.tags_classifier = MLPClassifier(tol=0.01, random_state=0)
|
||||
self.tags_classifier = MLPClassifier(tol=0.01)
|
||||
self.tags_classifier.fit(data_vectorized, labels_tags_vectorized)
|
||||
else:
|
||||
self.tags_classifier = None
|
||||
@@ -409,12 +380,8 @@ class DocumentClassifier:
|
||||
notify(
|
||||
f"Training correspondent classifier ({num_correspondents} correspondent(s))...",
|
||||
)
|
||||
self.correspondent_classifier = MLPClassifier(tol=0.01, random_state=0)
|
||||
self.correspondent_classifier.fit(
|
||||
data_vectorized,
|
||||
labels_correspondent,
|
||||
sample_weight=compute_sample_weight("balanced", labels_correspondent),
|
||||
)
|
||||
self.correspondent_classifier = MLPClassifier(tol=0.01)
|
||||
self.correspondent_classifier.fit(data_vectorized, labels_correspondent)
|
||||
else:
|
||||
self.correspondent_classifier = None
|
||||
logger.debug(
|
||||
@@ -426,12 +393,8 @@ class DocumentClassifier:
|
||||
notify(
|
||||
f"Training document type classifier ({num_document_types} type(s))...",
|
||||
)
|
||||
self.document_type_classifier = MLPClassifier(tol=0.01, random_state=0)
|
||||
self.document_type_classifier.fit(
|
||||
data_vectorized,
|
||||
labels_document_type,
|
||||
sample_weight=compute_sample_weight("balanced", labels_document_type),
|
||||
)
|
||||
self.document_type_classifier = MLPClassifier(tol=0.01)
|
||||
self.document_type_classifier.fit(data_vectorized, labels_document_type)
|
||||
else:
|
||||
self.document_type_classifier = None
|
||||
logger.debug(
|
||||
@@ -443,11 +406,10 @@ class DocumentClassifier:
|
||||
"Training storage paths classifier...",
|
||||
)
|
||||
notify(f"Training storage path classifier ({num_storage_paths} path(s))...")
|
||||
self.storage_path_classifier = MLPClassifier(tol=0.01, random_state=0)
|
||||
self.storage_path_classifier = MLPClassifier(tol=0.01)
|
||||
self.storage_path_classifier.fit(
|
||||
data_vectorized,
|
||||
labels_storage_path,
|
||||
sample_weight=compute_sample_weight("balanced", labels_storage_path),
|
||||
)
|
||||
else:
|
||||
self.storage_path_classifier = None
|
||||
@@ -584,24 +546,24 @@ class DocumentClassifier:
|
||||
def predict_correspondent(self, content: str) -> int | None:
|
||||
if self.correspondent_classifier:
|
||||
X = self._vectorize(content)
|
||||
predicted_id = _predict_with_threshold(
|
||||
self.correspondent_classifier,
|
||||
X,
|
||||
settings.CLASSIFIER_MATCH_THRESHOLD,
|
||||
)
|
||||
return predicted_id
|
||||
return None
|
||||
correspondent_id = self.correspondent_classifier.predict(X)
|
||||
if correspondent_id != -1:
|
||||
return correspondent_id
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
def predict_document_type(self, content: str) -> int | None:
|
||||
if self.document_type_classifier:
|
||||
X = self._vectorize(content)
|
||||
predicted_id = _predict_with_threshold(
|
||||
self.document_type_classifier,
|
||||
X,
|
||||
settings.CLASSIFIER_MATCH_THRESHOLD,
|
||||
)
|
||||
return predicted_id
|
||||
return None
|
||||
document_type_id = self.document_type_classifier.predict(X)
|
||||
if document_type_id != -1:
|
||||
return document_type_id
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
def predict_tags(self, content: str) -> list[int]:
|
||||
from sklearn.utils.multiclass import type_of_target
|
||||
@@ -627,10 +589,10 @@ class DocumentClassifier:
|
||||
def predict_storage_path(self, content: str) -> int | None:
|
||||
if self.storage_path_classifier:
|
||||
X = self._vectorize(content)
|
||||
predicted_id = _predict_with_threshold(
|
||||
self.storage_path_classifier,
|
||||
X,
|
||||
settings.CLASSIFIER_MATCH_THRESHOLD,
|
||||
)
|
||||
return predicted_id
|
||||
return None
|
||||
storage_path_id = self.storage_path_classifier.predict(X)
|
||||
if storage_path_id != -1:
|
||||
return storage_path_id
|
||||
else:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
@@ -173,6 +173,179 @@ def set_permissions_for_object(
|
||||
)
|
||||
|
||||
|
||||
def _resolve_permissions(codenames: set[str], ctype: ContentType) -> list[Permission]:
|
||||
"""
|
||||
Resolves `codenames` to Permission rows, raising like the single-object
|
||||
assign_perm() this bulk path replaces does (via a `.get()` internally)
|
||||
if any codename doesn't exist -- e.g. a client-supplied action name that
|
||||
was never validated (BulkEditObjectsSerializer._validate_permissions
|
||||
calls validate_set_permissions() only for its side-effecting id checks
|
||||
and discards the filtered dict it returns, so an unrecognized action key
|
||||
reaches this function as-is). A plain `.filter()` with no existence
|
||||
check would otherwise silently build zero rows and no-op instead of
|
||||
reporting the bad input.
|
||||
"""
|
||||
permission_objs = list(
|
||||
Permission.objects.filter(content_type=ctype, codename__in=codenames),
|
||||
)
|
||||
missing = codenames - {p.codename for p in permission_objs}
|
||||
if missing:
|
||||
raise Permission.DoesNotExist(
|
||||
f"Permission matching query does not exist for codename(s): "
|
||||
f"{', '.join(sorted(missing))}",
|
||||
)
|
||||
return permission_objs
|
||||
|
||||
|
||||
def _apply_bulk_permission_entry(
|
||||
*,
|
||||
perm_model: type[UserObjectPermission] | type[GroupObjectPermission],
|
||||
identity_model: type[User] | type[Group],
|
||||
identity_field: str,
|
||||
ids: list[int],
|
||||
codename: str,
|
||||
permission_objs: list[Permission],
|
||||
ctype: ContentType,
|
||||
object_pks: list[str],
|
||||
merge: bool,
|
||||
) -> None:
|
||||
# Only the ids are needed to build permission rows (via `<field>_id=`),
|
||||
# so avoid fetching full User/Group rows for identities that may not
|
||||
# even end up being granted anything new.
|
||||
add_ids = set(
|
||||
identity_model.objects.filter(id__in=ids).values_list("id", flat=True),
|
||||
)
|
||||
|
||||
if not merge:
|
||||
existing_ids = set(
|
||||
perm_model.objects.filter(
|
||||
content_type=ctype,
|
||||
object_pk__in=object_pks,
|
||||
permission__codename=codename,
|
||||
)
|
||||
.values_list(f"{identity_field}_id", flat=True)
|
||||
.distinct(),
|
||||
)
|
||||
remove_ids = existing_ids - add_ids
|
||||
if remove_ids:
|
||||
perm_model.objects.filter(
|
||||
content_type=ctype,
|
||||
object_pk__in=object_pks,
|
||||
permission__codename=codename,
|
||||
**{f"{identity_field}_id__in": remove_ids},
|
||||
).delete()
|
||||
|
||||
if not add_ids:
|
||||
return
|
||||
|
||||
rows = [
|
||||
perm_model(
|
||||
content_type=ctype,
|
||||
object_pk=pk,
|
||||
permission=permission_obj,
|
||||
**{f"{identity_field}_id": identity_id},
|
||||
)
|
||||
for permission_obj in permission_objs
|
||||
for pk in object_pks
|
||||
for identity_id in add_ids
|
||||
]
|
||||
# ignore_conflicts skips only rows that already exist as an exact
|
||||
# (identity, permission, object) match -- the same de-dup the
|
||||
# underlying (user|group, permission, object_pk) unique constraint
|
||||
# already enforces for the single-object assign_perm() this replaces,
|
||||
# so it doesn't change what counts as "already granted". batch_size
|
||||
# caps how many rows go into a single INSERT statement.
|
||||
perm_model.objects.bulk_create(rows, ignore_conflicts=True, batch_size=1000)
|
||||
|
||||
|
||||
def set_permissions_for_objects(
|
||||
permissions: dict,
|
||||
model: type[Model],
|
||||
pks: QuerySet | list,
|
||||
*,
|
||||
merge: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Bulk equivalent of set_permissions_for_object: applies the same
|
||||
permission changes to every object identified by `pks` at once.
|
||||
|
||||
Takes a model + pks (rather than model instances) deliberately -- the
|
||||
permission rows built below only ever need `pk`, `content_type`, and
|
||||
identity ids, so callers shouldn't have to fetch full rows (with every
|
||||
other field) just to hand them to this function.
|
||||
|
||||
Deliberately does not use guardian's queryset/list-aware assign_perm:
|
||||
passing a list as the object routes to bulk_assign_perm, which skips
|
||||
creating a direct permission row for anyone who already has the
|
||||
permission via ANY group membership (it checks
|
||||
ObjectPermissionChecker.has_perm, which is group-inheritance-aware) --
|
||||
unlike the single-object assign_perm this replaces, which always
|
||||
ensures a direct row via get_or_create regardless of group-derived
|
||||
access. Losing that guarantee would mean a later revocation of the
|
||||
group's grant silently strips access an admin explicitly asked to be
|
||||
direct. Bulk-creating rows straight against the permission models
|
||||
instead (see _apply_bulk_permission_entry) preserves the original
|
||||
always-create-a-direct-row semantics while still batching every object
|
||||
and every identity into one query per action, rather than one query per
|
||||
(object, user) pair.
|
||||
"""
|
||||
object_pks = [str(pk) for pk in pks]
|
||||
if not object_pks: # pragma: no cover
|
||||
return
|
||||
|
||||
model_name = model.__name__.lower()
|
||||
ctype = ContentType.objects.get_for_model(model)
|
||||
|
||||
# Every action is resolved up front, before anything is written, so an
|
||||
# unrecognized action name (see _resolve_permissions) aborts the whole
|
||||
# call instead of leaving the actions ahead of it already applied --
|
||||
# BulkEditObjectsSerializer lets unknown keys through and its view turns
|
||||
# the exception into a 400, so a half-applied change would otherwise be
|
||||
# reported to the client as a failure.
|
||||
permissions_by_action: dict[str, list[Permission]] = {}
|
||||
for action, entry in permissions.items():
|
||||
if "users" not in entry and "groups" not in entry:
|
||||
continue
|
||||
implied_codenames = {f"{action}_{model_name}"}
|
||||
if action == "change":
|
||||
# change gives view too
|
||||
implied_codenames.add(f"view_{model_name}")
|
||||
permissions_by_action[action] = _resolve_permissions(
|
||||
implied_codenames,
|
||||
ctype,
|
||||
)
|
||||
|
||||
for action, entry in permissions.items():
|
||||
codename = f"{action}_{model_name}"
|
||||
permission_objs = permissions_by_action.get(action, [])
|
||||
|
||||
if "users" in entry:
|
||||
_apply_bulk_permission_entry(
|
||||
perm_model=UserObjectPermission,
|
||||
identity_model=User,
|
||||
identity_field="user",
|
||||
ids=entry["users"],
|
||||
codename=codename,
|
||||
permission_objs=permission_objs,
|
||||
ctype=ctype,
|
||||
object_pks=object_pks,
|
||||
merge=merge,
|
||||
)
|
||||
|
||||
if "groups" in entry:
|
||||
_apply_bulk_permission_entry(
|
||||
perm_model=GroupObjectPermission,
|
||||
identity_model=Group,
|
||||
identity_field="group",
|
||||
ids=entry["groups"],
|
||||
codename=codename,
|
||||
permission_objs=permission_objs,
|
||||
ctype=ctype,
|
||||
object_pks=object_pks,
|
||||
merge=merge,
|
||||
)
|
||||
|
||||
|
||||
def permitted_object_ids(
|
||||
user: User | None,
|
||||
model: type[Model],
|
||||
|
||||
@@ -1256,26 +1256,20 @@ class DocumentSerializer(
|
||||
if "tags" in validated_data
|
||||
else []
|
||||
)
|
||||
tags_being_added = Tag.objects.filter(id__in=tag_ids_being_added)
|
||||
required_by_add_tags = set(tags_being_added)
|
||||
for tag in tags_being_added:
|
||||
required_by_add_tags.update(tag.get_ancestors())
|
||||
|
||||
# Remove its descendants too, except any that is being added in this same update
|
||||
tags_to_remove = set()
|
||||
for tag in Tag.objects.filter(is_inbox_tag=True):
|
||||
if tag in required_by_add_tags:
|
||||
continue
|
||||
tags_to_remove.add(tag)
|
||||
tags_to_remove.update(tag.get_descendants())
|
||||
|
||||
inbox_tags_not_being_added = Tag.objects.filter(is_inbox_tag=True).exclude(
|
||||
id__in=tag_ids_being_added,
|
||||
)
|
||||
if "tags" in validated_data:
|
||||
validated_data["tags"] = [
|
||||
tag for tag in validated_data["tags"] if tag not in tags_to_remove
|
||||
tag
|
||||
for tag in validated_data["tags"]
|
||||
if tag not in inbox_tags_not_being_added
|
||||
]
|
||||
else:
|
||||
validated_data["tags"] = [
|
||||
tag for tag in instance.tags.all() if tag not in tags_to_remove
|
||||
tag
|
||||
for tag in instance.tags.all()
|
||||
if tag not in inbox_tags_not_being_added
|
||||
]
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
|
||||
@@ -2,10 +2,15 @@ import datetime
|
||||
import json
|
||||
from unittest import mock
|
||||
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import Permission
|
||||
from django.contrib.auth.models import User
|
||||
from django.db import connection
|
||||
from django.test import override_settings
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from guardian.shortcuts import assign_perm
|
||||
from guardian.shortcuts import get_groups_with_perms
|
||||
from guardian.shortcuts import get_users_with_perms
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
@@ -842,6 +847,66 @@ class TestBulkEditObjects(APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(StoragePath.objects.count(), 0)
|
||||
|
||||
def test_bulk_objects_set_permissions_batched_across_object_count(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Many tags are being bulk-edited to set permissions at once
|
||||
WHEN:
|
||||
- bulk_edit_objects API endpoint is called with set_permissions
|
||||
operation over a small batch vs. a much larger one
|
||||
THEN:
|
||||
- Permissions are applied correctly at both scales
|
||||
- Query count does not grow with the number of tags, i.e. each
|
||||
user/group is applied across all tags with one batched call
|
||||
rather than one call per (tag, identity) pair
|
||||
"""
|
||||
group1 = Group.objects.create(name="perm-group")
|
||||
permissions = {
|
||||
"view": {"users": [self.user1.id, self.user2.id], "groups": [group1.id]},
|
||||
"change": {"users": [self.user1.id], "groups": [group1.id]},
|
||||
}
|
||||
|
||||
def run_with_n_tags(n: int) -> int:
|
||||
tags = [Tag.objects.create(name=f"perm-tag-{n}-{i}") for i in range(n)]
|
||||
with CaptureQueriesContext(connection) as ctx:
|
||||
response = self.client.post(
|
||||
"/api/bulk_edit_objects/",
|
||||
json.dumps(
|
||||
{
|
||||
"objects": [t.id for t in tags],
|
||||
"object_type": "tags",
|
||||
"operation": "set_permissions",
|
||||
"permissions": permissions,
|
||||
"merge": False,
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
for tag in tags:
|
||||
self.assertEqual(get_users_with_perms(tag).count(), 2)
|
||||
self.assertEqual(get_groups_with_perms(tag).count(), 1)
|
||||
return len(ctx.captured_queries)
|
||||
|
||||
small_batch_queries = run_with_n_tags(5)
|
||||
large_batch_queries = run_with_n_tags(50)
|
||||
|
||||
# A tolerance rather than equality, matching the N+1 check in
|
||||
# test_views.py: bulk_create's batch_size caps rows per INSERT, so a
|
||||
# large enough selection does legitimately add statements, and the
|
||||
# per-process ContentType cache makes the first run carry an extra
|
||||
# query. Neither can hide a regression to per-object assignment,
|
||||
# which would be ~10x the small-batch count here.
|
||||
self.assertLessEqual(
|
||||
large_batch_queries,
|
||||
small_batch_queries + 5,
|
||||
"Permission assignment appears to scale with object count: "
|
||||
f"{small_batch_queries} queries for 5 tags vs. "
|
||||
f"{large_batch_queries} for 50",
|
||||
)
|
||||
|
||||
def test_bulk_objects_delete_all_filtered(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -5,8 +5,11 @@ from unittest import mock
|
||||
|
||||
import pikepdf
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import Permission
|
||||
from django.contrib.auth.models import User
|
||||
from django.db import connection
|
||||
from django.test import TestCase
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from guardian.shortcuts import assign_perm
|
||||
from guardian.shortcuts import get_groups_with_perms
|
||||
from guardian.shortcuts import get_users_with_perms
|
||||
@@ -19,6 +22,7 @@ from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import set_permissions_for_objects
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
|
||||
|
||||
@@ -515,6 +519,178 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
|
||||
)
|
||||
self.assertEqual(groups_with_perms.count(), 2)
|
||||
|
||||
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
|
||||
def test_set_permissions_batched_across_document_count(
|
||||
self,
|
||||
m,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Many documents are being bulk-edited to set permissions at once
|
||||
WHEN:
|
||||
- set_permissions runs over a small batch vs. a much larger one
|
||||
THEN:
|
||||
- Permissions are applied correctly at both scales
|
||||
- Query count does not grow with the number of documents, i.e.
|
||||
each user/group is applied across all documents with one
|
||||
batched call rather than one call per (document, identity)
|
||||
pair
|
||||
"""
|
||||
permissions = {
|
||||
"view": {
|
||||
"users": [self.user1.id, self.user2.id],
|
||||
"groups": [self.group2.id],
|
||||
},
|
||||
"change": {
|
||||
"users": [self.user1.id],
|
||||
"groups": [self.group2.id],
|
||||
},
|
||||
}
|
||||
|
||||
def run_with_n_documents(n: int) -> int:
|
||||
docs = [
|
||||
Document.objects.create(checksum=f"perm-{n}-{i}", title=f"perm-{n}-{i}")
|
||||
for i in range(n)
|
||||
]
|
||||
with CaptureQueriesContext(connection) as ctx:
|
||||
bulk_edit.set_permissions(
|
||||
[doc.id for doc in docs],
|
||||
set_permissions=permissions,
|
||||
owner=self.owner,
|
||||
merge=False,
|
||||
)
|
||||
for doc in docs:
|
||||
self.assertEqual(get_users_with_perms(doc).count(), 2)
|
||||
self.assertEqual(get_groups_with_perms(doc).count(), 1)
|
||||
return len(ctx.captured_queries)
|
||||
|
||||
small_batch_queries = run_with_n_documents(5)
|
||||
large_batch_queries = run_with_n_documents(50)
|
||||
|
||||
# A tolerance rather than equality, matching the N+1 check in
|
||||
# test_views.py: bulk_create's batch_size caps rows per INSERT, so a
|
||||
# large enough selection does legitimately add statements, and the
|
||||
# per-process ContentType cache makes the first run carry an extra
|
||||
# query. Neither can hide a regression to per-document assignment,
|
||||
# which would be ~10x the small-batch count here.
|
||||
self.assertLessEqual(
|
||||
large_batch_queries,
|
||||
small_batch_queries + 5,
|
||||
"Permission assignment appears to scale with document count: "
|
||||
f"{small_batch_queries} queries for 5 documents vs. "
|
||||
f"{large_batch_queries} for 50",
|
||||
)
|
||||
|
||||
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
|
||||
def test_set_permissions_grants_direct_perm_even_if_already_granted_via_group(
|
||||
self,
|
||||
m,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A user already has view access to a document via group
|
||||
membership, with no direct grant of their own
|
||||
WHEN:
|
||||
- set_permissions explicitly grants that same user direct view
|
||||
access via bulk_edit
|
||||
THEN:
|
||||
- A direct permission grant is created for the user, not skipped
|
||||
because they already have equivalent access via the group
|
||||
|
||||
Regression test: guardian's queryset-aware assign_perm() (routed to
|
||||
when the target is a list/queryset) skips creating a direct row for
|
||||
anyone whose ObjectPermissionChecker.has_perm() already returns True
|
||||
-- which includes group-derived access. The single-object assign_perm
|
||||
this bulk path replaces has no such check; it always ensures a
|
||||
direct row via get_or_create. Losing that guarantee would mean
|
||||
revoking the group's grant later silently strips access that was
|
||||
supposed to be explicit.
|
||||
"""
|
||||
self.doc1.owner = self.user1
|
||||
self.doc1.save()
|
||||
self.user1.groups.add(self.group1)
|
||||
assign_perm("view_document", self.group1, self.doc1)
|
||||
|
||||
bulk_edit.set_permissions(
|
||||
[self.doc1.id],
|
||||
set_permissions={
|
||||
"view": {"users": [self.user1.id], "groups": []},
|
||||
},
|
||||
merge=True,
|
||||
)
|
||||
|
||||
direct_users = get_users_with_perms(
|
||||
self.doc1,
|
||||
only_with_perms_in=["view_document"],
|
||||
with_group_users=False,
|
||||
)
|
||||
self.assertIn(self.user1, direct_users)
|
||||
|
||||
def test_set_permissions_for_objects_raises_for_unknown_action(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An unrecognized permission action name with users to grant it
|
||||
to
|
||||
WHEN:
|
||||
- set_permissions_for_objects is called
|
||||
THEN:
|
||||
- Permission.DoesNotExist is raised, not a silent no-op
|
||||
|
||||
Regression test: the endpoint that calls this
|
||||
(BulkEditObjectPermissionsView) never actually validates action
|
||||
names against the raw client-supplied permissions dict --
|
||||
BulkEditObjectsSerializer._validate_permissions calls
|
||||
validate_set_permissions() only for its side-effecting user/group id
|
||||
checks and discards the filtered dict it returns -- so a bogus
|
||||
action key reaches this function as-is. Resolving the Permission via
|
||||
a bare `.filter()` (which returns empty instead of raising) would
|
||||
silently drop the grant and report success.
|
||||
"""
|
||||
with self.assertRaises(Permission.DoesNotExist):
|
||||
set_permissions_for_objects(
|
||||
{"not_a_real_action": {"users": [self.user1.id], "groups": []}},
|
||||
Document,
|
||||
[self.doc1.pk],
|
||||
)
|
||||
|
||||
def test_set_permissions_for_objects_unknown_action_applies_nothing(
|
||||
self,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A permissions dict with a valid action ordered ahead of an
|
||||
unrecognized one
|
||||
WHEN:
|
||||
- set_permissions_for_objects is called
|
||||
THEN:
|
||||
- Permission.DoesNotExist is raised
|
||||
- The valid action ahead of it is not applied either
|
||||
|
||||
Every action is resolved before any row is written, so a bad action
|
||||
name cannot leave a half-applied change behind. That matters because
|
||||
BulkEditObjectsView turns this exception into a 400: without the
|
||||
up-front resolution the client would be told the request failed
|
||||
while the leading action had already been committed.
|
||||
"""
|
||||
with self.assertRaises(Permission.DoesNotExist):
|
||||
set_permissions_for_objects(
|
||||
{
|
||||
"view": {"users": [self.user1.id], "groups": []},
|
||||
"not_a_real_action": {"users": [self.user1.id], "groups": []},
|
||||
},
|
||||
Document,
|
||||
[self.doc1.pk],
|
||||
)
|
||||
|
||||
self.assertNotIn(
|
||||
self.user1,
|
||||
get_users_with_perms(
|
||||
self.doc1,
|
||||
only_with_perms_in=["view_document"],
|
||||
with_group_users=False,
|
||||
),
|
||||
)
|
||||
|
||||
@mock.patch("documents.models.Document.delete")
|
||||
def test_delete_documents_old_uuid_field(self, m) -> None:
|
||||
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
|
||||
|
||||
@@ -3,7 +3,6 @@ import warnings
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from django.conf import settings
|
||||
from django.test import TestCase
|
||||
@@ -12,7 +11,6 @@ from django.test import override_settings
|
||||
from documents.classifier import ClassifierModelCorruptError
|
||||
from documents.classifier import DocumentClassifier
|
||||
from documents.classifier import IncompatibleClassifierVersionError
|
||||
from documents.classifier import _predict_with_threshold
|
||||
from documents.classifier import load_classifier
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
@@ -627,103 +625,6 @@ class TestClassifier(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(self.classifier.predict_storage_path(doc1.content), sp.pk)
|
||||
self.assertIsNone(self.classifier.predict_storage_path(doc2.content))
|
||||
|
||||
def test_predict_rejects_prediction_below_match_threshold(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Classifiers trained against test data with confident predictions
|
||||
WHEN:
|
||||
- CLASSIFIER_MATCH_THRESHOLD exceeds the model's confidence
|
||||
THEN:
|
||||
- Every predict_* method discards the match in favor of no match
|
||||
"""
|
||||
c1 = Correspondent.objects.create(
|
||||
name="c1",
|
||||
matching_algorithm=Correspondent.MATCH_AUTO,
|
||||
)
|
||||
dt1 = DocumentType.objects.create(
|
||||
name="dt1",
|
||||
matching_algorithm=DocumentType.MATCH_AUTO,
|
||||
)
|
||||
sp1 = StoragePath.objects.create(
|
||||
name="sp1",
|
||||
matching_algorithm=StoragePath.MATCH_AUTO,
|
||||
)
|
||||
|
||||
doc1 = Document.objects.create(
|
||||
title="doc1",
|
||||
content="this is a document from c1",
|
||||
correspondent=c1,
|
||||
document_type=dt1,
|
||||
storage_path=sp1,
|
||||
checksum="A",
|
||||
)
|
||||
Document.objects.create(
|
||||
title="doc2",
|
||||
content="this is a document from no one",
|
||||
checksum="B",
|
||||
)
|
||||
|
||||
self.classifier.train()
|
||||
|
||||
predictors = {
|
||||
"correspondent": self.classifier.predict_correspondent,
|
||||
"document_type": self.classifier.predict_document_type,
|
||||
"storage_path": self.classifier.predict_storage_path,
|
||||
}
|
||||
# No real prediction can reach a confidence this high, so this
|
||||
# isolates the threshold check from the model's actual output.
|
||||
with override_settings(CLASSIFIER_MATCH_THRESHOLD=0.999999):
|
||||
for name, predict in predictors.items():
|
||||
with self.subTest(field=name):
|
||||
self.assertIsNone(predict(doc1.content))
|
||||
|
||||
def test_train_uses_balanced_sample_weight(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A training set with correspondents, document types and storage paths
|
||||
WHEN:
|
||||
- The classifier is trained
|
||||
THEN:
|
||||
- Each MLP classifier is fit with balanced sample weights, so that
|
||||
over-represented classes don't dominate predictions
|
||||
"""
|
||||
c1 = Correspondent.objects.create(
|
||||
name="c1",
|
||||
matching_algorithm=Correspondent.MATCH_AUTO,
|
||||
)
|
||||
dt1 = DocumentType.objects.create(
|
||||
name="dt1",
|
||||
matching_algorithm=DocumentType.MATCH_AUTO,
|
||||
)
|
||||
sp1 = StoragePath.objects.create(
|
||||
name="sp1",
|
||||
matching_algorithm=StoragePath.MATCH_AUTO,
|
||||
)
|
||||
|
||||
Document.objects.create(
|
||||
title="doc1",
|
||||
content="this is a document from c1",
|
||||
correspondent=c1,
|
||||
document_type=dt1,
|
||||
storage_path=sp1,
|
||||
checksum="A",
|
||||
)
|
||||
Document.objects.create(
|
||||
title="doc2",
|
||||
content="this is a document from no one",
|
||||
checksum="B",
|
||||
)
|
||||
|
||||
with mock.patch(
|
||||
"sklearn.utils.class_weight.compute_sample_weight",
|
||||
return_value=None,
|
||||
) as mocked_compute_sample_weight:
|
||||
self.classifier.train()
|
||||
|
||||
self.assertEqual(mocked_compute_sample_weight.call_count, 3)
|
||||
for call in mocked_compute_sample_weight.call_args_list:
|
||||
self.assertEqual(call.args[0], "balanced")
|
||||
|
||||
def test_one_tag_predict(self) -> None:
|
||||
t1 = Tag.objects.create(name="t1", matching_algorithm=Tag.MATCH_AUTO, pk=12)
|
||||
|
||||
@@ -909,52 +810,6 @@ class TestClassifier(DirectoriesMixin, TestCase):
|
||||
load_classifier(raise_exception=True)
|
||||
|
||||
|
||||
class _StubProbaClassifier:
|
||||
"""
|
||||
A fake scikit-learn classifier exposing just enough of the API for
|
||||
`_predict_with_threshold`: `classes_` and `predict_proba`.
|
||||
"""
|
||||
|
||||
def __init__(self, classes: list[int], probabilities: list[float]) -> None:
|
||||
self.classes_ = np.array(classes)
|
||||
self._probabilities = np.array([probabilities])
|
||||
|
||||
def predict_proba(self, X) -> np.ndarray:
|
||||
return self._probabilities
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("classes", "probabilities", "threshold", "expected"),
|
||||
[
|
||||
# confident prediction above the threshold is returned
|
||||
([-1, 3], [0.1, 0.9], 0.6, 3),
|
||||
# prediction below the threshold is discarded
|
||||
([-1, 3], [0.45, 0.55], 0.6, None),
|
||||
# boundary: exactly at the threshold is accepted, not discarded
|
||||
([-1, 3], [0.4, 0.6], 0.6, 3),
|
||||
# the winning class is the "no match" pseudo-class, regardless of its
|
||||
# own confidence
|
||||
([-1, 3], [0.99, 0.01], 0.0, None),
|
||||
# threshold of 0.0 disables the confidence check entirely
|
||||
([-1, 3], [0.45, 0.55], 0.0, 3),
|
||||
],
|
||||
)
|
||||
def test_predict_with_threshold(classes, probabilities, threshold, expected) -> None:
|
||||
classifier = _StubProbaClassifier(classes, probabilities)
|
||||
result = _predict_with_threshold(classifier, X=None, threshold=threshold)
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_classifier_match_threshold_default() -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- No PAPERLESS_CLASSIFIER_MATCH_THRESHOLD environment variable is set
|
||||
THEN:
|
||||
- The classifier match threshold defaults to 0.6
|
||||
"""
|
||||
assert settings.CLASSIFIER_MATCH_THRESHOLD == 0.6
|
||||
|
||||
|
||||
def test_preprocess_content() -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -2,7 +2,6 @@ from unittest import mock
|
||||
|
||||
from django.contrib.auth.models import Permission
|
||||
from django.contrib.auth.models import User
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents import bulk_edit
|
||||
@@ -109,44 +108,6 @@ class TestTagHierarchy(DirectoriesMixin, APITestCase):
|
||||
self.document.refresh_from_db()
|
||||
assert self.document.tags.count() == 0
|
||||
|
||||
def test_remove_inbox_tags_removes_nested_children(self) -> None:
|
||||
inbox = Tag.objects.create(name="Inbox", is_inbox_tag=True)
|
||||
nested = Tag.objects.create(name="Nested", tn_parent=inbox)
|
||||
self.document.add_nested_tags([nested])
|
||||
|
||||
resp = self.client.patch(
|
||||
f"/api/documents/{self.document.pk}/",
|
||||
{"title": "new title", "remove_inbox_tags": True},
|
||||
format="json",
|
||||
)
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
self.document.refresh_from_db()
|
||||
assert self.document.tags.count() == 0
|
||||
|
||||
# A subsequent save must not re-add the inbox tag as an ancestor
|
||||
resp = self.client.patch(
|
||||
f"/api/documents/{self.document.pk}/",
|
||||
{"title": "another title", "tags": [], "remove_inbox_tags": True},
|
||||
format="json",
|
||||
)
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
self.document.refresh_from_db()
|
||||
assert self.document.tags.count() == 0
|
||||
|
||||
def test_remove_inbox_tags_keeps_inbox_when_nested_child_added(self) -> None:
|
||||
inbox = Tag.objects.create(name="Inbox", is_inbox_tag=True)
|
||||
nested = Tag.objects.create(name="Nested", tn_parent=inbox)
|
||||
self.document.add_nested_tags([inbox])
|
||||
|
||||
self.client.patch(
|
||||
f"/api/documents/{self.document.pk}/",
|
||||
{"tags": [nested.pk], "remove_inbox_tags": True},
|
||||
format="json",
|
||||
)
|
||||
self.document.refresh_from_db()
|
||||
tags = set(self.document.tags.values_list("pk", flat=True))
|
||||
assert tags == {inbox.pk, nested.pk}
|
||||
|
||||
def test_bulk_edit_respects_hierarchy(self) -> None:
|
||||
bulk_edit.add_tag([self.document.pk], self.child.pk)
|
||||
self.document.refresh_from_db()
|
||||
|
||||
@@ -179,7 +179,7 @@ from documents.permissions import has_perms_owner_aware
|
||||
from documents.permissions import has_system_status_permission
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.permissions import set_permissions_for_object
|
||||
from documents.permissions import set_permissions_for_objects
|
||||
from documents.permissions import user_is_unrestricted
|
||||
from documents.plugins.date_parsing import get_date_parser
|
||||
from documents.schema import generate_object_with_permissions_schema
|
||||
@@ -4967,12 +4967,12 @@ class BulkEditObjectsView(PassUserMixin):
|
||||
qs_owner_update.update(owner=owner)
|
||||
|
||||
if "permissions" in serializer.validated_data:
|
||||
for obj in qs:
|
||||
set_permissions_for_object(
|
||||
permissions=permissions,
|
||||
object=obj,
|
||||
merge=merge,
|
||||
)
|
||||
set_permissions_for_objects(
|
||||
permissions=permissions,
|
||||
model=object_class,
|
||||
pks=qs.values_list("pk", flat=True),
|
||||
merge=merge,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
|
||||
@@ -2,7 +2,7 @@ msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: paperless-ngx\n"
|
||||
"Report-Msgid-Bugs-To: \n"
|
||||
"POT-Creation-Date: 2026-09-09 22:41+0000\n"
|
||||
"POT-Creation-Date: 2026-09-09 16:03+0000\n"
|
||||
"PO-Revision-Date: 2022-02-17 04:17\n"
|
||||
"Last-Translator: \n"
|
||||
"Language-Team: English\n"
|
||||
@@ -2258,151 +2258,151 @@ msgstr ""
|
||||
msgid "paperless application settings"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:569
|
||||
#: paperless/settings/__init__.py:562
|
||||
msgid "English (US)"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:570
|
||||
#: paperless/settings/__init__.py:563
|
||||
msgid "Arabic"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:571
|
||||
#: paperless/settings/__init__.py:564
|
||||
msgid "Afrikaans"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:572
|
||||
#: paperless/settings/__init__.py:565
|
||||
msgid "Belarusian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:573
|
||||
#: paperless/settings/__init__.py:566
|
||||
msgid "Bulgarian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:574
|
||||
#: paperless/settings/__init__.py:567
|
||||
msgid "Catalan"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:575
|
||||
#: paperless/settings/__init__.py:568
|
||||
msgid "Czech"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:576
|
||||
#: paperless/settings/__init__.py:569
|
||||
msgid "Danish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:577
|
||||
#: paperless/settings/__init__.py:570
|
||||
msgid "German"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:578
|
||||
#: paperless/settings/__init__.py:571
|
||||
msgid "Greek"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:579
|
||||
#: paperless/settings/__init__.py:572
|
||||
msgid "English (GB)"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:580
|
||||
#: paperless/settings/__init__.py:573
|
||||
msgid "Spanish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:581
|
||||
#: paperless/settings/__init__.py:574
|
||||
msgid "Persian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:582
|
||||
#: paperless/settings/__init__.py:575
|
||||
msgid "Finnish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:583
|
||||
#: paperless/settings/__init__.py:576
|
||||
msgid "French"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:584
|
||||
#: paperless/settings/__init__.py:577
|
||||
msgid "Hungarian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:585
|
||||
#: paperless/settings/__init__.py:578
|
||||
msgid "Indonesian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:586
|
||||
#: paperless/settings/__init__.py:579
|
||||
msgid "Italian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:587
|
||||
#: paperless/settings/__init__.py:580
|
||||
msgid "Japanese"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:588
|
||||
#: paperless/settings/__init__.py:581
|
||||
msgid "Korean"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:589
|
||||
#: paperless/settings/__init__.py:582
|
||||
msgid "Luxembourgish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:590
|
||||
#: paperless/settings/__init__.py:583
|
||||
msgid "Norwegian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:591
|
||||
#: paperless/settings/__init__.py:584
|
||||
msgid "Dutch"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:592
|
||||
#: paperless/settings/__init__.py:585
|
||||
msgid "Polish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:593
|
||||
#: paperless/settings/__init__.py:586
|
||||
msgid "Portuguese (Brazil)"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:594
|
||||
#: paperless/settings/__init__.py:587
|
||||
msgid "Portuguese"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:595
|
||||
#: paperless/settings/__init__.py:588
|
||||
msgid "Romanian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:596
|
||||
#: paperless/settings/__init__.py:589
|
||||
msgid "Russian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:597
|
||||
#: paperless/settings/__init__.py:590
|
||||
msgid "Slovak"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:598
|
||||
#: paperless/settings/__init__.py:591
|
||||
msgid "Slovenian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:599
|
||||
#: paperless/settings/__init__.py:592
|
||||
msgid "Serbian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:600
|
||||
#: paperless/settings/__init__.py:593
|
||||
msgid "Swedish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:601
|
||||
#: paperless/settings/__init__.py:594
|
||||
msgid "Turkish"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:602
|
||||
#: paperless/settings/__init__.py:595
|
||||
msgid "Ukrainian"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:603
|
||||
#: paperless/settings/__init__.py:596
|
||||
msgid "Vietnamese"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:604
|
||||
#: paperless/settings/__init__.py:597
|
||||
msgid "Chinese Simplified"
|
||||
msgstr ""
|
||||
|
||||
#: paperless/settings/__init__.py:605
|
||||
#: paperless/settings/__init__.py:598
|
||||
msgid "Chinese Traditional"
|
||||
msgstr ""
|
||||
|
||||
|
||||
@@ -96,13 +96,6 @@ MODEL_FILE = get_path_from_env(
|
||||
"PAPERLESS_MODEL_FILE",
|
||||
DATA_DIR / "classification_model.pickle",
|
||||
)
|
||||
|
||||
# Minimum confidence (0.0-1.0) for the ML classifier to assign a correspondent,
|
||||
# document type, or storage path. 0.0 disables the threshold.
|
||||
CLASSIFIER_MATCH_THRESHOLD: Final[float] = get_float_from_env(
|
||||
"PAPERLESS_CLASSIFIER_MATCH_THRESHOLD",
|
||||
0.6,
|
||||
)
|
||||
LLM_INDEX_DIR = DATA_DIR / "llm_index"
|
||||
LLM_INDEX_LOCK = LLM_INDEX_DIR / "index.lock"
|
||||
# Cross-process read/write lock guarding the LLM index compaction/migration
|
||||
|
||||
Reference in New Issue
Block a user