Compare commits

..
Author SHA1 Message Date
stumpylog ec7745e71f Minor improvements from a Claude review 2026-09-02 08:14:41 -07:00
stumpylog 98e87d91ad Fix: skip vector store document id filter for unrestricted chat users
ChatStreamingView built an IN filter from every permitted document id
for the "chat over all documents" case, which exceeds the vector
store's SQLite bound-parameter safety limit on installs with more
than ~32700 documents, silently returning no context. For a user who
can see every document (an active superuser), that filter never
narrows anything, so skip it and let the retriever search the whole
index instead.
2026-09-02 08:14:23 -07:00
GitHub Actions 912c6eb52e Auto translate strings 2026-09-01 22:07:30 +00:00
shamoon 73ef14f37a Fix/chore: refactor some signal-backed conversion technical debt (#13902) 2026-09-01 15:05:58 -07:00
75 changed files with 1106 additions and 800 deletions
+1 -1
View File
@@ -61,7 +61,7 @@ def replace_with_symlinks(
total_duplicates = 0
space_saved = 0
for file_list in duplicate_groups.values():
for file_hash, file_list in duplicate_groups.items():
# Keep the first file as the original, replace others with symlinks
original_file = file_list[0]
duplicates = file_list[1:]
+46 -99
View File
@@ -186,117 +186,64 @@ line-ending = "lf"
# https://docs.astral.sh/ruff/rules/
select = [ "E4", "E7", "E9", "F" ]
extend-select = [
"ASYNC", # https://docs.astral.sh/ruff/rules/#flake8-async-async
"B002", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
"B003",
"B004",
"B005",
"B006",
"B008",
"B009",
"B010",
"B012",
"B013",
"B014",
"B015",
"B016",
"B017",
"B018",
"B019",
"B020",
"B021",
"B022",
"B023",
"B025",
"B026",
"B029",
"B030",
"B031",
"B032",
"B033",
"B035",
"B039",
"C4", # https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4
"COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
"D419", # https://docs.astral.sh/ruff/rules/#pydocstyle-d
"DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
"DTZ", # https://docs.astral.sh/ruff/rules/#flake8-datetimez-dtz
"EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
"FA", # https://docs.astral.sh/ruff/rules/#flake8-future-annotations-fa
"FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt
"FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly
"FURB", # https://docs.astral.sh/ruff/rules/#refurb-furb
"G", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"I", # https://docs.astral.sh/ruff/rules/#isort-i
"ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
"INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
"INT", # https://docs.astral.sh/ruff/rules/#flake8-gettext-int
"ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
"LOG", # https://docs.astral.sh/ruff/rules/#flake8-logging-log
"N999", # https://docs.astral.sh/ruff/rules/#pep8-naming-n
"PERF101", # https://docs.astral.sh/ruff/rules/#perflint-perf
"PERF102",
"PERF402",
"PGH005", # https://docs.astral.sh/ruff/rules/#pygrep-hooks-pgh
"PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
"PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLE", # https://docs.astral.sh/ruff/rules/#error-ple
"PLR0124", # https://docs.astral.sh/ruff/rules/#refactor-plr
"PLR0133",
"PLR0206",
"PLR0402",
"PLR1704",
"PLR1708",
"PLR1711",
"PLR1716",
"PLR1722",
"PLR1730",
"PLR1733",
"PLR1736",
"PLR2044",
"PLW", # https://docs.astral.sh/ruff/rules/#warning-plw
"PT010", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
"PT014",
"PT020",
"PT025",
"PT026",
"PT031",
"PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
"Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
"RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
"RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
"S102", # https://docs.astral.sh/ruff/rules/#flake8-bandit-s
"S110",
"S112",
"S113",
"SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
"T100", # https://docs.astral.sh/ruff/rules/#flake8-debugger-t10
"T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
"TC", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tc
"TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
"TRY002", # https://docs.astral.sh/ruff/rules/#tryceratops-try
"TRY004",
"TRY201",
"TRY203",
"TRY401",
"UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
"W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
"YTT", # https://docs.astral.sh/ruff/rules/#flake8-2020-ytt
"COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
"DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
"EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
"FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt
"FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly
"G201", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"I", # https://docs.astral.sh/ruff/rules/#isort-i
"ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
"INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
"ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
"PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
"PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLE", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
"Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
"RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
"RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
"SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
"T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
"TC", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tc
"TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
"UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
"W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
]
ignore = [
"DJ001",
"PLC0415",
"RUF012",
"SIM105",
"G004", # Logging statement uses f-string - good to do, but a large diff
]
# Migrations
per-file-ignores."*/migrations/*.py" = []
per-file-ignores."*/migrations/*.py" = [
"E501",
"SIM",
"T201",
]
# Testing
per-file-ignores."*/tests/*.py" = [
"DTZ",
"E501",
"SIM117",
]
per-file-ignores.".github/scripts/*.py" = [
"E501",
"INP001",
"SIM117",
]
# Docker specific
per-file-ignores."docker/rootfs/usr/local/bin/wait-for-redis.py" = [
"INP001",
"T201",
]
per-file-ignores."docker/wait-for-redis.py" = [
"INP001",
"T201",
]
per-file-ignores."src/documents/models.py" = [
"SIM115",
]
isort.force-single-line = true
[tool.codespell]
+140 -140
View File
File diff suppressed because it is too large Load Diff
@@ -41,6 +41,8 @@ export class TrashComponent
private modalService = inject(NgbModal)
private settingsService = inject(SettingsService)
private router = inject(Router)
private readonly emptyTrashDelaySetting =
this.settingsService.getSignal<number>(SETTINGS_KEYS.EMPTY_TRASH_DELAY)
readonly documentsInTrash = signal<Document[]>([])
readonly selectedDocuments = signal<Set<number>>(new Set())
@@ -200,8 +202,7 @@ export class TrashComponent
}
getDaysRemaining(document: Document): number {
this.settingsService.trackChanges()
const delay = this.settingsService.get(SETTINGS_KEYS.EMPTY_TRASH_DELAY)
const delay = this.emptyTrashDelaySetting()
const diff = new Date().getTime() - new Date(document.deleted_at).getTime()
const days = Math.ceil(diff / (1000 * 3600 * 24))
return delay - days
@@ -193,6 +193,23 @@ describe('AppFrameComponent', () => {
expect(savedViewSpy).toHaveBeenCalled()
})
it('should update reinitialized signal-backed settings without manual change detection', async () => {
settingsService.initializeSettings().subscribe()
httpTestingController
.expectOne(`${environment.apiBaseUrl}ui_settings/`)
.flush({
settings: { app_title: 'Reactive title' },
user: {},
permissions: [],
})
await fixture.whenStable()
expect(
fixture.nativeElement.querySelector('.brand-title').textContent
).toBe('Reactive title')
})
it('should check for update if enabled', () => {
const updateCheckSpy = jest.spyOn(remoteVersionService, 'checkForUpdates')
updateCheckSpy.mockImplementation(() => {
@@ -98,6 +98,29 @@ export class AppFrameComponent
readonly isMenuCollapsed = signal(true)
readonly slimSidebarAnimating = signal(false)
readonly mobileSearchHidden = signal(false)
private readonly versionSetting = this.settingsService.getSignal<string>(
SETTINGS_KEYS.VERSION
)
private readonly appTitleSetting = this.settingsService.getSignal<string>(
SETTINGS_KEYS.APP_TITLE
)
private readonly appLogoSetting = this.settingsService.getSignal<string>(
SETTINGS_KEYS.APP_LOGO
)
private readonly slimSidebarSetting = this.settingsService.getSignal<boolean>(
SETTINGS_KEYS.SLIM_SIDEBAR
)
private readonly attributesSectionsCollapsedSetting =
this.settingsService.getSignal<CollapsibleSection[]>(
SETTINGS_KEYS.ATTRIBUTES_SECTIONS_COLLAPSED
)
private readonly aiEnabledSetting = this.settingsService.getSignal<boolean>(
SETTINGS_KEYS.AI_ENABLED
)
private readonly sidebarViewsShowCountSetting =
this.settingsService.getSignal<boolean>(
SETTINGS_KEYS.SIDEBAR_VIEWS_SHOW_COUNT
)
private lastScrollY: number = 0
constructor() {
@@ -191,33 +214,23 @@ export class AppFrameComponent
}
get versionString(): string {
this.settingsService.trackChanges()
return `${environment.appTitle} v${this.settingsService.get(SETTINGS_KEYS.VERSION)}${environment.tag === 'prod' ? '' : ` #${environment.tag}`}`
return `${environment.appTitle} v${this.versionSetting()}${environment.tag === 'prod' ? '' : ` #${environment.tag}`}`
}
get appTitle(): string {
this.settingsService.trackChanges()
return (
this.settingsService.get(SETTINGS_KEYS.APP_TITLE) || environment.appTitle
)
return this.appTitleSetting() || environment.appTitle
}
get customAppTitle(): string {
this.settingsService.trackChanges()
return this.settingsService.get(SETTINGS_KEYS.APP_TITLE)
return this.appTitleSetting()
}
get hasCustomBranding(): boolean {
this.settingsService.trackChanges()
return !!(
this.settingsService.get(SETTINGS_KEYS.APP_TITLE)?.length ||
this.settingsService.get(SETTINGS_KEYS.APP_LOGO)?.length
)
return !!(this.appTitleSetting()?.length || this.appLogoSetting()?.length)
}
get customAppLogo(): string {
this.settingsService.trackChanges()
const logo = this.settingsService.get(SETTINGS_KEYS.APP_LOGO)
const logo = this.appLogoSetting()
return logo?.length
? environment.apiBaseUrl.replace(/\/api\/$/, logo)
: null
@@ -262,8 +275,7 @@ export class AppFrameComponent
}
get slimSidebarEnabled(): boolean {
this.settingsService.trackChanges()
return this.settingsService.get(SETTINGS_KEYS.SLIM_SIDEBAR)
return this.slimSidebarSetting()
}
set slimSidebarEnabled(enabled: boolean) {
@@ -286,10 +298,9 @@ export class AppFrameComponent
}
get attributesSectionsCollapsed(): boolean {
this.settingsService.trackChanges()
return this.settingsService
.get(SETTINGS_KEYS.ATTRIBUTES_SECTIONS_COLLAPSED)
?.includes(CollapsibleSection.ATTRIBUTES)
return this.attributesSectionsCollapsedSetting()?.includes(
CollapsibleSection.ATTRIBUTES
)
}
set attributesSectionsCollapsed(collapsed: boolean) {
@@ -312,8 +323,7 @@ export class AppFrameComponent
}
get aiEnabled(): boolean {
this.settingsService.trackChanges()
return this.settingsService.get(SETTINGS_KEYS.AI_ENABLED)
return this.aiEnabledSetting()
}
@HostListener('window:resize')
@@ -480,9 +490,8 @@ export class AppFrameComponent
}
get showSidebarCounts(): boolean {
this.settingsService.trackChanges()
return (
this.settingsService.get(SETTINGS_KEYS.SIDEBAR_VIEWS_SHOW_COUNT) &&
this.sidebarViewsShowCountSetting() &&
!this.settingsService.organizingSidebarSavedViews()
)
}
@@ -81,6 +81,10 @@ export class GlobalSearchComponent implements OnInit {
private hotkeyService = inject(HotKeyService)
private settingsService = inject(SettingsService)
private locationStrategy = inject(LocationStrategy)
private readonly searchFullTypeSetting =
this.settingsService.getSignal<GlobalSearchType>(
SETTINGS_KEYS.SEARCH_FULL_TYPE
)
public DataType = DataType
readonly query = signal<string>(null)
@@ -97,11 +101,7 @@ export class GlobalSearchComponent implements OnInit {
@ViewChildren('secondaryButton') secondaryButtons: QueryList<ElementRef>
get useAdvancedForFullSearch(): boolean {
this.settingsService.trackChanges()
return (
this.settingsService.get(SETTINGS_KEYS.SEARCH_FULL_TYPE) ===
GlobalSearchType.ADVANCED
)
return this.searchFullTypeSetting() === GlobalSearchType.ADVANCED
}
constructor() {
@@ -196,6 +196,16 @@ describe('WorkflowEditDialogComponent', () => {
fixture.detectChanges()
})
function setActionSettings({
email = true,
remoteOcr = true,
ai = true,
} = {}) {
settingsService.set(SETTINGS_KEYS.EMAIL_ENABLED, email)
settingsService.set(SETTINGS_KEYS.REMOTE_OCR_CONFIGURED, remoteOcr)
settingsService.set(SETTINGS_KEYS.AI_ENABLED, ai)
}
it('should support create and edit modes, support adding triggers and actions on new workflow', () => {
component.dialogMode.set(EditDialogMode.CREATE)
const createTitleSpy = jest.spyOn(component, 'getCreateTitle')
@@ -218,7 +228,7 @@ describe('WorkflowEditDialogComponent', () => {
})
it('should return source options, type options, type name, schedule date field options', () => {
jest.spyOn(settingsService, 'get').mockReturnValue(true)
setActionSettings()
component.ngOnInit()
expect(component.sourceOptions).toEqual(DOCUMENT_SOURCE_OPTIONS)
expect(component.triggerTypeOptions).toEqual(WORKFLOW_TYPE_OPTIONS)
@@ -242,7 +252,7 @@ describe('WorkflowEditDialogComponent', () => {
)
// Email, remote OCR and AI all disabled
jest.spyOn(settingsService, 'get').mockReturnValue(false)
setActionSettings({ email: false, remoteOcr: false, ai: false })
component.ngOnInit()
expect(component.actionTypeOptions).toEqual(
WORKFLOW_ACTION_OPTIONS.filter(
@@ -255,7 +265,7 @@ describe('WorkflowEditDialogComponent', () => {
})
it('should offer remote OCR only for consumption workflows', () => {
jest.spyOn(settingsService, 'get').mockReturnValue(true)
setActionSettings()
// A consumption trigger makes the action reachable
component.object = {
@@ -285,7 +295,7 @@ describe('WorkflowEditDialogComponent', () => {
})
it('should offer remote OCR on a trigger added to a new workflow', () => {
jest.spyOn(settingsService, 'get').mockReturnValue(true)
setActionSettings()
component.ngOnInit()
// Nothing for the action to apply to yet
@@ -311,7 +321,7 @@ describe('WorkflowEditDialogComponent', () => {
})
it('should keep remote OCR listed when an action already uses it', () => {
jest.spyOn(settingsService, 'get').mockReturnValue(true)
setActionSettings()
// Otherwise changing the trigger would silently blank the selection
component.object = {
@@ -329,9 +339,7 @@ describe('WorkflowEditDialogComponent', () => {
})
it('should not offer remote OCR when no engine is configured', () => {
jest
.spyOn(settingsService, 'get')
.mockImplementation((key) => key !== SETTINGS_KEYS.REMOTE_OCR_CONFIGURED)
setActionSettings({ remoteOcr: false })
component.object = {
name: 'Workflow 1',
@@ -348,7 +356,7 @@ describe('WorkflowEditDialogComponent', () => {
})
it('should offer apply AI suggestions unless every trigger is consumption', () => {
jest.spyOn(settingsService, 'get').mockReturnValue(true)
setActionSettings()
// Consumption runs before the document has been parsed, so there would be
// no content to make suggestions from
@@ -382,7 +390,7 @@ describe('WorkflowEditDialogComponent', () => {
})
it('should keep apply AI suggestions listed when an action already uses it', () => {
jest.spyOn(settingsService, 'get').mockReturnValue(true)
setActionSettings()
// Otherwise changing the trigger would silently blank the selection
component.object = {
@@ -400,9 +408,7 @@ describe('WorkflowEditDialogComponent', () => {
})
it('should not offer apply AI suggestions when AI is disabled', () => {
jest
.spyOn(settingsService, 'get')
.mockImplementation((key) => key !== SETTINGS_KEYS.AI_ENABLED)
setActionSettings({ ai: false })
component.object = {
name: 'Workflow 1',
@@ -537,6 +537,13 @@ export class WorkflowEditDialogComponent
readonly dateCustomFields = computed(() =>
this.customFields()?.filter((f) => f.data_type === CustomFieldDataType.Date)
)
private readonly emailEnabledSetting =
this.settingsService.getSignal<boolean>(SETTINGS_KEYS.EMAIL_ENABLED)
private readonly remoteOcrConfiguredSetting =
this.settingsService.getSignal<boolean>(SETTINGS_KEYS.REMOTE_OCR_CONFIGURED)
private readonly aiEnabledSetting = this.settingsService.getSignal<boolean>(
SETTINGS_KEYS.AI_ENABLED
)
expandedItem: number = null
@@ -589,7 +596,7 @@ export class WorkflowEditDialogComponent
private getAllowedActionTypes() {
let allowed = WORKFLOW_ACTION_OPTIONS
if (!this.settingsService.get(SETTINGS_KEYS.EMAIL_ENABLED)) {
if (!this.emailEnabledSetting()) {
allowed = allowed.filter((a) => a.id !== WorkflowActionType.Email)
}
@@ -597,7 +604,7 @@ export class WorkflowEditDialogComponent
// offered for workflows that run at consumption.
const formWorkflow: Workflow = this.objectForm?.value
const remoteOcrUsable =
this.settingsService.get(SETTINGS_KEYS.REMOTE_OCR_CONFIGURED) &&
this.remoteOcrConfiguredSetting() &&
(formWorkflow?.triggers?.some(
(trigger) => trigger.type === WorkflowTriggerType.Consumption
) ||
@@ -612,7 +619,7 @@ export class WorkflowEditDialogComponent
// once every trigger is consumption, so it stays offered on a workflow
// that has no triggers yet.
const aiSuggestionsUsable =
this.settingsService.get(SETTINGS_KEYS.AI_ENABLED) &&
this.aiEnabledSetting() &&
(!formWorkflow?.triggers?.length ||
formWorkflow.triggers.some(
(trigger) => trigger.type !== WorkflowTriggerType.Consumption
@@ -1362,7 +1369,6 @@ export class WorkflowEditDialogComponent
}
get actionTypeOptions() {
this.settingsService.trackChanges()
// Computed on read rather than cached
return this.getAllowedActionTypes()
}
@@ -839,7 +839,9 @@ describe('FilterableDropdownComponent & FilterableDropdownSelectionModel', () =>
selectionModel.items = [memoRoot]
selectionModel.documentCounts = [{ id: memoRoot.id, document_count: 9 }]
const getRootDocCount = (selectionModel as any).createRootDocCounter()
const getRootDocCount = (selectionModel as any).createRootDocCounter(
selectionModel.items
)
expect(getRootDocCount(memoRoot.id)).toEqual(9)
selectionModel.documentCounts = []
@@ -855,7 +857,9 @@ describe('FilterableDropdownComponent & FilterableDropdownSelectionModel', () =>
selectionModel.items = [rootWithoutSelection]
selectionModel.documentCounts = []
const getRootDocCount = (selectionModel as any).createRootDocCounter()
const getRootDocCount = (selectionModel as any).createRootDocCounter(
selectionModel.items
)
expect(getRootDocCount(rootWithoutSelection.id)).toEqual(4)
})
@@ -865,7 +869,9 @@ describe('FilterableDropdownComponent & FilterableDropdownSelectionModel', () =>
selectionModel.items = [rootWithoutCounts]
selectionModel.documentCounts = []
const getRootDocCount = (selectionModel as any).createRootDocCounter()
const getRootDocCount = (selectionModel as any).createRootDocCounter(
selectionModel.items
)
expect(getRootDocCount(rootWithoutCounts.id)).toEqual(0)
})
@@ -966,7 +972,7 @@ describe('FilterableDropdownComponent & FilterableDropdownSelectionModel', () =>
component.selectionModel['temporarySelectionStates'].set(id, state)
const changedSpy = jest.spyOn(component.selectionModel.changed, 'next')
component.selectionModel.exclude(id)
expect(component.selectionModel.temporaryLogicalOperator).toBe(
expect(component.selectionModel.temporaryLogicalOperator()).toBe(
LogicalOperator.And
)
expect(component.selectionModel['temporarySelectionStates'].get(id)).toBe(
@@ -64,43 +64,56 @@ export class FilterableDropdownSelectionModel {
manyToOne = false
singleSelect = false
private _logicalOperator: LogicalOperator = LogicalOperator.And
temporaryLogicalOperator: LogicalOperator = this._logicalOperator
private _intersection: Intersection = Intersection.Include
temporaryIntersection: Intersection = this._intersection
private _documentCounts: SelectionDataItem[] = []
private readonly _logicalOperator = signal(LogicalOperator.And)
readonly temporaryLogicalOperator = signal(LogicalOperator.And)
private readonly _intersection = signal(Intersection.Include)
readonly temporaryIntersection = signal(Intersection.Include)
private readonly _documentCounts = signal<SelectionDataItem[]>([])
private readonly _items = signal<MatchingModel[]>([])
private readonly _selectionStates = signal(
new Map<number, ToggleableItemState>()
)
private readonly _temporarySelectionStates = signal(
new Map<number, ToggleableItemState>()
)
public documentCountSortingEnabled = false
private get selectionStates(): ReadonlyMap<number, ToggleableItemState> {
return this._selectionStates()
}
private get temporarySelectionStates(): ReadonlyMap<
number,
ToggleableItemState
> {
return this._temporarySelectionStates()
}
public set documentCounts(counts: SelectionDataItem[]) {
this._documentCounts = counts
this._documentCounts.set(counts)
if (this.documentCountSortingEnabled) {
this.sortItems()
this._items.set(this.sortItems(this.items))
}
}
private _items: MatchingModel[] = []
get items(): MatchingModel[] {
return this._items
return this._items()
}
set items(items: MatchingModel[]) {
if (items) {
this._items = Array.from(items)
this.sortItems()
this.setNullItem()
this._items.set(this.withNullItem(this.sortItems(Array.from(items))))
}
}
private setNullItem() {
private withNullItem(items: MatchingModel[]): MatchingModel[] {
if (this.manyToOne && this.logicalOperator === LogicalOperator.Or) {
if (this._items[0]?.id === null) {
this._items.shift()
}
return
return items[0]?.id === null ? items.slice(1) : items
}
const item = {
const nullItem = {
name: $localize`:Filter drop down element to filter for documents with no correspondent/type/tag assigned:Not assigned`,
id:
this.manyToOne || this.intersection === Intersection.Include
@@ -108,22 +121,17 @@ export class FilterableDropdownSelectionModel {
: NEGATIVE_NULL_FILTER_VALUE,
}
if (
this._items[0]?.id === null ||
this._items[0]?.id === NEGATIVE_NULL_FILTER_VALUE
) {
this._items[0] = item
} else if (this._items) {
this._items.unshift(item)
}
return items[0]?.id === null || items[0]?.id === NEGATIVE_NULL_FILTER_VALUE
? [nullItem, ...items.slice(1)]
: [nullItem, ...items]
}
constructor(manyToOne: boolean = false) {
this.manyToOne = manyToOne
}
private sortItems() {
this._items.sort((a, b) => {
private sortItems(items: MatchingModel[]): MatchingModel[] {
const sorted = [...items].sort((a, b) => {
if (
(a.id == null && b.id != null) ||
(a.id == NEGATIVE_NULL_FILTER_VALUE &&
@@ -154,13 +162,13 @@ export class FilterableDropdownSelectionModel {
) {
return -1
} else if (
this._documentCounts.length &&
this._documentCounts().length &&
this.getDocumentCount(b.id) === 0 &&
this.getDocumentCount(a.id) > this.getDocumentCount(b.id)
) {
return -1
} else if (
this._documentCounts.length &&
this._documentCounts().length &&
this.getDocumentCount(a.id) === 0 &&
this.getDocumentCount(a.id) < this.getDocumentCount(b.id)
) {
@@ -170,15 +178,11 @@ export class FilterableDropdownSelectionModel {
}
})
if (this._documentCounts.length) {
this.promoteBranchesWithDocumentCounts()
}
return this._documentCounts().length
? this.promoteBranchesWithDocumentCounts(sorted)
: sorted
}
private selectionStates = new Map<number, ToggleableItemState>()
private temporarySelectionStates = new Map<number, ToggleableItemState>()
getSelectedItems() {
return this.items.filter(
(i) =>
@@ -194,30 +198,33 @@ export class FilterableDropdownSelectionModel {
}
set(id: number, state: ToggleableItemState, fireEvent = true) {
const states = new Map(this.temporarySelectionStates)
if (state == ToggleableItemState.NotSelected) {
this.temporarySelectionStates.delete(id)
states.delete(id)
} else {
this.temporarySelectionStates.set(id, state)
states.set(id, state)
}
this._temporarySelectionStates.set(states)
if (fireEvent) {
this.changed.next(this)
}
}
toggle(id: number, fireEvent = true) {
let state = this.temporarySelectionStates.get(id)
const states = new Map(this.temporarySelectionStates)
let state = states.get(id)
if (
state == undefined ||
(state != ToggleableItemState.Selected &&
state != ToggleableItemState.Excluded)
) {
if (this.manyToOne || this.singleSelect) {
this.temporarySelectionStates.set(id, ToggleableItemState.Selected)
states.set(id, ToggleableItemState.Selected)
if (this.singleSelect) {
for (let key of this.temporarySelectionStates.keys()) {
for (let key of states.keys()) {
if (key != id) {
this.temporarySelectionStates.delete(key)
states.delete(key)
}
}
}
@@ -233,25 +240,26 @@ export class FilterableDropdownSelectionModel {
) {
newState = ToggleableItemState.NotSelected
}
this.temporarySelectionStates.set(id, newState)
states.set(id, newState)
}
} else if (
state == ToggleableItemState.Selected ||
state == ToggleableItemState.Excluded
) {
this.temporarySelectionStates.delete(id)
this.clearDescendantSelections(id)
states.delete(id)
this.clearDescendantSelections(states, id)
}
if (!id) {
for (let key of this.temporarySelectionStates.keys()) {
for (let key of states.keys()) {
if (key) {
this.temporarySelectionStates.delete(key)
states.delete(key)
}
}
} else {
this.temporarySelectionStates.delete(null)
states.delete(null)
}
this._temporarySelectionStates.set(states)
if (fireEvent) {
this.changed.next(this)
@@ -259,20 +267,21 @@ export class FilterableDropdownSelectionModel {
}
exclude(id: number, fireEvent: boolean = true) {
let state = this.temporarySelectionStates.get(id)
const states = new Map(this.temporarySelectionStates)
let state = states.get(id)
if (id && (state == null || state != ToggleableItemState.Excluded)) {
this.temporaryLogicalOperator = this._logicalOperator = this.manyToOne
? LogicalOperator.And
: LogicalOperator.Or
const operator = this.manyToOne ? LogicalOperator.And : LogicalOperator.Or
this.temporaryLogicalOperator.set(operator)
this._logicalOperator.set(operator)
if (this.manyToOne || this.singleSelect) {
this.temporarySelectionStates.set(id, ToggleableItemState.Excluded)
this.clearDescendantSelections(id)
states.set(id, ToggleableItemState.Excluded)
this.clearDescendantSelections(states, id)
if (this.singleSelect) {
for (let key of this.temporarySelectionStates.keys()) {
for (let key of states.keys()) {
if (key != id) {
this.temporarySelectionStates.delete(key)
states.delete(key)
}
}
}
@@ -287,17 +296,18 @@ export class FilterableDropdownSelectionModel {
) {
newState = ToggleableItemState.NotSelected
}
this.temporarySelectionStates.set(id, newState)
states.set(id, newState)
if (newState == ToggleableItemState.Excluded) {
this.clearDescendantSelections(id)
this.clearDescendantSelections(states, id)
}
}
} else if (!id || state == ToggleableItemState.Excluded) {
this.temporarySelectionStates.delete(id)
states.delete(id)
if (id) {
this.clearDescendantSelections(id)
this.clearDescendantSelections(states, id)
}
}
this._temporarySelectionStates.set(states)
if (fireEvent) {
this.changed.next(this)
@@ -308,9 +318,12 @@ export class FilterableDropdownSelectionModel {
return this.selectionStates.get(id) || ToggleableItemState.NotSelected
}
private clearDescendantSelections(id: number) {
private clearDescendantSelections(
states: Map<number, ToggleableItemState>,
id: number
) {
for (const descendantID of this.getDescendantIDs(id)) {
this.temporarySelectionStates.delete(descendantID)
states.delete(descendantID)
}
}
@@ -320,7 +333,7 @@ export class FilterableDropdownSelectionModel {
while (queue.length) {
const parentID = queue.shift()
for (const item of this._items) {
for (const item of this.items) {
if (
typeof item?.id === 'number' &&
typeof (item as any)['parent'] === 'number' &&
@@ -336,12 +349,12 @@ export class FilterableDropdownSelectionModel {
}
get logicalOperator(): LogicalOperator {
return this.temporaryLogicalOperator
return this.temporaryLogicalOperator()
}
set logicalOperator(operator: LogicalOperator) {
this.temporaryLogicalOperator = operator
this.setNullItem()
this.temporaryLogicalOperator.set(operator)
this._items.set(this.withNullItem(this.items))
}
toggleOperator() {
@@ -349,12 +362,12 @@ export class FilterableDropdownSelectionModel {
}
get intersection(): Intersection {
return this.temporaryIntersection
return this.temporaryIntersection()
}
set intersection(intersection: Intersection) {
this.temporaryIntersection = intersection
this.setNullItem()
this.temporaryIntersection.set(intersection)
this._items.set(this.withNullItem(this.items))
}
toggleIntersection() {
@@ -364,18 +377,20 @@ export class FilterableDropdownSelectionModel {
? ToggleableItemState.Selected
: ToggleableItemState.Excluded
this.temporarySelectionStates.forEach((state, key) => {
const states = new Map(this.temporarySelectionStates)
states.forEach((state, key) => {
if (key === null && this.intersection === Intersection.Exclude) {
this.temporarySelectionStates.set(NEGATIVE_NULL_FILTER_VALUE, newState)
states.set(NEGATIVE_NULL_FILTER_VALUE, newState)
} else if (
key === NEGATIVE_NULL_FILTER_VALUE &&
this.intersection === Intersection.Include
) {
this.temporarySelectionStates.set(null, newState)
states.set(null, newState)
} else {
this.temporarySelectionStates.set(key, newState)
states.set(key, newState)
}
})
this._temporarySelectionStates.set(states)
this.changed.next(this)
}
@@ -395,10 +410,12 @@ export class FilterableDropdownSelectionModel {
}
clear(fireEvent = true) {
this.temporarySelectionStates.clear()
this.temporaryLogicalOperator = this._logicalOperator = LogicalOperator.And
this.temporaryIntersection = this._intersection = Intersection.Include
this.setNullItem()
this._temporarySelectionStates.set(new Map())
this.temporaryLogicalOperator.set(LogicalOperator.And)
this._logicalOperator.set(LogicalOperator.And)
this.temporaryIntersection.set(Intersection.Include)
this._intersection.set(Intersection.Include)
this._items.set(this.withNullItem(this.items))
if (fireEvent) {
this.changed.next(this)
}
@@ -419,9 +436,9 @@ export class FilterableDropdownSelectionModel {
)
) {
return true
} else if (this.temporaryLogicalOperator !== this._logicalOperator) {
} else if (this.temporaryLogicalOperator() !== this._logicalOperator()) {
return true
} else if (this.temporaryIntersection !== this._intersection) {
} else if (this.temporaryIntersection() !== this._intersection()) {
return true
} else {
return false
@@ -438,23 +455,29 @@ export class FilterableDropdownSelectionModel {
}
getDocumentCount(id: number) {
return this._documentCounts.find((c) => c.id === id)?.document_count
return this._documentCounts().find((c) => c.id === id)?.document_count
}
private promoteBranchesWithDocumentCounts() {
const parentById = this.buildParentById()
private promoteBranchesWithDocumentCounts(
items: MatchingModel[]
): MatchingModel[] {
const parentById = this.buildParentById(items)
const findRootId = this.createRootFinder(parentById)
const getRootDocCount = this.createRootDocCounter()
const summaries = this.buildBranchSummaries(findRootId, getRootDocCount)
const getRootDocCount = this.createRootDocCounter(items)
const summaries = this.buildBranchSummaries(
items,
findRootId,
getRootDocCount
)
const orderedBranches = this.orderBranchesByPriority(summaries)
this._items = orderedBranches.flatMap((summary) => summary.items)
return orderedBranches.flatMap((summary) => summary.items)
}
private buildParentById(): Map<number, number | null> {
private buildParentById(items: MatchingModel[]): Map<number, number | null> {
const parentById = new Map<number, number | null>()
for (const item of this._items) {
for (const item of items) {
if (typeof item?.id === 'number') {
const parentValue = (item as any)['parent']
parentById.set(
@@ -492,7 +515,9 @@ export class FilterableDropdownSelectionModel {
return findRootId
}
private createRootDocCounter(): (rootId: number) => number {
private createRootDocCounter(
items: MatchingModel[]
): (rootId: number) => number {
const docCountMemo = new Map<number, number>()
return (rootId: number): number => {
@@ -507,7 +532,7 @@ export class FilterableDropdownSelectionModel {
return explicit
}
const rootItem = this._items.find((i) => i.id === rootId)
const rootItem = items.find((i) => i.id === rootId)
const fallback =
typeof (rootItem as any)?.['document_count'] === 'number'
? (rootItem as any)['document_count']
@@ -519,12 +544,13 @@ export class FilterableDropdownSelectionModel {
}
private buildBranchSummaries(
items: MatchingModel[],
findRootId: (id: number) => number,
getRootDocCount: (rootId: number) => number
): Map<string, BranchSummary> {
const summaries = new Map<string, BranchSummary>()
for (const [index, item] of this._items.entries()) {
for (const [index, item] of items.entries()) {
const { key, special, rootId } = this.describeBranchItem(
item,
index,
@@ -616,28 +642,23 @@ export class FilterableDropdownSelectionModel {
}
init(map: Map<number, ToggleableItemState>) {
this.temporarySelectionStates = map
this._temporarySelectionStates.set(new Map(map))
this.apply()
}
apply() {
this.selectionStates.clear()
this.temporarySelectionStates.forEach((value, key) => {
this.selectionStates.set(key, value)
})
this._logicalOperator = this.temporaryLogicalOperator
this._intersection = this.temporaryIntersection
this.sortItems()
this._selectionStates.set(new Map(this.temporarySelectionStates))
this._logicalOperator.set(this.temporaryLogicalOperator())
this._intersection.set(this.temporaryIntersection())
this._items.set(this.sortItems(this.items))
}
reset(complete: boolean = false) {
this.temporarySelectionStates.clear()
if (complete) {
this.selectionStates.clear()
this._selectionStates.set(new Map())
this._temporarySelectionStates.set(new Map())
} else {
this.selectionStates.forEach((value, key) => {
this.temporarySelectionStates.set(key, value)
})
this._temporarySelectionStates.set(new Map(this.selectionStates))
}
}
@@ -7,7 +7,7 @@
<div class="list-group list-group-flush">
<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.NONE)" [disabled]="disabled">
<div class="selected-icon me-1">
@if (selectionModel.ownerFilter === OwnerFilterType.NONE) {
@if (selectionModel.ownerFilter() === OwnerFilterType.NONE) {
<i-bs width="1em" height="1em" name="check"></i-bs>
}
</div>
@@ -17,7 +17,7 @@
</button>
<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.SELF)" [disabled]="disabled">
<div class="selected-icon me-1">
@if (selectionModel.ownerFilter === OwnerFilterType.SELF) {
@if (selectionModel.ownerFilter() === OwnerFilterType.SELF) {
<i-bs width="1em" height="1em" name="check"></i-bs>
}
</div>
@@ -27,7 +27,7 @@
</button>
<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.NOT_SELF)" [disabled]="disabled">
<div class="selected-icon me-1">
@if (selectionModel.ownerFilter === OwnerFilterType.NOT_SELF) {
@if (selectionModel.ownerFilter() === OwnerFilterType.NOT_SELF) {
<i-bs width="1em" height="1em" name="check"></i-bs>
}
</div>
@@ -37,7 +37,7 @@
</button>
<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.SHARED_BY_ME)" [disabled]="disabled">
<div class="selected-icon me-1">
@if (selectionModel.ownerFilter === OwnerFilterType.SHARED_BY_ME) {
@if (selectionModel.ownerFilter() === OwnerFilterType.SHARED_BY_ME) {
<i-bs width="1em" height="1em" name="check"></i-bs>
}
</div>
@@ -47,7 +47,7 @@
</button>
<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.UNOWNED)" [disabled]="disabled">
<div class="selected-icon me-1">
@if (selectionModel.ownerFilter === OwnerFilterType.UNOWNED) {
@if (selectionModel.ownerFilter() === OwnerFilterType.UNOWNED) {
<i-bs width="1em" height="1em" name="check"></i-bs>
}
</div>
@@ -57,7 +57,7 @@
</button>
<button *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.User }" class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" [disabled]="disabled">
<div class="selected-icon me-1">
@if (selectionModel.ownerFilter === OwnerFilterType.OTHERS) {
@if (selectionModel.ownerFilter() === OwnerFilterType.OTHERS) {
<i-bs width="1em" height="1em" name="check"></i-bs>
}
</div>
@@ -65,7 +65,8 @@
<ng-select
name="user"
class="user-select small"
[(ngModel)]="selectionModel.includeUsers"
[ngModel]="selectionModel.includeUsers()"
(ngModelChange)="selectionModel.includeUsers.set($event)"
[disabled]="disabled"
[clearable]="false"
[items]="users()"
@@ -78,10 +79,10 @@
</ng-select>
</div>
</button>
@if (selectionModel.ownerFilter === OwnerFilterType.NONE || selectionModel.ownerFilter === OwnerFilterType.NOT_SELF) {
@if (selectionModel.ownerFilter() === OwnerFilterType.NONE || selectionModel.ownerFilter() === OwnerFilterType.NOT_SELF) {
<div class="list-group-item list-group-item-action d-flex align-items-center p-2 ps-3 border-bottom-0 border-start-0 border-end-0">
<div class="form-check form-switch w-100">
<input type="checkbox" class="form-check-input" id="hideUnowned" [(ngModel)]="this.selectionModel.hideUnowned" (change)="onChange()" [disabled]="disabled">
<input type="checkbox" class="form-check-input" id="hideUnowned" [ngModel]="selectionModel.hideUnowned()" (ngModelChange)="selectionModel.hideUnowned.set($event)" (change)="onChange()" [disabled]="disabled">
<label class="form-check-label w-100" for="hideUnowned"><small i18n>Hide unowned</small></label>
</div>
</div>
@@ -90,56 +90,56 @@ describe('PermissionsFilterDropdownComponent', () => {
component.setFilter(OwnerFilterType.OTHERS)
expect(component.isActive).toBeTruthy()
component.setFilter(OwnerFilterType.NONE)
component.selectionModel.hideUnowned = true
component.selectionModel.hideUnowned.set(true)
expect(component.isActive).toBeTruthy()
})
it('should describe concrete user filters honestly', () => {
component.selectionModel.ownerFilter = OwnerFilterType.SELF
component.selectionModel.userID = 1
component.selectionModel.ownerFilter.set(OwnerFilterType.SELF)
component.selectionModel.userID.set(1)
expect(component.ownerFilterLabel).toEqual('Owned by user1')
component.selectionModel.ownerFilter = OwnerFilterType.NOT_SELF
component.selectionModel.excludeUsers = [1]
component.selectionModel.ownerFilter.set(OwnerFilterType.NOT_SELF)
component.selectionModel.excludeUsers.set([1])
expect(component.ownerExclusionFilterLabel).toEqual('Not owned by user1')
component.selectionModel.ownerFilter = OwnerFilterType.SHARED_BY_ME
component.selectionModel.userID = 1
component.selectionModel.ownerFilter.set(OwnerFilterType.SHARED_BY_ME)
component.selectionModel.userID.set(1)
expect(component.sharedByFilterLabel).toEqual('Shared by user1')
})
it('should describe concrete filters when usernames are unavailable', () => {
component.selectionModel.ownerFilter = OwnerFilterType.SELF
component.selectionModel.userID = 99
component.selectionModel.ownerFilter.set(OwnerFilterType.SELF)
component.selectionModel.userID.set(99)
expect(component.ownerFilterLabel).toEqual('Owned by another user')
component.selectionModel.ownerFilter = OwnerFilterType.NOT_SELF
component.selectionModel.excludeUsers = [99]
component.selectionModel.ownerFilter.set(OwnerFilterType.NOT_SELF)
component.selectionModel.excludeUsers.set([99])
expect(component.ownerExclusionFilterLabel).toEqual(
'Not owned by another user'
)
component.selectionModel.excludeUsers = [98, 99]
component.selectionModel.excludeUsers.set([98, 99])
expect(component.ownerExclusionFilterLabel).toEqual(
'Not owned by selected users'
)
component.selectionModel.ownerFilter = OwnerFilterType.SHARED_BY_ME
component.selectionModel.userID = 99
component.selectionModel.ownerFilter.set(OwnerFilterType.SHARED_BY_ME)
component.selectionModel.userID.set(99)
expect(component.sharedByFilterLabel).toEqual('Shared by another user')
})
it('should retain relative labels for filters bound to the current user', () => {
component.selectionModel.userID = currentUserID
component.selectionModel.userID.set(currentUserID)
expect(component.ownerFilterLabel).toEqual('My documents')
expect(component.sharedByFilterLabel).toEqual('Shared by me')
component.selectionModel.excludeUsers = [currentUserID]
component.selectionModel.excludeUsers.set([currentUserID])
expect(component.ownerExclusionFilterLabel).toEqual('Shared with me')
})
it('should retain relative labels for inactive filter choices', () => {
component.selectionModel.ownerFilter = OwnerFilterType.NONE
component.selectionModel.ownerFilter.set(OwnerFilterType.NONE)
expect(component.ownerFilterLabel).toEqual('My documents')
expect(component.ownerExclusionFilterLabel).toEqual('Shared with me')
@@ -148,32 +148,41 @@ describe('PermissionsFilterDropdownComponent', () => {
it('should support reset', () => {
component.setFilter(OwnerFilterType.OTHERS)
expect(component.selectionModel.ownerFilter).not.toEqual(
expect(component.selectionModel.ownerFilter()).not.toEqual(
OwnerFilterType.NONE
)
component.reset()
expect(component.selectionModel.ownerFilter).toEqual(OwnerFilterType.NONE)
expect(component.selectionModel.ownerFilter()).toEqual(OwnerFilterType.NONE)
})
it('should toggle owner filter type when users selected', () => {
component.selectionModel.ownerFilter = OwnerFilterType.NONE
component.selectionModel.ownerFilter.set(OwnerFilterType.NONE)
// this would normally be done by select component
component.selectionModel.includeUsers = [12]
component.selectionModel.includeUsers.set([12])
component.onUserSelect()
expect(component.selectionModel.ownerFilter).toEqual(OwnerFilterType.OTHERS)
expect(component.selectionModel.ownerFilter()).toEqual(
OwnerFilterType.OTHERS
)
// this would normally be done by select component
component.selectionModel.includeUsers = null
component.selectionModel.includeUsers.set(null)
component.onUserSelect()
expect(component.selectionModel.ownerFilter).toEqual(OwnerFilterType.NONE)
expect(component.selectionModel.ownerFilter()).toEqual(OwnerFilterType.NONE)
})
it('should emit a selection model depending on the type of owner filter set', () => {
component.selectionModel.ownerFilter = OwnerFilterType.NONE
const emitted = () => ({
excludeUsers: ownerFilterSetResult.excludeUsers(),
hideUnowned: ownerFilterSetResult.hideUnowned(),
includeUsers: ownerFilterSetResult.includeUsers(),
ownerFilter: ownerFilterSetResult.ownerFilter(),
userID: ownerFilterSetResult.userID(),
})
component.selectionModel.ownerFilter.set(OwnerFilterType.NONE)
component.setFilter(OwnerFilterType.SELF)
expect(ownerFilterSetResult).toEqual({
expect(emitted()).toEqual({
excludeUsers: [],
hideUnowned: false,
includeUsers: [],
@@ -182,7 +191,7 @@ describe('PermissionsFilterDropdownComponent', () => {
})
component.setFilter(OwnerFilterType.NOT_SELF)
expect(ownerFilterSetResult).toEqual({
expect(emitted()).toEqual({
excludeUsers: [currentUserID],
hideUnowned: false,
includeUsers: [],
@@ -191,7 +200,7 @@ describe('PermissionsFilterDropdownComponent', () => {
})
component.setFilter(OwnerFilterType.NONE)
expect(ownerFilterSetResult).toEqual({
expect(emitted()).toEqual({
excludeUsers: [],
hideUnowned: false,
includeUsers: [],
@@ -200,7 +209,7 @@ describe('PermissionsFilterDropdownComponent', () => {
})
component.setFilter(OwnerFilterType.SHARED_BY_ME)
expect(ownerFilterSetResult).toEqual({
expect(emitted()).toEqual({
excludeUsers: [],
hideUnowned: false,
includeUsers: [],
@@ -209,7 +218,7 @@ describe('PermissionsFilterDropdownComponent', () => {
})
component.setFilter(OwnerFilterType.UNOWNED)
expect(ownerFilterSetResult).toEqual({
expect(emitted()).toEqual({
excludeUsers: [],
hideUnowned: false,
includeUsers: [],
@@ -25,18 +25,18 @@ import { ComponentWithPermissions } from '../../with-permissions/with-permission
import { ClearableBadgeComponent } from '../clearable-badge/clearable-badge.component'
export class PermissionsSelectionModel {
ownerFilter: OwnerFilterType
hideUnowned: boolean
userID: number
includeUsers: number[]
excludeUsers: number[]
readonly ownerFilter = signal(OwnerFilterType.NONE)
readonly hideUnowned = signal(false)
readonly userID = signal<number>(null)
readonly includeUsers = signal<number[]>([])
readonly excludeUsers = signal<number[]>([])
clear() {
this.ownerFilter = OwnerFilterType.NONE
this.userID = null
this.hideUnowned = false
this.includeUsers = []
this.excludeUsers = []
this.ownerFilter.set(OwnerFilterType.NONE)
this.userID.set(null)
this.hideUnowned.set(false)
this.includeUsers.set([])
this.excludeUsers.set([])
}
}
@@ -84,33 +84,31 @@ export class PermissionsFilterDropdownComponent extends ComponentWithPermissions
readonly users = signal<User[]>([])
hideUnowned: boolean
get isActive(): boolean {
return (
this.selectionModel.ownerFilter !== OwnerFilterType.NONE ||
this.selectionModel.hideUnowned
this.selectionModel.ownerFilter() !== OwnerFilterType.NONE ||
this.selectionModel.hideUnowned()
)
}
get ownerFilterLabel(): string {
if (
this.selectionModel?.ownerFilter !== OwnerFilterType.SELF ||
this.selectionModel?.userID === this.settingsService.currentUser()?.id
this.selectionModel?.ownerFilter() !== OwnerFilterType.SELF ||
this.selectionModel?.userID() === this.settingsService.currentUser()?.id
) {
return $localize`My documents`
}
const username = this.getUsername(this.selectionModel?.userID)
const username = this.getUsername(this.selectionModel?.userID())
return username
? $localize`Owned by ${username}`
: $localize`Owned by another user`
}
get ownerExclusionFilterLabel(): string {
const excludedUsers = this.selectionModel?.excludeUsers ?? []
const excludedUsers = this.selectionModel?.excludeUsers() ?? []
if (
this.selectionModel?.ownerFilter !== OwnerFilterType.NOT_SELF ||
this.selectionModel?.ownerFilter() !== OwnerFilterType.NOT_SELF ||
(excludedUsers.length === 1 &&
excludedUsers[0] === this.settingsService.currentUser()?.id)
) {
@@ -130,13 +128,13 @@ export class PermissionsFilterDropdownComponent extends ComponentWithPermissions
get sharedByFilterLabel(): string {
if (
this.selectionModel?.ownerFilter !== OwnerFilterType.SHARED_BY_ME ||
this.selectionModel?.userID === this.settingsService.currentUser()?.id
this.selectionModel?.ownerFilter() !== OwnerFilterType.SHARED_BY_ME ||
this.selectionModel?.userID() === this.settingsService.currentUser()?.id
) {
return $localize`Shared by me`
}
const username = this.getUsername(this.selectionModel?.userID)
const username = this.getUsername(this.selectionModel?.userID())
return username
? $localize`Shared by ${username}`
: $localize`Shared by another user`
@@ -169,34 +167,36 @@ export class PermissionsFilterDropdownComponent extends ComponentWithPermissions
}
setFilter(type: OwnerFilterType) {
this.selectionModel.ownerFilter = type
if (this.selectionModel.ownerFilter === OwnerFilterType.SELF) {
this.selectionModel.includeUsers = []
this.selectionModel.excludeUsers = []
this.selectionModel.userID = this.settingsService.currentUser().id
this.selectionModel.hideUnowned = false
} else if (this.selectionModel.ownerFilter === OwnerFilterType.NOT_SELF) {
this.selectionModel.userID = null
this.selectionModel.includeUsers = []
this.selectionModel.excludeUsers = [this.settingsService.currentUser().id]
this.selectionModel.hideUnowned = false
} else if (this.selectionModel.ownerFilter === OwnerFilterType.NONE) {
this.selectionModel.userID = null
this.selectionModel.includeUsers = []
this.selectionModel.excludeUsers = []
this.selectionModel.hideUnowned = false
this.selectionModel.ownerFilter.set(type)
if (this.selectionModel.ownerFilter() === OwnerFilterType.SELF) {
this.selectionModel.includeUsers.set([])
this.selectionModel.excludeUsers.set([])
this.selectionModel.userID.set(this.settingsService.currentUser().id)
this.selectionModel.hideUnowned.set(false)
} else if (this.selectionModel.ownerFilter() === OwnerFilterType.NOT_SELF) {
this.selectionModel.userID.set(null)
this.selectionModel.includeUsers.set([])
this.selectionModel.excludeUsers.set([
this.settingsService.currentUser().id,
])
this.selectionModel.hideUnowned.set(false)
} else if (this.selectionModel.ownerFilter() === OwnerFilterType.NONE) {
this.selectionModel.userID.set(null)
this.selectionModel.includeUsers.set([])
this.selectionModel.excludeUsers.set([])
this.selectionModel.hideUnowned.set(false)
} else if (
this.selectionModel.ownerFilter === OwnerFilterType.SHARED_BY_ME
this.selectionModel.ownerFilter() === OwnerFilterType.SHARED_BY_ME
) {
this.selectionModel.userID = this.settingsService.currentUser()?.id
this.selectionModel.includeUsers = []
this.selectionModel.excludeUsers = []
this.selectionModel.hideUnowned = false
} else if (this.selectionModel.ownerFilter === OwnerFilterType.UNOWNED) {
this.selectionModel.userID = null
this.selectionModel.includeUsers = []
this.selectionModel.excludeUsers = []
this.selectionModel.hideUnowned = false
this.selectionModel.userID.set(this.settingsService.currentUser()?.id)
this.selectionModel.includeUsers.set([])
this.selectionModel.excludeUsers.set([])
this.selectionModel.hideUnowned.set(false)
} else if (this.selectionModel.ownerFilter() === OwnerFilterType.UNOWNED) {
this.selectionModel.userID.set(null)
this.selectionModel.includeUsers.set([])
this.selectionModel.excludeUsers.set([])
this.selectionModel.hideUnowned.set(false)
}
this.onChange()
}
@@ -206,11 +206,11 @@ export class PermissionsFilterDropdownComponent extends ComponentWithPermissions
}
onUserSelect() {
if (this.selectionModel.includeUsers?.length) {
this.selectionModel.ownerFilter = OwnerFilterType.OTHERS
} else {
this.selectionModel.ownerFilter = OwnerFilterType.NONE
}
this.selectionModel.ownerFilter.set(
this.selectionModel.includeUsers()?.length
? OwnerFilterType.OTHERS
: OwnerFilterType.NONE
)
this.onChange()
}
@@ -1209,24 +1209,53 @@ describe('DocumentDetailComponent', () => {
expect(fixture.debugElement.queryAll(By.css('textarea.rtl'))).not.toBeNull()
})
it('should display built-in pdf viewer if not disabled', () => {
it('should display built-in pdf viewer if not disabled', async () => {
initNormally()
component.document().archived_file_name = 'file.pdf'
component.document.update((document) => ({
...document,
archived_file_name: 'file.pdf',
}))
settingsService.set(SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER, false)
expect(component.useNativePdfViewer).toBeFalsy()
fixture.detectChanges()
await fixture.whenStable()
expect(fixture.debugElement.query(By.css('pngx-pdf-viewer'))).not.toBeNull()
})
it('should display native pdf viewer if enabled', () => {
initNormally()
component.document().archived_file_name = 'file.pdf'
component.document.update((document) => ({
...document,
archived_file_name: 'file.pdf',
}))
settingsService.set(SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER, true)
expect(component.useNativePdfViewer).toBeTruthy()
fixture.detectChanges()
expect(fixture.debugElement.query(By.css('object'))).not.toBeNull()
})
it('should reflect signal-backed document detail display settings', () => {
settingsService.set(SETTINGS_KEYS.DOCUMENT_EDITING_OVERLAY_THUMBNAIL, false)
settingsService.set(SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS, [
component.DocumentDetailFieldID.Correspondent,
])
expect(component.showThumbnailOverlay).toBeFalsy()
expect(
component.isFieldHidden(component.DocumentDetailFieldID.Correspondent)
).toBeTruthy()
expect(
component.isFieldHidden(component.DocumentDetailFieldID.DocumentType)
).toBeFalsy()
settingsService.set(SETTINGS_KEYS.DOCUMENT_EDITING_OVERLAY_THUMBNAIL, true)
settingsService.set(SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS, [])
expect(component.showThumbnailOverlay).toBeTruthy()
expect(
component.isFieldHidden(component.DocumentDetailFieldID.Correspondent)
).toBeFalsy()
})
it('should attempt to retrieve metadata', () => {
const metadataSpy = jest.spyOn(documentService, 'getMetadata')
metadataSpy.mockReturnValue(of({ has_archive_version: true }))
@@ -1685,7 +1714,10 @@ describe('DocumentDetailComponent', () => {
it('should change preview element by render type', () => {
initNormally()
component.document().archived_file_name = 'file.pdf'
component.document.update((document) => ({
...document,
archived_file_name: 'file.pdf',
}))
fixture.detectChanges()
expect(component.archiveContentRenderType).toEqual(
component.ContentRenderType.PDF
@@ -1694,8 +1726,11 @@ describe('DocumentDetailComponent', () => {
fixture.debugElement.query(By.css('pdf-viewer-container'))
).not.toBeUndefined()
component.document().archived_file_name = undefined
component.document().mime_type = 'text/plain'
component.document.update((document) => ({
...document,
archived_file_name: undefined,
mime_type: 'text/plain',
}))
fixture.detectChanges()
expect(component.archiveContentRenderType).toEqual(
component.ContentRenderType.Text
@@ -1704,7 +1739,10 @@ describe('DocumentDetailComponent', () => {
fixture.debugElement.query(By.css('div.preview-sticky'))
).not.toBeUndefined()
component.document().mime_type = 'image/jpeg'
component.document.update((document) => ({
...document,
mime_type: 'image/jpeg',
}))
fixture.detectChanges()
expect(component.archiveContentRenderType).toEqual(
component.ContentRenderType.Image
@@ -1712,9 +1750,12 @@ describe('DocumentDetailComponent', () => {
expect(
fixture.debugElement.query(By.css('.preview-sticky img'))
).not.toBeUndefined()
;((component.document().mime_type =
'application/vnd.openxmlformats-officedocument.wordprocessingml.document'),
fixture.detectChanges())
component.document.update((document) => ({
...document,
mime_type:
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
}))
fixture.detectChanges()
expect(component.archiveContentRenderType).toEqual(
component.ContentRenderType.Other
)
@@ -227,6 +227,19 @@ export class DocumentDetailComponent
private deviceDetectorService = inject(DeviceDetectorService)
private savedViewService = inject(SavedViewService)
private readonly websocketStatusService = inject(WebsocketStatusService)
private readonly useNativePdfViewerSetting = this.settings.getSignal<boolean>(
SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER
)
private readonly aiEnabledSetting = this.settings.getSignal<boolean>(
SETTINGS_KEYS.AI_ENABLED
)
private readonly showThumbnailOverlaySetting =
this.settings.getSignal<boolean>(
SETTINGS_KEYS.DOCUMENT_EDITING_OVERLAY_THUMBNAIL
)
private readonly hiddenFieldsSetting = this.settings.getSignal<
DocumentDetailFieldID[]
>(SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS)
@ViewChild('inputTitle')
titleInput: TextComponent
@@ -333,8 +346,7 @@ export class DocumentDetailComponent
}
get useNativePdfViewer(): boolean {
this.settings.trackChanges()
return this.settings.get(SETTINGS_KEYS.USE_NATIVE_PDF_VIEWER)
return this.useNativePdfViewerSetting()
}
get isMobile(): boolean {
@@ -342,12 +354,10 @@ export class DocumentDetailComponent
}
get aiEnabled(): boolean {
this.settings.trackChanges()
return this.settings.get(SETTINGS_KEYS.AI_ENABLED)
return this.aiEnabledSetting()
}
get archiveContentRenderType(): ContentRenderType {
this.settings.trackChanges()
const hasArchiveVersion =
this.metadata()?.has_archive_version ??
!!this.document()?.archived_file_name
@@ -359,22 +369,17 @@ export class DocumentDetailComponent
}
get originalContentRenderType(): ContentRenderType {
this.settings.trackChanges()
return this.getRenderType(
this.metadata()?.original_mime_type || this.document()?.mime_type
)
}
get showThumbnailOverlay(): boolean {
this.settings.trackChanges()
return this.settings.get(SETTINGS_KEYS.DOCUMENT_EDITING_OVERLAY_THUMBNAIL)
return this.showThumbnailOverlaySetting()
}
isFieldHidden(fieldId: DocumentDetailFieldID): boolean {
this.settings.trackChanges()
return this.settings
.get(SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS)
.includes(fieldId)
return this.hiddenFieldsSetting().includes(fieldId)
}
private getRenderType(mimeType: string): ContentRenderType {
@@ -121,6 +121,8 @@ export class DocumentListComponent
settingsService = inject(SettingsService)
private hotKeyService = inject(HotKeyService)
permissionService = inject(PermissionsService)
private readonly notesEnabledSetting =
this.settingsService.getSignal<boolean>(SETTINGS_KEYS.NOTES_ENABLED)
DisplayField = DisplayField
DisplayMode = DisplayMode
@@ -574,8 +576,7 @@ export class DocumentListComponent
}
get notesEnabled(): boolean {
this.settingsService.trackChanges()
return this.settingsService.get(SETTINGS_KEYS.NOTES_ENABLED)
return this.notesEnabledSetting()
}
resetFilters() {
@@ -621,6 +621,43 @@ describe('FilterEditorComponent', () => {
component.toggleTag(2) // coverage
})
it('should reflect ingested tag filter rules in the dropdown toggle', () => {
const dropdown = fixture.debugElement.query(
By.css('pngx-filterable-dropdown')
)
const toggle = dropdown.nativeElement.querySelector('#dropdown_tags')
expect(toggle.classList.contains('btn-primary')).toBeFalsy()
expect(
dropdown.nativeElement.querySelector('pngx-clearable-badge')
).toBeNull()
// switching to a view with a tag filter
component.filterRules = [
{
rule_type: FILTER_HAS_TAGS_ALL,
value: '2',
},
]
fixture.detectChanges()
expect(toggle.classList.contains('btn-primary')).toBeTruthy()
expect(
dropdown.nativeElement.querySelector('pngx-clearable-badge')
).not.toBeNull()
// and back to a view without one
component.filterRules = [
{
rule_type: FILTER_HAS_CORRESPONDENT_ANY,
value: '12',
},
]
fixture.detectChanges()
expect(toggle.classList.contains('btn-primary')).toBeFalsy()
expect(
dropdown.nativeElement.querySelector('pngx-clearable-badge')
).toBeNull()
})
it('should ingest filter rules for has any tags', () => {
expect(component.tagSelectionModel.getSelectedItems()).toHaveLength(0)
component.filterRules = [
@@ -1078,7 +1115,7 @@ describe('FilterEditorComponent', () => {
})
it('should ingest filter rules for owner', () => {
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
expect(component.permissionsSelectionModel.ownerFilter()).toEqual(
OwnerFilterType.NONE
)
component.filterRules = [
@@ -1087,15 +1124,38 @@ describe('FilterEditorComponent', () => {
value: '100',
},
]
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
expect(component.permissionsSelectionModel.ownerFilter()).toEqual(
OwnerFilterType.SELF
)
expect(component.permissionsSelectionModel.hideUnowned).toBeFalsy()
expect(component.permissionsSelectionModel.userID).toEqual(100)
expect(component.permissionsSelectionModel.hideUnowned()).toBeFalsy()
expect(component.permissionsSelectionModel.userID()).toEqual(100)
})
it('should reflect ingested owner filter rules in the dropdown toggle', () => {
const dropdown = fixture.debugElement.query(
By.css('pngx-permissions-filter-dropdown')
)
const toggle = dropdown.nativeElement.querySelector('button')
expect(toggle.classList.contains('btn-primary')).toBeFalsy()
// switching to a view with an owner filter
component.filterRules = [
{
rule_type: FILTER_OWNER,
value: '100',
},
]
fixture.detectChanges()
expect(toggle.classList.contains('btn-primary')).toBeTruthy()
// and back to a view without one
component.filterRules = []
fixture.detectChanges()
expect(toggle.classList.contains('btn-primary')).toBeFalsy()
})
it('should ingest filter rules for owner is others', () => {
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
expect(component.permissionsSelectionModel.ownerFilter()).toEqual(
OwnerFilterType.NONE
)
component.filterRules = [
@@ -1104,14 +1164,14 @@ describe('FilterEditorComponent', () => {
value: '50',
},
]
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
expect(component.permissionsSelectionModel.ownerFilter()).toEqual(
OwnerFilterType.OTHERS
)
expect(component.permissionsSelectionModel.includeUsers).toContain(50)
expect(component.permissionsSelectionModel.includeUsers()).toContain(50)
})
it('should ingest filter rules for owner does not include others', () => {
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
expect(component.permissionsSelectionModel.ownerFilter()).toEqual(
OwnerFilterType.NONE
)
component.filterRules = [
@@ -1120,14 +1180,14 @@ describe('FilterEditorComponent', () => {
value: '50',
},
]
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
expect(component.permissionsSelectionModel.ownerFilter()).toEqual(
OwnerFilterType.NOT_SELF
)
expect(component.permissionsSelectionModel.excludeUsers).toContain(50)
expect(component.permissionsSelectionModel.excludeUsers()).toContain(50)
})
it('should ingest filter rules for owner is null', () => {
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
expect(component.permissionsSelectionModel.ownerFilter()).toEqual(
OwnerFilterType.NONE
)
component.filterRules = [
@@ -1136,10 +1196,10 @@ describe('FilterEditorComponent', () => {
value: 'true',
},
]
expect(component.permissionsSelectionModel.ownerFilter).toEqual(
expect(component.permissionsSelectionModel.ownerFilter()).toEqual(
OwnerFilterType.UNOWNED
)
expect(component.permissionsSelectionModel.hideUnowned).toBeFalsy()
expect(component.permissionsSelectionModel.hideUnowned()).toBeFalsy()
})
it('should ingest filter rules for owner is not null', () => {
@@ -1149,14 +1209,14 @@ describe('FilterEditorComponent', () => {
value: 'false',
},
]
expect(component.permissionsSelectionModel.hideUnowned).toBeTruthy()
expect(component.permissionsSelectionModel.hideUnowned()).toBeTruthy()
component.filterRules = [
{
rule_type: FILTER_OWNER_ISNULL,
value: '0',
},
]
expect(component.permissionsSelectionModel.hideUnowned).toBeTruthy()
expect(component.permissionsSelectionModel.hideUnowned()).toBeTruthy()
})
it('should ingest filter rules for shared by me', () => {
@@ -1166,7 +1226,7 @@ describe('FilterEditorComponent', () => {
value: '2',
},
]
expect(component.permissionsSelectionModel.userID).toEqual(2)
expect(component.permissionsSelectionModel.userID()).toEqual(2)
})
// GET filterRules
@@ -1932,7 +1992,10 @@ describe('FilterEditorComponent', () => {
value: '1',
},
])
component.permissionsSelectionModel.excludeUsers.push(2)
component.permissionsSelectionModel.excludeUsers.update((users) => [
...users,
2,
])
fixture.detectChanges()
expect(component.filterRules).toEqual([
{
@@ -1982,8 +2045,11 @@ describe('FilterEditorComponent', () => {
// TODO: mock input in code
// userSelect.query(By.css('input')).nativeElement.value = '3'
// userSelect.triggerEventHandler('change')
component.permissionsSelectionModel.ownerFilter = OwnerFilterType.OTHERS
component.permissionsSelectionModel.includeUsers.push(3)
component.permissionsSelectionModel.ownerFilter.set(OwnerFilterType.OTHERS)
component.permissionsSelectionModel.includeUsers.update((users) => [
...users,
3,
])
fixture.detectChanges()
expect(component.filterRules).toEqual([
{
@@ -2003,7 +2069,7 @@ describe('FilterEditorComponent', () => {
ownerToggle.nativeElement.checked = true
// ownerToggle.triggerEventHandler('change')
// TODO: ngModel isn't doing this here
component.permissionsSelectionModel.hideUnowned = true
component.permissionsSelectionModel.hideUnowned.set(true)
fixture.detectChanges()
expect(component.filterRules).toEqual([
{
@@ -735,38 +735,50 @@ export class FilterEditorComponent
this._textFilter = rule.value
break
case FILTER_OWNER:
this.permissionsSelectionModel.ownerFilter = OwnerFilterType.SELF
this.permissionsSelectionModel.hideUnowned = false
this.permissionsSelectionModel.ownerFilter.set(OwnerFilterType.SELF)
this.permissionsSelectionModel.hideUnowned.set(false)
if (rule.value)
this.permissionsSelectionModel.userID = parseInt(rule.value, 10)
this.permissionsSelectionModel.userID.set(
Number.parseInt(rule.value, 10)
)
break
case FILTER_OWNER_ANY:
this.permissionsSelectionModel.ownerFilter = OwnerFilterType.OTHERS
this.permissionsSelectionModel.ownerFilter.set(OwnerFilterType.OTHERS)
if (rule.value)
this.permissionsSelectionModel.includeUsers.push(
parseInt(rule.value, 10)
)
this.permissionsSelectionModel.includeUsers.update((users) => [
...users,
Number.parseInt(rule.value, 10),
])
break
case FILTER_OWNER_DOES_NOT_INCLUDE:
this.permissionsSelectionModel.ownerFilter = OwnerFilterType.NOT_SELF
this.permissionsSelectionModel.ownerFilter.set(
OwnerFilterType.NOT_SELF
)
if (rule.value)
this.permissionsSelectionModel.excludeUsers.push(
parseInt(rule.value, 10)
)
this.permissionsSelectionModel.excludeUsers.update((users) => [
...users,
Number.parseInt(rule.value, 10),
])
break
case FILTER_SHARED_BY_USER:
this.permissionsSelectionModel.ownerFilter =
this.permissionsSelectionModel.ownerFilter.set(
OwnerFilterType.SHARED_BY_ME
)
if (rule.value)
this.permissionsSelectionModel.userID = parseInt(rule.value, 10)
this.permissionsSelectionModel.userID.set(
Number.parseInt(rule.value, 10)
)
break
case FILTER_OWNER_ISNULL:
if (rule.value === 'true' || rule.value === '1') {
this.permissionsSelectionModel.hideUnowned = false
this.permissionsSelectionModel.ownerFilter = OwnerFilterType.UNOWNED
this.permissionsSelectionModel.hideUnowned.set(false)
this.permissionsSelectionModel.ownerFilter.set(
OwnerFilterType.UNOWNED
)
} else {
this.permissionsSelectionModel.hideUnowned =
this.permissionsSelectionModel.hideUnowned.set(
rule.value === 'false' || rule.value === '0'
)
break
}
}
@@ -1074,34 +1086,35 @@ export class FilterEditorComponent
})
}
}
if (this.permissionsSelectionModel.ownerFilter == OwnerFilterType.SELF) {
if (this.permissionsSelectionModel.ownerFilter() == OwnerFilterType.SELF) {
filterRules.push({
rule_type: FILTER_OWNER,
value: this.permissionsSelectionModel.userID.toString(),
value: this.permissionsSelectionModel.userID().toString(),
})
} else if (
this.permissionsSelectionModel.ownerFilter == OwnerFilterType.NOT_SELF
this.permissionsSelectionModel.ownerFilter() == OwnerFilterType.NOT_SELF
) {
filterRules.push({
rule_type: FILTER_OWNER_DOES_NOT_INCLUDE,
value: this.permissionsSelectionModel.excludeUsers?.join(','),
value: this.permissionsSelectionModel.excludeUsers()?.join(','),
})
} else if (
this.permissionsSelectionModel.ownerFilter == OwnerFilterType.OTHERS
this.permissionsSelectionModel.ownerFilter() == OwnerFilterType.OTHERS
) {
filterRules.push({
rule_type: FILTER_OWNER_ANY,
value: this.permissionsSelectionModel.includeUsers?.join(','),
value: this.permissionsSelectionModel.includeUsers()?.join(','),
})
} else if (
this.permissionsSelectionModel.ownerFilter == OwnerFilterType.SHARED_BY_ME
this.permissionsSelectionModel.ownerFilter() ==
OwnerFilterType.SHARED_BY_ME
) {
filterRules.push({
rule_type: FILTER_SHARED_BY_USER,
value: this.permissionsSelectionModel.userID.toString(),
value: this.permissionsSelectionModel.userID().toString(),
})
} else if (
this.permissionsSelectionModel.ownerFilter == OwnerFilterType.UNOWNED
this.permissionsSelectionModel.ownerFilter() == OwnerFilterType.UNOWNED
) {
filterRules.push({
rule_type: FILTER_OWNER_ISNULL,
@@ -1109,7 +1122,7 @@ export class FilterEditorComponent
})
}
if (this.permissionsSelectionModel.hideUnowned) {
if (this.permissionsSelectionModel.hideUnowned()) {
filterRules.push({
rule_type: FILTER_OWNER_ISNULL,
value: 'false',
@@ -210,6 +210,48 @@ describe('SettingsService', () => {
expect(settingsService.get(SETTINGS_KEYS.THEME_COLOR)).toEqual('#000000')
})
it('provides stable signals that update when settings change', () => {
const req = httpTestingController.expectOne(
`${environment.apiBaseUrl}ui_settings/`
)
req.flush(ui_settings)
const notesEnabled = settingsService.getSignal<boolean>(
SETTINGS_KEYS.NOTES_ENABLED
)
expect(notesEnabled()).toBeTruthy()
expect(
settingsService.getSignal<boolean>(SETTINGS_KEYS.NOTES_ENABLED)
).toBe(notesEnabled)
settingsService.set(SETTINGS_KEYS.NOTES_ENABLED, false)
expect(notesEnabled()).toBeFalsy()
})
it('updates setting signals when settings are reinitialized', () => {
let req = httpTestingController.expectOne(
`${environment.apiBaseUrl}ui_settings/`
)
req.flush(ui_settings)
const appTitle = settingsService.getSignal<string>(SETTINGS_KEYS.APP_TITLE)
settingsService.initializeSettings().subscribe()
req = httpTestingController.expectOne(
`${environment.apiBaseUrl}ui_settings/`
)
req.flush({
...ui_settings,
settings: {
...ui_settings.settings,
app_title: 'Updated title',
},
})
expect(appTitle()).toBe('Updated title')
})
it('sets django cookie for languages', () => {
httpTestingController
.expectOne(`${environment.apiBaseUrl}ui_settings/`)
+16 -4
View File
@@ -2,6 +2,8 @@ import { HttpClient } from '@angular/common/http'
import {
DOCUMENT,
EventEmitter,
Signal,
computed,
inject,
Injectable,
LOCALE_ID,
@@ -297,6 +299,7 @@ export class SettingsService {
private settings: Record<string, any> = {}
private readonly settingsVersion = signal(0)
private readonly settingSignals = new Map<string, Signal<unknown>>()
readonly currentUser = signal<User>(undefined)
public settingsSaved: EventEmitter<any> = new EventEmitter()
@@ -326,10 +329,6 @@ export class SettingsService {
return !UNSAFE_OBJECT_KEYS.has(key)
}
public trackChanges(): void {
this.settingsVersion()
}
private assignSafeSettings(source: Record<string, any>) {
if (!source || typeof source !== 'object' || Array.isArray(source)) {
return
@@ -339,6 +338,7 @@ export class SettingsService {
if (!this.isSafeObjectKey(key)) continue
this.settings[key] = source[key]
}
this.settingsVersion.update((version) => version + 1)
}
// this is called by the app initializer in app.module
@@ -594,6 +594,18 @@ export class SettingsService {
}
}
getSignal<T = any>(key: string): Signal<T> {
let settingSignal = this.settingSignals.get(key)
if (!settingSignal) {
settingSignal = computed(() => {
this.settingsVersion()
return this.get(key)
})
this.settingSignals.set(key, settingSignal)
}
return settingSignal as Signal<T>
}
set(key: string, value: any) {
// parse key:key:key into nested object
let settingObj = this.settings
+10 -10
View File
@@ -507,8 +507,8 @@ def rotate(
logger.info(
f"Queued new rotated version for document {pair.root_doc.id} by {degrees} degrees",
)
except Exception:
logger.exception(f"Error rotating document {pair.root_doc.id}")
except Exception as e:
logger.exception(f"Error rotating document {pair.root_doc.id}: {e}")
return "OK"
@@ -554,9 +554,9 @@ def merge(
affected_docs.append(doc.id)
if handoff_asn is None and doc.archive_serial_number is not None:
handoff_asn = doc.archive_serial_number
except Exception:
except Exception as e:
logger.exception(
f"Error merging document {doc.id}, it will not be included in the merge",
f"Error merging document {doc.id}, it will not be included in the merge: {e}",
)
if len(affected_docs) == 0:
logger.warning("No documents were merged")
@@ -805,8 +805,8 @@ def split(
else:
group(consume_tasks).delay()
except Exception:
logger.exception(f"Error splitting document {doc.id}")
except Exception as e:
logger.exception(f"Error splitting document {doc.id}: {e}")
return "OK"
@@ -858,8 +858,8 @@ def delete_pages(
logger.info(
f"Queued new version for document {pair.root_doc.id} after deleting pages {pages}",
)
except Exception:
logger.exception(f"Error deleting pages from document {pair.root_doc.id}")
except Exception as e:
logger.exception(f"Error deleting pages from document {pair.root_doc.id}: {e}")
return "OK"
@@ -986,7 +986,7 @@ def edit_pdf(
group(consume_tasks).delay()
except Exception as e:
logger.exception(f"Error editing document {pair.root_doc.id}")
logger.exception(f"Error editing document {pair.root_doc.id}: {e}")
raise ValueError(
f"An error occurred while editing the document: {e}",
) from e
@@ -1097,7 +1097,7 @@ def remove_password(
except Exception as e:
logger.exception(
f"Error removing password from document {pair.root_doc.id}",
f"Error removing password from document {pair.root_doc.id}: {e}",
)
raise ValueError(
f"An error occurred while removing the password: {e}",
+7 -7
View File
@@ -72,8 +72,8 @@ def load_classifier(*, raise_exception: bool = False) -> DocumentClassifier | No
Path(settings.MODEL_FILE).unlink()
classifier = None
if raise_exception:
raise
except ClassifierModelCorruptError:
raise e
except ClassifierModelCorruptError as e:
# there's something wrong with the model file.
logger.exception(
"Unrecoverable error while loading document "
@@ -82,17 +82,17 @@ def load_classifier(*, raise_exception: bool = False) -> DocumentClassifier | No
Path(settings.MODEL_FILE).unlink()
classifier = None
if raise_exception:
raise
except OSError:
raise e
except OSError as e:
logger.exception("IO error while loading document classification model")
classifier = None
if raise_exception:
raise
except Exception: # pragma: no cover
raise e
except Exception as e: # pragma: no cover
logger.exception("Unknown error while loading document classification model")
classifier = None
if raise_exception:
raise
raise e
return classifier
+6 -4
View File
@@ -217,7 +217,7 @@ class ConsumerPluginMixin:
current_progress,
max_progress,
document_id=document_id,
owner_id=self.metadata.owner_id or None,
owner_id=self.metadata.owner_id if self.metadata.owner_id else None,
users_can_view=(self.metadata.view_users or [])
+ (self.metadata.change_users or []),
groups_can_view=(self.metadata.view_groups or [])
@@ -675,7 +675,9 @@ class ConsumerPlugin(
document=document,
logging_group=self.logging_group,
classifier=classifier,
original_file=self.unmodified_original or self.working_copy,
original_file=self.unmodified_original
if self.unmodified_original
else self.working_copy,
)
# After everything is in the database, copy the files into
@@ -856,7 +858,7 @@ class ConsumerPlugin(
else:
stats = Path(self.input_doc.original_file).stat()
create_date = timezone.make_aware(
datetime.datetime.fromtimestamp(stats.st_mtime), # noqa: DTZ006 - make_aware() requires a naive datetime
datetime.datetime.fromtimestamp(stats.st_mtime),
)
self.log.debug(f"Creation date from st_mtime: {create_date}")
@@ -970,7 +972,7 @@ class ConsumerPlugin(
try:
copy_basic_file_stats(source, target)
except Exception: # pragma: no cover
self.log.debug("Unable to copy file stats from %s to %s", source, target)
pass
class ConsumerPreflightPlugin(
+2 -4
View File
@@ -78,9 +78,7 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin)
stats = staging.stat()
# if the file is older than the timeout, we don't consider
# it valid
if (
dt.datetime.now(tz=dt.UTC).timestamp() - stats.st_mtime
) > TIMEOUT_SECONDS:
if (dt.datetime.now().timestamp() - stats.st_mtime) > TIMEOUT_SECONDS:
logger.warning("Outdated double sided staging file exists, deleting it")
staging.unlink()
else:
@@ -136,7 +134,7 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin)
shutil.move(pdf_file, staging)
# update access to modification time so we know if the file
# is outdated when another file gets uploaded
timestamp = dt.datetime.now(tz=dt.UTC).timestamp()
timestamp = dt.datetime.now().timestamp()
os.utime(staging, (timestamp, timestamp))
logger.info(
"Got scan with odd numbered pages of double-sided scan, moved it to %s",
+1 -1
View File
@@ -734,7 +734,7 @@ class CustomFieldQueryParser:
)
# Check if any of the requested IDs are missing.
missing_ids = set(value) - {link.document_id for link in links}
missing_ids = set(value) - set(link.document_id for link in links)
if missing_ids:
# The result should be an empty set in this case.
return Q(id__in=[])
@@ -631,25 +631,23 @@ class Command(BaseCommand):
):
# Process each change
for change_type, path in changes:
resolved_path = Path(path).resolve()
path = Path(path).resolve()
if change_type == Change.deleted:
# Consumed (or otherwise removed); a later file
# reusing this name must not be skipped as
# already-queued.
queued.discard(resolved_path)
if not resolved_path.is_file():
queued.discard(path)
if not path.is_file():
continue
if resolved_path in queued:
if path in queued:
# Already queued and awaiting consumption; a stray
# event (NAS metadata touch, AV scan, etc.) while
# the file sits on disk mid-consumption must not
# cause it to be queued a second time (GH #13511).
logger.debug(
f"Ignoring event for queued file: {resolved_path}",
)
logger.debug(f"Ignoring event for queued file: {path}")
continue
logger.debug(f"Event: {change_type.name} for {resolved_path}")
tracker.track(resolved_path, change_type)
logger.debug(f"Event: {change_type.name} for {path}")
tracker.track(path, change_type)
# Check for stable files
for stable_path in tracker.get_stable_files():
+1 -7
View File
@@ -30,10 +30,6 @@ if TYPE_CHECKING:
logger = logging.getLogger("paperless.matching")
class UnsupportedWorkflowTriggerTypeError(Exception):
pass
def log_reason(
matching_model: MatchingModel | WorkflowTrigger,
document: Document,
@@ -695,9 +691,7 @@ def document_matches_workflow(
)
else:
# New trigger types need to be explicitly checked above
raise UnsupportedWorkflowTriggerTypeError(
f"Trigger type {trigger_type} not yet supported",
)
raise Exception(f"Trigger type {trigger_type} not yet supported")
if trigger_matched:
logger.info(f"Document matched {trigger} from {workflow}")
@@ -75,7 +75,7 @@ def recompute_checksums(apps, schema_editor):
if updated_fields:
batch.append(doc)
processed += 1 # noqa: SIM113
processed += 1
if len(batch) >= _BATCH_SIZE:
Document.objects.bulk_update(batch, ["checksum", "archive_checksum"])
+1 -1
View File
@@ -377,7 +377,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
from documents.versioning import versions_newest_first
if hasattr(self, "effective_content"):
return self.effective_content
return getattr(self, "effective_content")
if self.root_document_id is not None or self.pk is None:
return self.content
+2 -2
View File
@@ -41,7 +41,7 @@ def get_default_file_extension(mime_type: str) -> str:
return supported[mime_type]
ext = mimetypes.guess_extension(mime_type)
return ext or ""
return ext if ext else ""
def is_file_ext_supported(ext: str) -> bool:
@@ -110,7 +110,7 @@ def run_convert(
args += ["-define", "pdf:use-cropbox=true"] if use_cropbox else []
args += [str(input_file), str(output_file)]
logger.debug("Execute: %s", " ".join(args), extra={"group": logging_group})
logger.debug("Execute: " + " ".join(args), extra={"group": logging_group})
try:
run_subprocess(args, environment, logger)
@@ -43,8 +43,8 @@ def _discover_parser_class() -> type[DateParserPluginBase]:
valid_plugins.append(ep)
else:
logger.warning(f"Plugin {ep.name} does not subclass DateParser.")
except Exception:
logger.exception(f"Unable to load date parser plugin {ep.name}")
except Exception as e:
logger.exception(f"Unable to load date parser plugin {ep.name}: {e}")
if not valid_plugins:
return RegexDateParserPlugin
+2 -2
View File
@@ -91,8 +91,8 @@ class DateParserPluginBase(ABC):
},
locales=self.config.languages,
)
except Exception:
logger.exception(f"Error while parsing date string '{date_string}'")
except Exception as e:
logger.exception(f"Error while parsing date string '{date_string}': {e}")
return None
def _filter_date(
+6 -4
View File
@@ -59,10 +59,11 @@ def safe_regex_match(pattern: str, text: str, *, flags: int = 0):
try:
validate_regex_pattern(pattern)
compiled = regex.compile(pattern, flags=flags)
except (regex.error, ValueError):
except (regex.error, ValueError) as exc:
logger.exception(
"Error while processing regular expression %s",
"Error while processing regular expression %s: %s",
textwrap.shorten(pattern, width=80, placeholder=""),
exc,
)
return None
@@ -85,10 +86,11 @@ def safe_regex_sub(pattern: str, repl: str, text: str, *, flags: int = 0) -> str
try:
validate_regex_pattern(pattern)
compiled = regex.compile(pattern, flags=flags)
except (regex.error, ValueError):
except (regex.error, ValueError) as exc:
logger.exception(
"Error while processing regular expression %s",
"Error while processing regular expression %s: %s",
textwrap.shorten(pattern, width=80, placeholder=""),
exc,
)
return None
+2 -2
View File
@@ -1142,7 +1142,7 @@ def get_backend() -> TantivyBackend:
Returns:
Thread-safe singleton TantivyBackend instance
"""
global _backend, _backend_path # noqa: PLW0603 - module-level singleton, no class to hold this state
global _backend, _backend_path
current_path: Path = settings.INDEX_DIR
@@ -1173,7 +1173,7 @@ def reset_backend() -> None:
Forces creation of a new backend instance on the next get_backend() call.
Used for test isolation and when switching between different index directories.
"""
global _backend, _backend_path # noqa: PLW0603 - module-level singleton, no class to hold this state
global _backend, _backend_path
with _backend_lock:
if _backend is not None:
+1 -1
View File
@@ -240,7 +240,7 @@ def parse_user_query(
DEFAULT_SEARCH_FIELDS,
field_boosts=_FIELD_BOOSTS,
# (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
fuzzy_fields=dict.fromkeys(DEFAULT_SEARCH_FIELDS, (True, 1, True)),
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
)
# 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
+12 -11
View File
@@ -434,7 +434,7 @@ class OwnedObjectSerializer(
return set()
ctype = ContentType.objects.get_for_model(first_obj)
object_pks = [obj.pk for obj in objects]
object_pks = list(obj.pk for obj in objects)
pk_type = type(first_obj.pk)
def get_pks_for_permission_type(model):
@@ -730,7 +730,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
self.instance.clean()
except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e)
raise
raise e
finally:
self.instance.tn_parent = original_parent
else:
@@ -740,7 +740,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
temp.clean()
except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e)
raise
raise e
return super().validate(attrs)
@@ -1150,7 +1150,7 @@ class DocumentSerializer(
def to_representation(self, instance):
doc = super().to_representation(instance)
if "content" in self.fields and hasattr(instance, "effective_content"):
doc["content"] = instance.effective_content or ""
doc["content"] = getattr(instance, "effective_content") or ""
if self.truncate_content and "content" in self.fields:
doc["content"] = doc.get("content")[0:550]
return doc
@@ -1860,8 +1860,8 @@ class BulkEditSerializer(
if isinstance(custom_fields, dict):
try:
ids = [int(i[0]) for i in custom_fields.items()]
except Exception:
logger.exception("Error validating custom fields")
except Exception as e:
logger.exception(f"Error validating custom fields: {e}")
raise serializers.ValidationError(
f"{name} must be a list of integers or a dict of id:value pairs, see the log for details",
)
@@ -2059,12 +2059,13 @@ class BulkEditSerializer(
for doc in docs:
if "-" in doc:
pages.append(
list(
range(
[
x
for x in range(
int(doc.split("-")[0]),
int(doc.split("-")[1]) + 1,
),
),
)
],
)
else:
pages.append([int(doc)])
@@ -2925,7 +2926,7 @@ class ShareLinkBundleSerializer(OwnedObjectSerializer):
return share_link_bundle
def get_document_count(self, obj: ShareLinkBundle) -> int:
return obj.document_total or obj.documents.count()
return getattr(obj, "document_total") or obj.documents.count()
class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
+4 -5
View File
@@ -637,7 +637,7 @@ def update_filename_and_move_files(
# so this is not the end of the world.
# B: if moving the original file failed, nothing has changed
# anyway.
logger.exception("Error reverting document changes")
pass
# restore old values on the instance
instance.filename = old_filename
@@ -1102,11 +1102,10 @@ def _extract_input_data(
if v is None or k.startswith("_"):
continue
if isinstance(v, datetime.date):
override_dict[k] = v.isoformat()
v = v.isoformat()
elif isinstance(v, Path):
override_dict[k] = str(v)
else:
override_dict[k] = v
v = str(v)
override_dict[k] = v
if override_dict:
data["overrides"] = override_dict
return data
+7 -6
View File
@@ -217,9 +217,9 @@ def consume_file(
overrides.filename or input_doc.original_file.name,
self.request.id,
) as status_mgr,
TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir_name,
TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir,
):
tmp_dir = Path(tmp_dir_name)
tmp_dir = Path(tmp_dir)
msg = None
for plugin_class in plugins:
plugin_name = plugin_class.NAME
@@ -261,7 +261,7 @@ def consume_file(
)
except Exception as e:
logger.exception(f"{plugin_name} failed")
logger.exception(f"{plugin_name} failed: {e}")
status_mgr.send_progress(
ProgressStatusOptions.FAILED,
f"{e}",
@@ -495,8 +495,8 @@ def empty_trash(doc_ids=None) -> None:
content_type=ContentType.objects.get_for_model(Document),
object_id__in=deleted_document_ids,
).delete()
except Exception: # pragma: no cover
logger.exception("Error while emptying trash")
except Exception as e: # pragma: no cover
logger.exception(f"Error while emptying trash: {e}")
finally:
models.signals.post_delete.disconnect(
cleanup_document_deletion,
@@ -832,8 +832,9 @@ def build_share_link_bundle(bundle_id: int) -> None:
logger.info("Built share link bundle %s", bundle.pk)
except Exception as exc:
logger.exception(
"Failed to build share link bundle %s",
"Failed to build share link bundle %s: %s",
bundle_id,
exc,
)
bundle.status = ShareLinkBundle.Status.FAILED
bundle.last_error = {
-4
View File
@@ -78,10 +78,6 @@ class PlaceholderString(str):
def __ne__(self, other) -> bool:
return not self.__eq__(other)
def __hash__(self) -> int:
# Equal to both "-none-" and "none", so hash to a single canonical value
return hash("-none-")
NO_VALUE_PLACEHOLDER = PlaceholderString("-none-")
+3 -3
View File
@@ -138,9 +138,9 @@ def parse_w_workflow_placeholders(
# We're good!
return rendered_template
except UndefinedError:
except UndefinedError as e:
# The undefined class logs this already for us
raise
raise e
except TemplateSyntaxError as e:
logger.warning(f"Template syntax error in title generation: {e}")
except SecurityError as e:
@@ -150,5 +150,5 @@ def parse_w_workflow_placeholders(
logger.warning(
f"Invalid title format '{text}', workflow not applied: {e}",
)
raise
raise e
return None
@@ -296,7 +296,7 @@ class TestRegexDateParser:
# simulate parse failure for malformed input
if "99/99/9999" in date_string or "bad date" in date_string:
raise Exception("parse failed for malformed date") # noqa: TRY002 - simulates a generic parser failure
raise Exception("parse failed for malformed date")
return None
@@ -57,13 +57,13 @@ class MultiprocessCommand(PaperlessCommand):
def handle(self, *args, **options):
items = list(range(5))
results = list(
self.process_parallel(
_double_value,
items,
description="Processing...",
),
)
results = []
for result in self.process_parallel(
_double_value,
items,
description="Processing...",
):
results.append(result)
successes = sum(1 for r in results if r.success)
self.stdout.write(f"Successes: {successes}")
@@ -66,7 +66,7 @@ class TestWriteBatchLockRetry:
)
mock_sleep = mocker.patch(
"documents.search._backend.time.sleep",
side_effect=sleep_values.append,
side_effect=lambda s: sleep_values.append(s),
)
# Should not raise — 4th attempt succeeds
@@ -111,7 +111,7 @@ class TestWriteBatchLockRetry:
sleep_values: list[float] = []
mocker.patch(
"documents.search._backend.time.sleep",
side_effect=sleep_values.append,
side_effect=lambda s: sleep_values.append(s),
)
for _ in range(50):
sleep_values.clear()
+2 -2
View File
@@ -1003,8 +1003,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
for correspondent in response.data[field]:
self.assertEqual(correspondent["document_count"], 0)
self.assertCountEqual(
(c["id"] for c in response.data[field]),
(c["id"] for c in Entity.objects.values("id")),
map(lambda c: c["id"], response.data[field]),
map(lambda c: c["id"], Entity.objects.values("id")),
)
def test_api_selection_data(self) -> None:
+74
View File
@@ -1,11 +1,18 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from unittest import mock
import pytest
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User
from rest_framework import status
from rest_framework.test import APIClient
from rest_framework.test import APITestCase
if TYPE_CHECKING:
from pytest_mock import MockerFixture
class TestChatStreamingViewInputValidation(APITestCase):
def setUp(self) -> None:
@@ -42,3 +49,70 @@ class TestChatStreamingViewInputValidation(APITestCase):
format="json",
)
assert resp.status_code == status.HTTP_400_BAD_REQUEST
@pytest.mark.django_db
class TestChatStreamingViewUnrestrictedFlag:
"""The document id filter may only be skipped (``unrestricted=True``) for
a caller who can see every document, i.e. an active superuser.
"""
@pytest.fixture
def mocked_stream_chat(self, mocker: MockerFixture) -> mock.MagicMock:
"""AI enabled, with stream_chat_with_documents patched so the view
never touches the real vector store; returns the patched callable so
tests can inspect how it was called.
"""
mocker.patch("documents.views.AIConfig").return_value.ai_enabled = True
return mocker.patch(
"documents.views.stream_chat_with_documents",
return_value=iter(()),
)
@pytest.fixture
def viewer_client(self, user_client: APIClient, regular_user: User) -> APIClient:
"""The conftest regular-user client, additionally granted
view_document -- able to see every document without being a
superuser.
"""
regular_user.user_permissions.add(
*Permission.objects.filter(codename="view_document"),
)
return user_client
@pytest.mark.parametrize(
("client_fixture", "expected_unrestricted"),
[
pytest.param("admin_client", True, id="superuser_is_unrestricted"),
pytest.param("viewer_client", False, id="regular_user_is_restricted"),
],
)
def test_unrestricted_only_for_superuser(
self,
request: pytest.FixtureRequest,
mocked_stream_chat: mock.MagicMock,
client_fixture: str,
*,
expected_unrestricted: bool,
) -> None:
"""
GIVEN:
- A superuser, or a regular user holding view_document
WHEN:
- They post a chat question with no document_id
THEN:
- stream_chat_with_documents is called with unrestricted=True for
the superuser and unrestricted=False for the regular user, even
though that user can view every document
"""
client: APIClient = request.getfixturevalue(client_fixture)
client.post(
"/api/documents/chat/",
data={"q": "What's in these documents?"},
format="json",
)
assert (
mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
)
+2 -2
View File
@@ -18,8 +18,8 @@ class MockOpenIDProvider:
def get_brands(self):
default_servers = [
{"id": "yahoo", "name": "Yahoo", "openid_url": "http://me.yahoo.com"},
{"id": "hyves", "name": "Hyves", "openid_url": "http://hyves.nl"},
dict(id="yahoo", name="Yahoo", openid_url="http://me.yahoo.com"),
dict(id="hyves", name="Hyves", openid_url="http://hyves.nl"),
]
return default_servers
+2 -2
View File
@@ -205,12 +205,12 @@ class TestBarcode(
- Barcode is detected on page 1 (zero indexed)
"""
for test_filename in [
for test_file in [
"patch-code-t-middle-reverse.pdf",
"patch-code-t-middle-distorted.pdf",
"patch-code-t-middle-fuzzy.pdf",
]:
test_file = self.BARCODE_SAMPLE_DIR / test_filename
test_file = self.BARCODE_SAMPLE_DIR / test_file
with self.get_reader(test_file) as reader:
reader.detect()
+3 -3
View File
@@ -777,7 +777,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
sig.set.return_value.apply_async.side_effect = Exception("boom")
mock_consume_file.return_value = sig
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception
with self.assertRaises(Exception):
bulk_edit.merge(doc_ids, delete_originals=True)
self.doc1.refresh_from_db()
@@ -1318,7 +1318,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
sig.apply_async.side_effect = Exception("boom")
mock_chord.return_value = sig
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception
with self.assertRaises(Exception):
bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
self.doc2.refresh_from_db()
@@ -1430,7 +1430,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
{"page": 9999}, # invalid page, forces error during PDF load
]
with self.assertLogs("paperless.bulk_edit", level="ERROR"):
with self.assertRaises(ValueError):
with self.assertRaises(Exception):
bulk_edit.edit_pdf(doc_ids, operations)
mock_group.assert_not_called()
mock_consume_file.assert_not_called()
+1 -1
View File
@@ -806,7 +806,7 @@ class TestClassifier(DirectoriesMixin, TestCase):
Path(settings.MODEL_FILE).touch()
mock_load.side_effect = Exception()
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception
with self.assertRaises(Exception):
load_classifier(raise_exception=True)
+2 -2
View File
@@ -137,7 +137,7 @@ class FaultyParser(_BaseNewStyleParser):
class FaultyGenericExceptionParser(_BaseNewStyleParser):
def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:
raise Exception("Generic exception.") # noqa: TRY002 - deliberately not a ParseError
raise Exception("Generic exception.")
def fake_magic_from_file(file, *, mime=False): # NOSONAR
@@ -1356,7 +1356,7 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
script_calls = [
call
for call in m.call_args_list
if call.args and call.args[0] and call.args[0][0] != "pdftotext"
if call.args and call.args[0] and call.args[0][0] not in ("pdftotext",)
]
self.assertEqual(script_calls, [])
+1 -8
View File
@@ -44,7 +44,6 @@ from documents import tasks
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.matching import UnsupportedWorkflowTriggerTypeError
from documents.matching import document_matches_workflow
from documents.matching import existing_document_matches_workflow
from documents.matching import prefilter_documents_by_workflowtrigger
@@ -2852,13 +2851,7 @@ class TestWorkflows(
doc = Document.objects.create(
title="test",
)
self.assertRaises(
UnsupportedWorkflowTriggerTypeError,
document_matches_workflow,
doc,
w,
99,
)
self.assertRaises(Exception, document_matches_workflow, doc, w, 99)
def test_removal_action_document_updated_workflow(self) -> None:
"""
+5 -9
View File
@@ -21,32 +21,28 @@ def uri_validator(value: str, allowed_schemes: set[str] | None = None) -> None:
parts = urlparse(value)
if not parts.scheme:
raise ValidationError(
_("Unable to parse URI %(value)s, missing scheme"),
_(f"Unable to parse URI {value}, missing scheme"),
params={"value": value},
)
elif not parts.netloc and not parts.path:
raise ValidationError(
_("Unable to parse URI %(value)s, missing net location or path"),
_(f"Unable to parse URI {value}, missing net location or path"),
params={"value": value},
)
if allowed_schemes and parts.scheme not in allowed_schemes:
raise ValidationError(
_(
"URI scheme '%(scheme)s' is not allowed. Allowed schemes: %(allowed_schemes)s",
f"URI scheme '{parts.scheme}' is not allowed. Allowed schemes: {', '.join(allowed_schemes)}",
),
params={
"value": value,
"scheme": parts.scheme,
"allowed_schemes": ", ".join(allowed_schemes),
},
params={"value": value, "scheme": parts.scheme},
)
except ValidationError:
raise
except Exception as e:
raise ValidationError(
_("Unable to parse URI %(value)s"),
_(f"Unable to parse URI {value}"),
params={"value": value},
) from e
+30 -22
View File
@@ -180,6 +180,7 @@ from documents.permissions import has_system_status_permission
from documents.permissions import permitted_document_ids
from documents.permissions import permitted_object_ids
from documents.permissions import set_permissions_for_object
from documents.permissions import user_is_unrestricted
from documents.plugins.date_parsing import get_date_parser
from documents.schema import generate_object_with_permissions_schema
from documents.search import SearchHit
@@ -1449,7 +1450,7 @@ class DocumentViewSet(
try:
lang = detect(doc.content)
except Exception:
logger.debug("Unable to detect language for document %s", doc.pk)
pass
meta["lang"] = lang
return Response(meta)
@@ -1487,12 +1488,13 @@ class DocumentViewSet(
with get_date_parser() as date_parser:
gen = date_parser.parse(doc.filename, doc.content)
dates = sorted(
set(
itertools.islice(
{
i
for i in itertools.islice(
gen,
settings.NUMBER_OF_SUGGESTED_DATES,
),
),
)
},
)
resp_data = {
@@ -1580,16 +1582,21 @@ class DocumentViewSet(
except ValueError as exc:
logger.exception(
"Invalid AI configuration while generating suggestions for "
"document %s",
"document %s: %s",
doc.pk,
exc,
exc_info=True,
)
raise ValidationError(
{"ai": [_("Invalid AI configuration.")]},
) from exc
except LLMTimeoutError:
except LLMTimeoutError as exc:
logger.exception(
"AI backend timed out while generating suggestions for document %s",
"AI backend timed out while generating suggestions for "
"document %s: %s",
doc.pk,
exc,
exc_info=True,
)
return Response(
{"ai": [_("AI backend request timed out.")]},
@@ -2062,7 +2069,7 @@ class DocumentViewSet(
doc_name, doc_data = serializer.validated_data.get("document")
version_label = serializer.validated_data.get("version_label")
t = int(mktime(datetime.now().timetuple())) # noqa: DTZ005 - mktime() requires a local time tuple
t = int(mktime(datetime.now().timetuple()))
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
@@ -2323,10 +2330,12 @@ class ChatStreamingView(GenericAPIView[Any]):
return HttpResponseForbidden("Insufficient permissions")
documents = Document.objects.filter(pk=document.pk)
unrestricted = False
else:
documents = Document.objects.filter(
id__in=permitted_document_ids(request.user),
)
unrestricted = user_is_unrestricted(request.user)
output_language = get_llm_output_language(
ai_config=ai_config,
@@ -2337,6 +2346,7 @@ class ChatStreamingView(GenericAPIView[Any]):
stream_chat_with_documents(
query_str=question,
documents=documents,
unrestricted=unrestricted,
output_language=output_language,
),
content_type="text/event-stream",
@@ -3332,7 +3342,7 @@ class PostDocumentView(GenericAPIView[Any]):
cf = serializer.validated_data.get("custom_fields")
from_webui = serializer.validated_data.get("from_webui")
t = int(mktime(datetime.now().timetuple())) # noqa: DTZ005 - mktime() requires a local time tuple
t = int(mktime(datetime.now().timetuple()))
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
@@ -4142,7 +4152,7 @@ class UiSettingsView(GenericAPIView[Any]):
user_resp["last_name"] = user.last_name
# strip <app_label>.
roles = (re.sub(r"^\w+.", "", perm) for perm in user.get_all_permissions())
roles = map(lambda perm: re.sub(r"^\w+.", "", perm), user.get_all_permissions())
return Response(
{
"user": user_resp,
@@ -5180,11 +5190,11 @@ class SystemStatusView(PassUserMixin):
f"{m.app}.{m.name}"
for m in MigrationRecorder.Migration.objects.all().order_by("id")
]
except Exception: # pragma: no cover
except Exception as e: # pragma: no cover
applied_migrations = []
db_status = "ERROR"
logger.exception(
"System status detected a possible problem while connecting to the database",
f"System status detected a possible problem while connecting to the database: {e}",
)
db_error = "Error connecting to database, check logs for more detail."
@@ -5200,10 +5210,10 @@ class SystemStatusView(PassUserMixin):
try:
client.ping()
redis_status = "OK"
except Exception:
except Exception as e:
redis_status = "ERROR"
logger.exception(
"System status detected a possible problem while connecting to redis",
f"System status detected a possible problem while connecting to redis: {e}",
)
redis_error = "Error connecting to redis, check logs for more detail."
@@ -5233,10 +5243,10 @@ class SystemStatusView(PassUserMixin):
else:
celery_active = "WARNING"
celery_error = "Celery worker responded unexpectedly."
except Exception:
except Exception as e:
celery_active = "ERROR"
logger.exception(
"System status detected a possible problem while connecting to celery",
f"System status detected a possible problem while connecting to celery: {e}",
)
celery_error = "Error connecting to celery, check logs for more detail."
@@ -5251,15 +5261,13 @@ class SystemStatusView(PassUserMixin):
index_dir = settings.INDEX_DIR
mtimes = [p.stat().st_mtime for p in index_dir.iterdir() if p.is_file()]
index_last_modified = (
make_aware(datetime.fromtimestamp(max(mtimes))) # noqa: DTZ006 - make_aware() requires a naive datetime
if mtimes
else None
make_aware(datetime.fromtimestamp(max(mtimes))) if mtimes else None
)
except Exception:
except Exception as e:
index_status = "ERROR"
index_error = "Error opening index, check logs for more detail."
logger.exception(
"System status detected a possible problem while opening the index",
f"System status detected a possible problem while opening the index: {e}",
)
index_last_modified = None
+5 -5
View File
@@ -66,7 +66,7 @@ def build_workflow_action_context(
else None
)
filename = document.original_file or ""
filename = document.original_file if document.original_file else ""
return {
"title": overrides.title
if overrides and overrides.title
@@ -179,9 +179,9 @@ def execute_email_action(
f"Sent {n_messages} notification email(s) to {action.email.to}",
extra={"group": logging_group},
)
except Exception:
except Exception as e:
logger.exception(
"Error occurred sending notification email",
f"Error occurred sending notification email: {e}",
extra={"group": logging_group},
)
@@ -265,9 +265,9 @@ def execute_webhook_action(
f"Webhook to {action.webhook.url} queued",
extra={"group": logging_group},
)
except Exception:
except Exception as e:
logger.exception(
"Error occurred sending webhook",
f"Error occurred sending webhook: {e}",
extra={"group": logging_group},
)
+1 -1
View File
@@ -47,7 +47,7 @@ def resolve_date(dates: list[str]) -> date | None:
"""
for value in dates:
try:
return datetime.strptime(value, "%Y-%m-%d").date() # noqa: DTZ007 - only the calendar date is used, time/tz is discarded
return datetime.strptime(value, "%Y-%m-%d").date()
except (TypeError, ValueError):
logger.debug("Ignoring unparsable suggested date %s", value)
return None
+1 -1
View File
@@ -70,6 +70,6 @@ def send_webhook(
logger.error(
f"Failed attempt sending webhook to {url}: {e}",
)
raise
raise e
finally:
transport.close()
+1 -2
View File
@@ -241,7 +241,7 @@ def check_v3_minimum_upgrade_version(
return []
logger = logging.getLogger(__name__)
last_applied = max(applied) if applied else "(none)"
last_applied = sorted(applied)[-1] if applied else "(none)"
logger.error(
"V3 upgrade check failed: last applied documents migration is %r. "
"Expected '1075_workflowaction_order' (v2.20.15). "
@@ -341,7 +341,6 @@ def get_tesseract_langs():
proc = subprocess.run(
[shutil.which("tesseract"), "--list-langs"],
capture_output=True,
check=False,
)
# Decode bytes to string, split on newlines, trim out the header
+3 -3
View File
@@ -84,7 +84,7 @@ def get_parser_registry() -> ParserRegistry:
ParserRegistry
The shared registry singleton.
"""
global _registry, _discovery_complete # noqa: PLW0603 - module-level singleton, no class to hold this state
global _registry, _discovery_complete
with _lock:
if _registry is None:
@@ -113,7 +113,7 @@ def init_builtin_parsers() -> None:
-------
None
"""
global _registry # noqa: PLW0603 - module-level singleton, no class to hold this state
global _registry
with _lock:
if _registry is None:
@@ -137,7 +137,7 @@ def reset_parser_registry() -> None:
-------
None
"""
global _registry, _discovery_complete # noqa: PLW0603 - module-level singleton, no class to hold this state
global _registry, _discovery_complete
_registry = None
_discovery_complete = False
+2 -2
View File
@@ -78,7 +78,7 @@ class RemoteEngineConfig:
def engine_is_valid(self) -> bool:
"""Return True when the engine is known and fully configured."""
return (
self.engine == "azureai"
self.engine in ("azureai",)
and self.api_key is not None
and not (self.engine == "azureai" and self.endpoint is None)
)
@@ -505,7 +505,7 @@ class RemoteDocumentParser:
return result.content
except Exception as e:
logger.exception("Azure AI Vision parsing failed")
logger.exception("Azure AI Vision parsing failed: %s", e)
raise ParseError(f"Azure AI Vision parsing failed: {e}") from e
finally:
+3 -4
View File
@@ -306,9 +306,8 @@ def extract_pdf_metadata(
for key, value in meta.items():
if isinstance(value, list):
str_value = " ".join(str(e) for e in value)
else:
str_value = str(value)
value = " ".join(str(e) for e in value)
value = str(value)
try:
m = namespace_pattern.match(key)
@@ -330,7 +329,7 @@ def extract_pdf_metadata(
namespace=namespace,
prefix=meta.REVERSE_NS[namespace],
key=key_value,
value=str_value,
value=value,
),
)
except Exception as e:
+9 -14
View File
@@ -294,7 +294,7 @@ if _CHANNELS_BACKEND.startswith("channels_redis."):
###############################################################################
EMAIL_HOST: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST", "localhost")
EMAIL_PORT: Final[int] = get_int_from_env("PAPERLESS_EMAIL_PORT", 25)
EMAIL_PORT: Final[int] = int(os.getenv("PAPERLESS_EMAIL_PORT", 25))
EMAIL_HOST_USER: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_USER", "")
EMAIL_HOST_PASSWORD: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_PASSWORD", "")
DEFAULT_FROM_EMAIL: Final[str] = os.getenv("PAPERLESS_EMAIL_FROM", EMAIL_HOST_USER)
@@ -381,9 +381,8 @@ ACCOUNT_SESSION_REMEMBER = get_bool_from_env(
"True",
)
SESSION_EXPIRE_AT_BROWSER_CLOSE = not ACCOUNT_SESSION_REMEMBER
SESSION_COOKIE_AGE = get_int_from_env(
"PAPERLESS_SESSION_COOKIE_AGE",
60 * 60 * 24 * 7 * 3,
SESSION_COOKIE_AGE = int(
os.getenv("PAPERLESS_SESSION_COOKIE_AGE", 60 * 60 * 24 * 7 * 3),
)
# https://docs.djangoproject.com/en/5.1/ref/settings/#std-setting-SESSION_ENGINE
SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
@@ -396,6 +395,7 @@ if AUTO_LOGIN_USERNAME:
def _parse_remote_user_settings() -> str:
global MIDDLEWARE, AUTHENTICATION_BACKENDS, REST_FRAMEWORK
enable = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER")
enable_api = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER_API")
if enable or enable_api:
@@ -454,6 +454,7 @@ if ALLOWED_HOSTS != ["*"]:
def _parse_paperless_url():
global CSRF_TRUSTED_ORIGINS, CORS_ALLOWED_ORIGINS, ALLOWED_HOSTS
url = os.getenv("PAPERLESS_URL")
if url:
CSRF_TRUSTED_ORIGINS.append(url)
@@ -613,8 +614,8 @@ USE_TZ = True
LOGGING_DIR.mkdir(parents=True, exist_ok=True)
LOGROTATE_MAX_SIZE = get_int_from_env("PAPERLESS_LOGROTATE_MAX_SIZE", 1024 * 1024)
LOGROTATE_MAX_BACKUPS = get_int_from_env("PAPERLESS_LOGROTATE_MAX_BACKUPS", 20)
LOGROTATE_MAX_SIZE = os.getenv("PAPERLESS_LOGROTATE_MAX_SIZE", 1024 * 1024)
LOGROTATE_MAX_BACKUPS = os.getenv("PAPERLESS_LOGROTATE_MAX_BACKUPS", 20)
LOGGING = {
"version": 1,
@@ -810,15 +811,9 @@ IGNORABLE_FILES: Final[list[str]] = [
"Thumbs.db",
]
CONSUMER_POLLING_INTERVAL = get_float_from_env(
"PAPERLESS_CONSUMER_POLLING_INTERVAL",
0.0,
)
CONSUMER_POLLING_INTERVAL = float(os.getenv("PAPERLESS_CONSUMER_POLLING_INTERVAL", 0))
CONSUMER_STABILITY_DELAY = get_float_from_env(
"PAPERLESS_CONSUMER_STABILITY_DELAY",
5.0,
)
CONSUMER_STABILITY_DELAY = float(os.getenv("PAPERLESS_CONSUMER_STABILITY_DELAY", 5))
CONSUMER_DELETE_DUPLICATES = get_bool_from_env("PAPERLESS_CONSUMER_DELETE_DUPLICATES")
+1 -1
View File
@@ -111,7 +111,7 @@ def parse_dict_from_str(
return False
settings: dict[str, Any] = copy.deepcopy(defaults) if defaults else {}
_type_map = type_map or {}
_type_map = type_map if type_map else {}
if not env_str:
return settings
@@ -114,17 +114,17 @@ def test_cache_hit_when_enabled() -> None:
assert settings.CACHALOT_TIMEOUT == 1
# Read a table to populate the cache
list(Tag.objects.values_list("id", flat=True))
list(list(Tag.objects.values_list("id", flat=True)))
# Invalidate the cache then read the database, there should be DB hit
invalidate_db_cache()
with CaptureQueriesContext(connection) as ctx:
list(Tag.objects.values_list("id", flat=True))
list(list(Tag.objects.values_list("id", flat=True)))
assert len(ctx)
# Doing the same request again should hit the cache, not the DB
with CaptureQueriesContext(connection) as ctx:
list(Tag.objects.values_list("id", flat=True))
list(list(Tag.objects.values_list("id", flat=True)))
assert not len(ctx)
# Wait the end of TTL
@@ -133,7 +133,7 @@ def test_cache_hit_when_enabled() -> None:
# Read the DB again. The DB should be hit because the cache has expired
with CaptureQueriesContext(connection) as ctx:
list(Tag.objects.values_list("id", flat=True))
list(list(Tag.objects.values_list("id", flat=True)))
assert len(ctx)
# Invalidate the cache at the end of test
@@ -149,7 +149,7 @@ def test_cache_is_disabled_by_default() -> None:
# Read the table multiple times: the DB should always be hit without cache
for _ in range(3):
with CaptureQueriesContext(connection) as ctx:
list(Tag.objects.values_list("id", flat=True))
list(list(Tag.objects.values_list("id", flat=True)))
assert len(ctx)
# Invalidate the cache at the end of test
+1 -1
View File
@@ -59,7 +59,7 @@ def test_ocr_to_dateparser_languages_exception(
raise RuntimeError("Simulated error")
with caplog.at_level(logging.WARNING):
monkeypatch.setattr(utils, "LocaleDataLoader", DummyLoader)
monkeypatch.setattr(utils, "LocaleDataLoader", lambda: DummyLoader())
result = utils.ocr_to_dateparser_languages("eng+fra")
assert result == []
assert (
+16 -5
View File
@@ -95,22 +95,27 @@ def _format_chat_metadata_trailer(references: list[dict[str, int | str]]) -> str
def stream_chat_with_documents(
query_str: str,
documents: QuerySet[Document],
*,
unrestricted: bool = False,
output_language: str | None = None,
):
try:
yield from _stream_chat_with_documents(
query_str,
documents,
unrestricted=unrestricted,
output_language=output_language,
)
except Exception:
logger.exception("Failed to stream document chat response")
except Exception as e:
logger.exception("Failed to stream document chat response: %s", e)
yield CHAT_ERROR_MESSAGE
def _stream_chat_with_documents(
query_str: str,
documents: QuerySet[Document],
*,
unrestricted: bool = False,
output_language: str | None = None,
):
if not documents.exists():
@@ -123,9 +128,15 @@ def _stream_chat_with_documents(
from llama_index.core.retrievers import VectorIndexRetriever
config = AIConfig()
filters = _document_id_filters(
str(pk) for pk in documents.values_list("pk", flat=True)
)
if unrestricted:
# The caller can see every document, so an id filter would never narrow
# the search, only risk exceeding the vector store's bound parameter
# limit (_MAX_IN_VALUES in vector_store.py) on large installs.
filters = None
else:
filters = _document_id_filters(
str(pk) for pk in documents.values_list("pk", flat=True)
)
# Hold the shared read lock for the whole operation: the query engine
# retrieves from the vector store again during synthesis, so the connection
+1 -1
View File
@@ -164,7 +164,7 @@ def test_get_ai_document_classification_failure(mock_run_llm_query, mock_documen
"""
mock_run_llm_query.side_effect = Exception("LLM query failed")
with pytest.raises(Exception): # noqa: B017 - mock injects a bare Exception
with pytest.raises(Exception):
get_ai_document_classification(mock_document)
+66 -19
View File
@@ -1,4 +1,8 @@
from __future__ import annotations
import json
from typing import TYPE_CHECKING
from typing import Any
from unittest.mock import MagicMock
from unittest.mock import patch
@@ -18,6 +22,11 @@ from paperless_ai.chat import _build_chat_prompt
from paperless_ai.chat import _build_refine_prompt
from paperless_ai.chat import stream_chat_with_documents
if TYPE_CHECKING:
from pathlib import Path
import pytest_mock
@pytest.fixture(autouse=True)
def patch_embed_model():
@@ -312,6 +321,30 @@ def test_stream_chat_unexpected_failure_returns_generic_error(caplog) -> None:
@pytest.mark.django_db
class TestStreamChatRetrieval:
@pytest.fixture
def captured_filters(self, mocker: pytest_mock.MockerFixture) -> list[Any]:
"""Stub out the AI client and the retriever, capturing the ``filters``
kwarg of every VectorIndexRetriever construction.
VectorIndexRetriever is imported inside _stream_chat_with_documents,
so it is patched at the llama_index source for the lazy import to
pick it up.
"""
captured: list[Any] = []
retriever = mocker.MagicMock()
retriever.retrieve.return_value = []
def capture_retriever(*args, **kwargs) -> pytest_mock.MockType:
captured.append(kwargs.get("filters"))
return retriever
mocker.patch("paperless_ai.chat.AIClient")
mocker.patch(
"llama_index.core.retrievers.VectorIndexRetriever",
side_effect=capture_retriever,
)
return captured
def test_no_nodes_yields_no_content_message(
self,
temp_llm_index_dir,
@@ -329,9 +362,9 @@ class TestStreamChatRetrieval:
def test_chat_filter_contains_only_requested_document_ids(
self,
temp_llm_index_dir,
mock_embed_model,
mocker,
temp_llm_index_dir: Path,
mock_embed_model: pytest_mock.MockType,
captured_filters: list[Any],
) -> None:
"""The MetadataFilter passed to the retriever must be scoped to the
requested documents only content from other indexed documents must
@@ -342,22 +375,6 @@ class TestStreamChatRetrieval:
indexing.llm_index_add_or_update_document(included)
indexing.llm_index_add_or_update_document(excluded)
# VectorIndexRetriever is imported inside _stream_chat_with_documents;
# patch it at the llama_index source so the lazy import picks it up.
captured_filters = []
mock_retriever = mocker.MagicMock()
mock_retriever.retrieve.return_value = []
def capture_retriever(*args, **kwargs):
captured_filters.append(kwargs.get("filters"))
return mock_retriever
mocker.patch("paperless_ai.chat.AIClient")
mocker.patch(
"llama_index.core.retrievers.VectorIndexRetriever",
side_effect=capture_retriever,
)
list(
chat.stream_chat_with_documents(
"question?",
@@ -372,6 +389,36 @@ class TestStreamChatRetrieval:
assert str(included.pk) in filter_values
assert str(excluded.pk) not in filter_values
def test_unrestricted_chat_skips_document_id_filter(
self,
temp_llm_index_dir: Path,
mock_embed_model: pytest_mock.MockType,
captured_filters: list[Any],
) -> None:
"""
GIVEN:
- A document indexed in the vector store
WHEN:
- stream_chat_with_documents is called with unrestricted=True
THEN:
- The retriever receives no document id filter (filters=None), so
the whole index is searched instead of an IN-list that risks the
vector store's safety limit on large installs
"""
document = DocumentFactory.create(content="indexed document content")
indexing.llm_index_add_or_update_document(document)
list(
chat.stream_chat_with_documents(
"question?",
Document.objects.filter(pk=document.pk),
unrestricted=True,
),
)
assert captured_filters, "VectorIndexRetriever was never constructed"
assert captured_filters[0] is None
@pytest.mark.django_db
def test_get_document_references_only_queries_referenced_documents(
self,
@@ -21,6 +21,5 @@ class TestLazyAiImports:
capture_output=True,
text=True,
cwd=_SRC_DIR,
check=False,
)
assert result.returncode == 0, result.stdout + result.stderr
+8 -7
View File
@@ -7,6 +7,7 @@ import ssl
import tempfile
import traceback
import unicodedata
from datetime import date
from datetime import timedelta
from fnmatch import fnmatch
from pathlib import Path
@@ -405,7 +406,7 @@ def make_criterias(rule: MailRule, *, supports_gmail_labels: bool):
Returns criteria to be applied to MailBox.fetch for the given rule.
"""
maximum_age = timezone.localdate() - timedelta(days=rule.maximum_age)
maximum_age = date.today() - timedelta(days=rule.maximum_age)
criterias = {}
if rule.maximum_age > 0:
criterias["date_gte"] = maximum_age
@@ -722,9 +723,9 @@ class MailAccountHandler(LoggingMixin):
f"Rule {rule}: Stopping processing rules due to stop_processing flag",
)
break
except Exception:
except Exception as e:
self.log.exception(
f"Rule {rule}: Error while processing rule",
f"Rule {rule}: Error while processing rule: {e}",
)
except MailError:
raise
@@ -766,8 +767,8 @@ class MailAccountHandler(LoggingMixin):
self.log.info(f"Located folder: {folder_info.name}")
except Exception as e:
self.log.error(
"Exception during folder listing, unable to provide list folders: %s",
str(e),
"Exception during folder listing, unable to provide list folders: "
+ str(e),
)
raise MailError(
@@ -873,9 +874,9 @@ class MailAccountHandler(LoggingMixin):
total_processed_files += processed_files
mails_processed += 1
except Exception:
except Exception as e:
self.log.exception(
f"Rule {rule}: Error while processing mail {message.uid}",
f"Rule {rule}: Error while processing mail {message.uid}: {e}",
)
self.log.debug(f"Rule {rule}: Processed {mails_processed} matching mail(s)")
+1 -5
View File
@@ -11,10 +11,6 @@ from imap_tools import MailMessage
from documents.loggers import LoggingMixin
class MailDecryptionError(Exception):
pass
class MailMessagePreprocessor(abc.ABC):
"""
Defines the interface for preprocessors that alter messages before they are handled in MailAccountHandler
@@ -73,7 +69,7 @@ class MailMessageDecryptor(MailMessagePreprocessor, LoggingMixin):
f"Message decryption failed with status message "
f"{decrypted_raw_message.status}",
)
raise MailDecryptionError(
raise Exception(
f"Decryption failed: {decrypted_raw_message.status}, {decrypted_raw_message.stderr}",
)
self.log.debug("Message decrypted successfully.")
+1 -1
View File
@@ -50,7 +50,7 @@ class ProcessedMailFactory(DjangoModelFactory[ProcessedMail]):
rule = factory.SubFactory(MailRuleFactory)
folder = "INBOX"
uid = factory.Sequence(str)
uid = factory.Sequence(lambda n: str(n))
subject = factory.Faker("sentence", nb_words=4)
received = factory.LazyFunction(timezone.now)
processed = factory.LazyFunction(timezone.now)
+1 -1
View File
@@ -214,7 +214,7 @@ class BogusMailBox(AbstractContextManager):
)
self.messages = list(filter(lambda m: m.uid not in uid_list, self.messages))
else:
raise Exception # noqa: TRY002 - test double simulating a generic mailbox failure
raise Exception
def fake_magic_from_buffer(buffer, *, mime=False):
@@ -14,7 +14,6 @@ from imap_tools import MailMessage
from paperless_mail.mail import MailAccountHandler
from paperless_mail.models import MailRule
from paperless_mail.preprocessor import MailDecryptionError
from paperless_mail.preprocessor import MailMessageDecryptor
from paperless_mail.tests.factories import MailAccountFactory
from paperless_mail.tests.test_mail import TestMail
@@ -83,9 +82,7 @@ class MessageEncryptor:
armor=True,
)
if not encrypted_data.ok:
raise Exception( # noqa: TRY002 - test fixture setup, not production code
f"Encryption failed: {encrypted_data.stderr}",
)
raise Exception(f"Encryption failed: {encrypted_data.stderr}")
encrypted_email_content = encrypted_data.data
new_email = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
@@ -187,11 +184,7 @@ class TestMailMessageGpgDecryptor(TestMail):
EMAIL_GNUPG_HOME=empty_gpg_home,
):
message_decryptor = MailMessageDecryptor()
self.assertRaises(
MailDecryptionError,
message_decryptor.run,
encrypted_message,
)
self.assertRaises(Exception, message_decryptor.run, encrypted_message)
finally:
# Clean up the temporary GPG home used only by this test
try:
+2 -1
View File
@@ -1,3 +1,4 @@
import datetime
import logging
from datetime import timedelta
from http import HTTPStatus
@@ -86,7 +87,7 @@ class MailAccountViewSet(PassUserMixin, ModelViewSet[MailAccount]):
@action(methods=["post"], detail=False)
def test(self, request):
logger = logging.getLogger("paperless_mail")
request.data["name"] = timezone.now().isoformat()
request.data["name"] = datetime.datetime.now().isoformat()
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
existing_account = None