Compare commits

..
Author SHA1 Message Date
shamoon 2c329255a9 email and webhooks too 2026-10-26 21:07:00 -07:00
shamoon 7f7c7cc3c7 Support custom fields in workflow title templates 2026-10-26 21:02:00 -07:00
Jan Kleineandshamoon 7d8147c21d Performance: reduce tasks badge load times (#14552)
---------

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
2026-10-26 14:23:33 +00:00
18 changed files with 253 additions and 517 deletions

No files matched your search

-11
View File
@@ -162,17 +162,6 @@ including `gt` (>), `gte` (>=), `lt` (<), `lte` (<=), and `range`.
Lastly, document link fields support a `contains` operator that behaves
like a "is superset of" check.
Date fields also accept relative expressions supported by
[dateparser](https://dateparser.readthedocs.io/en/stable/), such as `today`,
`in 30 days`, or `1 month ago`. For example, dates within the next 30 days:
```json
["expiry", "range", ["today", "in 30 days"]]
```
Expressions resolve each time the query runs, using `PAPERLESS_TIME_ZONE` and
`PAPERLESS_DATE_PARSER_LANGUAGES` (automatic language detection when unset).
### `/api/search/autocomplete/`
Get auto completions for a partial search term.
+1
View File
@@ -726,6 +726,7 @@ applied. You can use the following placeholders in the template with any trigger
- `{{added_time}}`: added time in HH:MM format
- `{{original_filename}}`: original file name without extension
- `{{filename}}`: current file name without extension (for "added" workflows this may not be final yet, you can use `{{original_filename}}`)
- `{{custom_fields}}`: Custom field names mapped to their type and value, as in [file name templates](advanced_usage.md#additional-variables). For example, `{{ custom_fields.Invoice.value }}` or `{{ custom_fields | get_cf_value("Invoice", "") }}` for a fallback when the field is missing. In title assignments, fields assigned by the same action are available.
- `{{doc_title}}`: current document title (cannot be used in title assignment)
The following placeholders are only available for "added" or "updated" triggers
@@ -19,7 +19,6 @@ import {
PaperlessTaskTriggerSource,
PaperlessTaskType,
} from 'src/app/data/paperless-task'
import { Results } from 'src/app/data/results'
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
import { PermissionsGuard } from 'src/app/guards/permissions.guard'
import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
@@ -153,11 +152,6 @@ const tasks: PaperlessTask[] = [
},
]
const paginatedTasks: Results<PaperlessTask> = {
count: tasks.length,
results: tasks,
}
const sectionCountResponse = {
all: 7,
needs_attention: 2,
@@ -204,7 +198,7 @@ describe('TasksComponent', () => {
}).compileComponents()
tasksService = TestBed.inject(TasksService)
reloadSpy = jest.spyOn(tasksService, 'reload')
reloadSpy = jest.spyOn(tasksService, 'reload').mockImplementation(() => {})
httpTestingController = TestBed.inject(HttpTestingController)
modalService = TestBed.inject(NgbModal)
router = TestBed.inject(Router)
@@ -214,15 +208,6 @@ describe('TasksComponent', () => {
jest.useFakeTimers()
fixture.detectChanges()
httpTestingController
.expectOne(
(req) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
)
.flush(paginatedTasks)
httpTestingController
.expectOne(
(req) =>
@@ -344,11 +344,11 @@
ngbPopover="Tasks" i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end"
container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
<i-bs class="me-2" name="list-task"></i-bs><span class="nav-link-label"><ng-container i18n>Tasks</ng-container></span>
@if (tasksService.needsAttentionTasks.length > 0 && !slimSidebarEnabled) {
<span class="badge bg-danger ms-2 d-inline flex-shrink-0">{{tasksService.needsAttentionTasks.length}}</span>
@if (tasksService.needsAttentionTasksCount > 0 && !slimSidebarEnabled) {
<span class="badge bg-danger ms-2 d-inline flex-shrink-0">{{tasksService.needsAttentionTasksCount}}</span>
}
@if (tasksService.needsAttentionTasks.length > 0 && slimSidebarEnabled) {
<span class="badge bg-danger position-absolute top-0 end-0 d-none d-md-block">{{tasksService.needsAttentionTasks.length}}</span>
@if (tasksService.needsAttentionTasksCount > 0 && slimSidebarEnabled) {
<span class="badge bg-danger position-absolute top-0 end-0 d-none d-md-block">{{tasksService.needsAttentionTasksCount}}</span>
}
</a>
</li>
@@ -588,8 +588,8 @@ describe('AppFrameComponent', () => {
it('should show tasks badge for needs-attention tasks', () => {
jest
.spyOn(tasksService, 'needsAttentionTasks', 'get')
.mockReturnValue([{} as any, {} as any])
.spyOn(tasksService, 'needsAttentionTasksCount', 'get')
.mockReturnValue(2)
fixture.detectChanges()
@@ -33,54 +33,21 @@
<ng-template #comparisonValueTemplate let-atom="atom">
@if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Date) {
@let relativeDate = getRelativeDate(atom);
<div class="d-flex gap-2 w-100 ms-0 mt-1">
<fieldset class="btn-group btn-group-sm flex-shrink-0 date-mode" aria-label="Date mode" i18n-aria-label>
<input type="radio" class="btn-check" [id]="atom.id + '_fixed'" [name]="atom.id + '_dateMode'" [checked]="!relativeDate" (change)="toggleRelativeDate(atom)" [disabled]="disabled" aria-label="Fixed date" i18n-aria-label>
<label class="btn btn-outline-secondary text-nowrap" [for]="atom.id + '_fixed'" title="Fixed date" i18n-title>
<i-bs name="calendar"></i-bs><span class="d-none d-md-inline ms-2" i18n>Fixed</span>
</label>
<input type="radio" class="btn-check" [id]="atom.id + '_relative'" [name]="atom.id + '_dateMode'" [checked]="!!relativeDate" (change)="toggleRelativeDate(atom)" [disabled]="disabled" aria-label="Relative date" i18n-aria-label>
<label class="btn btn-outline-secondary text-nowrap" [for]="atom.id + '_relative'" title="Relative date" i18n-title>
<i-bs name="clock-history"></i-bs><span class="d-none d-md-inline ms-2" i18n>Relative</span>
</label>
</fieldset>
<div class="input-group input-group-sm flex-grow-1">
@if (relativeDate) {
@if (relativeDate.unit.includes('{n}')) {
<input class="form-control w-25 flex-grow-0" type="number" min="0" aria-label="Amount" i18n-aria-label
[ngModel]="relativeDate.amount"
(ngModelChange)="updateRelativeDate(atom, { amount: $event })"
[ngModelOptions]="{standalone: true}" />
}
<label class="visually-hidden" [for]="atom.id + '_unit'" i18n>Unit</label>
<select class="form-select" [id]="atom.id + '_unit'"
[ngModel]="relativeDate.unit"
(ngModelChange)="updateRelativeDate(atom, { unit: $event })"
[ngModelOptions]="{standalone: true}">
@for (unit of relativeDateUnits; track unit.id) {
<option [ngValue]="unit.id">{{unit.label}}</option>
}
</select>
} @else {
<input class="form-control" placeholder="yyyy-mm-dd"
[(ngModel)]="atom.value"
[ngModelOptions]="{standalone: true}"
ngbDatepicker
#d="ngbDatepicker"
[footerTemplate]="datePickerFooterTemplate" />
<button class="btn btn-sm btn-outline-secondary" (click)="d.toggle()" type="button" aria-label="Open date picker" i18n-aria-label>
<i-bs name="calendar-event"></i-bs>
</button>
<ng-template #datePickerFooterTemplate>
<div class="btn-group-xs border-top p-2 d-flex">
<button type="button" class="btn btn-primary" (click)="atom.value = today; d.close()" i18n>Today</button>
<button type="button" class="btn btn-secondary ms-auto" (click)="d.close()" i18n>Close</button>
</div>
</ng-template>
}
<input class="form-control" placeholder="yyyy-mm-dd"
[(ngModel)]="atom.value"
[ngModelOptions]="{standalone: true}"
ngbDatepicker
#d="ngbDatepicker"
[footerTemplate]="datePickerFooterTemplate" />
<button class="btn btn-sm btn-outline-secondary rounded-end" (click)="d.toggle()" type="button" aria-label="Open date picker" i18n-aria-label>
<i-bs name="calendar-event"></i-bs>
</button>
<ng-template #datePickerFooterTemplate>
<div class="btn-group-xs border-top p-2 d-flex">
<button type="button" class="btn btn-primary" (click)="atom.value = today; d.close()" i18n>Today</button>
<button type="button" class="btn btn-secondary ms-auto" (click)="d.close()" i18n>Close</button>
</div>
</div>
</ng-template>
} @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Float || getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Integer) {
<input class="w-25 form-control rounded-end" type="number" [(ngModel)]="atom.value" [ngModelOptions]="{standalone: true}" [disabled]="disabled">
} @else if (getCustomFieldByID(atom.field)?.data_type === CustomFieldDataType.Boolean) {
@@ -116,7 +83,6 @@
</ng-template>
<ng-template #queryAtom let-atom="atom">
<label class="visually-hidden" [for]="atom.id + '_operator'" i18n>Operator</label>
<div class="input-group input-group-sm">
<ng-select #fieldSelects
class="paperless-input-select"
@@ -130,7 +96,7 @@
[appendTo]="selectAppendTo"
(mousedown)="$event.stopImmediatePropagation()"
></ng-select>
<select class="w-25 form-select" [id]="atom.id + '_operator'" [class.rounded-end]="hasDateValueRow(atom)" [(ngModel)]="atom.operator" [ngModelOptions]="{standalone: true}" [disabled]="disabled">
<select class="w-25 form-select" [(ngModel)]="atom.operator" [ngModelOptions]="{standalone: true}" [disabled]="disabled">
@for (operator of getOperatorsForField(atom.field); track operator.label) {
<option [ngValue]="operator.value">{{operator.label}}</option>
}
@@ -185,10 +151,10 @@
<input class="w-25 form-control rounded-end" type="text" [(ngModel)]="atom.value" [ngModelOptions]="{standalone: true}" [disabled]="disabled">
}
}
<button class="btn btn-link btn-sm text-danger pe-0" type="button" (click)="removeElement(atom)" [disabled]="disabled" aria-label="Remove query" i18n-aria-label>
<i-bs name="x-circle"></i-bs>
</button>
</div>
<button class="btn btn-link btn-sm text-danger pe-0 align-self-center" type="button" (click)="removeElement(atom)" [disabled]="disabled" aria-label="Remove query" i18n-aria-label>
<i-bs name="x-circle"></i-bs>
</button>
</ng-template>
<ng-template #queryExpression let-expression="expression">
@@ -5,11 +5,6 @@
}
}
:host {
--ng-select-border-radius: var(--bs-border-radius);
--ng-select-height: calc(1.5em + 0.5rem + calc(var(--bs-border-width) * 2));
}
:host ::ng-deep .ng-select-container {
border-top-right-radius: 0 !important;
border-bottom-right-radius: 0 !important;
@@ -46,13 +41,3 @@
min-width: 140px;
}
}
.date-mode .btn {
--bs-btn-color: var(--bs-secondary-color);
--bs-btn-hover-bg: transparent;
--bs-btn-hover-color: var(--bs-body-color);
--bs-btn-hover-border-color: var(--bs-border-color);
--bs-btn-active-bg: transparent;
--bs-btn-active-color: var(--bs-primary);
--bs-btn-active-border-color: var(--bs-border-color);
}
@@ -3,7 +3,7 @@ import { provideHttpClientTesting } from '@angular/common/http/testing'
import { LOCALE_ID } from '@angular/core'
import { ComponentFixture, TestBed } from '@angular/core/testing'
import { FormsModule, ReactiveFormsModule } from '@angular/forms'
import { NgbDateAdapter, NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
import { NgbDropdownModule } from '@ng-bootstrap/ng-bootstrap'
import { NgSelectModule } from '@ng-select/ng-select'
import { allIcons, NgxBootstrapIconsModule } from 'ngx-bootstrap-icons'
import { of } from 'rxjs'
@@ -11,7 +11,6 @@ import { CustomField, CustomFieldDataType } from 'src/app/data/custom-field'
import {
CUSTOM_FIELD_QUERY_OPERATORS_BY_GROUP,
CustomFieldQueryLogicalOperator,
CustomFieldQueryOperator,
CustomFieldQueryOperatorGroups,
} from 'src/app/data/custom-field-query'
import { CustomFieldsService } from 'src/app/services/rest/custom-fields.service'
@@ -20,7 +19,6 @@ import {
CustomFieldQueryElement,
CustomFieldQueryExpression,
} from 'src/app/utils/custom-field-query-element'
import { ISODateAdapter } from 'src/app/utils/ngb-iso-date-adapter'
import {
CustomFieldQueriesModel,
CustomFieldsQueryDropdownComponent,
@@ -50,12 +48,6 @@ const customFields = [
data_type: CustomFieldDataType.Monetary,
extra_data: { default_currency: 'EUR' },
},
{
id: 4,
name: 'Test Date Field',
data_type: CustomFieldDataType.Date,
extra_data: {},
},
]
describe('CustomFieldsQueryDropdownComponent', () => {
@@ -77,7 +69,6 @@ describe('CustomFieldsQueryDropdownComponent', () => {
provideHttpClient(withInterceptorsFromDi()),
provideHttpClientTesting(),
{ provide: LOCALE_ID, useValue: 'de' },
{ provide: NgbDateAdapter, useClass: ISODateAdapter },
],
}).compileComponents()
@@ -246,74 +237,6 @@ describe('CustomFieldsQueryDropdownComponent', () => {
expect(component.selectionModel.queries.length).toBe(1)
})
it('should build relative date values from amount and unit', async () => {
component.useDropdown = false
const atom = new CustomFieldQueryAtom([
4,
CustomFieldQueryOperator.GreaterThanOrEqual,
'2026-10-22',
])
component.selectionModel.addAtom(atom)
fixture.detectChanges()
await fixture.whenStable()
const query = (selector: string) =>
fixture.nativeElement.querySelector(selector)
expect(query('[ngbDatepicker]')).not.toBeNull()
expect(query('input[id$="_fixed"]').checked).toBe(true)
query('input[id$="_relative"]').click()
fixture.detectChanges()
await fixture.whenStable()
expect(atom.value).toBeNull()
const amount: HTMLInputElement = query('input[aria-label="Amount"]')
const unit: HTMLSelectElement = query('select[id$="_unit"]')
amount.value = '30'
amount.dispatchEvent(new Event('input'))
expect(atom.serialize()).toEqual([4, 'gte', '30 days ago'])
unit.selectedIndex = 6
unit.dispatchEvent(new Event('change'))
expect(atom.serialize()).toEqual([4, 'gte', 'in 30 weeks'])
unit.selectedIndex = 0
unit.dispatchEvent(new Event('change'))
fixture.detectChanges()
expect(atom.serialize()).toEqual([4, 'gte', 'today'])
expect(query('input[aria-label="Amount"]')).toBeNull()
query('input[id$="_fixed"]').click()
fixture.detectChanges()
expect(atom.value).toBeNull()
expect(query('[ngbDatepicker]')).not.toBeNull()
})
it('should restore a saved relative date value', async () => {
component.useDropdown = false
const atom = new CustomFieldQueryAtom([
4,
CustomFieldQueryOperator.Exact,
'in 2 months',
])
component.selectionModel.addAtom(atom)
fixture.detectChanges()
await fixture.whenStable()
expect(component.getRelativeDate(atom)).toEqual({
amount: 2,
unit: 'in {n} months',
})
expect(
fixture.nativeElement.querySelector('input[aria-label="Amount"]').value
).toBe('2')
const today = new CustomFieldQueryAtom([
4,
CustomFieldQueryOperator.Exact,
'today',
])
expect(component.getRelativeDate(today)).toEqual({
amount: null,
unit: 'today',
})
})
describe('CustomFieldQueriesModel', () => {
let model: CustomFieldQueriesModel
@@ -49,26 +49,6 @@ import { LoadingComponentWithPermissions } from '../../loading-component/loading
import { ClearableBadgeComponent } from '../clearable-badge/clearable-badge.component'
import { DocumentLinkComponent } from '../input/document-link/document-link.component'
interface RelativeDate {
// The expression sent to the API, with {n} standing in for the amount
amount: number | null
unit: string
}
function parseRelativeDate(
value: string,
unit: string
): RelativeDate | undefined {
const [prefix, suffix] = unit.split('{n}')
if (suffix === undefined) {
return value === unit ? { amount: null, unit } : undefined
}
if (!value.startsWith(prefix) || !value.endsWith(suffix)) return undefined
const amount = value.slice(prefix.length, value.length - suffix.length)
if (!amount || !Number.isInteger(+amount) || +amount < 0) return undefined
return { amount: +amount, unit }
}
export class CustomFieldQueriesModel {
private readonly _queries = signal<CustomFieldQueryElement[]>([])
private rootSubscriptions: Subscription[] = []
@@ -325,67 +305,6 @@ export class CustomFieldsQueryDropdownComponent extends LoadingComponentWithPerm
public readonly today: string = new Date().toLocaleDateString('en-CA')
public readonly relativeDateUnits = [
{ id: 'today', label: $localize`Today` },
{ id: '{n} days ago', label: $localize`days ago` },
{ id: '{n} weeks ago', label: $localize`weeks ago` },
{ id: '{n} months ago', label: $localize`months ago` },
{ id: '{n} years ago', label: $localize`years ago` },
{ id: 'in {n} days', label: $localize`days from now` },
{ id: 'in {n} weeks', label: $localize`weeks from now` },
{ id: 'in {n} months', label: $localize`months from now` },
{ id: 'in {n} years', label: $localize`years from now` },
]
private readonly relativeDates = new Map<string, RelativeDate>()
getRelativeDate(atom: CustomFieldQueryAtom): RelativeDate | undefined {
if (!this.relativeDates.has(atom.id) && typeof atom.value === 'string') {
for (const { id } of this.relativeDateUnits) {
const relativeDate = parseRelativeDate(atom.value, id)
if (relativeDate) {
this.relativeDates.set(atom.id, relativeDate)
break
}
}
}
return this.relativeDates.get(atom.id)
}
hasDateValueRow(atom: CustomFieldQueryAtom): boolean {
return (
this.getCustomFieldByID(atom.field)?.data_type ===
CustomFieldDataType.Date &&
atom.operator !== CustomFieldQueryOperator.Exists &&
atom.operator !== CustomFieldQueryOperator.IsNull
)
}
toggleRelativeDate(atom: CustomFieldQueryAtom) {
if (this.getRelativeDate(atom)) {
this.relativeDates.delete(atom.id)
atom.value = null
} else {
this.relativeDates.set(atom.id, { amount: null, unit: '{n} days ago' })
this.updateRelativeDate(atom, {})
}
}
updateRelativeDate(
atom: CustomFieldQueryAtom,
change: Partial<RelativeDate>
) {
const { amount, unit } = Object.assign(
this.relativeDates.get(atom.id),
change
)
if (!unit.includes('{n}')) {
atom.value = unit
} else {
atom.value = amount === null ? null : unit.replace('{n}', `${amount}`)
}
}
public customFieldSearchFn = (term: string, field: CustomField): boolean =>
matchesSearchText(field?.name, term)
+22 -82
View File
@@ -9,11 +9,7 @@ import {
} from '@angular/common/http/testing'
import { TestBed } from '@angular/core/testing'
import { environment } from 'src/environments/environment'
import {
PaperlessTaskStatus,
PaperlessTaskTriggerSource,
PaperlessTaskType,
} from '../data/paperless-task'
import { PaperlessTaskType } from '../data/paperless-task'
import { TasksService } from './tasks.service'
describe('TasksService', () => {
@@ -38,23 +34,23 @@ describe('TasksService', () => {
httpTestingController.verify()
})
it('calls tasks api endpoint on reload', () => {
it('calls tasks status counts api endpoint on reload', () => {
tasksService.reload()
const req = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
expect(req.request.method).toEqual('GET')
req.flush({ count: 0, results: [] })
req.flush({ all: 5, needs_attention: 3, in_progress: 1, completed: 1 })
expect(tasksService.needsAttentionTasksCount).toEqual(3)
})
it('cancels an in-progress reload when reloading again', () => {
tasksService.reload()
const staleReload = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/`
req.url === `${environment.apiBaseUrl}tasks/status_counts/`
)
tasksService.reload()
@@ -62,9 +58,9 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/`
req.url === `${environment.apiBaseUrl}tasks/status_counts/`
)
.flush({ count: 0, results: [] })
.flush({ all: 0, needs_attention: 0, in_progress: 0, completed: 0 })
})
it('continues reloading after a reload request fails', () => {
@@ -72,7 +68,7 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/`
req.url === `${environment.apiBaseUrl}tasks/status_counts/`
)
.flush('error', { status: 500, statusText: 'error' })
@@ -82,16 +78,16 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/`
req.url === `${environment.apiBaseUrl}tasks/status_counts/`
)
.flush({ count: 0, results: [] })
.flush({ all: 0, needs_attention: 0, in_progress: 0, completed: 0 })
})
it('reloads after dismissing a task while a reload is already in progress', () => {
tasksService.reload()
const staleReload = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
@@ -104,12 +100,12 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
.flush({ count: 0, results: [] })
.flush({ all: 50, needs_attention: 42, in_progress: 3, completed: 5 })
expect(tasksService.needsAttentionTasks).toHaveLength(0)
expect(tasksService.needsAttentionTasksCount).toEqual(42)
})
it('calls acknowledge_tasks api endpoint on dismiss and reloads', () => {
@@ -126,11 +122,10 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
.flush({ count: 0, results: [] })
.flush({ all: 0, needs_attention: 0, in_progress: 0, completed: 0 })
})
it('calls acknowledge_tasks api endpoint on dismiss all and reloads', () => {
@@ -147,65 +142,10 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
.flush({ count: 0, results: [] })
})
it('includes revoked tasks in needs attention', () => {
const mockTasks = [
{
task_type: PaperlessTaskType.SanityCheck,
trigger_source: PaperlessTaskTriggerSource.System,
status: PaperlessTaskStatus.Failure,
acknowledged: false,
task_id: '1235',
input_data: {},
date_created: new Date(),
related_document_ids: [],
},
{
task_type: PaperlessTaskType.MailFetch,
trigger_source: PaperlessTaskTriggerSource.Scheduled,
status: PaperlessTaskStatus.Revoked,
acknowledged: false,
task_id: '1236',
input_data: {},
date_created: new Date(),
related_document_ids: [],
},
{
task_type: PaperlessTaskType.EmptyTrash,
trigger_source: PaperlessTaskTriggerSource.Manual,
status: PaperlessTaskStatus.Success,
acknowledged: false,
task_id: '1238',
input_data: {},
date_created: new Date(),
related_document_ids: [],
},
]
tasksService.reload()
const req = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
)
req.flush({ count: mockTasks.length, results: mockTasks })
expect(tasksService.needsAttentionTasks).toHaveLength(2)
expect(tasksService.needsAttentionTasks.map((task) => task.status)).toEqual(
expect.arrayContaining([
PaperlessTaskStatus.Failure,
PaperlessTaskStatus.Revoked,
])
)
.flush({ all: 0, needs_attention: 0, in_progress: 0, completed: 0 })
})
it('supports running tasks', () => {
+23 -33
View File
@@ -5,14 +5,12 @@ import {
catchError,
finalize,
first,
map,
switchMap,
takeUntil,
tap,
} from 'rxjs/operators'
import {
PaperlessTask,
PaperlessTaskStatus,
PaperlessTaskStatusCounts,
PaperlessTaskType,
} from 'src/app/data/paperless-task'
@@ -27,50 +25,42 @@ export class TasksService {
private baseUrl: string = environment.apiBaseUrl
private endpoint: string = 'tasks'
private readonly defaultReloadPageSize = 1000
public loading: boolean = false
private readonly tasks = signal<PaperlessTask[]>([])
private readonly tasksStatusCounts = signal<PaperlessTaskStatusCounts>({
all: 0,
needs_attention: 0,
in_progress: 0,
completed: 0,
})
private readonly reloadNotifier = new Subject<void>()
private unsubscribeNotifer: Subject<any> = new Subject()
constructor() {
this.reloadNotifier
.pipe(
switchMap(() => {
this.loading = true
return this.http
.get<Results<PaperlessTask>>(`${this.baseUrl}${this.endpoint}/`, {
params: {
acknowledged: 'false',
page_size: this.defaultReloadPageSize,
},
})
.pipe(
map((response) => response.results),
takeUntil(this.unsubscribeNotifer),
catchError(() => EMPTY),
finalize(() => {
this.loading = false
})
)
})
)
.subscribe((tasks) => {
this.tasks.set(tasks)
.pipe(switchMap(() => this.fetchUnacknowledgedTasksCounts()))
.subscribe((taskStatusCounts) => {
this.tasksStatusCounts.set(taskStatusCounts)
})
}
public get needsAttentionTasks(): PaperlessTask[] {
return this.tasks().filter((t) =>
[PaperlessTaskStatus.Failure, PaperlessTaskStatus.Revoked].includes(
t.status
)
private fetchUnacknowledgedTasksCounts(): Observable<PaperlessTaskStatusCounts> {
this.loading = true
return this.statusCounts({ acknowledged: false }).pipe(
takeUntil(this.unsubscribeNotifer),
catchError(() => EMPTY),
finalize(() => {
this.loading = false
})
)
}
public get needsAttentionTasksCount(): number {
return this.tasksStatusCounts().needs_attention
}
public reload() {
this.reloadNotifier.next()
}
@@ -105,7 +95,7 @@ export class TasksService {
public dismissTasks(task_ids: Set<number>): Observable<any> {
return this.http
.post(`${this.baseUrl}tasks/acknowledge/`, {
.post(`${this.baseUrl}${this.endpoint}/acknowledge/`, {
tasks: [...task_ids],
})
.pipe(
@@ -119,7 +109,7 @@ export class TasksService {
public dismissAllTasks(): Observable<any> {
return this.http
.post(`${this.baseUrl}tasks/acknowledge/`, {
.post(`${this.baseUrl}${this.endpoint}/acknowledge/`, {
all: true,
})
.pipe(
+13
View File
@@ -847,6 +847,19 @@ class ConsumerPlugin(
local_added,
self.filename,
self.filename,
custom_fields=[
CustomFieldInstance(
field=field,
**{
CustomFieldInstance.get_value_field_name(
field.data_type,
): self.metadata.custom_fields[field.pk],
},
)
for field in CustomField.objects.filter(
pk__in=self.metadata.custom_fields or {},
)
],
)
def _store(
-53
View File
@@ -11,8 +11,6 @@ from decimal import InvalidOperation
from typing import TYPE_CHECKING
from typing import Any
import dateparser
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db.models import Case
from django.db.models import CharField
@@ -27,7 +25,6 @@ from django.db.models import Value
from django.db.models import When
from django.db.models.functions import Cast
from django.db.models.functions import NullIf
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from django_filters import DateFilter
from django_filters.rest_framework import BooleanFilter
@@ -60,7 +57,6 @@ from documents.versioning import annotate_effective_content
if TYPE_CHECKING:
from collections.abc import Callable
from datetime import datetime
CHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"]
ID_KWARGS = ["in", "exact"]
@@ -326,51 +322,6 @@ class MonetaryAmountField(serializers.Field):
return str(value)
class DateQueryField(serializers.DateField):
"""
Accepts either a fixed date or a relative date expression using `dateparser` and
resolves it against the provided reference datetime.
"""
default_error_messages = {
"expression": _(
'Expected a date or a relative date such as "today" or "in 30 days".',
),
}
def __init__(self, *, reference_datetime: datetime, **kwargs):
super().__init__(**kwargs)
self.reference_datetime = reference_datetime
def to_internal_value(self, value):
try:
# Let the parent class handle fixed-date formats and string validation first
return super().to_internal_value(value)
except serializers.ValidationError:
if not isinstance(value, str):
raise
try:
parsed = dateparser.parse(
value,
# The UI always builds English expressions
locales=[*settings.DATE_PARSER_LANGUAGES, "en"]
if settings.DATE_PARSER_LANGUAGES
else None,
settings={
"PARSERS": ["relative-time"],
"RELATIVE_BASE": self.reference_datetime,
"TIMEZONE": str(self.reference_datetime.tzinfo),
"RETURN_AS_TIMEZONE_AWARE": True,
},
)
except (ValueError, OverflowError):
self.fail("expression")
if parsed is None:
self.fail("expression")
return parsed.astimezone(self.reference_datetime.tzinfo).date()
class SelectField(serializers.CharField):
def __init__(self, custom_field: CustomField) -> None:
self._options = custom_field.extra_data["select_options"]
@@ -512,8 +463,6 @@ class CustomFieldQueryParser:
raise serializers.ValidationError(
{self._validation_prefix: [_("Value must be valid JSON.")]},
)
# Save the current local time as the reference for the query
self._reference_datetime = timezone.localtime()
return (
self._parse_expr(expr, validation_prefix=self._validation_prefix),
self._annotations,
@@ -700,8 +649,6 @@ class CustomFieldQueryParser:
):
# DateField admits queries in the form of `year__exact`, etc. These take integers.
field = serializers.IntegerField()
elif custom_field.data_type == CustomField.FieldDataType.DATE:
field = DateQueryField(reference_datetime=self._reference_datetime)
elif custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
# We can be more specific here and make sure the value is a list.
field = serializers.ListField(child=serializers.IntegerField())
+6
View File
@@ -1,4 +1,5 @@
import logging
from collections.abc import Iterable
from datetime import date
from datetime import datetime
from pathlib import Path
@@ -13,7 +14,9 @@ from jinja2 import make_logging_undefined
from jinja2.meta import find_undeclared_variables
from jinja2.sandbox import SecurityError
from documents.models import CustomFieldInstance
from documents.templating.environment import _template_environment
from documents.templating.filepath import get_custom_fields_context
from documents.templating.filters import format_datetime
from documents.templating.filters import localize_date
@@ -56,6 +59,7 @@ _known_placeholder_names = {
"doc_title",
"doc_url",
"doc_id",
"custom_fields",
}
@@ -86,6 +90,7 @@ def parse_w_workflow_placeholders(
doc_title: str | None = None,
doc_url: str | None = None,
doc_id: int | None = None,
custom_fields: Iterable[CustomFieldInstance] = (),
) -> str:
"""
Available title placeholders for Workflows depend on what has already been assigned,
@@ -108,6 +113,7 @@ def parse_w_workflow_placeholders(
"original_filename": Path(original_filename).stem,
"filename": Path(filename).stem,
}
formatting.update(get_custom_fields_context(custom_fields))
if created is not None:
formatting.update(
{
@@ -2,12 +2,8 @@ import json
import types
from collections.abc import Callable
from datetime import date
from unittest.mock import patch
from urllib.parse import quote
import time_machine
from django.test import override_settings
from django.utils import timezone
from rest_framework.test import APITestCase
from documents.models import CustomField
@@ -447,65 +443,6 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
),
)
@override_settings(TIME_ZONE="America/Los_Angeles")
def test_relative_dates(self) -> None:
# Reevaluate the same mixed relative/fixed range on different local dates.
for today in (date(2024, 8, 22), date(2024, 11, 15)):
with (
self.subTest(today=today),
time_machine.travel(f"{today}T23:30:00-07:00", tick=False),
patch(
"documents.filters.timezone.localtime",
wraps=timezone.localtime,
) as localtime,
):
self._assert_query_match_predicate(
["date_field", "range", ["today", "2024-11-15"]],
lambda document, today=today: (
"date_field" in document
and document["date_field"] is not None
and today <= document["date_field"] <= date(2024, 11, 15)
),
)
localtime.assert_called_once_with()
def test_relative_date_validation(self) -> None:
for value, keyword in (
("not a date", "date"),
("January 2024", "date"),
("today + " + "9" * 5000 + " days", "date"),
(True, "format"),
):
with self.subTest(value=str(value)[:50]):
self._assert_validation_error(
json.dumps(["date_field", "exact", value]),
["custom_field_query", "2"],
keyword,
)
@time_machine.travel("2024-08-22T12:00:00Z", tick=False)
def test_relative_date_languages(self) -> None:
for locales in (["de"], ["zh-Hans", "de"], None):
with (
self.subTest(locales=locales),
override_settings(
DATE_PARSER_LANGUAGES=locales,
),
):
self._assert_query_match_predicate(
["date_field", "exact", "heute"],
lambda document: (
"date_field" in document
and document["date_field"] == date(2024, 8, 22)
),
)
with override_settings(DATE_PARSER_LANGUAGES=["en"]):
self._assert_validation_error(
json.dumps(["date_field", "exact", "heute"]),
["custom_field_query", "2"],
"date",
)
def test_gt_monetary(self) -> None:
self._assert_query_match_predicate(
["monetary_field", "gt", "99"],
+110
View File
@@ -1846,6 +1846,50 @@ class TestWorkflows(
expected_str = f"Document storage path {doc.storage_path} does not match {trigger.filter_has_storage_path}"
self.assertIn(expected_str, cm.output[1])
def test_title_custom_fields(self) -> None:
from documents.consumer import ConsumerPlugin
from documents.templating.workflows import validate_workflow_template
field = CustomField.objects.create(
name="Invoice",
data_type=CustomField.FieldDataType.STRING,
)
template = '{{ custom_fields.Invoice.value }} {{ custom_fields | get_cf_value("Missing", "fallback") }}'
validate_workflow_template(template)
consumer = mock.Mock(
filename="invoice.pdf",
metadata=DocumentMetadataOverrides(custom_fields={field.pk: "INV-123"}),
)
self.assertEqual(
ConsumerPlugin._parse_title_placeholders(consumer, template),
"INV-123 fallback",
)
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
action = WorkflowAction.objects.create(assign_title=template)
doc = Document.objects.create(title="Original")
action.assign_custom_fields.add(field)
action.assign_custom_fields_values = {str(field.pk): "INV-123"}
action.save()
workflow = Workflow.objects.create(name="Custom field title", order=0)
workflow.triggers.add(trigger)
workflow.actions.add(action)
document_consumption_finished.send(sender=self.__class__, document=doc)
doc.refresh_from_db()
self.assertEqual(doc.title, "INV-123 fallback")
# Existing fields remain available when the action assigns no custom fields.
action.assign_custom_fields.clear()
CustomFieldInstance.objects.filter(document=doc, field=field).update(
value_text="INV-456",
)
document_consumption_finished.send(sender=self.__class__, document=doc)
doc.refresh_from_db()
self.assertEqual(doc.title, "INV-456 fallback")
def test_document_added_invalid_title_placeholders(self) -> None:
"""
GIVEN:
@@ -5120,6 +5164,72 @@ class TestWorkflows(
self.assertEqual(Document.objects.count(), 0)
@pytest.mark.django_db
@pytest.mark.parametrize("use_overrides", [False, True])
def test_custom_fields_in_email_and_webhook_templates(
use_overrides,
settings,
tmp_path,
):
from documents.workflows.actions import build_workflow_action_context
from documents.workflows.actions import execute_email_action
from documents.workflows.actions import execute_webhook_action
settings.EMAIL_ENABLED = True
field = CustomField.objects.create(
name="Invoice",
data_type=CustomField.FieldDataType.STRING,
)
if use_overrides:
invoice = tmp_path / "invoice.pdf"
invoice.write_bytes(b"%PDF-1.7")
document = ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=invoice,
)
overrides = DocumentMetadataOverrides(custom_fields={field.pk: "INV-123"})
else:
document = Document.objects.create(title="Invoice")
CustomFieldInstance.objects.create(
document=document,
field=field,
value_text="INV-123",
)
overrides = None
context = build_workflow_action_context(document, overrides)
template = '{{ custom_fields.Invoice.value }} {{ custom_fields | get_cf_value("Missing", "fallback") }}'
action = WorkflowAction(
email=WorkflowActionEmail(
subject=template,
body=template,
to="test@example.com",
),
webhook=WorkflowActionWebhook(
url="https://example.com",
body=template,
),
)
with mock.patch("documents.workflows.actions.send_email") as send_email:
execute_email_action(
action,
document,
context,
None,
Path("invoice.pdf"),
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
assert send_email.call_args.kwargs["subject"] == "INV-123 fallback"
assert send_email.call_args.kwargs["body"] == "INV-123 fallback"
with mock.patch("documents.workflows.actions.send_webhook.apply_async") as send:
for use_params in (False, True):
action.webhook.use_params = use_params
action.webhook.params = {"invoice": template}
execute_webhook_action(action, document, context, None, Path("invoice.pdf"))
assert send.call_args.kwargs["kwargs"]["data"] == (
{"invoice": "INV-123 fallback"} if use_params else "INV-123 fallback"
)
class TestWebhookSend:
def test_send_webhook_data_or_json(
self,
+22
View File
@@ -11,6 +11,8 @@ from documents.data_models import DocumentMetadataOverrides
from documents.mail import EmailAttachment
from documents.mail import send_email
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import WorkflowAction
@@ -48,6 +50,9 @@ def build_workflow_action_context(
"added": timezone.localtime(document.added),
"created": document.created,
"id": document.pk,
"custom_fields": CustomFieldInstance.global_objects.filter(
document=document,
).select_related("field"),
}
correspondent_obj = (
@@ -80,6 +85,19 @@ def build_workflow_action_context(
"added": timezone.localtime(timezone.now()),
"created": overrides.created if overrides else None,
"id": "",
"custom_fields": [
CustomFieldInstance(
field=field,
**{
CustomFieldInstance.get_value_field_name(
field.data_type,
): overrides.custom_fields[field.pk],
},
)
for field in CustomField.objects.filter(
pk__in=overrides.custom_fields or {},
)
],
}
@@ -116,6 +134,7 @@ def execute_email_action(
context["title"],
context["doc_url"],
context["id"],
custom_fields=context["custom_fields"],
)
if action.email.subject
else ""
@@ -133,6 +152,7 @@ def execute_email_action(
context["title"],
context["doc_url"],
context["id"],
custom_fields=context["custom_fields"],
)
if action.email.body
else ""
@@ -211,6 +231,7 @@ def execute_webhook_action(
context["title"],
context["doc_url"],
context["id"],
custom_fields=context["custom_fields"],
)
except Exception as e:
logger.error(
@@ -230,6 +251,7 @@ def execute_webhook_action(
context["title"],
context["doc_url"],
context["id"],
custom_fields=context["custom_fields"],
)
headers = {}
if action.webhook.headers:
+30 -27
View File
@@ -43,6 +43,33 @@ def apply_assignment_to_document(
if action.assign_owner:
document.owner = action.assign_owner
if action.has_assign_custom_fields:
for field in action.assign_custom_fields.all():
value_field_name = CustomFieldInstance.get_value_field_name(
data_type=field.data_type,
)
args = {
value_field_name: action.assign_custom_fields_values.get(
str(field.pk),
None,
),
}
# for some reason update_or_create doesn't work here
instance = CustomFieldInstance.objects.filter(
field=field,
document=document,
).first()
# empty string is indistinguishable from no value in the UI
if instance and args[value_field_name] not in (None, ""):
setattr(instance, value_field_name, args[value_field_name])
instance.save()
elif not instance:
CustomFieldInstance.objects.create(
**args,
field=field,
document=document,
)
if action.assign_title:
try:
title = parse_w_workflow_placeholders(
@@ -57,6 +84,9 @@ def apply_assignment_to_document(
"", # dont pass the title to avoid recursion
"", # no urls in titles
document.pk,
custom_fields=CustomFieldInstance.global_objects.filter(
document=document,
).select_related("field"),
)
if title:
document.title = title
@@ -91,33 +121,6 @@ def apply_assignment_to_document(
merge=True,
)
if action.has_assign_custom_fields:
for field in action.assign_custom_fields.all():
value_field_name = CustomFieldInstance.get_value_field_name(
data_type=field.data_type,
)
args = {
value_field_name: action.assign_custom_fields_values.get(
str(field.pk),
None,
),
}
# for some reason update_or_create doesn't work here
instance = CustomFieldInstance.objects.filter(
field=field,
document=document,
).first()
# empty string is indistinguishable from no value in the UI
if instance and args[value_field_name] not in (None, ""):
setattr(instance, value_field_name, args[value_field_name])
instance.save()
elif not instance:
CustomFieldInstance.objects.create(
**args,
field=field,
document=document,
)
def apply_assignment_to_overrides(
action: WorkflowAction,