Compare commits

...
Author SHA1 Message Date
shamoon 2c329255a9 email and webhooks too 2026-10-26 21:07:00 -07:00
shamoon 7f7c7cc3c7 Support custom fields in workflow title templates 2026-10-26 21:02:00 -07:00
Jan Kleineandshamoon 7d8147c21d Performance: reduce tasks badge load times (#14552)
---------

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
2026-10-26 14:23:33 +00:00
11 changed files with 234 additions and 164 deletions

No files matched your search

+1
View File
@@ -726,6 +726,7 @@ applied. You can use the following placeholders in the template with any trigger
- `{{added_time}}`: added time in HH:MM format
- `{{original_filename}}`: original file name without extension
- `{{filename}}`: current file name without extension (for "added" workflows this may not be final yet, you can use `{{original_filename}}`)
- `{{custom_fields}}`: Custom field names mapped to their type and value, as in [file name templates](advanced_usage.md#additional-variables). For example, `{{ custom_fields.Invoice.value }}` or `{{ custom_fields | get_cf_value("Invoice", "") }}` for a fallback when the field is missing. In title assignments, fields assigned by the same action are available.
- `{{doc_title}}`: current document title (cannot be used in title assignment)
The following placeholders are only available for "added" or "updated" triggers
@@ -19,7 +19,6 @@ import {
PaperlessTaskTriggerSource,
PaperlessTaskType,
} from 'src/app/data/paperless-task'
import { Results } from 'src/app/data/results'
import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
import { PermissionsGuard } from 'src/app/guards/permissions.guard'
import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
@@ -153,11 +152,6 @@ const tasks: PaperlessTask[] = [
},
]
const paginatedTasks: Results<PaperlessTask> = {
count: tasks.length,
results: tasks,
}
const sectionCountResponse = {
all: 7,
needs_attention: 2,
@@ -204,7 +198,7 @@ describe('TasksComponent', () => {
}).compileComponents()
tasksService = TestBed.inject(TasksService)
reloadSpy = jest.spyOn(tasksService, 'reload')
reloadSpy = jest.spyOn(tasksService, 'reload').mockImplementation(() => {})
httpTestingController = TestBed.inject(HttpTestingController)
modalService = TestBed.inject(NgbModal)
router = TestBed.inject(Router)
@@ -214,15 +208,6 @@ describe('TasksComponent', () => {
jest.useFakeTimers()
fixture.detectChanges()
httpTestingController
.expectOne(
(req) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
)
.flush(paginatedTasks)
httpTestingController
.expectOne(
(req) =>
@@ -344,11 +344,11 @@
ngbPopover="Tasks" i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end"
container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
<i-bs class="me-2" name="list-task"></i-bs><span class="nav-link-label"><ng-container i18n>Tasks</ng-container></span>
@if (tasksService.needsAttentionTasks.length > 0 && !slimSidebarEnabled) {
<span class="badge bg-danger ms-2 d-inline flex-shrink-0">{{tasksService.needsAttentionTasks.length}}</span>
@if (tasksService.needsAttentionTasksCount > 0 && !slimSidebarEnabled) {
<span class="badge bg-danger ms-2 d-inline flex-shrink-0">{{tasksService.needsAttentionTasksCount}}</span>
}
@if (tasksService.needsAttentionTasks.length > 0 && slimSidebarEnabled) {
<span class="badge bg-danger position-absolute top-0 end-0 d-none d-md-block">{{tasksService.needsAttentionTasks.length}}</span>
@if (tasksService.needsAttentionTasksCount > 0 && slimSidebarEnabled) {
<span class="badge bg-danger position-absolute top-0 end-0 d-none d-md-block">{{tasksService.needsAttentionTasksCount}}</span>
}
</a>
</li>
@@ -588,8 +588,8 @@ describe('AppFrameComponent', () => {
it('should show tasks badge for needs-attention tasks', () => {
jest
.spyOn(tasksService, 'needsAttentionTasks', 'get')
.mockReturnValue([{} as any, {} as any])
.spyOn(tasksService, 'needsAttentionTasksCount', 'get')
.mockReturnValue(2)
fixture.detectChanges()
+22 -82
View File
@@ -9,11 +9,7 @@ import {
} from '@angular/common/http/testing'
import { TestBed } from '@angular/core/testing'
import { environment } from 'src/environments/environment'
import {
PaperlessTaskStatus,
PaperlessTaskTriggerSource,
PaperlessTaskType,
} from '../data/paperless-task'
import { PaperlessTaskType } from '../data/paperless-task'
import { TasksService } from './tasks.service'
describe('TasksService', () => {
@@ -38,23 +34,23 @@ describe('TasksService', () => {
httpTestingController.verify()
})
it('calls tasks api endpoint on reload', () => {
it('calls tasks status counts api endpoint on reload', () => {
tasksService.reload()
const req = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
expect(req.request.method).toEqual('GET')
req.flush({ count: 0, results: [] })
req.flush({ all: 5, needs_attention: 3, in_progress: 1, completed: 1 })
expect(tasksService.needsAttentionTasksCount).toEqual(3)
})
it('cancels an in-progress reload when reloading again', () => {
tasksService.reload()
const staleReload = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/`
req.url === `${environment.apiBaseUrl}tasks/status_counts/`
)
tasksService.reload()
@@ -62,9 +58,9 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/`
req.url === `${environment.apiBaseUrl}tasks/status_counts/`
)
.flush({ count: 0, results: [] })
.flush({ all: 0, needs_attention: 0, in_progress: 0, completed: 0 })
})
it('continues reloading after a reload request fails', () => {
@@ -72,7 +68,7 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/`
req.url === `${environment.apiBaseUrl}tasks/status_counts/`
)
.flush('error', { status: 500, statusText: 'error' })
@@ -82,16 +78,16 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/`
req.url === `${environment.apiBaseUrl}tasks/status_counts/`
)
.flush({ count: 0, results: [] })
.flush({ all: 0, needs_attention: 0, in_progress: 0, completed: 0 })
})
it('reloads after dismissing a task while a reload is already in progress', () => {
tasksService.reload()
const staleReload = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
@@ -104,12 +100,12 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
.flush({ count: 0, results: [] })
.flush({ all: 50, needs_attention: 42, in_progress: 3, completed: 5 })
expect(tasksService.needsAttentionTasks).toHaveLength(0)
expect(tasksService.needsAttentionTasksCount).toEqual(42)
})
it('calls acknowledge_tasks api endpoint on dismiss and reloads', () => {
@@ -126,11 +122,10 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
.flush({ count: 0, results: [] })
.flush({ all: 0, needs_attention: 0, in_progress: 0, completed: 0 })
})
it('calls acknowledge_tasks api endpoint on dismiss all and reloads', () => {
@@ -147,65 +142,10 @@ describe('TasksService', () => {
httpTestingController
.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false'
)
.flush({ count: 0, results: [] })
})
it('includes revoked tasks in needs attention', () => {
const mockTasks = [
{
task_type: PaperlessTaskType.SanityCheck,
trigger_source: PaperlessTaskTriggerSource.System,
status: PaperlessTaskStatus.Failure,
acknowledged: false,
task_id: '1235',
input_data: {},
date_created: new Date(),
related_document_ids: [],
},
{
task_type: PaperlessTaskType.MailFetch,
trigger_source: PaperlessTaskTriggerSource.Scheduled,
status: PaperlessTaskStatus.Revoked,
acknowledged: false,
task_id: '1236',
input_data: {},
date_created: new Date(),
related_document_ids: [],
},
{
task_type: PaperlessTaskType.EmptyTrash,
trigger_source: PaperlessTaskTriggerSource.Manual,
status: PaperlessTaskStatus.Success,
acknowledged: false,
task_id: '1238',
input_data: {},
date_created: new Date(),
related_document_ids: [],
},
]
tasksService.reload()
const req = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '1000'
)
req.flush({ count: mockTasks.length, results: mockTasks })
expect(tasksService.needsAttentionTasks).toHaveLength(2)
expect(tasksService.needsAttentionTasks.map((task) => task.status)).toEqual(
expect.arrayContaining([
PaperlessTaskStatus.Failure,
PaperlessTaskStatus.Revoked,
])
)
.flush({ all: 0, needs_attention: 0, in_progress: 0, completed: 0 })
})
it('supports running tasks', () => {
+23 -33
View File
@@ -5,14 +5,12 @@ import {
catchError,
finalize,
first,
map,
switchMap,
takeUntil,
tap,
} from 'rxjs/operators'
import {
PaperlessTask,
PaperlessTaskStatus,
PaperlessTaskStatusCounts,
PaperlessTaskType,
} from 'src/app/data/paperless-task'
@@ -27,50 +25,42 @@ export class TasksService {
private baseUrl: string = environment.apiBaseUrl
private endpoint: string = 'tasks'
private readonly defaultReloadPageSize = 1000
public loading: boolean = false
private readonly tasks = signal<PaperlessTask[]>([])
private readonly tasksStatusCounts = signal<PaperlessTaskStatusCounts>({
all: 0,
needs_attention: 0,
in_progress: 0,
completed: 0,
})
private readonly reloadNotifier = new Subject<void>()
private unsubscribeNotifer: Subject<any> = new Subject()
constructor() {
this.reloadNotifier
.pipe(
switchMap(() => {
this.loading = true
return this.http
.get<Results<PaperlessTask>>(`${this.baseUrl}${this.endpoint}/`, {
params: {
acknowledged: 'false',
page_size: this.defaultReloadPageSize,
},
})
.pipe(
map((response) => response.results),
takeUntil(this.unsubscribeNotifer),
catchError(() => EMPTY),
finalize(() => {
this.loading = false
})
)
})
)
.subscribe((tasks) => {
this.tasks.set(tasks)
.pipe(switchMap(() => this.fetchUnacknowledgedTasksCounts()))
.subscribe((taskStatusCounts) => {
this.tasksStatusCounts.set(taskStatusCounts)
})
}
public get needsAttentionTasks(): PaperlessTask[] {
return this.tasks().filter((t) =>
[PaperlessTaskStatus.Failure, PaperlessTaskStatus.Revoked].includes(
t.status
)
private fetchUnacknowledgedTasksCounts(): Observable<PaperlessTaskStatusCounts> {
this.loading = true
return this.statusCounts({ acknowledged: false }).pipe(
takeUntil(this.unsubscribeNotifer),
catchError(() => EMPTY),
finalize(() => {
this.loading = false
})
)
}
public get needsAttentionTasksCount(): number {
return this.tasksStatusCounts().needs_attention
}
public reload() {
this.reloadNotifier.next()
}
@@ -105,7 +95,7 @@ export class TasksService {
public dismissTasks(task_ids: Set<number>): Observable<any> {
return this.http
.post(`${this.baseUrl}tasks/acknowledge/`, {
.post(`${this.baseUrl}${this.endpoint}/acknowledge/`, {
tasks: [...task_ids],
})
.pipe(
@@ -119,7 +109,7 @@ export class TasksService {
public dismissAllTasks(): Observable<any> {
return this.http
.post(`${this.baseUrl}tasks/acknowledge/`, {
.post(`${this.baseUrl}${this.endpoint}/acknowledge/`, {
all: true,
})
.pipe(
+13
View File
@@ -847,6 +847,19 @@ class ConsumerPlugin(
local_added,
self.filename,
self.filename,
custom_fields=[
CustomFieldInstance(
field=field,
**{
CustomFieldInstance.get_value_field_name(
field.data_type,
): self.metadata.custom_fields[field.pk],
},
)
for field in CustomField.objects.filter(
pk__in=self.metadata.custom_fields or {},
)
],
)
def _store(
+6
View File
@@ -1,4 +1,5 @@
import logging
from collections.abc import Iterable
from datetime import date
from datetime import datetime
from pathlib import Path
@@ -13,7 +14,9 @@ from jinja2 import make_logging_undefined
from jinja2.meta import find_undeclared_variables
from jinja2.sandbox import SecurityError
from documents.models import CustomFieldInstance
from documents.templating.environment import _template_environment
from documents.templating.filepath import get_custom_fields_context
from documents.templating.filters import format_datetime
from documents.templating.filters import localize_date
@@ -56,6 +59,7 @@ _known_placeholder_names = {
"doc_title",
"doc_url",
"doc_id",
"custom_fields",
}
@@ -86,6 +90,7 @@ def parse_w_workflow_placeholders(
doc_title: str | None = None,
doc_url: str | None = None,
doc_id: int | None = None,
custom_fields: Iterable[CustomFieldInstance] = (),
) -> str:
"""
Available title placeholders for Workflows depend on what has already been assigned,
@@ -108,6 +113,7 @@ def parse_w_workflow_placeholders(
"original_filename": Path(original_filename).stem,
"filename": Path(filename).stem,
}
formatting.update(get_custom_fields_context(custom_fields))
if created is not None:
formatting.update(
{
+110
View File
@@ -1846,6 +1846,50 @@ class TestWorkflows(
expected_str = f"Document storage path {doc.storage_path} does not match {trigger.filter_has_storage_path}"
self.assertIn(expected_str, cm.output[1])
def test_title_custom_fields(self) -> None:
from documents.consumer import ConsumerPlugin
from documents.templating.workflows import validate_workflow_template
field = CustomField.objects.create(
name="Invoice",
data_type=CustomField.FieldDataType.STRING,
)
template = '{{ custom_fields.Invoice.value }} {{ custom_fields | get_cf_value("Missing", "fallback") }}'
validate_workflow_template(template)
consumer = mock.Mock(
filename="invoice.pdf",
metadata=DocumentMetadataOverrides(custom_fields={field.pk: "INV-123"}),
)
self.assertEqual(
ConsumerPlugin._parse_title_placeholders(consumer, template),
"INV-123 fallback",
)
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
action = WorkflowAction.objects.create(assign_title=template)
doc = Document.objects.create(title="Original")
action.assign_custom_fields.add(field)
action.assign_custom_fields_values = {str(field.pk): "INV-123"}
action.save()
workflow = Workflow.objects.create(name="Custom field title", order=0)
workflow.triggers.add(trigger)
workflow.actions.add(action)
document_consumption_finished.send(sender=self.__class__, document=doc)
doc.refresh_from_db()
self.assertEqual(doc.title, "INV-123 fallback")
# Existing fields remain available when the action assigns no custom fields.
action.assign_custom_fields.clear()
CustomFieldInstance.objects.filter(document=doc, field=field).update(
value_text="INV-456",
)
document_consumption_finished.send(sender=self.__class__, document=doc)
doc.refresh_from_db()
self.assertEqual(doc.title, "INV-456 fallback")
def test_document_added_invalid_title_placeholders(self) -> None:
"""
GIVEN:
@@ -5120,6 +5164,72 @@ class TestWorkflows(
self.assertEqual(Document.objects.count(), 0)
@pytest.mark.django_db
@pytest.mark.parametrize("use_overrides", [False, True])
def test_custom_fields_in_email_and_webhook_templates(
use_overrides,
settings,
tmp_path,
):
from documents.workflows.actions import build_workflow_action_context
from documents.workflows.actions import execute_email_action
from documents.workflows.actions import execute_webhook_action
settings.EMAIL_ENABLED = True
field = CustomField.objects.create(
name="Invoice",
data_type=CustomField.FieldDataType.STRING,
)
if use_overrides:
invoice = tmp_path / "invoice.pdf"
invoice.write_bytes(b"%PDF-1.7")
document = ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=invoice,
)
overrides = DocumentMetadataOverrides(custom_fields={field.pk: "INV-123"})
else:
document = Document.objects.create(title="Invoice")
CustomFieldInstance.objects.create(
document=document,
field=field,
value_text="INV-123",
)
overrides = None
context = build_workflow_action_context(document, overrides)
template = '{{ custom_fields.Invoice.value }} {{ custom_fields | get_cf_value("Missing", "fallback") }}'
action = WorkflowAction(
email=WorkflowActionEmail(
subject=template,
body=template,
to="test@example.com",
),
webhook=WorkflowActionWebhook(
url="https://example.com",
body=template,
),
)
with mock.patch("documents.workflows.actions.send_email") as send_email:
execute_email_action(
action,
document,
context,
None,
Path("invoice.pdf"),
WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
assert send_email.call_args.kwargs["subject"] == "INV-123 fallback"
assert send_email.call_args.kwargs["body"] == "INV-123 fallback"
with mock.patch("documents.workflows.actions.send_webhook.apply_async") as send:
for use_params in (False, True):
action.webhook.use_params = use_params
action.webhook.params = {"invoice": template}
execute_webhook_action(action, document, context, None, Path("invoice.pdf"))
assert send.call_args.kwargs["kwargs"]["data"] == (
{"invoice": "INV-123 fallback"} if use_params else "INV-123 fallback"
)
class TestWebhookSend:
def test_send_webhook_data_or_json(
self,
+22
View File
@@ -11,6 +11,8 @@ from documents.data_models import DocumentMetadataOverrides
from documents.mail import EmailAttachment
from documents.mail import send_email
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import WorkflowAction
@@ -48,6 +50,9 @@ def build_workflow_action_context(
"added": timezone.localtime(document.added),
"created": document.created,
"id": document.pk,
"custom_fields": CustomFieldInstance.global_objects.filter(
document=document,
).select_related("field"),
}
correspondent_obj = (
@@ -80,6 +85,19 @@ def build_workflow_action_context(
"added": timezone.localtime(timezone.now()),
"created": overrides.created if overrides else None,
"id": "",
"custom_fields": [
CustomFieldInstance(
field=field,
**{
CustomFieldInstance.get_value_field_name(
field.data_type,
): overrides.custom_fields[field.pk],
},
)
for field in CustomField.objects.filter(
pk__in=overrides.custom_fields or {},
)
],
}
@@ -116,6 +134,7 @@ def execute_email_action(
context["title"],
context["doc_url"],
context["id"],
custom_fields=context["custom_fields"],
)
if action.email.subject
else ""
@@ -133,6 +152,7 @@ def execute_email_action(
context["title"],
context["doc_url"],
context["id"],
custom_fields=context["custom_fields"],
)
if action.email.body
else ""
@@ -211,6 +231,7 @@ def execute_webhook_action(
context["title"],
context["doc_url"],
context["id"],
custom_fields=context["custom_fields"],
)
except Exception as e:
logger.error(
@@ -230,6 +251,7 @@ def execute_webhook_action(
context["title"],
context["doc_url"],
context["id"],
custom_fields=context["custom_fields"],
)
headers = {}
if action.webhook.headers:
+30 -27
View File
@@ -43,6 +43,33 @@ def apply_assignment_to_document(
if action.assign_owner:
document.owner = action.assign_owner
if action.has_assign_custom_fields:
for field in action.assign_custom_fields.all():
value_field_name = CustomFieldInstance.get_value_field_name(
data_type=field.data_type,
)
args = {
value_field_name: action.assign_custom_fields_values.get(
str(field.pk),
None,
),
}
# for some reason update_or_create doesn't work here
instance = CustomFieldInstance.objects.filter(
field=field,
document=document,
).first()
# empty string is indistinguishable from no value in the UI
if instance and args[value_field_name] not in (None, ""):
setattr(instance, value_field_name, args[value_field_name])
instance.save()
elif not instance:
CustomFieldInstance.objects.create(
**args,
field=field,
document=document,
)
if action.assign_title:
try:
title = parse_w_workflow_placeholders(
@@ -57,6 +84,9 @@ def apply_assignment_to_document(
"", # dont pass the title to avoid recursion
"", # no urls in titles
document.pk,
custom_fields=CustomFieldInstance.global_objects.filter(
document=document,
).select_related("field"),
)
if title:
document.title = title
@@ -91,33 +121,6 @@ def apply_assignment_to_document(
merge=True,
)
if action.has_assign_custom_fields:
for field in action.assign_custom_fields.all():
value_field_name = CustomFieldInstance.get_value_field_name(
data_type=field.data_type,
)
args = {
value_field_name: action.assign_custom_fields_values.get(
str(field.pk),
None,
),
}
# for some reason update_or_create doesn't work here
instance = CustomFieldInstance.objects.filter(
field=field,
document=document,
).first()
# empty string is indistinguishable from no value in the UI
if instance and args[value_field_name] not in (None, ""):
setattr(instance, value_field_name, args[value_field_name])
instance.save()
elif not instance:
CustomFieldInstance.objects.create(
**args,
field=field,
document=document,
)
def apply_assignment_to_overrides(
action: WorkflowAction,