Fix (beta): move task filtering to backend fully (#12956)

This commit is contained in:
shamoon
2026-06-07 15:45:15 -07:00
committed by GitHub
parent 6f8e39c2e0
commit c3459d8f62
9 changed files with 547 additions and 11 deletions
@@ -84,7 +84,7 @@
<button class="btn btn-sm btn-outline-primary" ngbDropdownToggle>{{filterTargetName}}</button>
<div class="dropdown-menu shadow" ngbDropdownMenu>
@for (t of filterTargets; track t.id) {
<button ngbDropdownItem [class.active]="filterTargetID === t.id" (click)="filterTargetID = t.id">{{t.name}}</button>
<button ngbDropdownItem [class.active]="filterTargetID === t.id" (click)="setFilterTarget(t.id)">{{t.name}}</button>
}
</div>
</div>
@@ -29,7 +29,11 @@ import { ToastService } from 'src/app/services/toast.service'
import { environment } from 'src/environments/environment'
import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component'
import { PageHeaderComponent } from '../../common/page-header/page-header.component'
import { TasksComponent, TaskSection } from './tasks.component'
import {
TaskFilterTargetID,
TasksComponent,
TaskSection,
} from './tasks.component'
const tasks: PaperlessTask[] = [
{
@@ -154,6 +158,13 @@ const paginatedTasks: Results<PaperlessTask> = {
results: tasks,
}
const sectionCountResponse = {
all: 7,
needs_attention: 2,
in_progress: 3,
completed: 2,
}
describe('TasksComponent', () => {
let component: TasksComponent
let fixture: ComponentFixture<TasksComponent>
@@ -221,6 +232,15 @@ describe('TasksComponent', () => {
req.params.get('page') === '1'
)
.flush(paginatedTasks)
httpTestingController
.expectOne(
(req) =>
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false' &&
!req.params.has('status')
)
.flush(sectionCountResponse)
})
it('should display task sections with counts', () => {
@@ -328,6 +348,74 @@ describe('TasksComponent', () => {
expect(pagination).not.toBeNull()
})
it('should apply the selected section to the server-side task query', () => {
component.setSection(TaskSection.NeedsAttention)
const req = httpTestingController.expectOne(
(request) =>
request.url === `${environment.apiBaseUrl}tasks/` &&
request.params.get('page') === '1' &&
request.params.get('page_size') === '25' &&
request.params.get('acknowledged') === 'false' &&
request.params.getAll('status').includes(PaperlessTaskStatus.Failure) &&
request.params.getAll('status').includes(PaperlessTaskStatus.Revoked)
)
req.flush({ count: 2, results: [tasks[0], tasks[1]] })
expect(component.totalTasks).toBe(2)
})
it('should apply task type and trigger source filters to the server-side task query', () => {
component.setTaskType(PaperlessTaskType.SanityCheck)
httpTestingController
.expectOne(
(request) =>
request.url === `${environment.apiBaseUrl}tasks/` &&
request.params.get('page_size') === '25' &&
request.params.get('task_type') === PaperlessTaskType.SanityCheck
)
.flush({ count: 1, results: [tasks[6]] })
component.setTriggerSource(PaperlessTaskTriggerSource.System)
httpTestingController
.expectOne(
(request) =>
request.url === `${environment.apiBaseUrl}tasks/` &&
request.params.get('page_size') === '25' &&
request.params.get('task_type') === PaperlessTaskType.SanityCheck &&
request.params.get('trigger_source') ===
PaperlessTaskTriggerSource.System
)
.flush({ count: 1, results: [tasks[6]] })
})
it('should apply text filters to the server-side task query', () => {
component.filterText = 'invoice'
jest.advanceTimersByTime(150)
httpTestingController
.expectOne(
(request) =>
request.url === `${environment.apiBaseUrl}tasks/` &&
request.params.get('page_size') === '25' &&
request.params.get('name') === 'invoice'
)
.flush({ count: 1, results: [tasks[0]] })
component.setFilterTarget(TaskFilterTargetID.Result)
httpTestingController
.expectOne(
(request) =>
request.url === `${environment.apiBaseUrl}tasks/` &&
request.params.get('page_size') === '25' &&
request.params.get('result') === 'invoice'
)
.flush({ count: 0, results: [] })
})
it('should load a different task page when pagination changes', () => {
component.setPage(2)
@@ -351,6 +439,27 @@ describe('TasksComponent', () => {
expect(component.pagedTasks).toEqual([tasks[0]])
})
it('should not replace section counts with current-page counts', () => {
component.setPage(2)
httpTestingController
.expectOne(
(req) =>
req.url === `${environment.apiBaseUrl}tasks/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('page_size') === '25' &&
req.params.get('page') === '2'
)
.flush({
count: 30,
results: [tasks[0]],
})
expect(component.sectionCount(TaskSection.NeedsAttention)).toBe(2)
expect(component.sectionCount(TaskSection.InProgress)).toBe(3)
expect(component.sectionCount(TaskSection.Completed)).toBe(2)
})
it('should expose stable task type options and disable empty ones', () => {
expect(component.taskTypeOptions.map((option) => option.value)).toContain(
PaperlessTaskType.TrainClassifier
@@ -714,6 +823,9 @@ describe('TasksComponent', () => {
})
it('should keep clearing selection independent from resetting filters', () => {
component.resetFilter()
expect(component.filterText).toBe('')
component.setTaskType(PaperlessTaskType.ConsumeFile)
component.toggleSelected(tasks[0])
expect(component.selectedTasks.size).toBe(1)
@@ -40,7 +40,7 @@ export enum TaskSection {
Completed = 'completed',
}
enum TaskFilterTargetID {
export enum TaskFilterTargetID {
Name,
Result,
}
@@ -167,6 +167,12 @@ export class TasksComponent
public readonly pageSize = 25
public page: number = 1
public totalTasks: number = 0
public sectionCounts: Record<TaskSection, number> = {
[TaskSection.All]: 0,
[TaskSection.NeedsAttention]: 0,
[TaskSection.InProgress]: 0,
[TaskSection.Completed]: 0,
}
public pagedTasks: PaperlessTask[] = []
public selectedSection: TaskSection = TaskSection.All
public selectedTaskType: PaperlessTaskType | null = null
@@ -282,6 +288,7 @@ export class TasksComponent
.subscribe((query) => {
this._filterText = query
this.clearSelection()
this.reloadPage(true)
})
}
@@ -470,9 +477,7 @@ export class TasksComponent
}
sectionCount(section: TaskSection): number {
return this.pagedTasks.filter((task) =>
this.taskBelongsToSection(task, section)
).length
return this.sectionCounts[section]
}
sectionShowsResults(section: TaskSection): boolean {
@@ -482,16 +487,27 @@ export class TasksComponent
setSection(section: TaskSection) {
this.selectedSection = section
this.clearSelection()
this.reloadPage(true)
}
setTaskType(taskType: PaperlessTaskType | null) {
this.selectedTaskType = taskType
this.clearSelection()
this.reloadPage(true)
}
setTriggerSource(triggerSource: PaperlessTaskTriggerSource | null) {
this.selectedTriggerSource = triggerSource
this.clearSelection()
this.reloadPage(true)
}
setFilterTarget(filterTargetID: TaskFilterTargetID) {
this.filterTargetID = filterTargetID
if (this._filterText.length) {
this.clearSelection()
this.reloadPage(true)
}
}
taskTypeOptionCount(taskType: PaperlessTaskType | null): number {
@@ -529,19 +545,32 @@ export class TasksComponent
}
public resetFilter() {
if (!this._filterText.length) {
return
}
this._filterText = ''
this.clearSelection()
this.reloadPage(true)
}
public resetFilters() {
const hadFilter = this.isFiltered
this.selectedTaskType = null
this.selectedTriggerSource = null
this.resetFilter()
this._filterText = ''
this.clearSelection()
if (hadFilter) {
this.reloadPage(true)
}
}
filterInputKeyup(event: KeyboardEvent) {
if (event.key == 'Enter') {
this._filterText = (event.target as HTMLInputElement).value
this.clearSelection()
this.reloadPage(true)
} else if (event.key === 'Escape') {
this.resetFilter()
}
@@ -630,19 +659,86 @@ export class TasksComponent
)
}
private reloadSectionCounts() {
this.tasksService
.statusCounts(this.getParamsForSection(TaskSection.All))
.pipe(first(), takeUntil(this.unsubscribeNotifier))
.subscribe((counts) => {
this.sectionCounts[TaskSection.All] = counts.all
this.sectionCounts[TaskSection.NeedsAttention] = counts.needs_attention
this.sectionCounts[TaskSection.InProgress] = counts.in_progress
this.sectionCounts[TaskSection.Completed] = counts.completed
})
}
private getParamsForSection(
section: TaskSection
): Record<string, string | number | boolean | readonly string[]> {
const params: Record<
string,
string | number | boolean | readonly string[]
> = {
acknowledged: false,
}
const statuses = this.statusesForSection(section)
if (statuses.length) {
params.status = statuses
}
if (this.selectedTaskType !== null) {
params.task_type = this.selectedTaskType
}
if (this.selectedTriggerSource !== null) {
params.trigger_source = this.selectedTriggerSource
}
if (this._filterText.length) {
params[
this.filterTargetID === TaskFilterTargetID.Name ? 'name' : 'result'
] = this._filterText
}
return params
}
private statusesForSection(section: TaskSection): PaperlessTaskStatus[] {
switch (section) {
case TaskSection.NeedsAttention:
return [PaperlessTaskStatus.Failure, PaperlessTaskStatus.Revoked]
case TaskSection.InProgress:
return [PaperlessTaskStatus.Pending, PaperlessTaskStatus.Started]
case TaskSection.Completed:
return [PaperlessTaskStatus.Success]
default:
return []
}
}
private reloadPage(resetToFirstPage: boolean = false) {
if (resetToFirstPage) {
this.page = 1
}
this.reloadSectionCounts()
this.loading = true
this.tasksService
.list(this.page, this.pageSize, { acknowledged: false })
.list(
this.page,
this.pageSize,
this.getParamsForSection(this.selectedSection)
)
.pipe(first(), takeUntil(this.unsubscribeNotifier))
.subscribe({
next: (result) => {
this.pagedTasks = result.results
this.totalTasks = result.count
this.sectionCounts[TaskSection.All] = result.count
if (this.selectedSection !== TaskSection.All) {
this.sectionCounts[this.selectedSection] = result.count
}
this.loading = false
if (
this.page > 1 &&
+7
View File
@@ -64,3 +64,10 @@ export interface PaperlessTaskSummary {
last_success: Date | null
last_failure: Date | null
}
export interface PaperlessTaskStatusCounts {
all: number
needs_attention: number
in_progress: number
completed: number
}
@@ -242,4 +242,34 @@ describe('TasksService', () => {
task_id: 'abc-123',
})
})
it('loads filtered task status counts', () => {
tasksService
.statusCounts({
acknowledged: false,
task_type: PaperlessTaskType.ConsumeFile,
})
.subscribe((res) => {
expect(res).toEqual({
all: 10,
needs_attention: 2,
in_progress: 3,
completed: 5,
})
})
const req = httpTestingController.expectOne(
(req: HttpRequest<unknown>) =>
req.url === `${environment.apiBaseUrl}tasks/status_counts/` &&
req.params.get('acknowledged') === 'false' &&
req.params.get('task_type') === PaperlessTaskType.ConsumeFile
)
expect(req.request.method).toEqual('GET')
req.flush({
all: 10,
needs_attention: 2,
in_progress: 3,
completed: 5,
})
})
})
+13 -1
View File
@@ -5,6 +5,7 @@ import { first, map, takeUntil, tap } from 'rxjs/operators'
import {
PaperlessTask,
PaperlessTaskStatus,
PaperlessTaskStatusCounts,
PaperlessTaskType,
} from 'src/app/data/paperless-task'
import { Results } from 'src/app/data/results'
@@ -88,7 +89,7 @@ export class TasksService {
public list(
page: number,
pageSize: number,
extraParams?: Record<string, string | number | boolean>
extraParams?: Record<string, string | number | boolean | readonly string[]>
): Observable<Results<PaperlessTask>> {
return this.http.get<Results<PaperlessTask>>(
`${this.baseUrl}${this.endpoint}/`,
@@ -102,6 +103,17 @@ export class TasksService {
)
}
public statusCounts(
extraParams?: Record<string, string | number | boolean | readonly string[]>
): Observable<PaperlessTaskStatusCounts> {
return this.http.get<PaperlessTaskStatusCounts>(
`${this.baseUrl}${this.endpoint}/status_counts/`,
{
params: extraParams,
}
)
}
public dismissTasks(task_ids: Set<number>): Observable<any> {
return this.http
.post(`${this.baseUrl}tasks/acknowledge/`, {
+63 -1
View File
@@ -28,6 +28,7 @@ from django.db.models.functions import Cast
from django.utils.translation import gettext_lazy as _
from django_filters import DateFilter
from django_filters.rest_framework import BooleanFilter
from django_filters.rest_framework import CharFilter
from django_filters.rest_framework import DateTimeFilter
from django_filters.rest_framework import Filter
from django_filters.rest_framework import FilterSet
@@ -900,6 +901,16 @@ class ShareLinkBundleFilterSet(FilterSet):
class PaperlessTaskFilterSet(FilterSet):
name = CharFilter(
method="filter_name",
label="Name",
)
result = CharFilter(
method="filter_result",
label="Result",
)
task_type = MultipleChoiceFilter(
choices=PaperlessTask.TaskType.choices,
label="Task Type",
@@ -939,7 +950,58 @@ class PaperlessTaskFilterSet(FilterSet):
class Meta:
model = PaperlessTask
fields = ["task_type", "trigger_source", "status", "acknowledged", "owner"]
fields = [
"task_type",
"trigger_source",
"status",
"acknowledged",
"owner",
"name",
"result",
]
def filter_name(self, queryset, name, value):
if not value:
return queryset
matching_task_types = [
task_type
for task_type, label in PaperlessTask.TaskType.choices
if value.lower() in str(label).lower()
]
matching_trigger_sources = [
trigger_source
for trigger_source, label in PaperlessTask.TriggerSource.choices
if value.lower() in str(label).lower()
]
return queryset.filter(
Q(input_data__filename__icontains=value)
| Q(task_type__in=matching_task_types)
| Q(trigger_source__in=matching_trigger_sources),
)
def filter_result(self, queryset, name, value):
if not value:
return queryset
query = Q(result_data__reason__icontains=value) | Q(
result_data__error_message__icontains=value,
)
try:
numeric_value = int(value)
except (TypeError, ValueError):
pass
else:
query |= Q(result_data__document_id=numeric_value) | Q(
result_data__duplicate_of=numeric_value,
)
if "duplicate" in value.lower():
query |= Q(result_data__duplicate_of__isnull=False)
return queryset.filter(query)
def filter_is_complete(self, queryset, name, value):
if value:
+160
View File
@@ -18,6 +18,7 @@ from guardian.shortcuts import assign_perm
from rest_framework import status
from rest_framework.test import APIClient
from documents.filters import PaperlessTaskFilterSet
from documents.models import PaperlessTask
from documents.tests.factories import DocumentFactory
from documents.tests.factories import PaperlessTaskFactory
@@ -169,6 +170,165 @@ class TestGetTasksV10:
PaperlessTask.Status.STARTED,
}
def test_filter_by_task_name(self, admin_client: APIClient) -> None:
"""?name= searches task filenames, task types, and trigger sources."""
filename_task = PaperlessTaskFactory(input_data={"filename": "invoice-123.pdf"})
type_task = PaperlessTaskFactory(task_type=PaperlessTask.TaskType.SANITY_CHECK)
source_task = PaperlessTaskFactory(
trigger_source=PaperlessTask.TriggerSource.EMAIL_CONSUME,
)
PaperlessTaskFactory(input_data={"filename": "unrelated.pdf"})
response = admin_client.get(ENDPOINT, {"name": "invoice"})
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
assert response.data["results"][0]["task_id"] == filename_task.task_id
response = admin_client.get(ENDPOINT, {"name": "sanity"})
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
assert response.data["results"][0]["task_id"] == type_task.task_id
response = admin_client.get(ENDPOINT, {"name": "email"})
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
assert response.data["results"][0]["task_id"] == source_task.task_id
def test_filter_by_task_result(self, admin_client: APIClient) -> None:
"""?result= searches common structured task result messages."""
reason_task = PaperlessTaskFactory(result_data={"reason": "Manual review"})
error_task = PaperlessTaskFactory(
result_data={"error_message": "Duplicate detected"},
)
document_task = PaperlessTaskFactory(result_data={"document_id": 321})
duplicate_task = PaperlessTaskFactory(result_data={"duplicate_of": 123})
PaperlessTaskFactory(result_data={"reason": "unrelated"})
response = admin_client.get(ENDPOINT, {"result": "manual"})
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
assert response.data["results"][0]["task_id"] == reason_task.task_id
response = admin_client.get(ENDPOINT, {"result": "duplicate"})
assert response.status_code == status.HTTP_200_OK
returned_ids = {task["task_id"] for task in response.data["results"]}
assert returned_ids == {error_task.task_id, duplicate_task.task_id}
response = admin_client.get(ENDPOINT, {"result": "321"})
assert response.status_code == status.HTTP_200_OK
assert response.data["count"] == 1
assert response.data["results"][0]["task_id"] == document_task.task_id
def test_empty_task_name_and_result_filters(self) -> None:
"""Empty name/result values leave the queryset unchanged."""
PaperlessTaskFactory.create_batch(2)
queryset = PaperlessTask.objects.all()
filterset = PaperlessTaskFilterSet()
assert filterset.filter_name(queryset, "name", "").count() == 2
assert filterset.filter_result(queryset, "result", "").count() == 2
def test_status_counts_respects_filters(self, admin_client: APIClient) -> None:
"""status_counts/ returns section counts for the filtered task queryset."""
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.FAILURE,
input_data={"filename": "invoice-a.pdf"},
)
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.REVOKED,
input_data={"filename": "invoice-b.pdf"},
)
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.PENDING,
input_data={"filename": "invoice-c.pdf"},
)
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.STARTED,
input_data={"filename": "invoice-d.pdf"},
)
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.SUCCESS,
input_data={"filename": "invoice-e.pdf"},
)
PaperlessTaskFactory(
acknowledged=True,
status=PaperlessTask.Status.SUCCESS,
input_data={"filename": "invoice-acknowledged.pdf"},
)
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.SUCCESS,
input_data={"filename": "unrelated.pdf"},
)
response = admin_client.get(
f"{ENDPOINT}status_counts/",
{"acknowledged": "false", "name": "invoice"},
)
assert response.status_code == status.HTTP_200_OK
assert response.data == {
"all": 5,
"needs_attention": 2,
"in_progress": 2,
"completed": 1,
}
def test_status_counts_ignores_section_filters(
self,
admin_client: APIClient,
) -> None:
"""status_counts/ ignores status-like filters for the sections it counts."""
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.FAILURE,
input_data={"filename": "invoice-a.pdf"},
)
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.PENDING,
input_data={"filename": "invoice-b.pdf"},
)
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.SUCCESS,
input_data={"filename": "invoice-c.pdf"},
)
PaperlessTaskFactory(
acknowledged=False,
status=PaperlessTask.Status.FAILURE,
input_data={"filename": "unrelated.pdf"},
)
response = admin_client.get(
f"{ENDPOINT}status_counts/",
{
"acknowledged": "false",
"name": "invoice",
"status": PaperlessTask.Status.FAILURE,
"is_complete": "false",
},
)
assert response.status_code == status.HTTP_200_OK
assert response.data == {
"all": 3,
"needs_attention": 1,
"in_progress": 1,
"completed": 1,
}
def test_default_ordering_is_newest_first(self, admin_client: APIClient) -> None:
"""Tasks are returned in descending date_created order (newest first)."""
base = timezone.now()
+58 -1
View File
@@ -4011,7 +4011,7 @@ class RemoteVersionView(GenericAPIView[Any]):
class _TasksViewSetSchema(AutoSchema):
_UNPAGINATED_ACTIONS = frozenset({"summary", "active"})
_UNPAGINATED_ACTIONS = frozenset({"summary", "active", "status_counts"})
def _get_paginator(self):
if getattr(self.view, "action", None) in self._UNPAGINATED_ACTIONS:
@@ -4071,6 +4071,19 @@ class _TasksViewSetSchema(AutoSchema):
),
],
),
status_counts=extend_schema(
responses={
200: inline_serializer(
name="TaskStatusCounts",
fields={
"all": serializers.IntegerField(),
"needs_attention": serializers.IntegerField(),
"in_progress": serializers.IntegerField(),
"completed": serializers.IntegerField(),
},
),
},
),
active=extend_schema(
description="Currently pending and running tasks (capped at 50).",
responses={200: TaskSerializerV10(many=True)},
@@ -4124,6 +4137,7 @@ class TasksViewSet(ReadOnlyModelViewSet[PaperlessTask]):
PaperlessTask.TaskType.SANITY_CHECK: (sanity_check, {"raise_on_error": False}),
PaperlessTask.TaskType.LLM_INDEX: (llmindex_index, {"rebuild": False}),
}
_STATUS_COUNT_EXCLUDED_FILTERS = frozenset({"status", "is_complete"})
def get_serializer_class(self):
# v9: use backwards-compatible serializer with old field names
@@ -4164,6 +4178,21 @@ class TasksViewSet(ReadOnlyModelViewSet[PaperlessTask]):
queryset = queryset.filter(task_id=task_id)
return queryset
def get_status_count_queryset(self):
"""Apply task filters except the status dimensions represented by the counts."""
query_params = self.request.query_params.copy()
for param in self._STATUS_COUNT_EXCLUDED_FILTERS:
query_params.pop(param, None)
filterset = self.filterset_class(
data=query_params,
queryset=self.get_queryset(),
request=self.request,
)
if not filterset.is_valid():
raise ValidationError(filterset.errors)
return filterset.qs
@action(
methods=["post"],
detail=False,
@@ -4233,6 +4262,34 @@ class TasksViewSet(ReadOnlyModelViewSet[PaperlessTask]):
serializer = TaskSummarySerializer(data, many=True)
return Response(serializer.data)
@action(methods=["get"], detail=False)
def status_counts(self, request):
"""Aggregated task counts for task UI sections."""
queryset = self.get_status_count_queryset()
counts = queryset.aggregate(
all=Count("id"),
needs_attention=Count(
"id",
filter=Q(
status__in=[
PaperlessTask.Status.FAILURE,
PaperlessTask.Status.REVOKED,
],
),
),
in_progress=Count(
"id",
filter=Q(
status__in=[
PaperlessTask.Status.PENDING,
PaperlessTask.Status.STARTED,
],
),
),
completed=Count("id", filter=Q(status=PaperlessTask.Status.SUCCESS)),
)
return Response(counts)
@action(methods=["get"], detail=False)
def active(self, request):
"""Currently pending and running tasks (capped at 50)."""