diff --git a/src-ui/src/app/data/paperless-task.ts b/src-ui/src/app/data/paperless-task.ts index f44b5a6f4..aa3390c96 100644 --- a/src-ui/src/app/data/paperless-task.ts +++ b/src-ui/src/app/data/paperless-task.ts @@ -4,10 +4,14 @@ export enum PaperlessTaskType { ConsumeFile = 'consume_file', TrainClassifier = 'train_classifier', SanityCheck = 'sanity_check', - IndexOptimize = 'index_optimize', - IndexRebuild = 'index_rebuild', MailFetch = 'mail_fetch', LlmIndex = 'llm_index', + EmptyTrash = 'empty_trash', + CheckWorkflows = 'check_workflows', + BulkUpdate = 'bulk_update', + ReprocessDocument = 'reprocess_document', + BuildShareLink = 'build_share_link', + BulkDelete = 'bulk_delete', } export enum PaperlessTaskTriggerSource { diff --git a/src/documents/migrations/0019_task_system_redesign.py b/src/documents/migrations/0019_task_system_redesign.py index 1651a60fc..5fcb7b54b 100644 --- a/src/documents/migrations/0019_task_system_redesign.py +++ b/src/documents/migrations/0019_task_system_redesign.py @@ -47,7 +47,7 @@ class Migration(migrations.Migration): "task_id", models.CharField( help_text="Celery task ID", - max_length=255, + max_length=72, unique=True, verbose_name="Task ID", ), @@ -59,12 +59,14 @@ class Migration(migrations.Migration): ("consume_file", "Consume File"), ("train_classifier", "Train Classifier"), ("sanity_check", "Sanity Check"), - ("index_optimize", "Index Optimize"), ("mail_fetch", "Mail Fetch"), ("llm_index", "LLM Index"), ("empty_trash", "Empty Trash"), ("check_workflows", "Check Workflows"), - ("cleanup_share_links", "Cleanup Share Links"), + ("bulk_update", "Bulk Update"), + ("reprocess_document", "Reprocess Document"), + ("build_share_link", "Build Share Link"), + ("bulk_delete", "Bulk Delete"), ], db_index=True, help_text="The kind of work being performed", diff --git a/src/documents/models.py b/src/documents/models.py index c77eef0d1..62f528f74 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -682,12 +682,14 @@ class PaperlessTask(ModelWithOwner): CONSUME_FILE = "consume_file", _("Consume File") TRAIN_CLASSIFIER = "train_classifier", _("Train Classifier") SANITY_CHECK = "sanity_check", _("Sanity Check") - INDEX_OPTIMIZE = "index_optimize", _("Index Optimize") MAIL_FETCH = "mail_fetch", _("Mail Fetch") LLM_INDEX = "llm_index", _("LLM Index") EMPTY_TRASH = "empty_trash", _("Empty Trash") CHECK_WORKFLOWS = "check_workflows", _("Check Workflows") - CLEANUP_SHARE_LINKS = "cleanup_share_links", _("Cleanup Share Links") + BULK_UPDATE = "bulk_update", _("Bulk Update") + REPROCESS_DOCUMENT = "reprocess_document", _("Reprocess Document") + BUILD_SHARE_LINK = "build_share_link", _("Build Share Link") + BULK_DELETE = "bulk_delete", _("Bulk Delete") COMPLETE_STATUSES = ( Status.SUCCESS, @@ -706,7 +708,7 @@ class PaperlessTask(ModelWithOwner): # Identification task_id = models.CharField( - max_length=255, + max_length=72, unique=True, verbose_name=_("Task ID"), help_text=_("Celery task ID"), diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 81a5925b0..3a205f746 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -2483,8 +2483,8 @@ class TaskSerializerV9(serializers.ModelSerializer): to work unchanged. """ - # v9 field: task_name -> task_type - task_name = serializers.CharField(source="task_type", read_only=True) + # v9 field: task_name -> task_type (with value remapping for renamed tasks) + task_name = serializers.SerializerMethodField() # v9 field: task_file_name -> input_data.filename task_file_name = serializers.SerializerMethodField() @@ -2492,6 +2492,9 @@ class TaskSerializerV9(serializers.ModelSerializer): # v9 field: type -> trigger_source (mapped to old enum labels) type = serializers.SerializerMethodField() + # v9 field: status -> uppercase Celery state strings + status = serializers.SerializerMethodField() + # v9 field: result -> result_message (with legacy format fallback) result = serializers.CharField( source="result_message", @@ -2524,31 +2527,61 @@ class TaskSerializerV9(serializers.ModelSerializer): ) read_only_fields = fields + _TASK_TYPE_TO_V9_NAME = { + PaperlessTask.TaskType.SANITY_CHECK: "check_sanity", + PaperlessTask.TaskType.LLM_INDEX: "llmindex_update", + } + + def get_task_name(self, obj: PaperlessTask) -> str: + return self._TASK_TYPE_TO_V9_NAME.get(obj.task_type, obj.task_type) + def get_task_file_name(self, obj: PaperlessTask) -> str | None: if not obj.input_data: return None return obj.input_data.get("filename") + _STATUS_TO_V9 = { + PaperlessTask.Status.PENDING: "PENDING", + PaperlessTask.Status.STARTED: "STARTED", + PaperlessTask.Status.SUCCESS: "SUCCESS", + PaperlessTask.Status.FAILURE: "FAILURE", + PaperlessTask.Status.REVOKED: "REVOKED", + } + + def get_status(self, obj: PaperlessTask) -> str: + return self._STATUS_TO_V9.get(obj.status, obj.status.upper()) + _TRIGGER_SOURCE_TO_V9_TYPE = { - PaperlessTask.TriggerSource.SCHEDULED: "SCHEDULED_TASK", - PaperlessTask.TriggerSource.SYSTEM: "AUTO_TASK", + PaperlessTask.TriggerSource.SCHEDULED: "scheduled_task", + PaperlessTask.TriggerSource.SYSTEM: "auto_task", # Email and folder-consumer documents are system-initiated, not manually triggered - PaperlessTask.TriggerSource.EMAIL_CONSUME: "AUTO_TASK", - PaperlessTask.TriggerSource.FOLDER_CONSUME: "AUTO_TASK", + PaperlessTask.TriggerSource.EMAIL_CONSUME: "auto_task", + PaperlessTask.TriggerSource.FOLDER_CONSUME: "auto_task", } def get_type(self, obj: PaperlessTask) -> str: - return self._TRIGGER_SOURCE_TO_V9_TYPE.get(obj.trigger_source, "MANUAL_TASK") + return self._TRIGGER_SOURCE_TO_V9_TYPE.get(obj.trigger_source, "manual_task") def get_related_document(self, obj: PaperlessTask) -> int | None: ids = obj.related_document_ids return ids[0] if ids else None - def get_duplicate_documents(self, obj: PaperlessTask) -> list[int]: + def get_duplicate_documents( + self, + obj: PaperlessTask, + ) -> list[dict[str, Any]]: if not obj.result_data: return [] dup_of = obj.result_data.get("duplicate_of") - return [dup_of] if dup_of is not None else [] + if dup_of is None: + return [] + return list( + Document.global_objects.filter(pk=dup_of).values( + "id", + "title", + "deleted_at", + ), + ) class TaskSummarySerializer(serializers.Serializer): diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 2256cc48c..b6f2886d4 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -1007,12 +1007,14 @@ TRACKED_TASKS: dict[str, PaperlessTask.TaskType] = { "documents.tasks.consume_file": PaperlessTask.TaskType.CONSUME_FILE, "documents.tasks.train_classifier": PaperlessTask.TaskType.TRAIN_CLASSIFIER, "documents.tasks.sanity_check": PaperlessTask.TaskType.SANITY_CHECK, - "documents.tasks.index_optimize": PaperlessTask.TaskType.INDEX_OPTIMIZE, "documents.tasks.llmindex_index": PaperlessTask.TaskType.LLM_INDEX, "documents.tasks.empty_trash": PaperlessTask.TaskType.EMPTY_TRASH, "documents.tasks.check_scheduled_workflows": PaperlessTask.TaskType.CHECK_WORKFLOWS, - "documents.tasks.cleanup_expired_share_link_bundles": PaperlessTask.TaskType.CLEANUP_SHARE_LINKS, "paperless_mail.tasks.process_mail_accounts": PaperlessTask.TaskType.MAIL_FETCH, + "documents.tasks.bulk_update_documents": PaperlessTask.TaskType.BULK_UPDATE, + "documents.tasks.update_document_content_maybe_archive_file": PaperlessTask.TaskType.REPROCESS_DOCUMENT, + "documents.tasks.build_share_link_bundle": PaperlessTask.TaskType.BUILD_SHARE_LINK, + "documents.bulk_edit.delete": PaperlessTask.TaskType.BULK_DELETE, } _CELERY_STATE_TO_STATUS: dict[str, PaperlessTask.Status] = { @@ -1202,6 +1204,8 @@ def task_prerun_handler(sender=None, task_id=None, task=None, **kwargs) -> None: """ if task_id is None: # pragma: no cover return + if task and task.name not in TRACKED_TASKS: + return try: close_old_connections() PaperlessTask.objects.filter(task_id=task_id).update( @@ -1227,12 +1231,14 @@ def task_postrun_handler( task_failure also fires when a task raises an exception, and it writes richer structured error data. To avoid a race where this handler overwrites that data, result_data and result_message are left untouched - when the final state is FAILURE — task_failure_handler owns those fields. + when the final state is FAILURE. task_failure_handler owns those fields. https://docs.celeryq.dev/en/stable/userguide/signals.html#task-postrun """ if task_id is None: # pragma: no cover return + if task and task.name not in TRACKED_TASKS: + return try: close_old_connections() @@ -1291,6 +1297,8 @@ def task_failure_handler( """ if task_id is None: # pragma: no cover return + if sender and sender.name not in TRACKED_TASKS: + return try: close_old_connections() @@ -1335,6 +1343,8 @@ def task_revoked_handler( task_id = request.id if request else None if task_id is None: # pragma: no cover return + if sender and sender.name not in TRACKED_TASKS: + return try: close_old_connections() PaperlessTask.objects.filter(task_id=task_id).update( diff --git a/src/documents/tests/factories.py b/src/documents/tests/factories.py index 0d59f6571..21d8bcb37 100644 --- a/src/documents/tests/factories.py +++ b/src/documents/tests/factories.py @@ -72,7 +72,7 @@ class PaperlessTaskFactory(DjangoModelFactory): class Meta: model = PaperlessTask - task_id = factory.LazyFunction(lambda: str(__import__("uuid").uuid4())) + task_id = factory.Faker("uuid4") task_type = PaperlessTask.TaskType.CONSUME_FILE trigger_source = PaperlessTask.TriggerSource.WEB_UI status = PaperlessTask.Status.PENDING diff --git a/src/documents/tests/test_api_tasks.py b/src/documents/tests/test_api_tasks.py index 369882493..4ff3f1c87 100644 --- a/src/documents/tests/test_api_tasks.py +++ b/src/documents/tests/test_api_tasks.py @@ -18,6 +18,7 @@ from rest_framework import status from rest_framework.test import APIClient from documents.models import PaperlessTask +from documents.tests.factories import DocumentFactory from documents.tests.factories import PaperlessTaskFactory pytestmark = pytest.mark.api @@ -237,40 +238,40 @@ class TestGetTasksV9: assert response.data[0]["task_file_name"] is None def test_type_scheduled_maps_to_scheduled_task(self, v9_client: APIClient) -> None: - """trigger_source=scheduled maps to type='SCHEDULED_TASK' in v9.""" + """trigger_source=scheduled maps to type='scheduled_task' in v9.""" PaperlessTaskFactory(trigger_source=PaperlessTask.TriggerSource.SCHEDULED) response = v9_client.get(ENDPOINT) assert response.status_code == status.HTTP_200_OK - assert response.data[0]["type"] == "SCHEDULED_TASK" + assert response.data[0]["type"] == "scheduled_task" def test_type_system_maps_to_auto_task(self, v9_client: APIClient) -> None: - """trigger_source=system maps to type='AUTO_TASK' in v9.""" + """trigger_source=system maps to type='auto_task' in v9.""" PaperlessTaskFactory(trigger_source=PaperlessTask.TriggerSource.SYSTEM) response = v9_client.get(ENDPOINT) assert response.status_code == status.HTTP_200_OK - assert response.data[0]["type"] == "AUTO_TASK" + assert response.data[0]["type"] == "auto_task" def test_type_web_ui_maps_to_manual_task(self, v9_client: APIClient) -> None: - """trigger_source=web_ui maps to type='MANUAL_TASK' in v9.""" + """trigger_source=web_ui maps to type='manual_task' in v9.""" PaperlessTaskFactory(trigger_source=PaperlessTask.TriggerSource.WEB_UI) response = v9_client.get(ENDPOINT) assert response.status_code == status.HTTP_200_OK - assert response.data[0]["type"] == "MANUAL_TASK" + assert response.data[0]["type"] == "manual_task" def test_type_manual_maps_to_manual_task(self, v9_client: APIClient) -> None: - """trigger_source=manual maps to type='MANUAL_TASK' in v9.""" + """trigger_source=manual maps to type='manual_task' in v9.""" PaperlessTaskFactory(trigger_source=PaperlessTask.TriggerSource.MANUAL) response = v9_client.get(ENDPOINT) assert response.status_code == status.HTTP_200_OK - assert response.data[0]["type"] == "MANUAL_TASK" + assert response.data[0]["type"] == "manual_task" def test_related_document_from_result_data_document_id( self, @@ -301,15 +302,20 @@ class TestGetTasksV9: def test_duplicate_documents_from_result_data(self, v9_client: APIClient) -> None: """duplicate_documents includes duplicate_of from result_data in v9.""" + doc = DocumentFactory.create(title="Duplicate Target") PaperlessTaskFactory( status=PaperlessTask.Status.SUCCESS, - result_data={"duplicate_of": 55}, + result_data={"duplicate_of": doc.pk}, ) response = v9_client.get(ENDPOINT) assert response.status_code == status.HTTP_200_OK - assert response.data[0]["duplicate_documents"] == [55] + dupes = response.data[0]["duplicate_documents"] + assert len(dupes) == 1 + assert dupes[0]["id"] == doc.pk + assert dupes[0]["title"] == doc.title + assert "deleted_at" in dupes[0] def test_duplicate_documents_empty_when_no_result_data( self, @@ -323,6 +329,88 @@ class TestGetTasksV9: assert response.status_code == status.HTTP_200_OK assert response.data[0]["duplicate_documents"] == [] + def test_status_remapped_to_uppercase(self, v9_client: APIClient) -> None: + """v9 status values are uppercase Celery state strings.""" + PaperlessTaskFactory(status=PaperlessTask.Status.SUCCESS) + PaperlessTaskFactory(status=PaperlessTask.Status.PENDING) + PaperlessTaskFactory(status=PaperlessTask.Status.FAILURE) + + response = v9_client.get(ENDPOINT) + + assert response.status_code == status.HTTP_200_OK + statuses = {t["status"] for t in response.data} + assert statuses == {"SUCCESS", "PENDING", "FAILURE"} + + def test_task_name_remap_sanity_check(self, v9_client: APIClient) -> None: + """v9 remaps task_type=sanity_check to task_name=check_sanity.""" + PaperlessTaskFactory(task_type=PaperlessTask.TaskType.SANITY_CHECK) + + response = v9_client.get(ENDPOINT) + + assert response.status_code == status.HTTP_200_OK + assert response.data[0]["task_name"] == "check_sanity" + + def test_task_name_remap_llm_index(self, v9_client: APIClient) -> None: + """v9 remaps task_type=llm_index to task_name=llmindex_update.""" + PaperlessTaskFactory(task_type=PaperlessTask.TaskType.LLM_INDEX) + + response = v9_client.get(ENDPOINT) + + assert response.status_code == status.HTTP_200_OK + assert response.data[0]["task_name"] == "llmindex_update" + + def test_type_email_consume_maps_to_auto_task(self, v9_client: APIClient) -> None: + """trigger_source=email_consume maps to type='auto_task' in v9.""" + PaperlessTaskFactory(trigger_source=PaperlessTask.TriggerSource.EMAIL_CONSUME) + + response = v9_client.get(ENDPOINT) + + assert response.status_code == status.HTTP_200_OK + assert response.data[0]["type"] == "auto_task" + + def test_type_folder_consume_maps_to_auto_task(self, v9_client: APIClient) -> None: + """trigger_source=folder_consume maps to type='auto_task' in v9.""" + PaperlessTaskFactory(trigger_source=PaperlessTask.TriggerSource.FOLDER_CONSUME) + + response = v9_client.get(ENDPOINT) + + assert response.status_code == status.HTTP_200_OK + assert response.data[0]["type"] == "auto_task" + + def test_filter_by_task_name_maps_old_value(self, v9_client: APIClient) -> None: + """?task_name=check_sanity maps to task_type=sanity_check in v9.""" + PaperlessTaskFactory(task_type=PaperlessTask.TaskType.SANITY_CHECK) + PaperlessTaskFactory(task_type=PaperlessTask.TaskType.CONSUME_FILE) + + response = v9_client.get(ENDPOINT, {"task_name": "check_sanity"}) + + assert response.status_code == status.HTTP_200_OK + assert len(response.data) == 1 + assert response.data[0]["task_name"] == "check_sanity" + + def test_v9_non_staff_sees_own_and_unowned_tasks( + self, + admin_user: User, + regular_user: User, + ) -> None: + """v9 non-staff users see their own tasks plus unowned tasks.""" + regular_user.user_permissions.add( + Permission.objects.get(codename="view_paperlesstask"), + ) + + client = APIClient() + client.force_authenticate(user=regular_user) + client.credentials(HTTP_ACCEPT=ACCEPT_V9) + + PaperlessTaskFactory(owner=admin_user) # other user, not visible + PaperlessTaskFactory(owner=None) # unowned, visible in v9 + PaperlessTaskFactory(owner=regular_user) # own task, visible + + response = client.get(ENDPOINT) + + assert response.status_code == status.HTTP_200_OK + assert len(response.data) == 2 + def test_filter_by_task_name_maps_to_task_type(self, v9_client: APIClient) -> None: """?task_name=consume_file filter maps to the task_type field for v9 compatibility.""" PaperlessTaskFactory(task_type=PaperlessTask.TaskType.CONSUME_FILE) @@ -335,15 +423,15 @@ class TestGetTasksV9: assert response.data[0]["task_name"] == "consume_file" def test_filter_by_type_maps_to_trigger_source(self, v9_client: APIClient) -> None: - """?type=SCHEDULED_TASK filter maps to trigger_source=scheduled for v9 compatibility.""" + """?type=scheduled_task filter maps to trigger_source=scheduled for v9 compatibility.""" PaperlessTaskFactory(trigger_source=PaperlessTask.TriggerSource.SCHEDULED) PaperlessTaskFactory(trigger_source=PaperlessTask.TriggerSource.WEB_UI) - response = v9_client.get(ENDPOINT, {"type": "SCHEDULED_TASK"}) + response = v9_client.get(ENDPOINT, {"type": "scheduled_task"}) assert response.status_code == status.HTTP_200_OK assert len(response.data) == 1 - assert response.data[0]["type"] == "SCHEDULED_TASK" + assert response.data[0]["type"] == "scheduled_task" @pytest.mark.django_db() diff --git a/src/documents/tests/test_task_signals.py b/src/documents/tests/test_task_signals.py index 2724067f9..9df6cf790 100644 --- a/src/documents/tests/test_task_signals.py +++ b/src/documents/tests/test_task_signals.py @@ -110,7 +110,7 @@ class TestBeforeTaskPublishHandler: assert task.trigger_source == PaperlessTask.TriggerSource.SYSTEM def test_ignores_untracked_task(self): - send_publish("documents.tasks.bulk_update_documents", ([1, 2],), {}) + send_publish("documents.tasks.some_untracked_task", (), {}) assert PaperlessTask.objects.count() == 0 def test_ignores_none_headers(self): diff --git a/src/documents/views.py b/src/documents/views.py index 4002fb4d6..73e5d1c1a 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -3789,11 +3789,17 @@ class TasksViewSet(ReadOnlyModelViewSet[PaperlessTask]): # Needed for drf-spectacular schema generation (get_queryset touches request.user) queryset = PaperlessTask.objects.none() + # v9 backwards compat: maps old task_name values to new task_type values + _V9_TASK_NAME_TO_TYPE = { + "check_sanity": PaperlessTask.TaskType.SANITY_CHECK, + "llmindex_update": PaperlessTask.TaskType.LLM_INDEX, + } + # v9 backwards compat: maps old "type" query param values to new TriggerSource _V9_TYPE_TO_TRIGGER_SOURCE = { - "AUTO_TASK": PaperlessTask.TriggerSource.SYSTEM, - "SCHEDULED_TASK": PaperlessTask.TriggerSource.SCHEDULED, - "MANUAL_TASK": PaperlessTask.TriggerSource.MANUAL, + "auto_task": PaperlessTask.TriggerSource.SYSTEM, + "scheduled_task": PaperlessTask.TriggerSource.SCHEDULED, + "manual_task": PaperlessTask.TriggerSource.MANUAL, } _RUNNABLE_TASKS = { @@ -3810,17 +3816,24 @@ class TasksViewSet(ReadOnlyModelViewSet[PaperlessTask]): return TaskSerializerV10 def get_queryset(self): - # Staff see all tasks; regular users see only tasks they own. - # Unowned tasks (system/scheduled) are admin-only. + # Staff see all tasks. + # v9 non-staff: own tasks + unowned tasks (preserves old behavior). + # v10 non-staff: own tasks only. + is_v9 = self.request.version and int(self.request.version) < 10 if self.request.user.is_staff: queryset = PaperlessTask.objects.all() + elif is_v9: + queryset = PaperlessTask.objects.filter( + Q(owner=self.request.user) | Q(owner__isnull=True), + ) else: queryset = PaperlessTask.objects.filter(owner=self.request.user) # v9 backwards compat: map old query params to new field names - if self.request.version and int(self.request.version) < 10: + if is_v9: task_name = self.request.query_params.get("task_name") if task_name is not None: - queryset = queryset.filter(task_type=task_name) + mapped = self._V9_TASK_NAME_TO_TYPE.get(task_name, task_name) + queryset = queryset.filter(task_type=mapped) task_type_old = self.request.query_params.get("type") if task_type_old is not None: new_source = self._V9_TYPE_TO_TRIGGER_SOURCE.get(task_type_old) @@ -3916,7 +3929,7 @@ class TasksViewSet(ReadOnlyModelViewSet[PaperlessTask]): @action(methods=["post"], detail=False, permission_classes=[IsAdminUser]) def run(self, request): - """Manually dispatch a background task. Superuser (admin) only.""" + """Manually dispatch a background task. Staff only.""" serializer = RunTaskSerializer(data=request.data) serializer.is_valid(raise_exception=True) task_type = serializer.validated_data.get("task_type")