Chorehancement: move all bulk operations to async hand-off (#14773)

This commit is contained in:
shamoon authored and GitHub committed 2026-11-17 00:23:44 +00:00
1 parent 28bb63824f
commit de75249e51
39 files changed
+1633 -421

No files matched your search

+29 -2
View File
@@ -427,6 +427,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
"Detected a possible incompatible database column. See https://docs.paperless-ngx.com/troubleshooting/#convert-uuid-field",
)
logger.error(f"Error deleting documents: {e!s}")
raise
return "OK"
@@ -474,9 +475,9 @@ def run_workflow(
"""
Manually run a single workflow against the given documents.
Done asynchronously via a Celery task.
Called from the tracked bulk operation task.
"""
run_workflows_for_documents.delay(
run_workflows_for_documents(
document_ids=doc_ids,
workflow_id=workflow,
filter_documents=filter_documents,
@@ -538,6 +539,7 @@ def rotate(
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
docs_by_root_id.setdefault(pair.root_doc.id, pair)
failed_ids = []
for pair in docs_by_root_id.values():
if pair.source_doc.mime_type != "application/pdf":
logger.warning(
@@ -572,8 +574,14 @@ def rotate(
f"Queued new rotated version for document {pair.root_doc.id} by {degrees} degrees",
)
except Exception as e:
failed_ids.append(pair.root_doc.id)
logger.exception(f"Error rotating document {pair.root_doc.id}: {e}")
if failed_ids:
raise RuntimeError(
f"Rotation failed for document(s): {failed_ids}. Check logs for details.",
)
return "OK"
@@ -684,6 +692,22 @@ def merge(
return "OK"
def validate_merge_as_versions_state(
documents: list[Document],
root_document_id: int,
) -> None:
if any(document.root_document_id is not None for document in documents):
raise ValueError("Only top-level documents can be merged as versions.")
source_ids = [
document.pk for document in documents if document.pk != root_document_id
]
if Document.global_objects.filter(root_document_id__in=source_ids).exists():
raise ValueError(
"Documents with existing versions cannot be merged into another document.",
)
def merge_as_versions(
doc_ids: list[int],
*,
@@ -701,6 +725,7 @@ def merge_as_versions(
)
documents_by_id = {document.id: document for document in documents}
validate_merge_as_versions_state(documents, root_document_id)
source_ids = [doc_id for doc_id in doc_ids if doc_id != root_document_id]
root_document = documents_by_id[root_document_id]
next_version_index = (
@@ -864,6 +889,7 @@ def split(
except Exception as e:
logger.exception(f"Error splitting document {doc.id}: {e}")
raise
return "OK"
@@ -909,6 +935,7 @@ def delete_pages(
)
except Exception as e:
logger.exception(f"Error deleting pages from document {pair.root_doc.id}: {e}")
raise
return "OK"
+65 -2
View File
@@ -1,4 +1,4 @@
"""Queued document metadata edits."""
"""Queued document bulk operations."""
from typing import Any
@@ -12,6 +12,7 @@ from rest_framework.exceptions import PermissionDenied
from documents import bulk_edit
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import PaperlessTask
from documents.permissions import has_document_operation_permissions
from documents.tasks import bulk_update_documents
@@ -25,7 +26,35 @@ MODIFIED_FIELDS = {
"modify_custom_fields": "custom_fields",
"set_permissions": None,
}
QUEUED_OPERATIONS = frozenset(MODIFIED_FIELDS)
LEGACY_MODIFIED_FIELDS = {"delete": "deleted_at", "reprocess": "checksum"}
QUEUED_OPERATIONS = frozenset(
(
*MODIFIED_FIELDS,
"delete",
"reprocess",
"run_workflow",
"rotate",
"merge",
"merge_as_versions",
"split",
"delete_pages",
"edit_pdf",
"remove_password",
),
)
USER_OPERATIONS = frozenset(
{
"split",
"merge",
"rotate",
"delete_pages",
"edit_pdf",
"remove_password",
"merge_as_versions",
},
)
# merge_as_versions doesn't queue any consume tasks
TRIGGER_SOURCE_OPERATIONS = USER_OPERATIONS - {"merge_as_versions"}
def snapshot_field(doc_ids: list[int], field: str) -> dict[int, Any]:
@@ -66,6 +95,7 @@ def execute_bulk_operation(
document_ids: list[int],
parameters: dict,
user_id: int,
trigger_source: str = PaperlessTask.TriggerSource.API_UPLOAD,
) -> dict:
if operation not in QUEUED_OPERATIONS:
raise ValueError(f"Unsupported bulk operation: {operation}")
@@ -79,6 +109,39 @@ def execute_bulk_operation(
):
raise PermissionDenied("Insufficient permissions to execute bulk operation")
if operation == "run_workflow" and not user.has_perm("documents.view_workflow"):
raise PermissionDenied("Insufficient permissions to run this workflow")
if operation not in MODIFIED_FIELDS:
execution_parameters = parameters.copy()
if operation in USER_OPERATIONS:
execution_parameters["user"] = user
if operation in TRIGGER_SOURCE_OPERATIONS:
execution_parameters["trigger_source"] = trigger_source
modified_field = LEGACY_MODIFIED_FIELDS.get(operation)
with set_actor(user):
old_values = (
snapshot_field(document_ids, modified_field)
if settings.AUDIT_LOG_ENABLED and modified_field
else {}
)
method(document_ids, **execution_parameters)
if settings.AUDIT_LOG_ENABLED and modified_field:
from auditlog.models import LogEntry
new_values = snapshot_field(document_ids, modified_field)
for doc in Document.objects.filter(pk__in=document_ids):
LogEntry.objects.log_create(
instance=doc,
changes={
modified_field: [old_values[doc.pk], new_values[doc.pk]],
},
action=LogEntry.Action.UPDATE,
actor=user,
additional_data={"reason": f"Bulk edit: {operation}"},
)
return {"operation": operation, "document_count": len(document_ids)}
affected_ids: list[int] = []
modified_field = MODIFIED_FIELDS[operation]
# Keep metadata writes and their audit entries together. Follow-up processing
+13 -13
View File
@@ -1688,6 +1688,7 @@ class MergeDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin
class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
root_document_id = serializers.IntegerField(required=True)
from_webui = serializers.BooleanField(required=False, default=False)
version_label = serializers.CharField(
required=False,
allow_blank=True,
@@ -1716,19 +1717,17 @@ class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
"root_document_id must be one of the selected documents.",
)
selected_documents = Document.objects.filter(id__in=documents)
if selected_documents.filter(root_document__isnull=False).exists():
raise serializers.ValidationError(
"Only top-level documents can be merged as versions.",
)
source_document_ids = set(documents) - {attrs["root_document_id"]}
if Document.global_objects.filter(
root_document_id__in=source_document_ids,
).exists():
raise serializers.ValidationError(
"Documents with existing versions cannot be merged into another document.",
# Rechecked under row locks during execution in case document state changes.
selected_documents = list(
Document.objects.filter(id__in=documents).only("id", "root_document"),
)
try:
bulk_edit.validate_merge_as_versions_state(
selected_documents,
attrs["root_document_id"],
)
except ValueError as error:
raise serializers.ValidationError(str(error)) from error
return attrs
@@ -1801,11 +1800,12 @@ class RemovePasswordDocumentsSerializer(
class DeleteDocumentsSerializer(DocumentSelectionSerializer):
pass
from_webui = serializers.BooleanField(required=False, default=False)
class ReprocessDocumentsSerializer(DocumentSelectionSerializer):
remote_ocr = serializers.BooleanField(required=False, default=False)
from_webui = serializers.BooleanField(required=False, default=False)
class BulkEditSerializer(
+2
View File
@@ -1206,6 +1206,8 @@ def before_task_publish_handler(
"document_count": len(task_kwargs["document_ids"]),
}
owner_id = task_kwargs["user_id"]
if task_kwargs["operation"] == "delete":
task_type = PaperlessTask.TaskType.BULK_DELETE
else:
input_data = _extract_input_data(task_type, task_kwargs)
owner_id = _extract_owner_id(task_type, task_kwargs)
+8 -1
View File
@@ -713,8 +713,9 @@ def run_workflows_for_documents(
logger.warning(
f"Manual workflow run requested for missing workflow {workflow_id}",
)
return
raise
failed_ids = []
for document in Document.objects.filter(
id__in=document_ids,
root_document__isnull=True,
@@ -732,10 +733,16 @@ def run_workflows_for_documents(
document=document,
)
except Exception:
failed_ids.append(document.pk)
logger.exception(
f"Error running workflow {workflow} on document {document.pk}",
)
if failed_ids:
raise RuntimeError(
f"Workflow {workflow_id} failed for document(s): {failed_ids}. Check logs for details.",
)
def update_document_parent_tags(tag: Tag, new_parent: Tag) -> None:
"""
+69 -9
View File
@@ -698,10 +698,10 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
def test_delete_documents_old_uuid_field(self, m) -> None:
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
doc_ids = [self.doc1.id, self.doc2.id, self.doc3.id]
bulk_edit.delete(doc_ids)
with self.assertLogs(level="WARNING") as cm:
bulk_edit.delete(doc_ids)
self.assertIn("possible incompatible database column", cm.output[0])
with self.assertRaisesRegex(Exception, "Data too long for column"):
bulk_edit.delete(doc_ids)
self.assertIn("possible incompatible database column", cm.output[0])
class TestPDFActions(DirectoriesMixin, TestCase):
@@ -1038,8 +1038,9 @@ class TestPDFActions(DirectoriesMixin, TestCase):
mock_consume_file.assert_not_called()
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.tasks.consume_file.s")
def test_split(self, mock_consume_file) -> None:
def test_split(self, mock_consume_file, mock_group) -> None:
"""
GIVEN:
- Existing documents
@@ -1060,6 +1061,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
self.assertIsNone(mock_consume_file.call_args.kwargs["overrides"].asn)
self.assertEqual(result, "OK")
mock_group.return_value.delay.assert_called_once()
@mock.patch("documents.bulk_edit.group")
@mock.patch("documents.pdf_ops.build_pdfs")
@@ -1168,8 +1170,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
sig.apply_async.side_effect = Exception("boom")
mock_chord.return_value = sig
result = bulk_edit.split(doc_ids, pages, delete_originals=True)
self.assertEqual(result, "OK")
with self.assertRaisesMessage(Exception, "boom"):
bulk_edit.split(doc_ids, pages, delete_originals=True)
self.doc2.refresh_from_db()
self.assertEqual(self.doc2.archive_serial_number, 222)
@@ -1191,7 +1193,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
pages = [[1, 2], [3]]
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
bulk_edit.split(doc_ids, pages)
with self.assertRaisesMessage(Exception, "Error building PDFs"):
bulk_edit.split(doc_ids, pages)
error_str = cm.output[0]
expected_str = "Error splitting document 2"
self.assertIn(expected_str, error_str)
@@ -1240,12 +1243,68 @@ class TestPDFActions(DirectoriesMixin, TestCase):
doc_ids = [self.doc2.id, self.doc3.id]
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
bulk_edit.rotate(doc_ids, 90)
with self.assertRaisesMessage(
RuntimeError,
f"Rotation failed for document(s): {doc_ids}",
):
bulk_edit.rotate(doc_ids, 90)
error_str = cm.output[0]
expected_str = "Error rotating document"
self.assertIn(expected_str, error_str)
self.assertEqual(mock_rotate_pdf.call_count, 2)
mock_consume_delay.assert_not_called()
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.rotate_pdf")
def test_rotate_continues_after_preparation_failure(
self,
mock_rotate_pdf,
mock_consume,
) -> None:
def rotate(source, destination, degrees):
if source == self.doc2.source_path:
raise RuntimeError("Invalid PDF")
shutil.copy(source, destination)
mock_rotate_pdf.side_effect = rotate
with self.assertRaisesMessage(
RuntimeError,
f"Rotation failed for document(s): [{self.doc2.id}]",
):
bulk_edit.rotate([self.doc2.id, self.doc3.id], 90)
self.assertEqual(mock_rotate_pdf.call_count, 2)
mock_consume.assert_called_once()
self.assertEqual(
mock_consume.call_args.kwargs["kwargs"]["input_doc"].root_document_id,
self.doc3.id,
)
@mock.patch("documents.tasks.consume_file.apply_async")
@mock.patch("documents.pdf_ops.rotate_pdf")
def test_rotate_continues_after_queue_failure(
self,
mock_rotate_pdf,
mock_consume,
) -> None:
mock_rotate_pdf.side_effect = lambda source, destination, degrees: shutil.copy(
source,
destination,
)
mock_consume.side_effect = [RuntimeError("Broker unavailable"), None]
with self.assertRaisesMessage(
RuntimeError,
f"Rotation failed for document(s): [{self.doc2.id}]",
):
bulk_edit.rotate([self.doc2.id, self.doc3.id], 90)
self.assertEqual(mock_rotate_pdf.call_count, 2)
self.assertEqual(mock_consume.call_count, 2)
self.assertEqual(
mock_consume.call_args.kwargs["kwargs"]["input_doc"].root_document_id,
self.doc3.id,
)
@mock.patch("documents.tasks.consume_file.apply_async")
def test_rotate_non_pdf(
self,
@@ -1375,7 +1434,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
pages = [1, 3]
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
bulk_edit.delete_pages(doc_ids, pages)
with self.assertRaisesMessage(Exception, "Error removing pages"):
bulk_edit.delete_pages(doc_ids, pages)
error_str = cm.output[0]
expected_str = "Error deleting pages from document"
self.assertIn(expected_str, error_str)
+170 -15
View File
@@ -54,6 +54,7 @@ def test_request_only_queues(bulk_setup, endpoint, operation, parameters):
"document_ids": [doc.pk],
"parameters": parameters,
"user_id": user.pk,
"trigger_source": "web_ui",
}
assert enqueue.call_args.kwargs["headers"]["trigger_source"] == "web_ui"
@@ -102,6 +103,122 @@ def test_permissions_rechecked_on_worker(bulk_setup):
assert doc.correspondent_id is None
@pytest.mark.django_db
def test_document_action_runs_with_user_in_worker(bulk_setup):
user, _, doc, _ = bulk_setup
with mock.patch("documents.bulk_edit.rotate") as rotate:
result = execute_bulk_operation(
"rotate",
[doc.pk],
{"degrees": 90},
user.pk,
trigger_source="web_ui",
)
rotate.assert_called_once_with(
[doc.pk],
degrees=90,
user=user,
trigger_source="web_ui",
)
assert result == {"operation": "rotate", "document_count": 1}
@pytest.mark.django_db
def test_document_action_queue_contains_only_serializable_parameters(bulk_setup):
user, client, doc, _ = bulk_setup
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
enqueue.return_value.id = "rotate-task"
response = client.post(
"/api/documents/rotate/",
{"documents": [doc.pk], "degrees": 90, "from_webui": True},
format="json",
)
assert response.data == {"result": "OK", "task_id": "rotate-task"}
assert enqueue.call_args.kwargs["kwargs"] == {
"operation": "rotate",
"document_ids": [doc.pk],
"parameters": {"degrees": 90, "source_mode": "latest_version"},
"user_id": user.pk,
"trigger_source": "web_ui",
}
@pytest.mark.django_db
def test_reprocess_keeps_bulk_audit_entry(bulk_setup, settings):
user, _, doc, _ = bulk_setup
settings.AUDIT_LOG_ENABLED = True
LogEntry.objects.all().delete()
with mock.patch("documents.bulk_edit.reprocess"):
execute_bulk_operation("reprocess", [doc.pk], {}, user.pk)
entry = LogEntry.objects.get_for_object(doc).get()
assert entry.changes_dict["checksum"] == [doc.checksum, doc.checksum]
assert entry.actor_id == user.pk
@pytest.mark.django_db
def test_workflow_runs_in_tracked_task_instead_of_dispatching_another(bulk_setup):
user, _, doc, _ = bulk_setup
with mock.patch("documents.bulk_edit.run_workflows_for_documents") as run:
execute_bulk_operation(
"run_workflow",
[doc.pk],
{"workflow": 7, "filter_documents": False},
user.pk,
)
run.assert_called_once_with(
document_ids=[doc.pk],
workflow_id=7,
filter_documents=False,
)
@pytest.mark.django_db
def test_delete_failure_propagates_from_worker(bulk_setup):
user, _, doc, _ = bulk_setup
with mock.patch(
"documents.models.Document.delete",
side_effect=RuntimeError("delete failed"),
):
with pytest.raises(RuntimeError, match="delete failed"):
execute_bulk_operation("delete", [doc.pk], {}, user.pk)
assert Document.objects.filter(pk=doc.pk).exists()
@pytest.mark.django_db
@pytest.mark.parametrize(
"operation,parameters,preparation",
[
("rotate", {"degrees": 90}, "rotate_pdf"),
("split", {"pages": [[1]]}, "build_pdfs"),
("delete_pages", {"pages": [1]}, "remove_pages"),
],
)
def test_preparation_failure_propagates_from_worker(
bulk_setup,
operation,
parameters,
preparation,
):
user, _, doc, _ = bulk_setup
doc.mime_type = "application/pdf"
doc.save()
with (
mock.patch(
f"documents.bulk_edit.pdf_ops.{preparation}",
side_effect=RuntimeError("preparation failed"),
),
mock.patch("documents.bulk_edit.consume_file.apply_async") as consume,
mock.patch("documents.bulk_edit.consume_file.s") as consume_signature,
pytest.raises(
RuntimeError,
match="Rotation failed" if operation == "rotate" else "preparation failed",
),
):
execute_bulk_operation(operation, [doc.pk], parameters, user.pk)
consume.assert_not_called()
consume_signature.assert_not_called()
@pytest.mark.django_db
def test_queue_failure_does_not_change_documents(bulk_setup):
_, client, doc, correspondent = bulk_setup
@@ -162,6 +279,22 @@ def test_tracked_outcome_records_initiator_and_result(bulk_setup, failed):
assert task.result_data == {"document_count": 1}
@pytest.mark.django_db
def test_queued_delete_is_tracked_as_bulk_delete(bulk_setup):
user, _, doc, _ = bulk_setup
before_task_publish_handler(
headers={"task": execute_bulk_operation.name, "id": "delete-task"},
body=(
(),
{"operation": "delete", "document_ids": [doc.pk], "user_id": user.pk},
{},
),
)
task = PaperlessTask.objects.get(task_id="delete-task")
assert task.task_type == PaperlessTask.TaskType.BULK_DELETE
assert task.owner_id == user.pk
@pytest.mark.django_db
def test_large_filtered_selection_is_one_task(bulk_setup):
_, client, _, _ = bulk_setup
@@ -284,24 +417,46 @@ async def test_completion_socket_is_private(anyio_backend):
@pytest.mark.django_db
@pytest.mark.parametrize("endpoint", ["delete", "bulk_edit"])
def test_trash_remains_synchronous(bulk_setup, endpoint):
_, client, doc, _ = bulk_setup
data = {"documents": [doc.pk]}
if endpoint == "bulk_edit":
data.update(method="delete", parameters={})
with (
mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue,
mock.patch("documents.search.get_backend"),
):
def test_legacy_trash_queues_without_deleting_in_request(bulk_setup):
user, client, doc, _ = bulk_setup
data = {"documents": [doc.pk], "method": "delete", "parameters": {}}
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
enqueue.return_value.id = "delete-task"
response = client.post(
f"/api/documents/{endpoint}/",
"/api/documents/bulk_edit/",
data,
format="json",
HTTP_ACCEPT="application/json; version=9",
)
assert response.status_code == 200
assert response.data == {"result": "OK"}
assert not Document.objects.filter(pk=doc.pk).exists()
assert Document.deleted_objects.filter(pk=doc.pk).exists()
enqueue.assert_not_called()
assert response.data == {"result": "OK", "task_id": "delete-task"}
assert Document.objects.filter(pk=doc.pk).exists()
assert enqueue.call_args.kwargs["kwargs"] == {
"operation": "delete",
"document_ids": [doc.pk],
"parameters": {},
"user_id": user.pk,
"trigger_source": "api_upload",
}
@pytest.mark.django_db
def test_dedicated_trash_queues_without_deleting_in_request(bulk_setup):
user, client, doc, _ = bulk_setup
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
enqueue.return_value.id = "delete-task"
response = client.post(
"/api/documents/delete/",
{"documents": [doc.pk]},
format="json",
)
assert response.status_code == 200
assert response.data == {"result": "OK", "task_id": "delete-task"}
assert Document.objects.filter(pk=doc.pk).exists()
assert enqueue.call_args.kwargs["kwargs"] == {
"operation": "delete",
"document_ids": [doc.pk],
"parameters": {},
"user_id": user.pk,
"trigger_source": "api_upload",
}
@@ -8,6 +8,7 @@ from rest_framework import status
from rest_framework.test import APITestCase
from documents.bulk_edit import merge_as_versions
from documents.bulk_operations import execute_bulk_operation
from documents.models import Document
from documents.serialisers import MergeDocumentsAsVersionsSerializer
from paperless_testing.factories import UserFactory
@@ -34,6 +35,7 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
{
"documents": [self.doc1.id, self.doc2.id, self.doc3.id],
"root_document_id": self.doc2.id,
"from_webui": False,
},
)
@@ -106,7 +108,6 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
def test_rejects_selected_version(self) -> None:
version = Document.objects.create(
checksum="D",
title="D",
root_document=self.doc1,
version_index=1,
)
@@ -116,54 +117,12 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
"root_document_id": self.doc2.id,
},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
serializer.errors["non_field_errors"][0],
"Only top-level documents can be merged as versions.",
)
def test_rejects_source_document_with_versions(self) -> None:
Document.objects.create(
checksum="D",
title="D",
root_document=self.doc1,
version_index=1,
)
serializer = MergeDocumentsAsVersionsSerializer(
data={
"documents": [self.doc1.id, self.doc2.id],
"root_document_id": self.doc2.id,
},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
serializer.errors["non_field_errors"][0],
"Documents with existing versions cannot be merged into another document.",
)
def test_rejects_source_document_with_trashed_versions(self) -> None:
version = Document.objects.create(
checksum="D",
title="D",
root_document=self.doc1,
version_index=1,
)
version.delete() # trashed, but still points at doc1
serializer = MergeDocumentsAsVersionsSerializer(
data={
"documents": [self.doc1.id, self.doc2.id],
"root_document_id": self.doc2.id,
},
)
self.assertFalse(serializer.is_valid())
self.assertEqual(
serializer.errors["non_field_errors"][0],
"Documents with existing versions cannot be merged into another document.",
)
def test_allows_root_document_with_versions(self) -> None:
Document.objects.create(
checksum="D",
@@ -182,6 +141,95 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
class TestMergeDocumentsAsVersions(TestCase):
def setUp(self) -> None:
self.doc1 = Document.objects.create(checksum="test-A", title="A")
self.doc2 = Document.objects.create(checksum="test-B", title="B")
def test_rejects_selected_version(self) -> None:
version = Document.objects.create(
checksum="D",
title="D",
root_document=self.doc1,
version_index=1,
)
with self.assertRaisesMessage(
ValueError,
"Only top-level documents can be merged as versions.",
):
merge_as_versions(
[version.id, self.doc2.id],
root_document_id=self.doc2.id,
)
def test_rejects_source_document_with_versions(self) -> None:
Document.objects.create(
checksum="D",
title="D",
root_document=self.doc1,
version_index=1,
)
with self.assertRaisesMessage(
ValueError,
"Documents with existing versions cannot be merged into another document.",
):
merge_as_versions(
[self.doc1.id, self.doc2.id],
root_document_id=self.doc2.id,
)
def test_rejects_source_document_with_trashed_versions(self) -> None:
version = Document.objects.create(
checksum="D",
title="D",
root_document=self.doc1,
version_index=1,
)
version.delete() # trashed, but still points at doc1
with self.assertRaisesMessage(
ValueError,
"Documents with existing versions cannot be merged into another document.",
):
merge_as_versions(
[self.doc1.id, self.doc2.id],
root_document_id=self.doc2.id,
)
@mock.patch("documents.bulk_edit.DocumentsStatusManager")
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.search.get_backend")
def test_rejects_source_that_gained_versions_after_validation(
self,
_get_backend_mock,
_bulk_update_mock,
_status_manager_mock,
) -> None:
destination = Document.objects.create(checksum="destination")
serializer = MergeDocumentsAsVersionsSerializer(
data={
"documents": [self.doc1.id, destination.id],
"root_document_id": destination.id,
},
)
self.assertTrue(serializer.is_valid(), serializer.errors)
merge_as_versions(
[self.doc1.id, self.doc2.id],
root_document_id=self.doc1.id,
)
with self.assertRaisesMessage(
ValueError,
"Documents with existing versions cannot be merged into another document.",
):
merge_as_versions(
serializer.validated_data["documents"],
root_document_id=serializer.validated_data["root_document_id"],
)
self.doc1.refresh_from_db()
self.doc2.refresh_from_db()
self.assertIsNone(self.doc1.root_document_id)
self.assertEqual(self.doc2.root_document_id, self.doc1.id)
@mock.patch("documents.bulk_edit.DocumentsStatusManager")
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
@mock.patch("documents.search.get_backend")
@@ -348,6 +396,17 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
owner=self.user,
)
self.client.force_authenticate(user=self.user)
enqueue = mock.patch(
"documents.views.execute_bulk_operation.apply_async",
side_effect=lambda *, kwargs, headers: self.run_operation(kwargs),
)
enqueue.start()
self.addCleanup(enqueue.stop)
@staticmethod
def run_operation(kwargs):
execute_bulk_operation(**kwargs)
return mock.Mock(id="test-bulk-operation")
@mock.patch("documents.views.bulk_edit.merge_as_versions")
def test_merges_documents_as_versions(self, merge_mock) -> None:
@@ -367,7 +426,10 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, {"result": "OK"})
self.assertEqual(
response.data,
{"result": "OK", "task_id": "test-bulk-operation"},
)
merge_mock.assert_called_once_with(
[self.doc1.id, self.doc2.id],
root_document_id=self.doc2.id,
@@ -440,27 +502,36 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
merge_mock.assert_not_called()
@mock.patch("documents.views.bulk_edit.merge_as_versions")
def test_rejects_source_document_with_versions(self, merge_mock) -> None:
Document.objects.create(
@mock.patch("documents.views.execute_bulk_operation.apply_async")
def test_rejects_source_document_with_versions_before_queueing(
self,
enqueue_mock,
) -> None:
version = Document.objects.create(
checksum="C",
title="C",
root_document=self.doc1,
version_index=1,
owner=self.user,
)
response = self.client.post(
"/api/documents/merge_as_versions/",
{
"documents": [self.doc1.id, self.doc2.id],
"root_document_id": self.doc2.id,
},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
merge_mock.assert_not_called()
for trashed in (False, True):
with self.subTest(trashed=trashed):
if trashed:
version.delete()
response = self.client.post(
"/api/documents/merge_as_versions/",
{
"documents": [self.doc1.id, self.doc2.id],
"root_document_id": self.doc2.id,
},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(
response.data["non_field_errors"][0],
"Documents with existing versions cannot be merged into another document.",
)
enqueue_mock.assert_not_called()
self.doc1.refresh_from_db()
self.assertIsNone(self.doc1.root_document_id)
+12 -3
View File
@@ -2333,9 +2333,12 @@ class TestWorkflows(
self.assertEqual(doc.owner, self.user2)
def test_manual_workflow_run_missing_workflow(self) -> None:
"""A missing workflow id logs a warning rather than raising."""
"""A workflow removed after queueing fails the task."""
doc = DocumentFactory()
with self.assertLogs("paperless.tasks", level="WARNING"):
with (
self.assertLogs("paperless.tasks", level="WARNING"),
self.assertRaises(Workflow.DoesNotExist),
):
tasks.run_workflows_for_documents([doc.pk], 9999)
@mock.patch("documents.tasks.send_websocket_document_updated")
@@ -2356,7 +2359,13 @@ class TestWorkflows(
mock_run_workflows.side_effect = side_effect
with self.assertLogs("paperless.tasks", level="ERROR"):
with (
self.assertLogs("paperless.tasks", level="ERROR"),
self.assertRaisesMessage(
RuntimeError,
f"failed for document(s): [{doc1.pk}]",
),
):
tasks.run_workflows_for_documents([doc1.pk, doc2.pk], workflow.pk)
# Both documents were attempted; the websocket update only fired for the
+35 -81
View File
@@ -107,9 +107,9 @@ from documents import bulk_edit
from documents.bulk_download import ArchiveOnlyStrategy
from documents.bulk_download import OriginalAndArchiveStrategy
from documents.bulk_download import OriginalsOnlyStrategy
from documents.bulk_operations import QUEUED_OPERATIONS
from documents.bulk_operations import TRIGGER_SOURCE_OPERATIONS
from documents.bulk_operations import USER_OPERATIONS
from documents.bulk_operations import execute_bulk_operation
from documents.bulk_operations import snapshot_field
from documents.caching import get_llm_suggestion_cache
from documents.caching import get_metadata_cache
from documents.caching import get_suggestion_cache
@@ -2974,38 +2974,30 @@ class DocumentSelectionMixin:
class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
permission_classes = (IsAuthenticated,)
parser_classes = (parsers.JSONParser,)
METHOD_NAMES_REQUIRING_USER = {
"split",
"merge",
"rotate",
"delete_pages",
"edit_pdf",
"remove_password",
"merge_as_versions",
}
# merge_as_versions doesn't queue any consume tasks
METHOD_NAMES_REQUIRING_TRIGGER_SOURCE = METHOD_NAMES_REQUIRING_USER - {
"merge_as_versions",
}
def _queue_bulk_operation(self, method, documents, parameters, from_webui):
if "update_callback" in parameters:
raise ValueError("Unsupported bulk operation parameter: update_callback")
signature(method).bind(documents, **parameters)
trigger_source = (
PaperlessTask.TriggerSource.WEB_UI
if from_webui
else PaperlessTask.TriggerSource.API_UPLOAD
)
final_parameters = parameters.copy()
if method.__name__ in USER_OPERATIONS:
final_parameters["user"] = self.request.user
if method.__name__ in TRIGGER_SOURCE_OPERATIONS:
final_parameters["trigger_source"] = trigger_source
signature(method).bind(documents, **final_parameters)
task = execute_bulk_operation.apply_async(
kwargs={
"operation": method.__name__,
"document_ids": documents,
"parameters": parameters,
"user_id": self.request.user.pk,
"trigger_source": trigger_source,
},
headers={
"trigger_source": (
PaperlessTask.TriggerSource.WEB_UI
if from_webui
else PaperlessTask.TriggerSource.API_UPLOAD
),
},
headers={"trigger_source": trigger_source},
)
return Response({"result": "OK", "task_id": task.id})
@@ -3129,15 +3121,6 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
user = self.request.user
from_webui = validated_data.get("from_webui", False)
if method.__name__ in self.METHOD_NAMES_REQUIRING_USER:
parameters["user"] = user
if method.__name__ in self.METHOD_NAMES_REQUIRING_TRIGGER_SOURCE:
parameters["trigger_source"] = (
PaperlessTask.TriggerSource.WEB_UI
if from_webui
else PaperlessTask.TriggerSource.API_UPLOAD
)
if not has_document_operation_permissions(
user=user,
documents=documents,
@@ -3147,8 +3130,12 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
return HttpResponseForbidden("Insufficient permissions")
try:
result = method(documents, **parameters)
return Response({"result": result})
return self._queue_bulk_operation(
method,
documents,
parameters,
from_webui,
)
except Exception as e:
logger.warning(f"An error occurred performing {operation_label}: {e!s}")
return HttpResponseBadRequest(
@@ -3169,19 +3156,13 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
name="BulkEditDocumentsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(required=False),
"task_id": serializers.CharField(),
},
),
},
),
)
class BulkEditView(DocumentOperationPermissionMixin):
LEGACY_MODIFIED_FIELDS = {
"delete": "deleted_at",
"reprocess": "checksum",
"remove_password": None,
}
serializer_class = BulkEditSerializer
def post(self, request, *args, **kwargs):
@@ -3211,14 +3192,6 @@ class BulkEditView(DocumentOperationPermissionMixin):
user=user,
validated_data=serializer.validated_data,
)
if method.__name__ in self.METHOD_NAMES_REQUIRING_USER:
parameters["user"] = user
if method.__name__ in self.METHOD_NAMES_REQUIRING_TRIGGER_SOURCE:
parameters["trigger_source"] = (
PaperlessTask.TriggerSource.WEB_UI
if from_webui
else PaperlessTask.TriggerSource.API_UPLOAD
)
if not has_document_operation_permissions(
user=user,
documents=documents,
@@ -3228,38 +3201,12 @@ class BulkEditView(DocumentOperationPermissionMixin):
return HttpResponseForbidden("Insufficient permissions")
try:
if method.__name__ in QUEUED_OPERATIONS:
return self._queue_bulk_operation(
method,
documents,
parameters,
from_webui,
)
modified_field = self.LEGACY_MODIFIED_FIELDS.get(method.__name__)
if settings.AUDIT_LOG_ENABLED and modified_field:
old_values = snapshot_field(documents, modified_field)
result = method(documents, **parameters)
if settings.AUDIT_LOG_ENABLED and modified_field:
new_values = snapshot_field(documents, modified_field)
for doc in Document.objects.filter(pk__in=documents):
LogEntry.objects.log_create(
instance=doc,
changes={
modified_field: [
old_values[doc.pk],
new_values[doc.pk],
],
},
action=LogEntry.Action.UPDATE,
actor=user,
additional_data={
"reason": f"Bulk edit: {method.__name__}",
},
)
return Response({"result": result})
return self._queue_bulk_operation(
method,
documents,
parameters,
from_webui,
)
except Exception as e:
logger.warning(f"An error occurred performing bulk edit: {e!s}")
return HttpResponseBadRequest(
@@ -3276,6 +3223,7 @@ class BulkEditView(DocumentOperationPermissionMixin):
name="RotateDocumentsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(),
},
),
},
@@ -3303,6 +3251,7 @@ class RotateDocumentsView(DocumentOperationPermissionMixin):
name="MergeDocumentsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(),
},
),
},
@@ -3330,6 +3279,7 @@ class MergeDocumentsView(DocumentOperationPermissionMixin):
name="MergeDocumentsAsVersionsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(),
},
),
},
@@ -3357,6 +3307,7 @@ class MergeDocumentsAsVersionsView(DocumentOperationPermissionMixin):
name="DeleteDocumentsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(),
},
),
},
@@ -3384,6 +3335,7 @@ class DeleteDocumentsView(DocumentOperationPermissionMixin):
name="ReprocessDocumentsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(),
},
),
},
@@ -3411,6 +3363,7 @@ class ReprocessDocumentsView(DocumentOperationPermissionMixin):
name="EditPdfDocumentsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(),
},
),
},
@@ -3438,6 +3391,7 @@ class EditPdfDocumentsView(DocumentOperationPermissionMixin):
name="RemovePasswordDocumentsResult",
fields={
"result": serializers.CharField(),
"task_id": serializers.CharField(),
},
),
},