mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-20 12:36:53 +00:00
Merge branch 'dev' into feature/ocr-templates
This commit is contained in:
163 files changed
+66973
-42649
No files matched your search
@@ -429,6 +429,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
|
||||
"Detected a possible incompatible database column. See https://docs.paperless-ngx.com/troubleshooting/#convert-uuid-field",
|
||||
)
|
||||
logger.error(f"Error deleting documents: {e!s}")
|
||||
raise
|
||||
|
||||
return "OK"
|
||||
|
||||
@@ -476,9 +477,9 @@ def run_workflow(
|
||||
"""
|
||||
Manually run a single workflow against the given documents.
|
||||
|
||||
Done asynchronously via a Celery task.
|
||||
Called from the tracked bulk operation task.
|
||||
"""
|
||||
run_workflows_for_documents.delay(
|
||||
run_workflows_for_documents(
|
||||
document_ids=doc_ids,
|
||||
workflow_id=workflow,
|
||||
filter_documents=filter_documents,
|
||||
@@ -540,6 +541,7 @@ def rotate(
|
||||
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
|
||||
docs_by_root_id.setdefault(pair.root_doc.id, pair)
|
||||
|
||||
failed_ids = []
|
||||
for pair in docs_by_root_id.values():
|
||||
if pair.source_doc.mime_type != "application/pdf":
|
||||
logger.warning(
|
||||
@@ -574,8 +576,14 @@ def rotate(
|
||||
f"Queued new rotated version for document {pair.root_doc.id} by {degrees} degrees",
|
||||
)
|
||||
except Exception as e:
|
||||
failed_ids.append(pair.root_doc.id)
|
||||
logger.exception(f"Error rotating document {pair.root_doc.id}: {e}")
|
||||
|
||||
if failed_ids:
|
||||
raise RuntimeError(
|
||||
f"Rotation failed for document(s): {failed_ids}. Check logs for details.",
|
||||
)
|
||||
|
||||
return "OK"
|
||||
|
||||
|
||||
@@ -686,6 +694,22 @@ def merge(
|
||||
return "OK"
|
||||
|
||||
|
||||
def validate_merge_as_versions_state(
|
||||
documents: list[Document],
|
||||
root_document_id: int,
|
||||
) -> None:
|
||||
if any(document.root_document_id is not None for document in documents):
|
||||
raise ValueError("Only top-level documents can be merged as versions.")
|
||||
|
||||
source_ids = [
|
||||
document.pk for document in documents if document.pk != root_document_id
|
||||
]
|
||||
if Document.global_objects.filter(root_document_id__in=source_ids).exists():
|
||||
raise ValueError(
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
)
|
||||
|
||||
|
||||
def merge_as_versions(
|
||||
doc_ids: list[int],
|
||||
*,
|
||||
@@ -703,6 +727,7 @@ def merge_as_versions(
|
||||
)
|
||||
documents_by_id = {document.id: document for document in documents}
|
||||
|
||||
validate_merge_as_versions_state(documents, root_document_id)
|
||||
source_ids = [doc_id for doc_id in doc_ids if doc_id != root_document_id]
|
||||
root_document = documents_by_id[root_document_id]
|
||||
next_version_index = (
|
||||
@@ -866,6 +891,7 @@ def split(
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error splitting document {doc.id}: {e}")
|
||||
raise
|
||||
|
||||
return "OK"
|
||||
|
||||
@@ -911,6 +937,7 @@ def delete_pages(
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(f"Error deleting pages from document {pair.root_doc.id}: {e}")
|
||||
raise
|
||||
|
||||
return "OK"
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Queued document metadata edits."""
|
||||
"""Queued document bulk operations."""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from typing import Any
|
||||
|
||||
from auditlog.context import set_actor
|
||||
@@ -12,6 +13,7 @@ from rest_framework.exceptions import PermissionDenied
|
||||
from documents import bulk_edit
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import PaperlessTask
|
||||
from documents.permissions import has_document_operation_permissions
|
||||
from documents.tasks import bulk_update_documents
|
||||
|
||||
@@ -25,7 +27,35 @@ MODIFIED_FIELDS = {
|
||||
"modify_custom_fields": "custom_fields",
|
||||
"set_permissions": None,
|
||||
}
|
||||
QUEUED_OPERATIONS = frozenset(MODIFIED_FIELDS)
|
||||
LEGACY_MODIFIED_FIELDS = {"delete": "deleted_at", "reprocess": "checksum"}
|
||||
QUEUED_OPERATIONS = frozenset(
|
||||
(
|
||||
*MODIFIED_FIELDS,
|
||||
"delete",
|
||||
"reprocess",
|
||||
"run_workflow",
|
||||
"rotate",
|
||||
"merge",
|
||||
"merge_as_versions",
|
||||
"split",
|
||||
"delete_pages",
|
||||
"edit_pdf",
|
||||
"remove_password",
|
||||
),
|
||||
)
|
||||
USER_OPERATIONS = frozenset(
|
||||
{
|
||||
"split",
|
||||
"merge",
|
||||
"rotate",
|
||||
"delete_pages",
|
||||
"edit_pdf",
|
||||
"remove_password",
|
||||
"merge_as_versions",
|
||||
},
|
||||
)
|
||||
# merge_as_versions doesn't queue any consume tasks
|
||||
TRIGGER_SOURCE_OPERATIONS = USER_OPERATIONS - {"merge_as_versions"}
|
||||
|
||||
|
||||
def snapshot_field(doc_ids: list[int], field: str) -> dict[int, Any]:
|
||||
@@ -66,6 +96,7 @@ def execute_bulk_operation(
|
||||
document_ids: list[int],
|
||||
parameters: dict,
|
||||
user_id: int,
|
||||
trigger_source: str = PaperlessTask.TriggerSource.API_UPLOAD,
|
||||
) -> dict:
|
||||
if operation not in QUEUED_OPERATIONS:
|
||||
raise ValueError(f"Unsupported bulk operation: {operation}")
|
||||
@@ -79,11 +110,46 @@ def execute_bulk_operation(
|
||||
):
|
||||
raise PermissionDenied("Insufficient permissions to execute bulk operation")
|
||||
|
||||
if operation == "run_workflow" and not user.has_perm("documents.view_workflow"):
|
||||
raise PermissionDenied("Insufficient permissions to run this workflow")
|
||||
|
||||
actor_context = set_actor(user) if settings.AUDIT_LOG_ENABLED else nullcontext()
|
||||
|
||||
if operation not in MODIFIED_FIELDS:
|
||||
execution_parameters = parameters.copy()
|
||||
if operation in USER_OPERATIONS:
|
||||
execution_parameters["user"] = user
|
||||
if operation in TRIGGER_SOURCE_OPERATIONS:
|
||||
execution_parameters["trigger_source"] = trigger_source
|
||||
modified_field = LEGACY_MODIFIED_FIELDS.get(operation)
|
||||
with actor_context:
|
||||
old_values = (
|
||||
snapshot_field(document_ids, modified_field)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field
|
||||
else {}
|
||||
)
|
||||
method(document_ids, **execution_parameters)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field:
|
||||
from auditlog.models import LogEntry
|
||||
|
||||
new_values = snapshot_field(document_ids, modified_field)
|
||||
for doc in Document.objects.filter(pk__in=document_ids):
|
||||
LogEntry.objects.log_create(
|
||||
instance=doc,
|
||||
changes={
|
||||
modified_field: [old_values[doc.pk], new_values[doc.pk]],
|
||||
},
|
||||
action=LogEntry.Action.UPDATE,
|
||||
actor=user,
|
||||
additional_data={"reason": f"Bulk edit: {operation}"},
|
||||
)
|
||||
return {"operation": operation, "document_count": len(document_ids)}
|
||||
|
||||
affected_ids: list[int] = []
|
||||
modified_field = MODIFIED_FIELDS[operation]
|
||||
# Keep metadata writes and their audit entries together. Follow-up processing
|
||||
# runs after commit, within this same tracked task.
|
||||
with set_actor(user), transaction.atomic():
|
||||
with actor_context, transaction.atomic():
|
||||
old_values = (
|
||||
snapshot_field(document_ids, modified_field)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field
|
||||
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import logging
|
||||
import re
|
||||
from fnmatch import fnmatch
|
||||
from fnmatch import translate as fnmatch_translate
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from rest_framework import serializers
|
||||
@@ -548,6 +547,48 @@ def existing_document_matches_workflow(
|
||||
return (True, None)
|
||||
|
||||
|
||||
def _glob_to_portable_regex(pattern: str) -> str:
|
||||
"""
|
||||
Translate an fnmatch-style glob into an anchored regex which is accepted by
|
||||
Python, PostgreSQL (ARE), and MariaDB (PCRE).
|
||||
|
||||
fnmatch.translate() can't be used, as its output (`(?s:...)\\Z`) uses syntax
|
||||
PostgreSQL rejects. Supports `*`, `?`, `[seq]` and `[!seq]`.
|
||||
"""
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
n = len(pattern)
|
||||
while i < n:
|
||||
c = pattern[i]
|
||||
i += 1
|
||||
if c == "*":
|
||||
out.append(".*")
|
||||
elif c == "?":
|
||||
out.append(".")
|
||||
elif c == "[":
|
||||
j = i
|
||||
if j < n and pattern[j] == "!":
|
||||
j += 1
|
||||
if j < n and pattern[j] == "]":
|
||||
j += 1
|
||||
while j < n and pattern[j] != "]":
|
||||
j += 1
|
||||
if j >= n:
|
||||
# No closing bracket, treat as a literal
|
||||
out.append(r"\[")
|
||||
else:
|
||||
body = pattern[i:j]
|
||||
i = j + 1
|
||||
negate = body.startswith("!")
|
||||
if negate:
|
||||
body = body[1:]
|
||||
body = "".join(f"\\{ch}" if ch in "\\[^" else ch for ch in body)
|
||||
out.append(f"[{'^' if negate else ''}{body}]")
|
||||
else:
|
||||
out.append(re.escape(c))
|
||||
return f"^{''.join(out)}$"
|
||||
|
||||
|
||||
def prefilter_documents_by_workflowtrigger(
|
||||
documents: QuerySet[Document],
|
||||
trigger: WorkflowTrigger,
|
||||
@@ -628,8 +669,9 @@ def prefilter_documents_by_workflowtrigger(
|
||||
documents = documents.annotate(**annotations).filter(custom_field_q)
|
||||
|
||||
if trigger.filter_filename:
|
||||
regex = fnmatch_translate(trigger.filter_filename).lstrip("^").rstrip("$")
|
||||
documents = documents.filter(original_filename__iregex=regex)
|
||||
documents = documents.filter(
|
||||
original_filename__iregex=_glob_to_portable_regex(trigger.filter_filename),
|
||||
)
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
@@ -789,6 +789,7 @@ def has_document_operation_permissions(
|
||||
bulk_edit.edit_pdf,
|
||||
bulk_edit.merge_as_versions,
|
||||
bulk_edit.remove_password,
|
||||
bulk_edit.run_workflow,
|
||||
]
|
||||
)
|
||||
or (
|
||||
|
||||
@@ -1692,6 +1692,7 @@ class MergeDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin
|
||||
|
||||
class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
|
||||
root_document_id = serializers.IntegerField(required=True)
|
||||
from_webui = serializers.BooleanField(required=False, default=False)
|
||||
version_label = serializers.CharField(
|
||||
required=False,
|
||||
allow_blank=True,
|
||||
@@ -1720,19 +1721,17 @@ class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
|
||||
"root_document_id must be one of the selected documents.",
|
||||
)
|
||||
|
||||
selected_documents = Document.objects.filter(id__in=documents)
|
||||
if selected_documents.filter(root_document__isnull=False).exists():
|
||||
raise serializers.ValidationError(
|
||||
"Only top-level documents can be merged as versions.",
|
||||
)
|
||||
|
||||
source_document_ids = set(documents) - {attrs["root_document_id"]}
|
||||
if Document.global_objects.filter(
|
||||
root_document_id__in=source_document_ids,
|
||||
).exists():
|
||||
raise serializers.ValidationError(
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
# Rechecked under row locks during execution in case document state changes.
|
||||
selected_documents = list(
|
||||
Document.objects.filter(id__in=documents).only("id", "root_document"),
|
||||
)
|
||||
try:
|
||||
bulk_edit.validate_merge_as_versions_state(
|
||||
selected_documents,
|
||||
attrs["root_document_id"],
|
||||
)
|
||||
except ValueError as error:
|
||||
raise serializers.ValidationError(str(error)) from error
|
||||
return attrs
|
||||
|
||||
|
||||
@@ -1805,11 +1804,12 @@ class RemovePasswordDocumentsSerializer(
|
||||
|
||||
|
||||
class DeleteDocumentsSerializer(DocumentSelectionSerializer):
|
||||
pass
|
||||
from_webui = serializers.BooleanField(required=False, default=False)
|
||||
|
||||
|
||||
class ReprocessDocumentsSerializer(DocumentSelectionSerializer):
|
||||
remote_ocr = serializers.BooleanField(required=False, default=False)
|
||||
from_webui = serializers.BooleanField(required=False, default=False)
|
||||
|
||||
|
||||
class RunZoneOcrSerializer(serializers.Serializer):
|
||||
|
||||
@@ -1206,6 +1206,8 @@ def before_task_publish_handler(
|
||||
"document_count": len(task_kwargs["document_ids"]),
|
||||
}
|
||||
owner_id = task_kwargs["user_id"]
|
||||
if task_kwargs["operation"] == "delete":
|
||||
task_type = PaperlessTask.TaskType.BULK_DELETE
|
||||
else:
|
||||
input_data = _extract_input_data(task_type, task_kwargs)
|
||||
owner_id = _extract_owner_id(task_type, task_kwargs)
|
||||
|
||||
@@ -741,8 +741,9 @@ def run_workflows_for_documents(
|
||||
logger.warning(
|
||||
f"Manual workflow run requested for missing workflow {workflow_id}",
|
||||
)
|
||||
return
|
||||
raise
|
||||
|
||||
failed_ids = []
|
||||
for document in Document.objects.filter(
|
||||
id__in=document_ids,
|
||||
root_document__isnull=True,
|
||||
@@ -760,10 +761,16 @@ def run_workflows_for_documents(
|
||||
document=document,
|
||||
)
|
||||
except Exception:
|
||||
failed_ids.append(document.pk)
|
||||
logger.exception(
|
||||
f"Error running workflow {workflow} on document {document.pk}",
|
||||
)
|
||||
|
||||
if failed_ids:
|
||||
raise RuntimeError(
|
||||
f"Workflow {workflow_id} failed for document(s): {failed_ids}. Check logs for details.",
|
||||
)
|
||||
|
||||
|
||||
def update_document_parent_tags(tag: Tag, new_parent: Tag) -> None:
|
||||
"""
|
||||
|
||||
@@ -704,11 +704,12 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.run_workflow")
|
||||
def test_run_workflow_requires_ownership(self, m) -> None:
|
||||
"""A non-owner without document ownership is forbidden (actions mutate docs)."""
|
||||
"""A non-owner with change permission cannot run mutating workflow actions."""
|
||||
self.setup_mock(m, "run_workflow")
|
||||
user = UserFactory(username="no_perms")
|
||||
user = UserFactory(username="non_owner")
|
||||
grant_all_global(user)
|
||||
doc = DocumentFactory(owner=UserFactory(username="other"))
|
||||
grant_object(user, doc, "change_document")
|
||||
workflow = self._make_manual_workflow()
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
|
||||
@@ -4,14 +4,18 @@ from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from asgiref.sync import async_to_sync
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.views import _iterate_in_sync_thread
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.permissions import grant_global
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
@@ -62,7 +66,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
),
|
||||
mock.patch(
|
||||
"paperless_ai.chat.stream_chat_with_documents",
|
||||
return_value=iter(chunks),
|
||||
return_value=(c for c in chunks),
|
||||
),
|
||||
):
|
||||
resp = self.client.post(
|
||||
@@ -74,7 +78,11 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
assert not resp.has_header("Content-Encoding")
|
||||
assert list(resp.streaming_content) == [c.encode() for c in chunks]
|
||||
|
||||
async def read_parts() -> list[bytes]:
|
||||
return [part async for part in resp.streaming_content]
|
||||
|
||||
assert async_to_sync(read_parts)() == [c.encode() for c in chunks]
|
||||
|
||||
def test_missing_question_is_rejected(self) -> None:
|
||||
with mock.patch(
|
||||
@@ -155,3 +163,74 @@ class TestChatStreamingViewUnrestrictedFlag:
|
||||
assert (
|
||||
mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
|
||||
)
|
||||
|
||||
|
||||
class TestIterateInSyncThread:
|
||||
@pytest.fixture
|
||||
def events(self) -> list[str]:
|
||||
return []
|
||||
|
||||
@pytest.fixture
|
||||
def chunks(self, events: list[str]) -> Generator[str, None, None]:
|
||||
def generate() -> Generator[str, None, None]:
|
||||
try:
|
||||
for chunk in ("first", "second"):
|
||||
events.append(f"produced {chunk}")
|
||||
yield chunk
|
||||
finally:
|
||||
events.append("closed")
|
||||
|
||||
return generate()
|
||||
|
||||
def test_yields_each_chunk_before_pulling_the_next(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The wrapper is consumed
|
||||
THEN:
|
||||
- Each chunk is delivered before the next one is generated,
|
||||
rather than the whole generator being drained up front
|
||||
"""
|
||||
|
||||
async def consume() -> None:
|
||||
async for chunk in _iterate_in_sync_thread(chunks):
|
||||
events.append(f"received {chunk}")
|
||||
|
||||
async_to_sync(consume)()
|
||||
|
||||
assert events == [
|
||||
"produced first",
|
||||
"received first",
|
||||
"produced second",
|
||||
"received second",
|
||||
"closed",
|
||||
]
|
||||
|
||||
def test_closing_early_closes_the_source_generator(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The consumer closes the wrapper after the first chunk, as
|
||||
happens when the client disconnects
|
||||
THEN:
|
||||
- The source generator is closed, so its cleanup runs
|
||||
"""
|
||||
|
||||
async def consume_one() -> None:
|
||||
wrapper = _iterate_in_sync_thread(chunks)
|
||||
await anext(wrapper)
|
||||
await wrapper.aclose()
|
||||
|
||||
async_to_sync(consume_one)()
|
||||
|
||||
assert events == ["produced first", "closed"]
|
||||
@@ -700,10 +700,10 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
|
||||
def test_delete_documents_old_uuid_field(self, m) -> None:
|
||||
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
|
||||
doc_ids = [self.doc1.id, self.doc2.id, self.doc3.id]
|
||||
bulk_edit.delete(doc_ids)
|
||||
with self.assertLogs(level="WARNING") as cm:
|
||||
bulk_edit.delete(doc_ids)
|
||||
self.assertIn("possible incompatible database column", cm.output[0])
|
||||
with self.assertRaisesRegex(Exception, "Data too long for column"):
|
||||
bulk_edit.delete(doc_ids)
|
||||
self.assertIn("possible incompatible database column", cm.output[0])
|
||||
|
||||
|
||||
class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
@@ -1040,8 +1040,9 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
|
||||
mock_consume_file.assert_not_called()
|
||||
|
||||
@mock.patch("documents.bulk_edit.group")
|
||||
@mock.patch("documents.tasks.consume_file.s")
|
||||
def test_split(self, mock_consume_file) -> None:
|
||||
def test_split(self, mock_consume_file, mock_group) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Existing documents
|
||||
@@ -1062,6 +1063,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
self.assertIsNone(mock_consume_file.call_args.kwargs["overrides"].asn)
|
||||
|
||||
self.assertEqual(result, "OK")
|
||||
mock_group.return_value.delay.assert_called_once()
|
||||
|
||||
@mock.patch("documents.bulk_edit.group")
|
||||
@mock.patch("documents.pdf_ops.build_pdfs")
|
||||
@@ -1170,8 +1172,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
sig.apply_async.side_effect = Exception("boom")
|
||||
mock_chord.return_value = sig
|
||||
|
||||
result = bulk_edit.split(doc_ids, pages, delete_originals=True)
|
||||
self.assertEqual(result, "OK")
|
||||
with self.assertRaisesMessage(Exception, "boom"):
|
||||
bulk_edit.split(doc_ids, pages, delete_originals=True)
|
||||
|
||||
self.doc2.refresh_from_db()
|
||||
self.assertEqual(self.doc2.archive_serial_number, 222)
|
||||
@@ -1193,7 +1195,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
pages = [[1, 2], [3]]
|
||||
|
||||
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
|
||||
bulk_edit.split(doc_ids, pages)
|
||||
with self.assertRaisesMessage(Exception, "Error building PDFs"):
|
||||
bulk_edit.split(doc_ids, pages)
|
||||
error_str = cm.output[0]
|
||||
expected_str = "Error splitting document 2"
|
||||
self.assertIn(expected_str, error_str)
|
||||
@@ -1242,12 +1245,68 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
doc_ids = [self.doc2.id, self.doc3.id]
|
||||
|
||||
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
|
||||
bulk_edit.rotate(doc_ids, 90)
|
||||
with self.assertRaisesMessage(
|
||||
RuntimeError,
|
||||
f"Rotation failed for document(s): {doc_ids}",
|
||||
):
|
||||
bulk_edit.rotate(doc_ids, 90)
|
||||
error_str = cm.output[0]
|
||||
expected_str = "Error rotating document"
|
||||
self.assertIn(expected_str, error_str)
|
||||
self.assertEqual(mock_rotate_pdf.call_count, 2)
|
||||
mock_consume_delay.assert_not_called()
|
||||
|
||||
@mock.patch("documents.tasks.consume_file.apply_async")
|
||||
@mock.patch("documents.pdf_ops.rotate_pdf")
|
||||
def test_rotate_continues_after_preparation_failure(
|
||||
self,
|
||||
mock_rotate_pdf,
|
||||
mock_consume,
|
||||
) -> None:
|
||||
def rotate(source, destination, degrees):
|
||||
if source == self.doc2.source_path:
|
||||
raise RuntimeError("Invalid PDF")
|
||||
shutil.copy(source, destination)
|
||||
|
||||
mock_rotate_pdf.side_effect = rotate
|
||||
with self.assertRaisesMessage(
|
||||
RuntimeError,
|
||||
f"Rotation failed for document(s): [{self.doc2.id}]",
|
||||
):
|
||||
bulk_edit.rotate([self.doc2.id, self.doc3.id], 90)
|
||||
|
||||
self.assertEqual(mock_rotate_pdf.call_count, 2)
|
||||
mock_consume.assert_called_once()
|
||||
self.assertEqual(
|
||||
mock_consume.call_args.kwargs["kwargs"]["input_doc"].root_document_id,
|
||||
self.doc3.id,
|
||||
)
|
||||
|
||||
@mock.patch("documents.tasks.consume_file.apply_async")
|
||||
@mock.patch("documents.pdf_ops.rotate_pdf")
|
||||
def test_rotate_continues_after_queue_failure(
|
||||
self,
|
||||
mock_rotate_pdf,
|
||||
mock_consume,
|
||||
) -> None:
|
||||
mock_rotate_pdf.side_effect = lambda source, destination, degrees: shutil.copy(
|
||||
source,
|
||||
destination,
|
||||
)
|
||||
mock_consume.side_effect = [RuntimeError("Broker unavailable"), None]
|
||||
with self.assertRaisesMessage(
|
||||
RuntimeError,
|
||||
f"Rotation failed for document(s): [{self.doc2.id}]",
|
||||
):
|
||||
bulk_edit.rotate([self.doc2.id, self.doc3.id], 90)
|
||||
|
||||
self.assertEqual(mock_rotate_pdf.call_count, 2)
|
||||
self.assertEqual(mock_consume.call_count, 2)
|
||||
self.assertEqual(
|
||||
mock_consume.call_args.kwargs["kwargs"]["input_doc"].root_document_id,
|
||||
self.doc3.id,
|
||||
)
|
||||
|
||||
@mock.patch("documents.tasks.consume_file.apply_async")
|
||||
def test_rotate_non_pdf(
|
||||
self,
|
||||
@@ -1377,7 +1436,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
pages = [1, 3]
|
||||
|
||||
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
|
||||
bulk_edit.delete_pages(doc_ids, pages)
|
||||
with self.assertRaisesMessage(Exception, "Error removing pages"):
|
||||
bulk_edit.delete_pages(doc_ids, pages)
|
||||
error_str = cm.output[0]
|
||||
expected_str = "Error deleting pages from document"
|
||||
self.assertIn(expected_str, error_str)
|
||||
|
||||
@@ -54,6 +54,7 @@ def test_request_only_queues(bulk_setup, endpoint, operation, parameters):
|
||||
"document_ids": [doc.pk],
|
||||
"parameters": parameters,
|
||||
"user_id": user.pk,
|
||||
"trigger_source": "web_ui",
|
||||
}
|
||||
assert enqueue.call_args.kwargs["headers"]["trigger_source"] == "web_ui"
|
||||
|
||||
@@ -102,6 +103,157 @@ def test_permissions_rechecked_on_worker(bulk_setup):
|
||||
assert doc.correspondent_id is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_document_action_runs_with_user_in_worker(bulk_setup):
|
||||
user, _, doc, _ = bulk_setup
|
||||
with mock.patch("documents.bulk_edit.rotate") as rotate:
|
||||
result = execute_bulk_operation(
|
||||
"rotate",
|
||||
[doc.pk],
|
||||
{"degrees": 90},
|
||||
user.pk,
|
||||
trigger_source="web_ui",
|
||||
)
|
||||
rotate.assert_called_once_with(
|
||||
[doc.pk],
|
||||
degrees=90,
|
||||
user=user,
|
||||
trigger_source="web_ui",
|
||||
)
|
||||
assert result == {"operation": "rotate", "document_count": 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_delete_task_without_audit_log(bulk_setup, settings):
|
||||
user, _, doc, _ = bulk_setup
|
||||
settings.AUDIT_LOG_ENABLED = False
|
||||
with mock.patch(
|
||||
"documents.bulk_operations.set_actor",
|
||||
side_effect=AssertionError("audit context must not be entered"),
|
||||
):
|
||||
result = execute_bulk_operation("delete", [doc.pk], {}, user.pk)
|
||||
assert result == {"operation": "delete", "document_count": 1}
|
||||
assert not Document.objects.filter(pk=doc.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_metadata_task_without_audit_log(bulk_setup, settings):
|
||||
user, _, doc, correspondent = bulk_setup
|
||||
settings.AUDIT_LOG_ENABLED = False
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.bulk_operations.set_actor",
|
||||
side_effect=AssertionError("audit context must not be entered"),
|
||||
),
|
||||
mock.patch("documents.bulk_operations.bulk_update_documents"),
|
||||
):
|
||||
result = execute_bulk_operation(
|
||||
"set_correspondent",
|
||||
[doc.pk],
|
||||
{"correspondent": correspondent.pk},
|
||||
user.pk,
|
||||
)
|
||||
doc.refresh_from_db()
|
||||
assert doc.correspondent_id == correspondent.pk
|
||||
assert result == {"operation": "set_correspondent", "document_count": 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_document_action_queue_contains_only_serializable_parameters(bulk_setup):
|
||||
user, client, doc, _ = bulk_setup
|
||||
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
|
||||
enqueue.return_value.id = "rotate-task"
|
||||
response = client.post(
|
||||
"/api/documents/rotate/",
|
||||
{"documents": [doc.pk], "degrees": 90, "from_webui": True},
|
||||
format="json",
|
||||
)
|
||||
assert response.data == {"result": "OK", "task_id": "rotate-task"}
|
||||
assert enqueue.call_args.kwargs["kwargs"] == {
|
||||
"operation": "rotate",
|
||||
"document_ids": [doc.pk],
|
||||
"parameters": {"degrees": 90, "source_mode": "latest_version"},
|
||||
"user_id": user.pk,
|
||||
"trigger_source": "web_ui",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_reprocess_keeps_bulk_audit_entry(bulk_setup, settings):
|
||||
user, _, doc, _ = bulk_setup
|
||||
settings.AUDIT_LOG_ENABLED = True
|
||||
LogEntry.objects.all().delete()
|
||||
with mock.patch("documents.bulk_edit.reprocess"):
|
||||
execute_bulk_operation("reprocess", [doc.pk], {}, user.pk)
|
||||
entry = LogEntry.objects.get_for_object(doc).get()
|
||||
assert entry.changes_dict["checksum"] == [doc.checksum, doc.checksum]
|
||||
assert entry.actor_id == user.pk
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_workflow_runs_in_tracked_task_instead_of_dispatching_another(bulk_setup):
|
||||
user, _, doc, _ = bulk_setup
|
||||
with mock.patch("documents.bulk_edit.run_workflows_for_documents") as run:
|
||||
execute_bulk_operation(
|
||||
"run_workflow",
|
||||
[doc.pk],
|
||||
{"workflow": 7, "filter_documents": False},
|
||||
user.pk,
|
||||
)
|
||||
run.assert_called_once_with(
|
||||
document_ids=[doc.pk],
|
||||
workflow_id=7,
|
||||
filter_documents=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_delete_failure_propagates_from_worker(bulk_setup):
|
||||
user, _, doc, _ = bulk_setup
|
||||
with mock.patch(
|
||||
"documents.models.Document.delete",
|
||||
side_effect=RuntimeError("delete failed"),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="delete failed"):
|
||||
execute_bulk_operation("delete", [doc.pk], {}, user.pk)
|
||||
assert Document.objects.filter(pk=doc.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize(
|
||||
"operation,parameters,preparation",
|
||||
[
|
||||
("rotate", {"degrees": 90}, "rotate_pdf"),
|
||||
("split", {"pages": [[1]]}, "build_pdfs"),
|
||||
("delete_pages", {"pages": [1]}, "remove_pages"),
|
||||
],
|
||||
)
|
||||
def test_preparation_failure_propagates_from_worker(
|
||||
bulk_setup,
|
||||
operation,
|
||||
parameters,
|
||||
preparation,
|
||||
):
|
||||
user, _, doc, _ = bulk_setup
|
||||
doc.mime_type = "application/pdf"
|
||||
doc.save()
|
||||
with (
|
||||
mock.patch(
|
||||
f"documents.bulk_edit.pdf_ops.{preparation}",
|
||||
side_effect=RuntimeError("preparation failed"),
|
||||
),
|
||||
mock.patch("documents.bulk_edit.consume_file.apply_async") as consume,
|
||||
mock.patch("documents.bulk_edit.consume_file.s") as consume_signature,
|
||||
pytest.raises(
|
||||
RuntimeError,
|
||||
match="Rotation failed" if operation == "rotate" else "preparation failed",
|
||||
),
|
||||
):
|
||||
execute_bulk_operation(operation, [doc.pk], parameters, user.pk)
|
||||
consume.assert_not_called()
|
||||
consume_signature.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_queue_failure_does_not_change_documents(bulk_setup):
|
||||
_, client, doc, correspondent = bulk_setup
|
||||
@@ -162,6 +314,22 @@ def test_tracked_outcome_records_initiator_and_result(bulk_setup, failed):
|
||||
assert task.result_data == {"document_count": 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_queued_delete_is_tracked_as_bulk_delete(bulk_setup):
|
||||
user, _, doc, _ = bulk_setup
|
||||
before_task_publish_handler(
|
||||
headers={"task": execute_bulk_operation.name, "id": "delete-task"},
|
||||
body=(
|
||||
(),
|
||||
{"operation": "delete", "document_ids": [doc.pk], "user_id": user.pk},
|
||||
{},
|
||||
),
|
||||
)
|
||||
task = PaperlessTask.objects.get(task_id="delete-task")
|
||||
assert task.task_type == PaperlessTask.TaskType.BULK_DELETE
|
||||
assert task.owner_id == user.pk
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_large_filtered_selection_is_one_task(bulk_setup):
|
||||
_, client, _, _ = bulk_setup
|
||||
@@ -284,24 +452,46 @@ async def test_completion_socket_is_private(anyio_backend):
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize("endpoint", ["delete", "bulk_edit"])
|
||||
def test_trash_remains_synchronous(bulk_setup, endpoint):
|
||||
_, client, doc, _ = bulk_setup
|
||||
data = {"documents": [doc.pk]}
|
||||
if endpoint == "bulk_edit":
|
||||
data.update(method="delete", parameters={})
|
||||
with (
|
||||
mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue,
|
||||
mock.patch("documents.search.get_backend"),
|
||||
):
|
||||
def test_legacy_trash_queues_without_deleting_in_request(bulk_setup):
|
||||
user, client, doc, _ = bulk_setup
|
||||
data = {"documents": [doc.pk], "method": "delete", "parameters": {}}
|
||||
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
|
||||
enqueue.return_value.id = "delete-task"
|
||||
response = client.post(
|
||||
f"/api/documents/{endpoint}/",
|
||||
"/api/documents/bulk_edit/",
|
||||
data,
|
||||
format="json",
|
||||
HTTP_ACCEPT="application/json; version=9",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.data == {"result": "OK"}
|
||||
assert not Document.objects.filter(pk=doc.pk).exists()
|
||||
assert Document.deleted_objects.filter(pk=doc.pk).exists()
|
||||
enqueue.assert_not_called()
|
||||
assert response.data == {"result": "OK", "task_id": "delete-task"}
|
||||
assert Document.objects.filter(pk=doc.pk).exists()
|
||||
assert enqueue.call_args.kwargs["kwargs"] == {
|
||||
"operation": "delete",
|
||||
"document_ids": [doc.pk],
|
||||
"parameters": {},
|
||||
"user_id": user.pk,
|
||||
"trigger_source": "api_upload",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_dedicated_trash_queues_without_deleting_in_request(bulk_setup):
|
||||
user, client, doc, _ = bulk_setup
|
||||
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
|
||||
enqueue.return_value.id = "delete-task"
|
||||
response = client.post(
|
||||
"/api/documents/delete/",
|
||||
{"documents": [doc.pk]},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.data == {"result": "OK", "task_id": "delete-task"}
|
||||
assert Document.objects.filter(pk=doc.pk).exists()
|
||||
assert enqueue.call_args.kwargs["kwargs"] == {
|
||||
"operation": "delete",
|
||||
"document_ids": [doc.pk],
|
||||
"parameters": {},
|
||||
"user_id": user.pk,
|
||||
"trigger_source": "api_upload",
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import fnmatch
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.matching import _glob_to_portable_regex
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pattern", "expected"),
|
||||
[
|
||||
pytest.param("", r"^$", id="empty"),
|
||||
pytest.param("abc", r"^abc$", id="literal"),
|
||||
pytest.param("*", r"^.*$", id="star"),
|
||||
pytest.param("?", r"^.$", id="question"),
|
||||
pytest.param("*.pdf", r"^.*\.pdf$", id="star-escapes-dot"),
|
||||
pytest.param("[abc]", r"^[abc]$", id="class"),
|
||||
pytest.param("[a-z]", r"^[a-z]$", id="class-range"),
|
||||
pytest.param("[!abc]", r"^[^abc]$", id="negated-class"),
|
||||
pytest.param("[]a]", r"^[]a]$", id="leading-close-bracket-is-member"),
|
||||
pytest.param("[!]a]", r"^[^]a]$", id="negated-leading-close-bracket"),
|
||||
pytest.param("[a^b]", r"^[a\^b]$", id="caret-escaped-in-class"),
|
||||
pytest.param("[a[b]", r"^[a\[b]$", id="open-bracket-escaped-in-class"),
|
||||
pytest.param("[a\\b]", r"^[a\\b]$", id="backslash-escaped-in-class"),
|
||||
pytest.param("[abc", r"^\[abc$", id="unclosed-class-literal"),
|
||||
pytest.param("[", r"^\[$", id="lone-open-bracket"),
|
||||
pytest.param("[!", r"^\[!$", id="unclosed-negation-literal"),
|
||||
pytest.param("[]", r"^\[\]$", id="unclosed-with-only-close-bracket"),
|
||||
pytest.param("a[0-9]?*", r"^a[0-9]..*$", id="mixed"),
|
||||
],
|
||||
)
|
||||
def test_translation(pattern: str, expected: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A glob pattern
|
||||
WHEN:
|
||||
- It is translated to a portable regex
|
||||
THEN:
|
||||
- The exact anchored regex is produced
|
||||
"""
|
||||
assert _glob_to_portable_regex(pattern) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern",
|
||||
["*", "?", "*.pdf", "a?c", "[abc]x", "[!abc]x", "[]a]", "[!]a]", "[a-c]*", "[a"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
["", "a", "abc", "x", "]", "]x", "file.pdf", "ax", "bx", "dx", "[a"],
|
||||
)
|
||||
def test_agrees_with_fnmatch(pattern: str, name: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A glob pattern and a name
|
||||
WHEN:
|
||||
- Matched via the portable regex and via stdlib fnmatch
|
||||
THEN:
|
||||
- Both agree
|
||||
"""
|
||||
regex = _glob_to_portable_regex(pattern)
|
||||
assert bool(re.search(regex, name)) == fnmatch.fnmatchcase(name, pattern)
|
||||
@@ -8,6 +8,7 @@ from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.bulk_edit import merge_as_versions
|
||||
from documents.bulk_operations import execute_bulk_operation
|
||||
from documents.models import Document
|
||||
from documents.serialisers import MergeDocumentsAsVersionsSerializer
|
||||
from paperless_testing.factories import UserFactory
|
||||
@@ -34,6 +35,7 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
|
||||
{
|
||||
"documents": [self.doc1.id, self.doc2.id, self.doc3.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
"from_webui": False,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -106,7 +108,6 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
|
||||
def test_rejects_selected_version(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
@@ -116,54 +117,12 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(serializer.is_valid())
|
||||
self.assertEqual(
|
||||
serializer.errors["non_field_errors"][0],
|
||||
"Only top-level documents can be merged as versions.",
|
||||
)
|
||||
|
||||
def test_rejects_source_document_with_versions(self) -> None:
|
||||
Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
serializer = MergeDocumentsAsVersionsSerializer(
|
||||
data={
|
||||
"documents": [self.doc1.id, self.doc2.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(serializer.is_valid())
|
||||
self.assertEqual(
|
||||
serializer.errors["non_field_errors"][0],
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
)
|
||||
|
||||
def test_rejects_source_document_with_trashed_versions(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
version.delete() # trashed, but still points at doc1
|
||||
serializer = MergeDocumentsAsVersionsSerializer(
|
||||
data={
|
||||
"documents": [self.doc1.id, self.doc2.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(serializer.is_valid())
|
||||
self.assertEqual(
|
||||
serializer.errors["non_field_errors"][0],
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
)
|
||||
|
||||
def test_allows_root_document_with_versions(self) -> None:
|
||||
Document.objects.create(
|
||||
checksum="D",
|
||||
@@ -182,6 +141,95 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
|
||||
|
||||
|
||||
class TestMergeDocumentsAsVersions(TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.doc1 = Document.objects.create(checksum="test-A", title="A")
|
||||
self.doc2 = Document.objects.create(checksum="test-B", title="B")
|
||||
|
||||
def test_rejects_selected_version(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Only top-level documents can be merged as versions.",
|
||||
):
|
||||
merge_as_versions(
|
||||
[version.id, self.doc2.id],
|
||||
root_document_id=self.doc2.id,
|
||||
)
|
||||
|
||||
def test_rejects_source_document_with_versions(self) -> None:
|
||||
Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
):
|
||||
merge_as_versions(
|
||||
[self.doc1.id, self.doc2.id],
|
||||
root_document_id=self.doc2.id,
|
||||
)
|
||||
|
||||
def test_rejects_source_document_with_trashed_versions(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
version.delete() # trashed, but still points at doc1
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
):
|
||||
merge_as_versions(
|
||||
[self.doc1.id, self.doc2.id],
|
||||
root_document_id=self.doc2.id,
|
||||
)
|
||||
|
||||
@mock.patch("documents.bulk_edit.DocumentsStatusManager")
|
||||
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
def test_rejects_source_that_gained_versions_after_validation(
|
||||
self,
|
||||
_get_backend_mock,
|
||||
_bulk_update_mock,
|
||||
_status_manager_mock,
|
||||
) -> None:
|
||||
destination = Document.objects.create(checksum="destination")
|
||||
serializer = MergeDocumentsAsVersionsSerializer(
|
||||
data={
|
||||
"documents": [self.doc1.id, destination.id],
|
||||
"root_document_id": destination.id,
|
||||
},
|
||||
)
|
||||
self.assertTrue(serializer.is_valid(), serializer.errors)
|
||||
|
||||
merge_as_versions(
|
||||
[self.doc1.id, self.doc2.id],
|
||||
root_document_id=self.doc1.id,
|
||||
)
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
):
|
||||
merge_as_versions(
|
||||
serializer.validated_data["documents"],
|
||||
root_document_id=serializer.validated_data["root_document_id"],
|
||||
)
|
||||
|
||||
self.doc1.refresh_from_db()
|
||||
self.doc2.refresh_from_db()
|
||||
self.assertIsNone(self.doc1.root_document_id)
|
||||
self.assertEqual(self.doc2.root_document_id, self.doc1.id)
|
||||
|
||||
@mock.patch("documents.bulk_edit.DocumentsStatusManager")
|
||||
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
@@ -348,6 +396,17 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
|
||||
owner=self.user,
|
||||
)
|
||||
self.client.force_authenticate(user=self.user)
|
||||
enqueue = mock.patch(
|
||||
"documents.views.execute_bulk_operation.apply_async",
|
||||
side_effect=lambda *, kwargs, headers: self.run_operation(kwargs),
|
||||
)
|
||||
enqueue.start()
|
||||
self.addCleanup(enqueue.stop)
|
||||
|
||||
@staticmethod
|
||||
def run_operation(kwargs):
|
||||
execute_bulk_operation(**kwargs)
|
||||
return mock.Mock(id="test-bulk-operation")
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.merge_as_versions")
|
||||
def test_merges_documents_as_versions(self, merge_mock) -> None:
|
||||
@@ -367,7 +426,10 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data, {"result": "OK"})
|
||||
self.assertEqual(
|
||||
response.data,
|
||||
{"result": "OK", "task_id": "test-bulk-operation"},
|
||||
)
|
||||
merge_mock.assert_called_once_with(
|
||||
[self.doc1.id, self.doc2.id],
|
||||
root_document_id=self.doc2.id,
|
||||
@@ -440,27 +502,36 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
merge_mock.assert_not_called()
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.merge_as_versions")
|
||||
def test_rejects_source_document_with_versions(self, merge_mock) -> None:
|
||||
Document.objects.create(
|
||||
@mock.patch("documents.views.execute_bulk_operation.apply_async")
|
||||
def test_rejects_source_document_with_versions_before_queueing(
|
||||
self,
|
||||
enqueue_mock,
|
||||
) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="C",
|
||||
title="C",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
owner=self.user,
|
||||
)
|
||||
|
||||
response = self.client.post(
|
||||
"/api/documents/merge_as_versions/",
|
||||
{
|
||||
"documents": [self.doc1.id, self.doc2.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
merge_mock.assert_not_called()
|
||||
for trashed in (False, True):
|
||||
with self.subTest(trashed=trashed):
|
||||
if trashed:
|
||||
version.delete()
|
||||
response = self.client.post(
|
||||
"/api/documents/merge_as_versions/",
|
||||
{
|
||||
"documents": [self.doc1.id, self.doc2.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(
|
||||
response.data["non_field_errors"][0],
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
)
|
||||
enqueue_mock.assert_not_called()
|
||||
self.doc1.refresh_from_db()
|
||||
self.assertIsNone(self.doc1.root_document_id)
|
||||
|
||||
|
||||
@@ -2333,9 +2333,12 @@ class TestWorkflows(
|
||||
self.assertEqual(doc.owner, self.user2)
|
||||
|
||||
def test_manual_workflow_run_missing_workflow(self) -> None:
|
||||
"""A missing workflow id logs a warning rather than raising."""
|
||||
"""A workflow removed after queueing fails the task."""
|
||||
doc = DocumentFactory()
|
||||
with self.assertLogs("paperless.tasks", level="WARNING"):
|
||||
with (
|
||||
self.assertLogs("paperless.tasks", level="WARNING"),
|
||||
self.assertRaises(Workflow.DoesNotExist),
|
||||
):
|
||||
tasks.run_workflows_for_documents([doc.pk], 9999)
|
||||
|
||||
@mock.patch("documents.tasks.send_websocket_document_updated")
|
||||
@@ -2356,7 +2359,13 @@ class TestWorkflows(
|
||||
|
||||
mock_run_workflows.side_effect = side_effect
|
||||
|
||||
with self.assertLogs("paperless.tasks", level="ERROR"):
|
||||
with (
|
||||
self.assertLogs("paperless.tasks", level="ERROR"),
|
||||
self.assertRaisesMessage(
|
||||
RuntimeError,
|
||||
f"failed for document(s): [{doc1.pk}]",
|
||||
),
|
||||
):
|
||||
tasks.run_workflows_for_documents([doc1.pk, doc2.pk], workflow.pk)
|
||||
|
||||
# Both documents were attempted; the websocket update only fired for the
|
||||
|
||||
+64
-88
@@ -8,7 +8,9 @@ import tempfile
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Generator
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
@@ -29,6 +31,7 @@ from urllib.parse import urlparse
|
||||
import httpx
|
||||
import magic
|
||||
import pathvalidate
|
||||
from asgiref.sync import sync_to_async
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import User
|
||||
@@ -109,9 +112,9 @@ from documents import bulk_edit
|
||||
from documents.bulk_download import ArchiveOnlyStrategy
|
||||
from documents.bulk_download import OriginalAndArchiveStrategy
|
||||
from documents.bulk_download import OriginalsOnlyStrategy
|
||||
from documents.bulk_operations import QUEUED_OPERATIONS
|
||||
from documents.bulk_operations import TRIGGER_SOURCE_OPERATIONS
|
||||
from documents.bulk_operations import USER_OPERATIONS
|
||||
from documents.bulk_operations import execute_bulk_operation
|
||||
from documents.bulk_operations import snapshot_field
|
||||
from documents.caching import get_llm_suggestion_cache
|
||||
from documents.caching import get_metadata_cache
|
||||
from documents.caching import get_suggestion_cache
|
||||
@@ -2512,6 +2515,24 @@ class DocumentViewSet(
|
||||
)
|
||||
|
||||
|
||||
async def _iterate_in_sync_thread(
|
||||
chunks: Generator[str, None, None],
|
||||
) -> AsyncIterator[str]:
|
||||
"""Serve a blocking sync generator to an ASGI server chunk by chunk.
|
||||
|
||||
Under ASGI, Django drains a sync iterator with ``sync_to_async(list)``
|
||||
before sending anything. Pull each chunk in the request's sync thread
|
||||
instead, where the generator's ORM access belongs.
|
||||
"""
|
||||
sentinel = object() # StopIteration can't cross an asyncio Future
|
||||
next_chunk = sync_to_async(next)
|
||||
try:
|
||||
while (chunk := await next_chunk(chunks, sentinel)) is not sentinel:
|
||||
yield chunk
|
||||
finally:
|
||||
await sync_to_async(chunks.close)()
|
||||
|
||||
|
||||
class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]):
|
||||
q = serializers.CharField(required=True, max_length=4000)
|
||||
document_id = serializers.IntegerField(required=False, allow_null=True)
|
||||
@@ -2570,16 +2591,17 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
user=request.user,
|
||||
)
|
||||
|
||||
response = StreamingHttpResponse(
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
return StreamingHttpResponse(
|
||||
_iterate_in_sync_thread(
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
),
|
||||
),
|
||||
content_type="text/event-stream",
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
@@ -3070,38 +3092,30 @@ class DocumentSelectionMixin:
|
||||
class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
permission_classes = (IsAuthenticated,)
|
||||
parser_classes = (parsers.JSONParser,)
|
||||
METHOD_NAMES_REQUIRING_USER = {
|
||||
"split",
|
||||
"merge",
|
||||
"rotate",
|
||||
"delete_pages",
|
||||
"edit_pdf",
|
||||
"remove_password",
|
||||
"merge_as_versions",
|
||||
}
|
||||
# merge_as_versions doesn't queue any consume tasks
|
||||
METHOD_NAMES_REQUIRING_TRIGGER_SOURCE = METHOD_NAMES_REQUIRING_USER - {
|
||||
"merge_as_versions",
|
||||
}
|
||||
|
||||
def _queue_bulk_operation(self, method, documents, parameters, from_webui):
|
||||
if "update_callback" in parameters:
|
||||
raise ValueError("Unsupported bulk operation parameter: update_callback")
|
||||
signature(method).bind(documents, **parameters)
|
||||
trigger_source = (
|
||||
PaperlessTask.TriggerSource.WEB_UI
|
||||
if from_webui
|
||||
else PaperlessTask.TriggerSource.API_UPLOAD
|
||||
)
|
||||
final_parameters = parameters.copy()
|
||||
if method.__name__ in USER_OPERATIONS:
|
||||
final_parameters["user"] = self.request.user
|
||||
if method.__name__ in TRIGGER_SOURCE_OPERATIONS:
|
||||
final_parameters["trigger_source"] = trigger_source
|
||||
signature(method).bind(documents, **final_parameters)
|
||||
task = execute_bulk_operation.apply_async(
|
||||
kwargs={
|
||||
"operation": method.__name__,
|
||||
"document_ids": documents,
|
||||
"parameters": parameters,
|
||||
"user_id": self.request.user.pk,
|
||||
"trigger_source": trigger_source,
|
||||
},
|
||||
headers={
|
||||
"trigger_source": (
|
||||
PaperlessTask.TriggerSource.WEB_UI
|
||||
if from_webui
|
||||
else PaperlessTask.TriggerSource.API_UPLOAD
|
||||
),
|
||||
},
|
||||
headers={"trigger_source": trigger_source},
|
||||
)
|
||||
return Response({"result": "OK", "task_id": task.id})
|
||||
|
||||
@@ -3225,15 +3239,6 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
user = self.request.user
|
||||
from_webui = validated_data.get("from_webui", False)
|
||||
|
||||
if method.__name__ in self.METHOD_NAMES_REQUIRING_USER:
|
||||
parameters["user"] = user
|
||||
if method.__name__ in self.METHOD_NAMES_REQUIRING_TRIGGER_SOURCE:
|
||||
parameters["trigger_source"] = (
|
||||
PaperlessTask.TriggerSource.WEB_UI
|
||||
if from_webui
|
||||
else PaperlessTask.TriggerSource.API_UPLOAD
|
||||
)
|
||||
|
||||
if not has_document_operation_permissions(
|
||||
user=user,
|
||||
documents=documents,
|
||||
@@ -3243,8 +3248,12 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
try:
|
||||
result = method(documents, **parameters)
|
||||
return Response({"result": result})
|
||||
return self._queue_bulk_operation(
|
||||
method,
|
||||
documents,
|
||||
parameters,
|
||||
from_webui,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"An error occurred performing {operation_label}: {e!s}")
|
||||
return HttpResponseBadRequest(
|
||||
@@ -3265,19 +3274,13 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
name="BulkEditDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(required=False),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
)
|
||||
class BulkEditView(DocumentOperationPermissionMixin):
|
||||
LEGACY_MODIFIED_FIELDS = {
|
||||
"delete": "deleted_at",
|
||||
"reprocess": "checksum",
|
||||
"remove_password": None,
|
||||
}
|
||||
|
||||
serializer_class = BulkEditSerializer
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
@@ -3307,14 +3310,6 @@ class BulkEditView(DocumentOperationPermissionMixin):
|
||||
user=user,
|
||||
validated_data=serializer.validated_data,
|
||||
)
|
||||
if method.__name__ in self.METHOD_NAMES_REQUIRING_USER:
|
||||
parameters["user"] = user
|
||||
if method.__name__ in self.METHOD_NAMES_REQUIRING_TRIGGER_SOURCE:
|
||||
parameters["trigger_source"] = (
|
||||
PaperlessTask.TriggerSource.WEB_UI
|
||||
if from_webui
|
||||
else PaperlessTask.TriggerSource.API_UPLOAD
|
||||
)
|
||||
if not has_document_operation_permissions(
|
||||
user=user,
|
||||
documents=documents,
|
||||
@@ -3324,38 +3319,12 @@ class BulkEditView(DocumentOperationPermissionMixin):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
try:
|
||||
if method.__name__ in QUEUED_OPERATIONS:
|
||||
return self._queue_bulk_operation(
|
||||
method,
|
||||
documents,
|
||||
parameters,
|
||||
from_webui,
|
||||
)
|
||||
modified_field = self.LEGACY_MODIFIED_FIELDS.get(method.__name__)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field:
|
||||
old_values = snapshot_field(documents, modified_field)
|
||||
|
||||
result = method(documents, **parameters)
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field:
|
||||
new_values = snapshot_field(documents, modified_field)
|
||||
for doc in Document.objects.filter(pk__in=documents):
|
||||
LogEntry.objects.log_create(
|
||||
instance=doc,
|
||||
changes={
|
||||
modified_field: [
|
||||
old_values[doc.pk],
|
||||
new_values[doc.pk],
|
||||
],
|
||||
},
|
||||
action=LogEntry.Action.UPDATE,
|
||||
actor=user,
|
||||
additional_data={
|
||||
"reason": f"Bulk edit: {method.__name__}",
|
||||
},
|
||||
)
|
||||
|
||||
return Response({"result": result})
|
||||
return self._queue_bulk_operation(
|
||||
method,
|
||||
documents,
|
||||
parameters,
|
||||
from_webui,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"An error occurred performing bulk edit: {e!s}")
|
||||
return HttpResponseBadRequest(
|
||||
@@ -3372,6 +3341,7 @@ class BulkEditView(DocumentOperationPermissionMixin):
|
||||
name="RotateDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3399,6 +3369,7 @@ class RotateDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="MergeDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3426,6 +3397,7 @@ class MergeDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="MergeDocumentsAsVersionsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3453,6 +3425,7 @@ class MergeDocumentsAsVersionsView(DocumentOperationPermissionMixin):
|
||||
name="DeleteDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3480,6 +3453,7 @@ class DeleteDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="ReprocessDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3541,6 +3515,7 @@ class RunZoneOcrDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="EditPdfDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3568,6 +3543,7 @@ class EditPdfDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="RemovePasswordDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user