mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-21 13:02:40 +00:00
Merge branch 'dev' into feature/ocr-templates
This commit is contained in:
163 files changed
+66973
-42649
No files matched your search
@@ -429,6 +429,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
|
||||
"Detected a possible incompatible database column. See https://docs.paperless-ngx.com/troubleshooting/#convert-uuid-field",
|
||||
)
|
||||
logger.error(f"Error deleting documents: {e!s}")
|
||||
raise
|
||||
|
||||
return "OK"
|
||||
|
||||
@@ -476,9 +477,9 @@ def run_workflow(
|
||||
"""
|
||||
Manually run a single workflow against the given documents.
|
||||
|
||||
Done asynchronously via a Celery task.
|
||||
Called from the tracked bulk operation task.
|
||||
"""
|
||||
run_workflows_for_documents.delay(
|
||||
run_workflows_for_documents(
|
||||
document_ids=doc_ids,
|
||||
workflow_id=workflow,
|
||||
filter_documents=filter_documents,
|
||||
@@ -540,6 +541,7 @@ def rotate(
|
||||
pair = _resolve_root_and_source_doc(doc, source_mode=source_mode)
|
||||
docs_by_root_id.setdefault(pair.root_doc.id, pair)
|
||||
|
||||
failed_ids = []
|
||||
for pair in docs_by_root_id.values():
|
||||
if pair.source_doc.mime_type != "application/pdf":
|
||||
logger.warning(
|
||||
@@ -574,8 +576,14 @@ def rotate(
|
||||
f"Queued new rotated version for document {pair.root_doc.id} by {degrees} degrees",
|
||||
)
|
||||
except Exception as e:
|
||||
failed_ids.append(pair.root_doc.id)
|
||||
logger.exception(f"Error rotating document {pair.root_doc.id}: {e}")
|
||||
|
||||
if failed_ids:
|
||||
raise RuntimeError(
|
||||
f"Rotation failed for document(s): {failed_ids}. Check logs for details.",
|
||||
)
|
||||
|
||||
return "OK"
|
||||
|
||||
|
||||
@@ -686,6 +694,22 @@ def merge(
|
||||
return "OK"
|
||||
|
||||
|
||||
def validate_merge_as_versions_state(
|
||||
documents: list[Document],
|
||||
root_document_id: int,
|
||||
) -> None:
|
||||
if any(document.root_document_id is not None for document in documents):
|
||||
raise ValueError("Only top-level documents can be merged as versions.")
|
||||
|
||||
source_ids = [
|
||||
document.pk for document in documents if document.pk != root_document_id
|
||||
]
|
||||
if Document.global_objects.filter(root_document_id__in=source_ids).exists():
|
||||
raise ValueError(
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
)
|
||||
|
||||
|
||||
def merge_as_versions(
|
||||
doc_ids: list[int],
|
||||
*,
|
||||
@@ -703,6 +727,7 @@ def merge_as_versions(
|
||||
)
|
||||
documents_by_id = {document.id: document for document in documents}
|
||||
|
||||
validate_merge_as_versions_state(documents, root_document_id)
|
||||
source_ids = [doc_id for doc_id in doc_ids if doc_id != root_document_id]
|
||||
root_document = documents_by_id[root_document_id]
|
||||
next_version_index = (
|
||||
@@ -866,6 +891,7 @@ def split(
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Error splitting document {doc.id}: {e}")
|
||||
raise
|
||||
|
||||
return "OK"
|
||||
|
||||
@@ -911,6 +937,7 @@ def delete_pages(
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(f"Error deleting pages from document {pair.root_doc.id}: {e}")
|
||||
raise
|
||||
|
||||
return "OK"
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Queued document metadata edits."""
|
||||
"""Queued document bulk operations."""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from typing import Any
|
||||
|
||||
from auditlog.context import set_actor
|
||||
@@ -12,6 +13,7 @@ from rest_framework.exceptions import PermissionDenied
|
||||
from documents import bulk_edit
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import PaperlessTask
|
||||
from documents.permissions import has_document_operation_permissions
|
||||
from documents.tasks import bulk_update_documents
|
||||
|
||||
@@ -25,7 +27,35 @@ MODIFIED_FIELDS = {
|
||||
"modify_custom_fields": "custom_fields",
|
||||
"set_permissions": None,
|
||||
}
|
||||
QUEUED_OPERATIONS = frozenset(MODIFIED_FIELDS)
|
||||
LEGACY_MODIFIED_FIELDS = {"delete": "deleted_at", "reprocess": "checksum"}
|
||||
QUEUED_OPERATIONS = frozenset(
|
||||
(
|
||||
*MODIFIED_FIELDS,
|
||||
"delete",
|
||||
"reprocess",
|
||||
"run_workflow",
|
||||
"rotate",
|
||||
"merge",
|
||||
"merge_as_versions",
|
||||
"split",
|
||||
"delete_pages",
|
||||
"edit_pdf",
|
||||
"remove_password",
|
||||
),
|
||||
)
|
||||
USER_OPERATIONS = frozenset(
|
||||
{
|
||||
"split",
|
||||
"merge",
|
||||
"rotate",
|
||||
"delete_pages",
|
||||
"edit_pdf",
|
||||
"remove_password",
|
||||
"merge_as_versions",
|
||||
},
|
||||
)
|
||||
# merge_as_versions doesn't queue any consume tasks
|
||||
TRIGGER_SOURCE_OPERATIONS = USER_OPERATIONS - {"merge_as_versions"}
|
||||
|
||||
|
||||
def snapshot_field(doc_ids: list[int], field: str) -> dict[int, Any]:
|
||||
@@ -66,6 +96,7 @@ def execute_bulk_operation(
|
||||
document_ids: list[int],
|
||||
parameters: dict,
|
||||
user_id: int,
|
||||
trigger_source: str = PaperlessTask.TriggerSource.API_UPLOAD,
|
||||
) -> dict:
|
||||
if operation not in QUEUED_OPERATIONS:
|
||||
raise ValueError(f"Unsupported bulk operation: {operation}")
|
||||
@@ -79,11 +110,46 @@ def execute_bulk_operation(
|
||||
):
|
||||
raise PermissionDenied("Insufficient permissions to execute bulk operation")
|
||||
|
||||
if operation == "run_workflow" and not user.has_perm("documents.view_workflow"):
|
||||
raise PermissionDenied("Insufficient permissions to run this workflow")
|
||||
|
||||
actor_context = set_actor(user) if settings.AUDIT_LOG_ENABLED else nullcontext()
|
||||
|
||||
if operation not in MODIFIED_FIELDS:
|
||||
execution_parameters = parameters.copy()
|
||||
if operation in USER_OPERATIONS:
|
||||
execution_parameters["user"] = user
|
||||
if operation in TRIGGER_SOURCE_OPERATIONS:
|
||||
execution_parameters["trigger_source"] = trigger_source
|
||||
modified_field = LEGACY_MODIFIED_FIELDS.get(operation)
|
||||
with actor_context:
|
||||
old_values = (
|
||||
snapshot_field(document_ids, modified_field)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field
|
||||
else {}
|
||||
)
|
||||
method(document_ids, **execution_parameters)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field:
|
||||
from auditlog.models import LogEntry
|
||||
|
||||
new_values = snapshot_field(document_ids, modified_field)
|
||||
for doc in Document.objects.filter(pk__in=document_ids):
|
||||
LogEntry.objects.log_create(
|
||||
instance=doc,
|
||||
changes={
|
||||
modified_field: [old_values[doc.pk], new_values[doc.pk]],
|
||||
},
|
||||
action=LogEntry.Action.UPDATE,
|
||||
actor=user,
|
||||
additional_data={"reason": f"Bulk edit: {operation}"},
|
||||
)
|
||||
return {"operation": operation, "document_count": len(document_ids)}
|
||||
|
||||
affected_ids: list[int] = []
|
||||
modified_field = MODIFIED_FIELDS[operation]
|
||||
# Keep metadata writes and their audit entries together. Follow-up processing
|
||||
# runs after commit, within this same tracked task.
|
||||
with set_actor(user), transaction.atomic():
|
||||
with actor_context, transaction.atomic():
|
||||
old_values = (
|
||||
snapshot_field(document_ids, modified_field)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field
|
||||
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import logging
|
||||
import re
|
||||
from fnmatch import fnmatch
|
||||
from fnmatch import translate as fnmatch_translate
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from rest_framework import serializers
|
||||
@@ -548,6 +547,48 @@ def existing_document_matches_workflow(
|
||||
return (True, None)
|
||||
|
||||
|
||||
def _glob_to_portable_regex(pattern: str) -> str:
|
||||
"""
|
||||
Translate an fnmatch-style glob into an anchored regex which is accepted by
|
||||
Python, PostgreSQL (ARE), and MariaDB (PCRE).
|
||||
|
||||
fnmatch.translate() can't be used, as its output (`(?s:...)\\Z`) uses syntax
|
||||
PostgreSQL rejects. Supports `*`, `?`, `[seq]` and `[!seq]`.
|
||||
"""
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
n = len(pattern)
|
||||
while i < n:
|
||||
c = pattern[i]
|
||||
i += 1
|
||||
if c == "*":
|
||||
out.append(".*")
|
||||
elif c == "?":
|
||||
out.append(".")
|
||||
elif c == "[":
|
||||
j = i
|
||||
if j < n and pattern[j] == "!":
|
||||
j += 1
|
||||
if j < n and pattern[j] == "]":
|
||||
j += 1
|
||||
while j < n and pattern[j] != "]":
|
||||
j += 1
|
||||
if j >= n:
|
||||
# No closing bracket, treat as a literal
|
||||
out.append(r"\[")
|
||||
else:
|
||||
body = pattern[i:j]
|
||||
i = j + 1
|
||||
negate = body.startswith("!")
|
||||
if negate:
|
||||
body = body[1:]
|
||||
body = "".join(f"\\{ch}" if ch in "\\[^" else ch for ch in body)
|
||||
out.append(f"[{'^' if negate else ''}{body}]")
|
||||
else:
|
||||
out.append(re.escape(c))
|
||||
return f"^{''.join(out)}$"
|
||||
|
||||
|
||||
def prefilter_documents_by_workflowtrigger(
|
||||
documents: QuerySet[Document],
|
||||
trigger: WorkflowTrigger,
|
||||
@@ -628,8 +669,9 @@ def prefilter_documents_by_workflowtrigger(
|
||||
documents = documents.annotate(**annotations).filter(custom_field_q)
|
||||
|
||||
if trigger.filter_filename:
|
||||
regex = fnmatch_translate(trigger.filter_filename).lstrip("^").rstrip("$")
|
||||
documents = documents.filter(original_filename__iregex=regex)
|
||||
documents = documents.filter(
|
||||
original_filename__iregex=_glob_to_portable_regex(trigger.filter_filename),
|
||||
)
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
@@ -789,6 +789,7 @@ def has_document_operation_permissions(
|
||||
bulk_edit.edit_pdf,
|
||||
bulk_edit.merge_as_versions,
|
||||
bulk_edit.remove_password,
|
||||
bulk_edit.run_workflow,
|
||||
]
|
||||
)
|
||||
or (
|
||||
|
||||
@@ -1692,6 +1692,7 @@ class MergeDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin
|
||||
|
||||
class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
|
||||
root_document_id = serializers.IntegerField(required=True)
|
||||
from_webui = serializers.BooleanField(required=False, default=False)
|
||||
version_label = serializers.CharField(
|
||||
required=False,
|
||||
allow_blank=True,
|
||||
@@ -1720,19 +1721,17 @@ class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
|
||||
"root_document_id must be one of the selected documents.",
|
||||
)
|
||||
|
||||
selected_documents = Document.objects.filter(id__in=documents)
|
||||
if selected_documents.filter(root_document__isnull=False).exists():
|
||||
raise serializers.ValidationError(
|
||||
"Only top-level documents can be merged as versions.",
|
||||
)
|
||||
|
||||
source_document_ids = set(documents) - {attrs["root_document_id"]}
|
||||
if Document.global_objects.filter(
|
||||
root_document_id__in=source_document_ids,
|
||||
).exists():
|
||||
raise serializers.ValidationError(
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
# Rechecked under row locks during execution in case document state changes.
|
||||
selected_documents = list(
|
||||
Document.objects.filter(id__in=documents).only("id", "root_document"),
|
||||
)
|
||||
try:
|
||||
bulk_edit.validate_merge_as_versions_state(
|
||||
selected_documents,
|
||||
attrs["root_document_id"],
|
||||
)
|
||||
except ValueError as error:
|
||||
raise serializers.ValidationError(str(error)) from error
|
||||
return attrs
|
||||
|
||||
|
||||
@@ -1805,11 +1804,12 @@ class RemovePasswordDocumentsSerializer(
|
||||
|
||||
|
||||
class DeleteDocumentsSerializer(DocumentSelectionSerializer):
|
||||
pass
|
||||
from_webui = serializers.BooleanField(required=False, default=False)
|
||||
|
||||
|
||||
class ReprocessDocumentsSerializer(DocumentSelectionSerializer):
|
||||
remote_ocr = serializers.BooleanField(required=False, default=False)
|
||||
from_webui = serializers.BooleanField(required=False, default=False)
|
||||
|
||||
|
||||
class RunZoneOcrSerializer(serializers.Serializer):
|
||||
|
||||
@@ -1206,6 +1206,8 @@ def before_task_publish_handler(
|
||||
"document_count": len(task_kwargs["document_ids"]),
|
||||
}
|
||||
owner_id = task_kwargs["user_id"]
|
||||
if task_kwargs["operation"] == "delete":
|
||||
task_type = PaperlessTask.TaskType.BULK_DELETE
|
||||
else:
|
||||
input_data = _extract_input_data(task_type, task_kwargs)
|
||||
owner_id = _extract_owner_id(task_type, task_kwargs)
|
||||
|
||||
@@ -741,8 +741,9 @@ def run_workflows_for_documents(
|
||||
logger.warning(
|
||||
f"Manual workflow run requested for missing workflow {workflow_id}",
|
||||
)
|
||||
return
|
||||
raise
|
||||
|
||||
failed_ids = []
|
||||
for document in Document.objects.filter(
|
||||
id__in=document_ids,
|
||||
root_document__isnull=True,
|
||||
@@ -760,10 +761,16 @@ def run_workflows_for_documents(
|
||||
document=document,
|
||||
)
|
||||
except Exception:
|
||||
failed_ids.append(document.pk)
|
||||
logger.exception(
|
||||
f"Error running workflow {workflow} on document {document.pk}",
|
||||
)
|
||||
|
||||
if failed_ids:
|
||||
raise RuntimeError(
|
||||
f"Workflow {workflow_id} failed for document(s): {failed_ids}. Check logs for details.",
|
||||
)
|
||||
|
||||
|
||||
def update_document_parent_tags(tag: Tag, new_parent: Tag) -> None:
|
||||
"""
|
||||
|
||||
@@ -704,11 +704,12 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.run_workflow")
|
||||
def test_run_workflow_requires_ownership(self, m) -> None:
|
||||
"""A non-owner without document ownership is forbidden (actions mutate docs)."""
|
||||
"""A non-owner with change permission cannot run mutating workflow actions."""
|
||||
self.setup_mock(m, "run_workflow")
|
||||
user = UserFactory(username="no_perms")
|
||||
user = UserFactory(username="non_owner")
|
||||
grant_all_global(user)
|
||||
doc = DocumentFactory(owner=UserFactory(username="other"))
|
||||
grant_object(user, doc, "change_document")
|
||||
workflow = self._make_manual_workflow()
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
|
||||
@@ -4,14 +4,18 @@ from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from asgiref.sync import async_to_sync
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APIClient
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.views import _iterate_in_sync_thread
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.permissions import grant_global
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from django.contrib.auth.models import User
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
@@ -62,7 +66,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
),
|
||||
mock.patch(
|
||||
"paperless_ai.chat.stream_chat_with_documents",
|
||||
return_value=iter(chunks),
|
||||
return_value=(c for c in chunks),
|
||||
),
|
||||
):
|
||||
resp = self.client.post(
|
||||
@@ -74,7 +78,11 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
|
||||
assert resp.status_code == status.HTTP_200_OK
|
||||
assert not resp.has_header("Content-Encoding")
|
||||
assert list(resp.streaming_content) == [c.encode() for c in chunks]
|
||||
|
||||
async def read_parts() -> list[bytes]:
|
||||
return [part async for part in resp.streaming_content]
|
||||
|
||||
assert async_to_sync(read_parts)() == [c.encode() for c in chunks]
|
||||
|
||||
def test_missing_question_is_rejected(self) -> None:
|
||||
with mock.patch(
|
||||
@@ -155,3 +163,74 @@ class TestChatStreamingViewUnrestrictedFlag:
|
||||
assert (
|
||||
mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
|
||||
)
|
||||
|
||||
|
||||
class TestIterateInSyncThread:
|
||||
@pytest.fixture
|
||||
def events(self) -> list[str]:
|
||||
return []
|
||||
|
||||
@pytest.fixture
|
||||
def chunks(self, events: list[str]) -> Generator[str, None, None]:
|
||||
def generate() -> Generator[str, None, None]:
|
||||
try:
|
||||
for chunk in ("first", "second"):
|
||||
events.append(f"produced {chunk}")
|
||||
yield chunk
|
||||
finally:
|
||||
events.append("closed")
|
||||
|
||||
return generate()
|
||||
|
||||
def test_yields_each_chunk_before_pulling_the_next(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The wrapper is consumed
|
||||
THEN:
|
||||
- Each chunk is delivered before the next one is generated,
|
||||
rather than the whole generator being drained up front
|
||||
"""
|
||||
|
||||
async def consume() -> None:
|
||||
async for chunk in _iterate_in_sync_thread(chunks):
|
||||
events.append(f"received {chunk}")
|
||||
|
||||
async_to_sync(consume)()
|
||||
|
||||
assert events == [
|
||||
"produced first",
|
||||
"received first",
|
||||
"produced second",
|
||||
"received second",
|
||||
"closed",
|
||||
]
|
||||
|
||||
def test_closing_early_closes_the_source_generator(
|
||||
self,
|
||||
chunks: Generator[str, None, None],
|
||||
events: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A sync generator wrapped for async iteration
|
||||
WHEN:
|
||||
- The consumer closes the wrapper after the first chunk, as
|
||||
happens when the client disconnects
|
||||
THEN:
|
||||
- The source generator is closed, so its cleanup runs
|
||||
"""
|
||||
|
||||
async def consume_one() -> None:
|
||||
wrapper = _iterate_in_sync_thread(chunks)
|
||||
await anext(wrapper)
|
||||
await wrapper.aclose()
|
||||
|
||||
async_to_sync(consume_one)()
|
||||
|
||||
assert events == ["produced first", "closed"]
|
||||
@@ -700,10 +700,10 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
|
||||
def test_delete_documents_old_uuid_field(self, m) -> None:
|
||||
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
|
||||
doc_ids = [self.doc1.id, self.doc2.id, self.doc3.id]
|
||||
bulk_edit.delete(doc_ids)
|
||||
with self.assertLogs(level="WARNING") as cm:
|
||||
bulk_edit.delete(doc_ids)
|
||||
self.assertIn("possible incompatible database column", cm.output[0])
|
||||
with self.assertRaisesRegex(Exception, "Data too long for column"):
|
||||
bulk_edit.delete(doc_ids)
|
||||
self.assertIn("possible incompatible database column", cm.output[0])
|
||||
|
||||
|
||||
class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
@@ -1040,8 +1040,9 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
|
||||
mock_consume_file.assert_not_called()
|
||||
|
||||
@mock.patch("documents.bulk_edit.group")
|
||||
@mock.patch("documents.tasks.consume_file.s")
|
||||
def test_split(self, mock_consume_file) -> None:
|
||||
def test_split(self, mock_consume_file, mock_group) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Existing documents
|
||||
@@ -1062,6 +1063,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
self.assertIsNone(mock_consume_file.call_args.kwargs["overrides"].asn)
|
||||
|
||||
self.assertEqual(result, "OK")
|
||||
mock_group.return_value.delay.assert_called_once()
|
||||
|
||||
@mock.patch("documents.bulk_edit.group")
|
||||
@mock.patch("documents.pdf_ops.build_pdfs")
|
||||
@@ -1170,8 +1172,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
sig.apply_async.side_effect = Exception("boom")
|
||||
mock_chord.return_value = sig
|
||||
|
||||
result = bulk_edit.split(doc_ids, pages, delete_originals=True)
|
||||
self.assertEqual(result, "OK")
|
||||
with self.assertRaisesMessage(Exception, "boom"):
|
||||
bulk_edit.split(doc_ids, pages, delete_originals=True)
|
||||
|
||||
self.doc2.refresh_from_db()
|
||||
self.assertEqual(self.doc2.archive_serial_number, 222)
|
||||
@@ -1193,7 +1195,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
pages = [[1, 2], [3]]
|
||||
|
||||
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
|
||||
bulk_edit.split(doc_ids, pages)
|
||||
with self.assertRaisesMessage(Exception, "Error building PDFs"):
|
||||
bulk_edit.split(doc_ids, pages)
|
||||
error_str = cm.output[0]
|
||||
expected_str = "Error splitting document 2"
|
||||
self.assertIn(expected_str, error_str)
|
||||
@@ -1242,12 +1245,68 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
doc_ids = [self.doc2.id, self.doc3.id]
|
||||
|
||||
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
|
||||
bulk_edit.rotate(doc_ids, 90)
|
||||
with self.assertRaisesMessage(
|
||||
RuntimeError,
|
||||
f"Rotation failed for document(s): {doc_ids}",
|
||||
):
|
||||
bulk_edit.rotate(doc_ids, 90)
|
||||
error_str = cm.output[0]
|
||||
expected_str = "Error rotating document"
|
||||
self.assertIn(expected_str, error_str)
|
||||
self.assertEqual(mock_rotate_pdf.call_count, 2)
|
||||
mock_consume_delay.assert_not_called()
|
||||
|
||||
@mock.patch("documents.tasks.consume_file.apply_async")
|
||||
@mock.patch("documents.pdf_ops.rotate_pdf")
|
||||
def test_rotate_continues_after_preparation_failure(
|
||||
self,
|
||||
mock_rotate_pdf,
|
||||
mock_consume,
|
||||
) -> None:
|
||||
def rotate(source, destination, degrees):
|
||||
if source == self.doc2.source_path:
|
||||
raise RuntimeError("Invalid PDF")
|
||||
shutil.copy(source, destination)
|
||||
|
||||
mock_rotate_pdf.side_effect = rotate
|
||||
with self.assertRaisesMessage(
|
||||
RuntimeError,
|
||||
f"Rotation failed for document(s): [{self.doc2.id}]",
|
||||
):
|
||||
bulk_edit.rotate([self.doc2.id, self.doc3.id], 90)
|
||||
|
||||
self.assertEqual(mock_rotate_pdf.call_count, 2)
|
||||
mock_consume.assert_called_once()
|
||||
self.assertEqual(
|
||||
mock_consume.call_args.kwargs["kwargs"]["input_doc"].root_document_id,
|
||||
self.doc3.id,
|
||||
)
|
||||
|
||||
@mock.patch("documents.tasks.consume_file.apply_async")
|
||||
@mock.patch("documents.pdf_ops.rotate_pdf")
|
||||
def test_rotate_continues_after_queue_failure(
|
||||
self,
|
||||
mock_rotate_pdf,
|
||||
mock_consume,
|
||||
) -> None:
|
||||
mock_rotate_pdf.side_effect = lambda source, destination, degrees: shutil.copy(
|
||||
source,
|
||||
destination,
|
||||
)
|
||||
mock_consume.side_effect = [RuntimeError("Broker unavailable"), None]
|
||||
with self.assertRaisesMessage(
|
||||
RuntimeError,
|
||||
f"Rotation failed for document(s): [{self.doc2.id}]",
|
||||
):
|
||||
bulk_edit.rotate([self.doc2.id, self.doc3.id], 90)
|
||||
|
||||
self.assertEqual(mock_rotate_pdf.call_count, 2)
|
||||
self.assertEqual(mock_consume.call_count, 2)
|
||||
self.assertEqual(
|
||||
mock_consume.call_args.kwargs["kwargs"]["input_doc"].root_document_id,
|
||||
self.doc3.id,
|
||||
)
|
||||
|
||||
@mock.patch("documents.tasks.consume_file.apply_async")
|
||||
def test_rotate_non_pdf(
|
||||
self,
|
||||
@@ -1377,7 +1436,8 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
pages = [1, 3]
|
||||
|
||||
with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
|
||||
bulk_edit.delete_pages(doc_ids, pages)
|
||||
with self.assertRaisesMessage(Exception, "Error removing pages"):
|
||||
bulk_edit.delete_pages(doc_ids, pages)
|
||||
error_str = cm.output[0]
|
||||
expected_str = "Error deleting pages from document"
|
||||
self.assertIn(expected_str, error_str)
|
||||
|
||||
@@ -54,6 +54,7 @@ def test_request_only_queues(bulk_setup, endpoint, operation, parameters):
|
||||
"document_ids": [doc.pk],
|
||||
"parameters": parameters,
|
||||
"user_id": user.pk,
|
||||
"trigger_source": "web_ui",
|
||||
}
|
||||
assert enqueue.call_args.kwargs["headers"]["trigger_source"] == "web_ui"
|
||||
|
||||
@@ -102,6 +103,157 @@ def test_permissions_rechecked_on_worker(bulk_setup):
|
||||
assert doc.correspondent_id is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_document_action_runs_with_user_in_worker(bulk_setup):
|
||||
user, _, doc, _ = bulk_setup
|
||||
with mock.patch("documents.bulk_edit.rotate") as rotate:
|
||||
result = execute_bulk_operation(
|
||||
"rotate",
|
||||
[doc.pk],
|
||||
{"degrees": 90},
|
||||
user.pk,
|
||||
trigger_source="web_ui",
|
||||
)
|
||||
rotate.assert_called_once_with(
|
||||
[doc.pk],
|
||||
degrees=90,
|
||||
user=user,
|
||||
trigger_source="web_ui",
|
||||
)
|
||||
assert result == {"operation": "rotate", "document_count": 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_delete_task_without_audit_log(bulk_setup, settings):
|
||||
user, _, doc, _ = bulk_setup
|
||||
settings.AUDIT_LOG_ENABLED = False
|
||||
with mock.patch(
|
||||
"documents.bulk_operations.set_actor",
|
||||
side_effect=AssertionError("audit context must not be entered"),
|
||||
):
|
||||
result = execute_bulk_operation("delete", [doc.pk], {}, user.pk)
|
||||
assert result == {"operation": "delete", "document_count": 1}
|
||||
assert not Document.objects.filter(pk=doc.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_metadata_task_without_audit_log(bulk_setup, settings):
|
||||
user, _, doc, correspondent = bulk_setup
|
||||
settings.AUDIT_LOG_ENABLED = False
|
||||
with (
|
||||
mock.patch(
|
||||
"documents.bulk_operations.set_actor",
|
||||
side_effect=AssertionError("audit context must not be entered"),
|
||||
),
|
||||
mock.patch("documents.bulk_operations.bulk_update_documents"),
|
||||
):
|
||||
result = execute_bulk_operation(
|
||||
"set_correspondent",
|
||||
[doc.pk],
|
||||
{"correspondent": correspondent.pk},
|
||||
user.pk,
|
||||
)
|
||||
doc.refresh_from_db()
|
||||
assert doc.correspondent_id == correspondent.pk
|
||||
assert result == {"operation": "set_correspondent", "document_count": 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_document_action_queue_contains_only_serializable_parameters(bulk_setup):
|
||||
user, client, doc, _ = bulk_setup
|
||||
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
|
||||
enqueue.return_value.id = "rotate-task"
|
||||
response = client.post(
|
||||
"/api/documents/rotate/",
|
||||
{"documents": [doc.pk], "degrees": 90, "from_webui": True},
|
||||
format="json",
|
||||
)
|
||||
assert response.data == {"result": "OK", "task_id": "rotate-task"}
|
||||
assert enqueue.call_args.kwargs["kwargs"] == {
|
||||
"operation": "rotate",
|
||||
"document_ids": [doc.pk],
|
||||
"parameters": {"degrees": 90, "source_mode": "latest_version"},
|
||||
"user_id": user.pk,
|
||||
"trigger_source": "web_ui",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_reprocess_keeps_bulk_audit_entry(bulk_setup, settings):
|
||||
user, _, doc, _ = bulk_setup
|
||||
settings.AUDIT_LOG_ENABLED = True
|
||||
LogEntry.objects.all().delete()
|
||||
with mock.patch("documents.bulk_edit.reprocess"):
|
||||
execute_bulk_operation("reprocess", [doc.pk], {}, user.pk)
|
||||
entry = LogEntry.objects.get_for_object(doc).get()
|
||||
assert entry.changes_dict["checksum"] == [doc.checksum, doc.checksum]
|
||||
assert entry.actor_id == user.pk
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_workflow_runs_in_tracked_task_instead_of_dispatching_another(bulk_setup):
|
||||
user, _, doc, _ = bulk_setup
|
||||
with mock.patch("documents.bulk_edit.run_workflows_for_documents") as run:
|
||||
execute_bulk_operation(
|
||||
"run_workflow",
|
||||
[doc.pk],
|
||||
{"workflow": 7, "filter_documents": False},
|
||||
user.pk,
|
||||
)
|
||||
run.assert_called_once_with(
|
||||
document_ids=[doc.pk],
|
||||
workflow_id=7,
|
||||
filter_documents=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_delete_failure_propagates_from_worker(bulk_setup):
|
||||
user, _, doc, _ = bulk_setup
|
||||
with mock.patch(
|
||||
"documents.models.Document.delete",
|
||||
side_effect=RuntimeError("delete failed"),
|
||||
):
|
||||
with pytest.raises(RuntimeError, match="delete failed"):
|
||||
execute_bulk_operation("delete", [doc.pk], {}, user.pk)
|
||||
assert Document.objects.filter(pk=doc.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize(
|
||||
"operation,parameters,preparation",
|
||||
[
|
||||
("rotate", {"degrees": 90}, "rotate_pdf"),
|
||||
("split", {"pages": [[1]]}, "build_pdfs"),
|
||||
("delete_pages", {"pages": [1]}, "remove_pages"),
|
||||
],
|
||||
)
|
||||
def test_preparation_failure_propagates_from_worker(
|
||||
bulk_setup,
|
||||
operation,
|
||||
parameters,
|
||||
preparation,
|
||||
):
|
||||
user, _, doc, _ = bulk_setup
|
||||
doc.mime_type = "application/pdf"
|
||||
doc.save()
|
||||
with (
|
||||
mock.patch(
|
||||
f"documents.bulk_edit.pdf_ops.{preparation}",
|
||||
side_effect=RuntimeError("preparation failed"),
|
||||
),
|
||||
mock.patch("documents.bulk_edit.consume_file.apply_async") as consume,
|
||||
mock.patch("documents.bulk_edit.consume_file.s") as consume_signature,
|
||||
pytest.raises(
|
||||
RuntimeError,
|
||||
match="Rotation failed" if operation == "rotate" else "preparation failed",
|
||||
),
|
||||
):
|
||||
execute_bulk_operation(operation, [doc.pk], parameters, user.pk)
|
||||
consume.assert_not_called()
|
||||
consume_signature.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_queue_failure_does_not_change_documents(bulk_setup):
|
||||
_, client, doc, correspondent = bulk_setup
|
||||
@@ -162,6 +314,22 @@ def test_tracked_outcome_records_initiator_and_result(bulk_setup, failed):
|
||||
assert task.result_data == {"document_count": 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_queued_delete_is_tracked_as_bulk_delete(bulk_setup):
|
||||
user, _, doc, _ = bulk_setup
|
||||
before_task_publish_handler(
|
||||
headers={"task": execute_bulk_operation.name, "id": "delete-task"},
|
||||
body=(
|
||||
(),
|
||||
{"operation": "delete", "document_ids": [doc.pk], "user_id": user.pk},
|
||||
{},
|
||||
),
|
||||
)
|
||||
task = PaperlessTask.objects.get(task_id="delete-task")
|
||||
assert task.task_type == PaperlessTask.TaskType.BULK_DELETE
|
||||
assert task.owner_id == user.pk
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_large_filtered_selection_is_one_task(bulk_setup):
|
||||
_, client, _, _ = bulk_setup
|
||||
@@ -284,24 +452,46 @@ async def test_completion_socket_is_private(anyio_backend):
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize("endpoint", ["delete", "bulk_edit"])
|
||||
def test_trash_remains_synchronous(bulk_setup, endpoint):
|
||||
_, client, doc, _ = bulk_setup
|
||||
data = {"documents": [doc.pk]}
|
||||
if endpoint == "bulk_edit":
|
||||
data.update(method="delete", parameters={})
|
||||
with (
|
||||
mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue,
|
||||
mock.patch("documents.search.get_backend"),
|
||||
):
|
||||
def test_legacy_trash_queues_without_deleting_in_request(bulk_setup):
|
||||
user, client, doc, _ = bulk_setup
|
||||
data = {"documents": [doc.pk], "method": "delete", "parameters": {}}
|
||||
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
|
||||
enqueue.return_value.id = "delete-task"
|
||||
response = client.post(
|
||||
f"/api/documents/{endpoint}/",
|
||||
"/api/documents/bulk_edit/",
|
||||
data,
|
||||
format="json",
|
||||
HTTP_ACCEPT="application/json; version=9",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.data == {"result": "OK"}
|
||||
assert not Document.objects.filter(pk=doc.pk).exists()
|
||||
assert Document.deleted_objects.filter(pk=doc.pk).exists()
|
||||
enqueue.assert_not_called()
|
||||
assert response.data == {"result": "OK", "task_id": "delete-task"}
|
||||
assert Document.objects.filter(pk=doc.pk).exists()
|
||||
assert enqueue.call_args.kwargs["kwargs"] == {
|
||||
"operation": "delete",
|
||||
"document_ids": [doc.pk],
|
||||
"parameters": {},
|
||||
"user_id": user.pk,
|
||||
"trigger_source": "api_upload",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_dedicated_trash_queues_without_deleting_in_request(bulk_setup):
|
||||
user, client, doc, _ = bulk_setup
|
||||
with mock.patch("documents.views.execute_bulk_operation.apply_async") as enqueue:
|
||||
enqueue.return_value.id = "delete-task"
|
||||
response = client.post(
|
||||
"/api/documents/delete/",
|
||||
{"documents": [doc.pk]},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.data == {"result": "OK", "task_id": "delete-task"}
|
||||
assert Document.objects.filter(pk=doc.pk).exists()
|
||||
assert enqueue.call_args.kwargs["kwargs"] == {
|
||||
"operation": "delete",
|
||||
"document_ids": [doc.pk],
|
||||
"parameters": {},
|
||||
"user_id": user.pk,
|
||||
"trigger_source": "api_upload",
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import fnmatch
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.matching import _glob_to_portable_regex
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pattern", "expected"),
|
||||
[
|
||||
pytest.param("", r"^$", id="empty"),
|
||||
pytest.param("abc", r"^abc$", id="literal"),
|
||||
pytest.param("*", r"^.*$", id="star"),
|
||||
pytest.param("?", r"^.$", id="question"),
|
||||
pytest.param("*.pdf", r"^.*\.pdf$", id="star-escapes-dot"),
|
||||
pytest.param("[abc]", r"^[abc]$", id="class"),
|
||||
pytest.param("[a-z]", r"^[a-z]$", id="class-range"),
|
||||
pytest.param("[!abc]", r"^[^abc]$", id="negated-class"),
|
||||
pytest.param("[]a]", r"^[]a]$", id="leading-close-bracket-is-member"),
|
||||
pytest.param("[!]a]", r"^[^]a]$", id="negated-leading-close-bracket"),
|
||||
pytest.param("[a^b]", r"^[a\^b]$", id="caret-escaped-in-class"),
|
||||
pytest.param("[a[b]", r"^[a\[b]$", id="open-bracket-escaped-in-class"),
|
||||
pytest.param("[a\\b]", r"^[a\\b]$", id="backslash-escaped-in-class"),
|
||||
pytest.param("[abc", r"^\[abc$", id="unclosed-class-literal"),
|
||||
pytest.param("[", r"^\[$", id="lone-open-bracket"),
|
||||
pytest.param("[!", r"^\[!$", id="unclosed-negation-literal"),
|
||||
pytest.param("[]", r"^\[\]$", id="unclosed-with-only-close-bracket"),
|
||||
pytest.param("a[0-9]?*", r"^a[0-9]..*$", id="mixed"),
|
||||
],
|
||||
)
|
||||
def test_translation(pattern: str, expected: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A glob pattern
|
||||
WHEN:
|
||||
- It is translated to a portable regex
|
||||
THEN:
|
||||
- The exact anchored regex is produced
|
||||
"""
|
||||
assert _glob_to_portable_regex(pattern) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern",
|
||||
["*", "?", "*.pdf", "a?c", "[abc]x", "[!abc]x", "[]a]", "[!]a]", "[a-c]*", "[a"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
["", "a", "abc", "x", "]", "]x", "file.pdf", "ax", "bx", "dx", "[a"],
|
||||
)
|
||||
def test_agrees_with_fnmatch(pattern: str, name: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A glob pattern and a name
|
||||
WHEN:
|
||||
- Matched via the portable regex and via stdlib fnmatch
|
||||
THEN:
|
||||
- Both agree
|
||||
"""
|
||||
regex = _glob_to_portable_regex(pattern)
|
||||
assert bool(re.search(regex, name)) == fnmatch.fnmatchcase(name, pattern)
|
||||
@@ -8,6 +8,7 @@ from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.bulk_edit import merge_as_versions
|
||||
from documents.bulk_operations import execute_bulk_operation
|
||||
from documents.models import Document
|
||||
from documents.serialisers import MergeDocumentsAsVersionsSerializer
|
||||
from paperless_testing.factories import UserFactory
|
||||
@@ -34,6 +35,7 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
|
||||
{
|
||||
"documents": [self.doc1.id, self.doc2.id, self.doc3.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
"from_webui": False,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -106,7 +108,6 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
|
||||
def test_rejects_selected_version(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
@@ -116,54 +117,12 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(serializer.is_valid())
|
||||
self.assertEqual(
|
||||
serializer.errors["non_field_errors"][0],
|
||||
"Only top-level documents can be merged as versions.",
|
||||
)
|
||||
|
||||
def test_rejects_source_document_with_versions(self) -> None:
|
||||
Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
serializer = MergeDocumentsAsVersionsSerializer(
|
||||
data={
|
||||
"documents": [self.doc1.id, self.doc2.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(serializer.is_valid())
|
||||
self.assertEqual(
|
||||
serializer.errors["non_field_errors"][0],
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
)
|
||||
|
||||
def test_rejects_source_document_with_trashed_versions(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
version.delete() # trashed, but still points at doc1
|
||||
serializer = MergeDocumentsAsVersionsSerializer(
|
||||
data={
|
||||
"documents": [self.doc1.id, self.doc2.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(serializer.is_valid())
|
||||
self.assertEqual(
|
||||
serializer.errors["non_field_errors"][0],
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
)
|
||||
|
||||
def test_allows_root_document_with_versions(self) -> None:
|
||||
Document.objects.create(
|
||||
checksum="D",
|
||||
@@ -182,6 +141,95 @@ class TestMergeDocumentsAsVersionsSerializer(TestCase):
|
||||
|
||||
|
||||
class TestMergeDocumentsAsVersions(TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.doc1 = Document.objects.create(checksum="test-A", title="A")
|
||||
self.doc2 = Document.objects.create(checksum="test-B", title="B")
|
||||
|
||||
def test_rejects_selected_version(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Only top-level documents can be merged as versions.",
|
||||
):
|
||||
merge_as_versions(
|
||||
[version.id, self.doc2.id],
|
||||
root_document_id=self.doc2.id,
|
||||
)
|
||||
|
||||
def test_rejects_source_document_with_versions(self) -> None:
|
||||
Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
):
|
||||
merge_as_versions(
|
||||
[self.doc1.id, self.doc2.id],
|
||||
root_document_id=self.doc2.id,
|
||||
)
|
||||
|
||||
def test_rejects_source_document_with_trashed_versions(self) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="D",
|
||||
title="D",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
)
|
||||
version.delete() # trashed, but still points at doc1
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
):
|
||||
merge_as_versions(
|
||||
[self.doc1.id, self.doc2.id],
|
||||
root_document_id=self.doc2.id,
|
||||
)
|
||||
|
||||
@mock.patch("documents.bulk_edit.DocumentsStatusManager")
|
||||
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
def test_rejects_source_that_gained_versions_after_validation(
|
||||
self,
|
||||
_get_backend_mock,
|
||||
_bulk_update_mock,
|
||||
_status_manager_mock,
|
||||
) -> None:
|
||||
destination = Document.objects.create(checksum="destination")
|
||||
serializer = MergeDocumentsAsVersionsSerializer(
|
||||
data={
|
||||
"documents": [self.doc1.id, destination.id],
|
||||
"root_document_id": destination.id,
|
||||
},
|
||||
)
|
||||
self.assertTrue(serializer.is_valid(), serializer.errors)
|
||||
|
||||
merge_as_versions(
|
||||
[self.doc1.id, self.doc2.id],
|
||||
root_document_id=self.doc1.id,
|
||||
)
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
):
|
||||
merge_as_versions(
|
||||
serializer.validated_data["documents"],
|
||||
root_document_id=serializer.validated_data["root_document_id"],
|
||||
)
|
||||
|
||||
self.doc1.refresh_from_db()
|
||||
self.doc2.refresh_from_db()
|
||||
self.assertIsNone(self.doc1.root_document_id)
|
||||
self.assertEqual(self.doc2.root_document_id, self.doc1.id)
|
||||
|
||||
@mock.patch("documents.bulk_edit.DocumentsStatusManager")
|
||||
@mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
@@ -348,6 +396,17 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
|
||||
owner=self.user,
|
||||
)
|
||||
self.client.force_authenticate(user=self.user)
|
||||
enqueue = mock.patch(
|
||||
"documents.views.execute_bulk_operation.apply_async",
|
||||
side_effect=lambda *, kwargs, headers: self.run_operation(kwargs),
|
||||
)
|
||||
enqueue.start()
|
||||
self.addCleanup(enqueue.stop)
|
||||
|
||||
@staticmethod
|
||||
def run_operation(kwargs):
|
||||
execute_bulk_operation(**kwargs)
|
||||
return mock.Mock(id="test-bulk-operation")
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.merge_as_versions")
|
||||
def test_merges_documents_as_versions(self, merge_mock) -> None:
|
||||
@@ -367,7 +426,10 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data, {"result": "OK"})
|
||||
self.assertEqual(
|
||||
response.data,
|
||||
{"result": "OK", "task_id": "test-bulk-operation"},
|
||||
)
|
||||
merge_mock.assert_called_once_with(
|
||||
[self.doc1.id, self.doc2.id],
|
||||
root_document_id=self.doc2.id,
|
||||
@@ -440,27 +502,36 @@ class TestMergeDocumentsAsVersionsAPI(APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
merge_mock.assert_not_called()
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.merge_as_versions")
|
||||
def test_rejects_source_document_with_versions(self, merge_mock) -> None:
|
||||
Document.objects.create(
|
||||
@mock.patch("documents.views.execute_bulk_operation.apply_async")
|
||||
def test_rejects_source_document_with_versions_before_queueing(
|
||||
self,
|
||||
enqueue_mock,
|
||||
) -> None:
|
||||
version = Document.objects.create(
|
||||
checksum="C",
|
||||
title="C",
|
||||
root_document=self.doc1,
|
||||
version_index=1,
|
||||
owner=self.user,
|
||||
)
|
||||
|
||||
response = self.client.post(
|
||||
"/api/documents/merge_as_versions/",
|
||||
{
|
||||
"documents": [self.doc1.id, self.doc2.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
merge_mock.assert_not_called()
|
||||
for trashed in (False, True):
|
||||
with self.subTest(trashed=trashed):
|
||||
if trashed:
|
||||
version.delete()
|
||||
response = self.client.post(
|
||||
"/api/documents/merge_as_versions/",
|
||||
{
|
||||
"documents": [self.doc1.id, self.doc2.id],
|
||||
"root_document_id": self.doc2.id,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(
|
||||
response.data["non_field_errors"][0],
|
||||
"Documents with existing versions cannot be merged into another document.",
|
||||
)
|
||||
enqueue_mock.assert_not_called()
|
||||
self.doc1.refresh_from_db()
|
||||
self.assertIsNone(self.doc1.root_document_id)
|
||||
|
||||
|
||||
@@ -2333,9 +2333,12 @@ class TestWorkflows(
|
||||
self.assertEqual(doc.owner, self.user2)
|
||||
|
||||
def test_manual_workflow_run_missing_workflow(self) -> None:
|
||||
"""A missing workflow id logs a warning rather than raising."""
|
||||
"""A workflow removed after queueing fails the task."""
|
||||
doc = DocumentFactory()
|
||||
with self.assertLogs("paperless.tasks", level="WARNING"):
|
||||
with (
|
||||
self.assertLogs("paperless.tasks", level="WARNING"),
|
||||
self.assertRaises(Workflow.DoesNotExist),
|
||||
):
|
||||
tasks.run_workflows_for_documents([doc.pk], 9999)
|
||||
|
||||
@mock.patch("documents.tasks.send_websocket_document_updated")
|
||||
@@ -2356,7 +2359,13 @@ class TestWorkflows(
|
||||
|
||||
mock_run_workflows.side_effect = side_effect
|
||||
|
||||
with self.assertLogs("paperless.tasks", level="ERROR"):
|
||||
with (
|
||||
self.assertLogs("paperless.tasks", level="ERROR"),
|
||||
self.assertRaisesMessage(
|
||||
RuntimeError,
|
||||
f"failed for document(s): [{doc1.pk}]",
|
||||
),
|
||||
):
|
||||
tasks.run_workflows_for_documents([doc1.pk, doc2.pk], workflow.pk)
|
||||
|
||||
# Both documents were attempted; the websocket update only fired for the
|
||||
|
||||
+64
-88
@@ -8,7 +8,9 @@ import tempfile
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Generator
|
||||
from dataclasses import asdict
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
@@ -29,6 +31,7 @@ from urllib.parse import urlparse
|
||||
import httpx
|
||||
import magic
|
||||
import pathvalidate
|
||||
from asgiref.sync import sync_to_async
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import Group
|
||||
from django.contrib.auth.models import User
|
||||
@@ -109,9 +112,9 @@ from documents import bulk_edit
|
||||
from documents.bulk_download import ArchiveOnlyStrategy
|
||||
from documents.bulk_download import OriginalAndArchiveStrategy
|
||||
from documents.bulk_download import OriginalsOnlyStrategy
|
||||
from documents.bulk_operations import QUEUED_OPERATIONS
|
||||
from documents.bulk_operations import TRIGGER_SOURCE_OPERATIONS
|
||||
from documents.bulk_operations import USER_OPERATIONS
|
||||
from documents.bulk_operations import execute_bulk_operation
|
||||
from documents.bulk_operations import snapshot_field
|
||||
from documents.caching import get_llm_suggestion_cache
|
||||
from documents.caching import get_metadata_cache
|
||||
from documents.caching import get_suggestion_cache
|
||||
@@ -2512,6 +2515,24 @@ class DocumentViewSet(
|
||||
)
|
||||
|
||||
|
||||
async def _iterate_in_sync_thread(
|
||||
chunks: Generator[str, None, None],
|
||||
) -> AsyncIterator[str]:
|
||||
"""Serve a blocking sync generator to an ASGI server chunk by chunk.
|
||||
|
||||
Under ASGI, Django drains a sync iterator with ``sync_to_async(list)``
|
||||
before sending anything. Pull each chunk in the request's sync thread
|
||||
instead, where the generator's ORM access belongs.
|
||||
"""
|
||||
sentinel = object() # StopIteration can't cross an asyncio Future
|
||||
next_chunk = sync_to_async(next)
|
||||
try:
|
||||
while (chunk := await next_chunk(chunks, sentinel)) is not sentinel:
|
||||
yield chunk
|
||||
finally:
|
||||
await sync_to_async(chunks.close)()
|
||||
|
||||
|
||||
class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]):
|
||||
q = serializers.CharField(required=True, max_length=4000)
|
||||
document_id = serializers.IntegerField(required=False, allow_null=True)
|
||||
@@ -2570,16 +2591,17 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
user=request.user,
|
||||
)
|
||||
|
||||
response = StreamingHttpResponse(
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
return StreamingHttpResponse(
|
||||
_iterate_in_sync_thread(
|
||||
stream_chat_with_documents(
|
||||
query_str=question,
|
||||
documents=documents,
|
||||
unrestricted=unrestricted,
|
||||
output_language=output_language,
|
||||
),
|
||||
),
|
||||
content_type="text/event-stream",
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
@@ -3070,38 +3092,30 @@ class DocumentSelectionMixin:
|
||||
class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
permission_classes = (IsAuthenticated,)
|
||||
parser_classes = (parsers.JSONParser,)
|
||||
METHOD_NAMES_REQUIRING_USER = {
|
||||
"split",
|
||||
"merge",
|
||||
"rotate",
|
||||
"delete_pages",
|
||||
"edit_pdf",
|
||||
"remove_password",
|
||||
"merge_as_versions",
|
||||
}
|
||||
# merge_as_versions doesn't queue any consume tasks
|
||||
METHOD_NAMES_REQUIRING_TRIGGER_SOURCE = METHOD_NAMES_REQUIRING_USER - {
|
||||
"merge_as_versions",
|
||||
}
|
||||
|
||||
def _queue_bulk_operation(self, method, documents, parameters, from_webui):
|
||||
if "update_callback" in parameters:
|
||||
raise ValueError("Unsupported bulk operation parameter: update_callback")
|
||||
signature(method).bind(documents, **parameters)
|
||||
trigger_source = (
|
||||
PaperlessTask.TriggerSource.WEB_UI
|
||||
if from_webui
|
||||
else PaperlessTask.TriggerSource.API_UPLOAD
|
||||
)
|
||||
final_parameters = parameters.copy()
|
||||
if method.__name__ in USER_OPERATIONS:
|
||||
final_parameters["user"] = self.request.user
|
||||
if method.__name__ in TRIGGER_SOURCE_OPERATIONS:
|
||||
final_parameters["trigger_source"] = trigger_source
|
||||
signature(method).bind(documents, **final_parameters)
|
||||
task = execute_bulk_operation.apply_async(
|
||||
kwargs={
|
||||
"operation": method.__name__,
|
||||
"document_ids": documents,
|
||||
"parameters": parameters,
|
||||
"user_id": self.request.user.pk,
|
||||
"trigger_source": trigger_source,
|
||||
},
|
||||
headers={
|
||||
"trigger_source": (
|
||||
PaperlessTask.TriggerSource.WEB_UI
|
||||
if from_webui
|
||||
else PaperlessTask.TriggerSource.API_UPLOAD
|
||||
),
|
||||
},
|
||||
headers={"trigger_source": trigger_source},
|
||||
)
|
||||
return Response({"result": "OK", "task_id": task.id})
|
||||
|
||||
@@ -3225,15 +3239,6 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
user = self.request.user
|
||||
from_webui = validated_data.get("from_webui", False)
|
||||
|
||||
if method.__name__ in self.METHOD_NAMES_REQUIRING_USER:
|
||||
parameters["user"] = user
|
||||
if method.__name__ in self.METHOD_NAMES_REQUIRING_TRIGGER_SOURCE:
|
||||
parameters["trigger_source"] = (
|
||||
PaperlessTask.TriggerSource.WEB_UI
|
||||
if from_webui
|
||||
else PaperlessTask.TriggerSource.API_UPLOAD
|
||||
)
|
||||
|
||||
if not has_document_operation_permissions(
|
||||
user=user,
|
||||
documents=documents,
|
||||
@@ -3243,8 +3248,12 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
try:
|
||||
result = method(documents, **parameters)
|
||||
return Response({"result": result})
|
||||
return self._queue_bulk_operation(
|
||||
method,
|
||||
documents,
|
||||
parameters,
|
||||
from_webui,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"An error occurred performing {operation_label}: {e!s}")
|
||||
return HttpResponseBadRequest(
|
||||
@@ -3265,19 +3274,13 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
name="BulkEditDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(required=False),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
)
|
||||
class BulkEditView(DocumentOperationPermissionMixin):
|
||||
LEGACY_MODIFIED_FIELDS = {
|
||||
"delete": "deleted_at",
|
||||
"reprocess": "checksum",
|
||||
"remove_password": None,
|
||||
}
|
||||
|
||||
serializer_class = BulkEditSerializer
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
@@ -3307,14 +3310,6 @@ class BulkEditView(DocumentOperationPermissionMixin):
|
||||
user=user,
|
||||
validated_data=serializer.validated_data,
|
||||
)
|
||||
if method.__name__ in self.METHOD_NAMES_REQUIRING_USER:
|
||||
parameters["user"] = user
|
||||
if method.__name__ in self.METHOD_NAMES_REQUIRING_TRIGGER_SOURCE:
|
||||
parameters["trigger_source"] = (
|
||||
PaperlessTask.TriggerSource.WEB_UI
|
||||
if from_webui
|
||||
else PaperlessTask.TriggerSource.API_UPLOAD
|
||||
)
|
||||
if not has_document_operation_permissions(
|
||||
user=user,
|
||||
documents=documents,
|
||||
@@ -3324,38 +3319,12 @@ class BulkEditView(DocumentOperationPermissionMixin):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
try:
|
||||
if method.__name__ in QUEUED_OPERATIONS:
|
||||
return self._queue_bulk_operation(
|
||||
method,
|
||||
documents,
|
||||
parameters,
|
||||
from_webui,
|
||||
)
|
||||
modified_field = self.LEGACY_MODIFIED_FIELDS.get(method.__name__)
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field:
|
||||
old_values = snapshot_field(documents, modified_field)
|
||||
|
||||
result = method(documents, **parameters)
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED and modified_field:
|
||||
new_values = snapshot_field(documents, modified_field)
|
||||
for doc in Document.objects.filter(pk__in=documents):
|
||||
LogEntry.objects.log_create(
|
||||
instance=doc,
|
||||
changes={
|
||||
modified_field: [
|
||||
old_values[doc.pk],
|
||||
new_values[doc.pk],
|
||||
],
|
||||
},
|
||||
action=LogEntry.Action.UPDATE,
|
||||
actor=user,
|
||||
additional_data={
|
||||
"reason": f"Bulk edit: {method.__name__}",
|
||||
},
|
||||
)
|
||||
|
||||
return Response({"result": result})
|
||||
return self._queue_bulk_operation(
|
||||
method,
|
||||
documents,
|
||||
parameters,
|
||||
from_webui,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"An error occurred performing bulk edit: {e!s}")
|
||||
return HttpResponseBadRequest(
|
||||
@@ -3372,6 +3341,7 @@ class BulkEditView(DocumentOperationPermissionMixin):
|
||||
name="RotateDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3399,6 +3369,7 @@ class RotateDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="MergeDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3426,6 +3397,7 @@ class MergeDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="MergeDocumentsAsVersionsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3453,6 +3425,7 @@ class MergeDocumentsAsVersionsView(DocumentOperationPermissionMixin):
|
||||
name="DeleteDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3480,6 +3453,7 @@ class DeleteDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="ReprocessDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3541,6 +3515,7 @@ class RunZoneOcrDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="EditPdfDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
@@ -3568,6 +3543,7 @@ class EditPdfDocumentsView(DocumentOperationPermissionMixin):
|
||||
name="RemovePasswordDocumentsResult",
|
||||
fields={
|
||||
"result": serializers.CharField(),
|
||||
"task_id": serializers.CharField(),
|
||||
},
|
||||
),
|
||||
},
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,23 @@
|
||||
from rest_framework import serializers
|
||||
|
||||
OBFUSCATED_PASSWORD_MASK = "*" * 10
|
||||
|
||||
|
||||
def is_obfuscated_password(value: str) -> bool:
|
||||
"""
|
||||
True if the value is only mask characters (or empty), meaning a client echoed
|
||||
the placeholder back rather than supplying a new secret
|
||||
"""
|
||||
return value.replace("*", "") == ""
|
||||
|
||||
|
||||
class ObfuscatedPasswordField(serializers.CharField):
|
||||
"""
|
||||
Sends a constant mask string instead of the password in the clear
|
||||
"""
|
||||
|
||||
def to_representation(self, value) -> str:
|
||||
return OBFUSCATED_PASSWORD_MASK
|
||||
|
||||
def to_internal_value(self, data):
|
||||
return data
|
||||
@@ -20,18 +20,19 @@ from rest_framework import serializers
|
||||
from rest_framework.authtoken.serializers import AuthTokenSerializer
|
||||
|
||||
from paperless.config import OCR_RESERVED_USER_ARGS
|
||||
from paperless.fields import ObfuscatedPasswordField
|
||||
from paperless.fields import is_obfuscated_password
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless.network import validate_outbound_http_url
|
||||
from paperless.validators import reject_dangerous_svg
|
||||
from paperless.validators import validate_raster_image
|
||||
from paperless_mail.serialisers import ObfuscatedPasswordField
|
||||
|
||||
logger = logging.getLogger("paperless.settings")
|
||||
|
||||
|
||||
class PasswordValidationMixin:
|
||||
def _has_real_password(self, value: str | None) -> bool:
|
||||
return bool(value) and value.replace("*", "") != ""
|
||||
return bool(value) and not is_obfuscated_password(value)
|
||||
|
||||
def validate_password(self, value: str) -> str:
|
||||
if not self._has_real_password(value):
|
||||
@@ -285,7 +286,7 @@ class ApplicationConfigurationSerializer(
|
||||
if data[field] == "":
|
||||
data[field] = None
|
||||
# Not a real value, don't overwrite the stored one
|
||||
elif len(data[field].replace("*", "")) == 0:
|
||||
elif is_obfuscated_password(data[field]):
|
||||
del data[field]
|
||||
return super().run_validation(data)
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import pytest
|
||||
|
||||
from paperless.fields import OBFUSCATED_PASSWORD_MASK
|
||||
from paperless.fields import ObfuscatedPasswordField
|
||||
from paperless.fields import is_obfuscated_password
|
||||
|
||||
|
||||
class TestObfuscatedPasswordField:
|
||||
@pytest.mark.parametrize(
|
||||
"secret",
|
||||
[
|
||||
"a",
|
||||
"short",
|
||||
"exactly10!",
|
||||
"a-much-longer-secret-that-should-not-leak-its-length",
|
||||
],
|
||||
)
|
||||
def test_representation_is_constant_regardless_of_length(self, secret: str) -> None:
|
||||
assert (
|
||||
ObfuscatedPasswordField().to_representation(secret)
|
||||
== OBFUSCATED_PASSWORD_MASK
|
||||
)
|
||||
|
||||
|
||||
class TestIsObfuscatedPassword:
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
("", True),
|
||||
("*", True),
|
||||
("******", True),
|
||||
(OBFUSCATED_PASSWORD_MASK, True),
|
||||
("real-secret", False),
|
||||
("***x***", False),
|
||||
(" ", False),
|
||||
],
|
||||
)
|
||||
def test_detects_placeholder(self, value: str, *, expected: bool) -> None:
|
||||
assert is_obfuscated_password(value) is expected
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import Final
|
||||
|
||||
__version__: Final[tuple[int, int, int]] = (3, 4, 2)
|
||||
__version__: Final[tuple[int, int, int]] = (3, 5, 0)
|
||||
# Version string like X.Y.Z
|
||||
__full_version_str__: Final[str] = ".".join(map(str, __version__))
|
||||
# Version string like X.Y
|
||||
|
||||
@@ -41,6 +41,7 @@ from documents.models import PaperlessTask
|
||||
from documents.permissions import PaperlessModelPermissions
|
||||
from documents.permissions import PaperlessObjectPermissions
|
||||
from documents.tasks import llmindex_index
|
||||
from paperless.fields import is_obfuscated_password
|
||||
from paperless.filters import GroupFilterSet
|
||||
from paperless.filters import UserFilterSet
|
||||
from paperless.models import ApplicationConfiguration
|
||||
@@ -260,7 +261,7 @@ class ProfileView(GenericAPIView[Any]):
|
||||
user = self.request.user if hasattr(self.request, "user") else None
|
||||
|
||||
password = serializer.validated_data.pop("password", None)
|
||||
if password and password.replace("*", ""):
|
||||
if password and not is_obfuscated_password(password):
|
||||
user.set_password(password)
|
||||
user.save()
|
||||
|
||||
|
||||
@@ -8,23 +8,13 @@ from documents.serialisers import CorrespondentField
|
||||
from documents.serialisers import DocumentTypeField
|
||||
from documents.serialisers import OwnedObjectSerializer
|
||||
from documents.serialisers import TagsField
|
||||
from paperless.fields import ObfuscatedPasswordField
|
||||
from paperless.fields import is_obfuscated_password
|
||||
from paperless_mail.models import MailAccount
|
||||
from paperless_mail.models import MailRule
|
||||
from paperless_mail.models import ProcessedMail
|
||||
|
||||
|
||||
class ObfuscatedPasswordField(serializers.CharField):
|
||||
"""
|
||||
Sends *** string instead of password in the clear
|
||||
"""
|
||||
|
||||
def to_representation(self, value) -> str:
|
||||
return "*" * max(10, len(value))
|
||||
|
||||
def to_internal_value(self, data):
|
||||
return data
|
||||
|
||||
|
||||
class MailAccountSerializer(OwnedObjectSerializer):
|
||||
password = ObfuscatedPasswordField()
|
||||
imap_port = serializers.IntegerField(required=True, allow_null=False)
|
||||
@@ -50,9 +40,8 @@ class MailAccountSerializer(OwnedObjectSerializer):
|
||||
]
|
||||
|
||||
def update(self, instance, validated_data):
|
||||
if (
|
||||
"password" in validated_data
|
||||
and len(validated_data.get("password").replace("*", "")) == 0
|
||||
if "password" in validated_data and is_obfuscated_password(
|
||||
validated_data["password"],
|
||||
):
|
||||
validated_data.pop("password")
|
||||
super().update(instance, validated_data)
|
||||
|
||||
@@ -30,6 +30,7 @@ from documents.permissions import PaperlessObjectPermissions
|
||||
from documents.permissions import has_perms_owner_aware
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.views import PassUserMixin
|
||||
from paperless.fields import is_obfuscated_password
|
||||
from paperless.views import StandardPagination
|
||||
from paperless_mail.filters import ProcessedMailFilterSet
|
||||
from paperless_mail.mail import MailError
|
||||
@@ -126,9 +127,8 @@ class MailAccountViewSet(PassUserMixin, ModelViewSet[MailAccount]):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
# account exists, use the password from there instead of ***
|
||||
if (
|
||||
len(serializer.validated_data.get("password").replace("*", "")) == 0
|
||||
and existing_account is not None
|
||||
if existing_account is not None and is_obfuscated_password(
|
||||
serializer.validated_data.get("password"),
|
||||
):
|
||||
serializer.validated_data["password"] = existing_account.password
|
||||
serializer.validated_data["account_type"] = existing_account.account_type
|
||||
|
||||
Reference in new issue
Block a user