Merge branch 'dev' into feature/centralized-share-links

This commit is contained in:
shamoon authored and GitHub committed 2026-09-15 16:45:26 -07:00
commit 7bec186e66
12 files changed
+1138 -248

No files matched your search

+129 -52
View File
@@ -13,12 +13,16 @@ if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Iterator
from datetime import datetime
from types import TracebackType
from typing import BinaryIO
from typing import Self
from numpy import ndarray
from django.conf import settings
from django.core.cache import cache
from django.core.cache import caches
from django.db.models import Prefetch
from documents.caching import CACHE_5_MINUTES
from documents.caching import CACHE_50_MINUTES
@@ -28,6 +32,7 @@ from documents.caching import CLASSIFIER_VERSION_KEY
from documents.caching import StoredLRUCache
from documents.models import Document
from documents.models import MatchingModel
from documents.models import Tag
from paperless.signed_pickle import SignedPickleError
from paperless.signed_pickle import signed_pickle_dumps
from paperless.signed_pickle import signed_pickle_loads
@@ -65,6 +70,52 @@ read_cache = caches["read-cache"]
RE_DIGIT = re.compile(r"\d")
RE_WORD = re.compile(r"\b[\w]+\b") # words that may contain digits
# Documents whose content is fetched per query while training
_CONTENT_CHUNK_SIZE = 1000
class _SignedFileWriter:
"""
Atomically writes a file made of an HMAC signature followed by the data,
signing the data as it streams to disk rather than holding it in memory.
The signature is only known once everything is written, so its space is
reserved at the start of the file and filled in on exit. The target is only
replaced once the file is complete; on error the partial file is removed.
"""
def __init__(self, target: Path, mac: hmac.HMAC) -> None:
self._target = target
self._temp = target.with_name(f"{target.name}.part")
self._mac = mac
self._file: BinaryIO
def __enter__(self) -> Self:
self._file = self._temp.open("wb")
self._file.write(bytes(self._mac.digest_size))
return self
def write(self, data: bytes | memoryview) -> int:
self._mac.update(data)
return self._file.write(data)
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
try:
with self._file:
if exc_type is None:
self._file.seek(0)
self._file.write(self._mac.digest())
if exc_type is None:
self._temp.rename(self._target)
finally:
# A no-op after a successful rename, otherwise removes the partial file
self._temp.unlink(missing_ok=True)
class IncompatibleClassifierVersionError(Exception):
def __init__(self, message: str, *args: object) -> None:
@@ -158,12 +209,14 @@ class DocumentClassifier:
).hexdigest()
@staticmethod
def _compute_hmac(data: bytes) -> bytes:
return hmac.new(
settings.SECRET_KEY.encode(),
data,
sha256,
).digest()
def _new_hmac() -> hmac.HMAC:
return hmac.new(settings.SECRET_KEY.encode(), digestmod=sha256)
@staticmethod
def _compute_hmac(data: bytes | memoryview) -> bytes:
mac = DocumentClassifier._new_hmac()
mac.update(data)
return mac.digest()
def load(self) -> None:
from sklearn.exceptions import InconsistentVersionWarning
@@ -173,8 +226,13 @@ class DocumentClassifier:
if len(raw) <= self.HMAC_SIZE:
raise ClassifierModelCorruptError
signature = raw[: self.HMAC_SIZE]
data = raw[self.HMAC_SIZE :]
# Slice through a memoryview so the (potentially multi-GB) payload is
# not copied; hmac and pickle both accept buffers directly.
# The whole file is still verified from memory before unpickling, rather
# than streamed from disk, so it cannot change between check and load.
view = memoryview(raw)
signature = view[: self.HMAC_SIZE]
data = view[self.HMAC_SIZE :]
if not hmac.compare_digest(signature, self._compute_hmac(data)):
raise ClassifierModelCorruptError
@@ -219,29 +277,24 @@ class DocumentClassifier:
raise IncompatibleClassifierVersionError("sklearn version update")
def save(self) -> None:
target_file: Path = settings.MODEL_FILE
target_file_temp: Path = target_file.with_suffix(".pickle.part")
data = pickle.dumps(
(
self.FORMAT_VERSION,
self.last_doc_change_time,
self.last_auto_type_hash,
self.data_vectorizer,
self.tags_binarizer,
self.tags_classifier,
self.correspondent_classifier,
self.document_type_classifier,
self.storage_path_classifier,
),
)
signature = self._compute_hmac(data)
with target_file_temp.open("wb") as f:
f.write(signature + data)
target_file_temp.rename(target_file)
# Stream to disk instead of building the payload in memory. Protocol 5+
# pickles numpy arrays without copying them (the default is 4 before 3.14).
with _SignedFileWriter(settings.MODEL_FILE, self._new_hmac()) as f:
pickle.dump(
(
self.FORMAT_VERSION,
self.last_doc_change_time,
self.last_auto_type_hash,
self.data_vectorizer,
self.tags_binarizer,
self.tags_classifier,
self.correspondent_classifier,
self.document_type_classifier,
self.storage_path_classifier,
),
f,
protocol=pickle.HIGHEST_PROTOCOL,
)
def train(
self,
@@ -250,29 +303,48 @@ class DocumentClassifier:
notify = status_callback if status_callback is not None else lambda _: None
# Get non-inbox documents
docs_queryset = (
Document.objects.exclude(
tags__is_inbox_tag=True,
)
.select_related("document_type", "correspondent", "storage_path")
.prefetch_related("tags")
.order_by("pk")
)
docs_queryset = Document.objects.exclude(
tags__is_inbox_tag=True,
).order_by("pk")
# No documents exit to train against
if docs_queryset.count() == 0:
doc_count = docs_queryset.count()
if doc_count == 0:
raise ValueError("No training data available.")
labels_tags = []
labels_correspondent = []
labels_document_type = []
labels_storage_path = []
# Content is fetched separately later, for exactly these documents in this
# order, so it never all has to be in memory at once
doc_pks: list[int] = []
latest_doc_change: datetime | None = None
# Step 1: Extract and preprocess training data from the database.
logger.debug("Gathering data from database...")
notify(f"Gathering data from {docs_queryset.count()} document(s)...")
notify(f"Gathering data from {doc_count} document(s)...")
hasher = sha256()
for doc in docs_queryset:
for doc in (
docs_queryset.defer("content")
.select_related("document_type", "correspondent", "storage_path")
.prefetch_related(
Prefetch(
"tags",
queryset=Tag.objects.filter(
matching_algorithm=MatchingModel.MATCH_AUTO,
)
.order_by("pk")
.only("pk"),
to_attr="auto_tags",
),
)
.iterator(chunk_size=2000)
):
doc_pks.append(doc.pk)
if latest_doc_change is None or doc.modified > latest_doc_change:
latest_doc_change = doc.modified
y = -1
dt = doc.document_type
if dt and dt.matching_algorithm == MatchingModel.MATCH_AUTO:
@@ -287,11 +359,7 @@ class DocumentClassifier:
hasher.update(y.to_bytes(4, "little", signed=True))
labels_correspondent.append(y)
tags: list[int] = list(
doc.tags.filter(matching_algorithm=MatchingModel.MATCH_AUTO)
.order_by("pk")
.values_list("pk", flat=True),
)
tags: list[int] = [tag.pk for tag in doc.auto_tags]
for tag in tags:
hasher.update(tag.to_bytes(4, "little", signed=True))
labels_tags.append(tags)
@@ -310,7 +378,6 @@ class DocumentClassifier:
# Check if retraining is actually required.
# A document has been updated since the classifier was trained
# New auto tags, types, correspondent, storage paths exist
latest_doc_change = docs_queryset.latest("modified").modified
if (
self.last_doc_change_time is not None
and self.last_doc_change_time >= latest_doc_change
@@ -337,7 +404,7 @@ class DocumentClassifier:
num_storage_paths: int = len(set(labels_storage_path) | {-1}) - 1
logger.debug(
f"{docs_queryset.count()} documents, {num_tags} tag(s), {num_correspondents} correspondent(s), "
f"{len(doc_pks)} documents, {num_tags} tag(s), {num_correspondents} correspondent(s), "
f"{num_document_types} document type(s). {num_storage_paths} storage path(s)",
)
@@ -359,10 +426,20 @@ class DocumentClassifier:
def content_generator() -> Iterator[str]:
"""
Generates the content for documents, but once at a time
Generates the content for documents, in the same order as the labels,
fetching it a chunk at a time
"""
for doc in docs_queryset:
yield self.preprocess_content(doc.content, shared_cache=False)
for start in range(0, len(doc_pks), _CONTENT_CHUNK_SIZE):
chunk = doc_pks[start : start + _CONTENT_CHUNK_SIZE]
docs = Document.objects.only("content").order_by().in_bulk(chunk)
for pk in chunk:
# A document deleted since its labels were gathered still
# needs a row, so labels and content stay aligned
doc = docs.get(pk)
yield self.preprocess_content(
doc.content if doc is not None else "",
shared_cache=False,
)
self.data_vectorizer = CountVectorizer(
analyzer="word",
+7 -11
View File
@@ -177,13 +177,10 @@ def _resolve_permissions(codenames: set[str], ctype: ContentType) -> list[Permis
"""
Resolves `codenames` to Permission rows, raising like the single-object
assign_perm() this bulk path replaces does (via a `.get()` internally)
if any codename doesn't exist -- e.g. a client-supplied action name that
was never validated (BulkEditObjectsSerializer._validate_permissions
calls validate_set_permissions() only for its side-effecting id checks
and discards the filtered dict it returns, so an unrecognized action key
reaches this function as-is). A plain `.filter()` with no existence
check would otherwise silently build zero rows and no-op instead of
reporting the bad input.
if any codename doesn't exist. SetPermissionsSerializer rejects unknown
action names at the API, but a caller passing one directly would
otherwise get a plain `.filter()` that silently builds zero rows and
no-ops instead of reporting the bad input.
"""
permission_objs = list(
Permission.objects.filter(content_type=ctype, codename__in=codenames),
@@ -298,10 +295,9 @@ def set_permissions_for_objects(
# Every action is resolved up front, before anything is written, so an
# unrecognized action name (see _resolve_permissions) aborts the whole
# call instead of leaving the actions ahead of it already applied --
# BulkEditObjectsSerializer lets unknown keys through and its view turns
# the exception into a 400, so a half-applied change would otherwise be
# reported to the client as a failure.
# call instead of leaving the actions ahead of it already applied.
# SetPermissionsSerializer rejects unknown actions at the API, so this
# guards any other caller.
permissions_by_action: dict[str, list[Permission]] = {}
for action, entry in permissions.items():
if "users" not in entry and "groups" not in entry:
+121 -121
View File
@@ -210,6 +210,9 @@ class MatchingModelSerializer(serializers.ModelSerializer[Any]):
return match
PERMISSION_ACTIONS = ("view", "change")
class SetPermissionsMixin:
def _validate_user_ids(self, user_ids):
users = User.objects.none()
@@ -232,12 +235,9 @@ class SetPermissionsMixin:
return groups
def validate_set_permissions(self, set_permissions=None):
permissions_dict = {
"view": {},
"change": {},
}
permissions_dict = {action: {} for action in PERMISSION_ACTIONS}
if set_permissions is not None:
for action in ["view", "change"]:
for action in PERMISSION_ACTIONS:
if action in set_permissions:
if "users" in set_permissions[action]:
users = set_permissions[action]["users"]
@@ -265,41 +265,31 @@ class SerializerWithPerms(serializers.Serializer[dict[str, Any]]):
super().__init__(*args, **kwargs)
@extend_schema_field(
field={
"type": "object",
"properties": {
"view": {
"type": "object",
"properties": {
"users": {
"type": "array",
"items": {"type": "integer"},
},
"groups": {
"type": "array",
"items": {"type": "integer"},
},
},
},
"change": {
"type": "object",
"properties": {
"users": {
"type": "array",
"items": {"type": "integer"},
},
"groups": {
"type": "array",
"items": {"type": "integer"},
},
},
},
},
},
)
class SetPermissionsSerializer(serializers.DictField):
pass
class PermissionSetSerializer(serializers.Serializer[dict[str, Any]]):
users = serializers.ListField(
child=serializers.IntegerField(),
required=False,
allow_null=True,
)
groups = serializers.ListField(
child=serializers.IntegerField(),
required=False,
allow_null=True,
)
class SetPermissionsSerializer(serializers.Serializer[dict[str, Any]]):
view = PermissionSetSerializer(required=False)
change = PermissionSetSerializer(required=False)
def to_internal_value(self, data):
if isinstance(data, dict):
unknown_keys = set(data) - set(PERMISSION_ACTIONS)
if unknown_keys:
raise serializers.ValidationError(
{key: "Unknown permission action." for key in sorted(unknown_keys)},
)
return super().to_internal_value(data)
class OwnedObjectSerializer(
@@ -470,7 +460,6 @@ class OwnedObjectSerializer(
set_permissions = SetPermissionsSerializer(
label="Set permissions",
allow_empty=True,
required=False,
write_only=True,
)
@@ -1680,6 +1669,13 @@ class SourceModeValidationMixin:
return source_mode
def _validate_rotation_degrees(degrees: int, field: str = "degrees") -> int:
# QPDF refuses any other angle, which would otherwise fail inside the task
if degrees % 90 != 0:
raise serializers.ValidationError(f"{field} must be a multiple of 90")
return degrees
class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidationMixin):
degrees = serializers.IntegerField(required=True)
source_mode = serializers.CharField(
@@ -1688,6 +1684,9 @@ class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidatio
)
from_webui = serializers.BooleanField(required=False, default=False)
def validate_degrees(self, value: int) -> int:
return _validate_rotation_degrees(value)
class MergeDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin):
metadata_document_id = serializers.IntegerField(
@@ -1749,8 +1748,21 @@ class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
return attrs
class PdfEditOperationSerializer(serializers.Serializer[dict[str, int]]):
page = serializers.IntegerField(min_value=1)
rotate = serializers.IntegerField(required=False)
doc = serializers.IntegerField(required=False, min_value=0)
def validate_rotate(self, value: int) -> int:
return _validate_rotation_degrees(value, field="rotate")
class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin):
operations = serializers.ListField(required=True, allow_empty=False)
operations = serializers.ListField(
child=PdfEditOperationSerializer(),
required=True,
allow_empty=False,
)
delete_original = serializers.BooleanField(required=False, default=False)
update_document = serializers.BooleanField(required=False, default=False)
include_metadata = serializers.BooleanField(required=False, default=True)
@@ -1768,18 +1780,9 @@ class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMix
)
operations = attrs["operations"]
if not isinstance(operations, list):
raise serializers.ValidationError("operations must be a list")
for op in operations:
if not isinstance(op, dict):
raise serializers.ValidationError("invalid operation entry")
if "page" not in op or not isinstance(op["page"], int):
raise serializers.ValidationError("page must be an integer")
if "rotate" in op and not isinstance(op["rotate"], int):
raise serializers.ValidationError("rotate must be an integer")
if "doc" in op and not isinstance(op["doc"], int):
raise serializers.ValidationError("doc must be an integer")
if any(op.get("doc", 0) >= len(operations) for op in operations):
raise serializers.ValidationError("doc index is out of bounds")
if attrs["update_document"]:
max_idx = max(op.get("doc", 0) for op in operations)
@@ -1788,16 +1791,10 @@ class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMix
"update_document only allowed with a single output document",
)
if any(
op.get("doc", 0) < 0 or op.get("doc", 0) >= len(operations)
for op in operations
):
raise serializers.ValidationError("doc index is out of bounds")
doc = Document.objects.get(id=documents[0])
if doc.page_count:
for op in operations:
if op["page"] < 1 or op["page"] > doc.page_count:
if op["page"] > doc.page_count:
raise serializers.ValidationError(
f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.",
)
@@ -2042,32 +2039,39 @@ class BulkEditSerializer(
else:
raise serializers.ValidationError("remove_custom_fields not specified")
def _validate_owner(self, owner):
ownerUser = User.objects.get(pk=owner)
if ownerUser is None:
raise serializers.ValidationError("Specified owner cannot be found")
return ownerUser
def _validate_owner(self, owner) -> User:
owner_field = serializers.PrimaryKeyRelatedField(queryset=User.objects.all())
try:
return owner_field.run_validation(owner)
except serializers.ValidationError as e:
raise serializers.ValidationError(
"Specified owner cannot be found",
) from e
def _validate_parameters_set_permissions(self, parameters) -> None:
if "set_permissions" not in parameters:
raise serializers.ValidationError("set_permissions not specified")
set_permissions = parameters["set_permissions"]
if set_permissions is not None:
set_permissions = SetPermissionsSerializer().run_validation(
set_permissions,
)
parameters["set_permissions"] = self.validate_set_permissions(
parameters["set_permissions"],
set_permissions,
)
if "owner" in parameters and parameters["owner"] is not None:
self._validate_owner(parameters["owner"])
parameters["owner"] = self._validate_owner(parameters["owner"]).pk
if "merge" not in parameters:
parameters["merge"] = False
def _validate_parameters_rotate(self, parameters) -> None:
try:
if (
"degrees" not in parameters
or not float(parameters["degrees"]).is_integer()
):
raise serializers.ValidationError("invalid rotation degrees")
except ValueError:
if "degrees" not in parameters:
raise serializers.ValidationError("invalid rotation degrees")
try:
degrees = serializers.IntegerField().run_validation(parameters["degrees"])
except serializers.ValidationError as e:
raise serializers.ValidationError("invalid rotation degrees") from e
parameters["degrees"] = _validate_rotation_degrees(degrees)
def _validate_source_mode(self, parameters) -> None:
source_mode = parameters.get(
@@ -2076,28 +2080,25 @@ class BulkEditSerializer(
)
parameters["source_mode"] = self.validate_source_mode(source_mode)
def _validate_parameters_split(self, parameters) -> None:
def _validate_parameters_split(self, parameters, document_id) -> None:
if "pages" not in parameters:
raise serializers.ValidationError("pages not specified")
try:
pages = []
docs = parameters["pages"].split(",")
for doc in docs:
if "-" in doc:
pages.append(
[
x
for x in range(
int(doc.split("-")[0]),
int(doc.split("-")[1]) + 1,
)
],
)
else:
pages.append([int(doc)])
parameters["pages"] = pages
except ValueError:
if not isinstance(parameters["pages"], str):
raise serializers.ValidationError("invalid pages specified")
page_count = Document.objects.get(id=document_id).page_count
pages = []
for group in parameters["pages"].split(","):
start, is_range, end = group.partition("-")
try:
first = int(start)
last = int(end) if is_range else first
except ValueError as e:
raise serializers.ValidationError("invalid pages specified") from e
# Bound the range before building it, a huge one would exhaust memory
if not 1 <= first <= last or (page_count and last > page_count):
raise serializers.ValidationError("invalid pages specified")
pages.append(list(range(first, last + 1)))
parameters["pages"] = pages
if "delete_originals" in parameters:
if not isinstance(parameters["delete_originals"], bool):
@@ -2128,19 +2129,18 @@ class BulkEditSerializer(
def _validate_parameters_edit_pdf(self, parameters, document_id) -> None:
if "operations" not in parameters:
raise serializers.ValidationError("operations not specified")
if not isinstance(parameters["operations"], list):
raise serializers.ValidationError("operations must be a list")
if not parameters["operations"]:
raise serializers.ValidationError("operations must not be empty")
for op in parameters["operations"]:
if not isinstance(op, dict):
raise serializers.ValidationError("invalid operation entry")
if "page" not in op or not isinstance(op["page"], int):
raise serializers.ValidationError("page must be an integer")
if "rotate" in op and not isinstance(op["rotate"], int):
raise serializers.ValidationError("rotate must be an integer")
if "doc" in op and not isinstance(op["doc"], int):
raise serializers.ValidationError("doc must be an integer")
operations_field = serializers.ListField(
child=PdfEditOperationSerializer(),
allow_empty=False,
)
try:
operations = operations_field.run_validation(parameters["operations"])
except serializers.ValidationError as e:
# Key the errors under "operations" so they match what the
# dedicated edit_pdf endpoint returns
raise serializers.ValidationError({"operations": e.detail}) from e
parameters["operations"] = operations
if "update_document" in parameters:
if not isinstance(parameters["update_document"], bool):
raise serializers.ValidationError("update_document must be a boolean")
@@ -2152,24 +2152,21 @@ class BulkEditSerializer(
else:
parameters["include_metadata"] = True
if any(op.get("doc", 0) >= len(operations) for op in operations):
raise serializers.ValidationError("doc index is out of bounds")
if parameters["update_document"]:
max_idx = max(op.get("doc", 0) for op in parameters["operations"])
max_idx = max(op.get("doc", 0) for op in operations)
if max_idx > 0:
raise serializers.ValidationError(
"update_document only allowed with a single output document",
)
if any(
op.get("doc", 0) < 0 or op.get("doc", 0) >= len(parameters["operations"])
for op in parameters["operations"]
):
raise serializers.ValidationError("doc index is out of bounds")
doc = Document.objects.get(id=document_id)
# doc existence is already validated
if doc.page_count:
for op in parameters["operations"]:
if op["page"] < 1 or op["page"] > doc.page_count:
for op in operations:
if op["page"] > doc.page_count:
raise serializers.ValidationError(
f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.",
)
@@ -2228,7 +2225,7 @@ class BulkEditSerializer(
raise serializers.ValidationError(
"Split method only supports one document",
)
self._validate_parameters_split(parameters)
self._validate_parameters_split(parameters, attrs["documents"][0])
elif method == bulk_edit.delete_pages:
if len(attrs["documents"]) > 1:
raise serializers.ValidationError(
@@ -3021,9 +3018,8 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
allow_null=True,
)
permissions = serializers.DictField(
permissions = SetPermissionsSerializer(
label="Set permissions",
allow_empty=False,
required=False,
write_only=True,
)
@@ -3059,8 +3055,8 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
)
return objects
def _validate_permissions(self, permissions) -> None:
self.validate_set_permissions(
def _validate_permissions(self, permissions) -> dict:
return self.validate_set_permissions(
permissions,
)
@@ -3084,7 +3080,11 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
if operation == "set_permissions":
permissions = attrs.get("permissions")
if permissions is not None:
self._validate_permissions(permissions)
if not permissions:
raise serializers.ValidationError(
"permissions must not be empty",
)
attrs["permissions"] = self._validate_permissions(permissions)
return attrs
@@ -194,6 +194,70 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
self.assertEqual(config.language, None)
self.assertEqual(config.barcode_tag_mapping, None)
def test_api_update_config_json_objects(self) -> None:
"""
GIVEN:
- API request to update app config with JSON objects for the
user_args and barcode_tag_mapping JSONFields
WHEN:
- API is called
THEN:
- Correct HTTP response
- Both objects are stored as sent
"""
user_args = {"unpaper_args": "--pre-rotate 90", "jobs": 2}
barcode_tag_mapping = {"TAG:(.*)": "\\g<1>", "ASN12.*": ""}
response = self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps(
{
"user_args": json.dumps(user_args),
"barcode_tag_mapping": json.dumps(barcode_tag_mapping),
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
config = ApplicationConfiguration.objects.first()
assert config is not None
self.assertEqual(config.user_args, user_args)
self.assertEqual(config.barcode_tag_mapping, barcode_tag_mapping)
def test_api_update_config_rejects_invalid_json_objects(self) -> None:
"""
GIVEN:
- API request to update app config with a JSON-encoded value that
is not an object for user_args or barcode_tag_mapping, or a
barcode_tag_mapping with a non-string substitute
WHEN:
- API is called
THEN:
- Request is rejected with a 400 naming the problem
- Config is not updated
"""
not_objects = (True, 1, [1, 2, 3], "not a dict")
cases = [
(field, value, b"must be a JSON object")
for field in ("user_args", "barcode_tag_mapping")
for value in not_objects
]
cases += [
("barcode_tag_mapping", {"TAG:(.*)": 5}, b"values must be strings"),
("barcode_tag_mapping", {"TAG:(.*)": None}, b"values must be strings"),
]
for field, value, expected_message in cases:
with self.subTest(field=field, value=value):
response = self.client.patch(
f"{self.ENDPOINT}1/",
json.dumps({field: json.dumps(value)}),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
config = ApplicationConfiguration.objects.first()
assert config is not None
self.assertIsNone(getattr(config, field))
def test_api_replace_app_logo(self) -> None:
"""
GIVEN:
+365 -9
View File
@@ -1165,6 +1165,150 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertIn(b"set_permissions not specified", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_rejects_malformed_set_permissions(self, m) -> None:
"""
GIVEN:
- A set_permissions bulk edit where set_permissions is not an
object, has a non-list users value, or has an unknown action
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400 describing the problem
- set_permissions is not called
"""
self.setup_mock(m, "set_permissions")
for set_permissions, expected_message in (
(False, b"Expected a dictionary"),
({"view": {"users": False}}, b"Expected a list"),
({"not_a_real_action": {"users": [1]}}, b"Unknown permission action"),
):
with self.subTest(set_permissions=set_permissions):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "set_permissions",
"parameters": {"set_permissions": set_permissions},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_rejects_invalid_owner(self, m) -> None:
"""
GIVEN:
- A set_permissions bulk edit with an owner that is a nonexistent
id, a boolean, a list, a dict, or a non-numeric string
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- set_permissions is not called
"""
self.setup_mock(m, "set_permissions")
for bad_owner in (
999999,
True,
["not", "an", "id"],
{"nested": "dict"},
"not-a-number",
):
with self.subTest(owner=bad_owner):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "set_permissions",
"parameters": {
"set_permissions": {
"view": {"users": [self.user.id]},
},
"owner": bad_owner,
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"Specified owner cannot be found", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_null_is_a_noop(self, m) -> None:
"""
GIVEN:
- A set_permissions bulk edit with set_permissions null and an
owner, i.e. a request that only changes the owner
WHEN:
- API to bulk edit is called
THEN:
- Request succeeds
- set_permissions receives no users or groups for any action
"""
self.setup_mock(m, "set_permissions")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "set_permissions",
"parameters": {
"set_permissions": None,
"owner": self.user.id,
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
self.assertEqual(
m.call_args.kwargs["set_permissions"],
{"view": {}, "change": {}},
)
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_passes_validated_owner_id(self, m) -> None:
"""
GIVEN:
- A set_permissions bulk edit with the owner id given as a string
WHEN:
- API to bulk edit is called
THEN:
- set_permissions receives the owner as an integer id
"""
self.setup_mock(m, "set_permissions")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "set_permissions",
"parameters": {
"set_permissions": {"view": {"users": [self.user.id]}},
"owner": str(self.user.id),
},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
self.assertEqual(m.call_args.kwargs["owner"], self.user.id)
@mock.patch("documents.serialisers.bulk_edit.set_permissions")
def test_set_permissions_merge(self, m) -> None:
self.setup_mock(m, "set_permissions")
@@ -1300,7 +1444,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.client.force_authenticate(user=user1)
permissions = {
"owner": user1.id,
"view": {"users": [user1.id]},
}
response = self.client.post(
@@ -1436,8 +1580,146 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
response = self.client.post(
"/api/documents/rotate/",
json.dumps(
{
"documents": [self.doc2.id, self.doc3.id],
"degrees": 45,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"degrees must be a multiple of 90", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.rotate")
def test_bulk_edit_rotate_rejects_invalid_degrees(self, m) -> None:
"""
GIVEN:
- A legacy rotate bulk edit with degrees that are null, not an
integer, or not a multiple of 90
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- rotate is not called
"""
self.setup_mock(m, "rotate")
for degrees, expected_message in (
(None, b"invalid rotation degrees"),
(True, b"invalid rotation degrees"),
("foo", b"invalid rotation degrees"),
(90.5, b"invalid rotation degrees"),
(45, b"degrees must be a multiple of 90"),
):
with self.subTest(degrees=degrees):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "rotate",
"parameters": {"degrees": degrees},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.rotate")
def test_bulk_edit_rotate_passes_integer_degrees(self, m) -> None:
"""
GIVEN:
- A legacy rotate bulk edit with degrees given as a string
WHEN:
- API to bulk edit is called
THEN:
- rotate receives the degrees as an integer
"""
self.setup_mock(m, "rotate")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "rotate",
"parameters": {"degrees": "-90"},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
self.assertEqual(m.call_args.kwargs["degrees"], -90)
@mock.patch("documents.serialisers.bulk_edit.split")
def test_bulk_edit_split_rejects_invalid_pages(self, m) -> None:
"""
GIVEN:
- A legacy split bulk edit of a 5 page document with pages that
are not a string, not numeric, zero, a reversed range, or past
the last page
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- split is not called
"""
self.setup_mock(m, "split")
for pages in (None, "", "a", "1-2-3", "0", "3-1", "1-6", "1-5000000"):
with self.subTest(pages=pages):
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "split",
"parameters": {"pages": pages},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"invalid pages specified", response.content)
m.assert_not_called()
@mock.patch("documents.serialisers.bulk_edit.split")
def test_bulk_edit_split_parses_pages(self, m) -> None:
"""
GIVEN:
- A legacy split bulk edit with single pages and ranges
WHEN:
- API to bulk edit is called
THEN:
- split receives one page list per comma separated group
"""
self.setup_mock(m, "split")
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "split",
"parameters": {"pages": "1,2-4,5"},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]])
@mock.patch("documents.views.bulk_edit.rotate")
def test_rotate_insufficient_permissions(self, m) -> None:
self.doc1.owner = User.objects.get(username="temp_admin")
@@ -1681,7 +1963,10 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"operations must not be empty", response.content)
self.assertEqual(
response.json(),
{"operations": ["This list may not be empty."]},
)
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_edit_pdf(self, m) -> None:
@@ -1751,7 +2036,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"invalid operation entry", response.content)
self.assertIn(b"Expected a dictionary", response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1764,7 +2049,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"page must be an integer", response.content)
self.assertIn(b"valid integer is required", response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1777,7 +2062,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"rotate must be an integer", response.content)
self.assertIn(b"valid integer is required", response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1790,9 +2075,14 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"doc must be an integer", response.content)
self.assertIn(b"valid integer is required", response.content)
for doc_index in (-1, 2**32):
# A negative index fails the field's min_value before the
# object-level bound against len(operations) is checked
for doc_index, expected_message in (
(-1, b"greater than or equal to 0"),
(2**32, b"doc index is out of bounds"),
):
with self.subTest(doc_index=doc_index):
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1805,7 +2095,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"doc index is out of bounds", response.content)
self.assertIn(expected_message, response.content)
response = self.client.post(
"/api/documents/edit_pdf/",
@@ -1813,7 +2103,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
{
"documents": [self.doc2.id],
"update_document": True,
"operations": [{"page": 1, "doc": 1}, {"page": 2, "doc": 2}],
"operations": [{"page": 1, "doc": 0}, {"page": 2, "doc": 1}],
},
),
content_type="application/json",
@@ -1838,6 +2128,72 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(b"Invalid source_mode", response.content)
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_edit_pdf_rejects_invalid_operation_values(self, m) -> None:
"""
GIVEN:
- An edit_pdf operation with a non-positive page, a boolean in
place of an integer, or a rotation that is not a multiple of 90
WHEN:
- API to edit the PDF is called
THEN:
- API returns HTTP 400 with the field error
- edit_pdf is not called
"""
self.setup_mock(m, "edit_pdf")
for operation, expected_message in (
({"page": 0}, b"greater than or equal to 1"),
({"page": True}, b"valid integer is required"),
({"page": 1, "rotate": True}, b"valid integer is required"),
({"page": 1, "doc": True}, b"valid integer is required"),
({"page": 1, "rotate": 45}, b"rotate must be a multiple of 90"),
):
with self.subTest(operation=operation):
response = self.client.post(
"/api/documents/edit_pdf/",
json.dumps(
{
"documents": [self.doc2.id],
"operations": [operation],
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn(expected_message, response.content)
m.assert_not_called()
def test_legacy_bulk_edit_keys_pdf_operation_errors_under_operations(
self,
) -> None:
"""
GIVEN:
- A legacy bulk_edit edit_pdf request with an invalid operation
WHEN:
- API to bulk edit is called
THEN:
- API returns HTTP 400
- The error is keyed under operations and the operation index,
matching the edit_pdf endpoint
"""
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"documents": [self.doc2.id],
"method": "edit_pdf",
"parameters": {"operations": [{"page": 1, "rotate": 45}]},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(
response.json(),
{"operations": {"0": {"rotate": ["rotate must be a multiple of 90"]}}},
)
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_edit_pdf_page_out_of_bounds(self, m) -> None:
self.setup_mock(m, "edit_pdf")
+104
View File
@@ -1499,6 +1499,110 @@ class TestBulkEditObjectPermissions(APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(Tag.objects.filter(pk=self.t1.id).exists())
def test_bulk_object_set_permissions_rejects_empty_permissions(self) -> None:
"""
GIVEN:
- Existing objects
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation and an empty permissions dict
THEN:
- Validation fails rather than silently applying a no-op
"""
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"permissions": {},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_bulk_object_set_permissions_rejects_non_dict_permissions(self) -> None:
"""
GIVEN:
- Existing objects
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation and a non-dict permissions value
THEN:
- Validation fails rather than crashing
"""
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"permissions": False,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_bulk_object_set_permissions_rejects_unknown_action(self) -> None:
"""
GIVEN:
- Existing objects
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation and an unrecognized permission action name
THEN:
- Validation fails rather than silently no-oping
"""
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"permissions": {"not_a_real_action": {"users": [self.user1.id]}},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def test_bulk_object_set_permissions_null_users_clears_users(self) -> None:
"""
GIVEN:
- An object a user has view permission on
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation, merge off, and an explicit null for the view users
THEN:
- Request succeeds and null is treated as an empty user list,
so the existing view permission is removed
"""
assign_perm("view_tag", self.user1, self.t1)
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"permissions": {"view": {"users": None}},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertNotIn(self.user1, get_users_with_perms(self.t1))
def test_bulk_edit_object_permissions_validation(self) -> None:
"""
GIVEN:
+3 -8
View File
@@ -636,14 +636,9 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
THEN:
- Permission.DoesNotExist is raised, not a silent no-op
Regression test: the endpoint that calls this
(BulkEditObjectPermissionsView) never actually validates action
names against the raw client-supplied permissions dict --
BulkEditObjectsSerializer._validate_permissions calls
validate_set_permissions() only for its side-effecting user/group id
checks and discards the filtered dict it returns -- so a bogus
action key reaches this function as-is. Resolving the Permission via
a bare `.filter()` (which returns empty instead of raising) would
The API rejects unknown action names before they get here, but a
direct caller could still pass one. Resolving the Permission via a
bare `.filter()` (which returns empty instead of raising) would
silently drop the grant and report success.
"""
with self.assertRaises(Permission.DoesNotExist):
+220
View File
@@ -1,13 +1,20 @@
import pickle
import re
import warnings
from datetime import UTC
from datetime import datetime
from pathlib import Path
from unittest import mock
import numpy as np
import pytest
from django.conf import settings
from django.db import connection
from django.test import TestCase
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
from pytest_django.fixtures import Settings
from pytest_mock import MockerFixture
from documents.classifier import ClassifierModelCorruptError
from documents.classifier import DocumentClassifier
@@ -20,6 +27,8 @@ from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import StoragePath
from documents.models import Tag
from documents.tests.factories import DocumentFactory
from documents.tests.factories import TagFactory
from documents.tests.utils import DirectoriesMixin
from paperless.signed_pickle import HMAC_SIZE
from paperless.signed_pickle import signed_pickle_dumps
@@ -909,6 +918,85 @@ class TestClassifier(DirectoriesMixin, TestCase):
load_classifier(raise_exception=True)
class TestClassifierSave:
@pytest.fixture
def model_file(self, tmp_path: Path, settings: Settings) -> Path:
settings.MODEL_FILE = tmp_path / "classifier.pickle"
return settings.MODEL_FILE
@pytest.fixture
def classifier(self) -> DocumentClassifier:
classifier = DocumentClassifier()
classifier.last_doc_change_time = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC)
classifier.last_auto_type_hash = b"\x01" * 32
return classifier
def test_save_writes_signed_pickle(
self,
model_file: Path,
classifier: DocumentClassifier,
) -> None:
"""
GIVEN:
- A classifier with training state
WHEN:
- The classifier is saved
THEN:
- The file is the HMAC of the pickled state followed by that pickle
- The pickle uses the highest protocol
- The saved state loads back into a new classifier
- No temporary file is left behind
"""
classifier.save()
raw = model_file.read_bytes()
signature = raw[: DocumentClassifier.HMAC_SIZE]
data = raw[DocumentClassifier.HMAC_SIZE :]
assert signature == DocumentClassifier._compute_hmac(data)
# A pickle opens with the PROTO opcode followed by the protocol number
assert data[:2] == bytes([pickle.PROTO[0], pickle.HIGHEST_PROTOCOL])
assert pickle.loads(data)[:3] == (
DocumentClassifier.FORMAT_VERSION,
classifier.last_doc_change_time,
classifier.last_auto_type_hash,
)
loaded = DocumentClassifier()
loaded.load()
assert loaded.last_doc_change_time == classifier.last_doc_change_time
assert loaded.last_auto_type_hash == classifier.last_auto_type_hash
assert not model_file.with_name(f"{model_file.name}.part").exists()
def test_save_failure_removes_partial_file(
self,
model_file: Path,
classifier: DocumentClassifier,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- An existing classifier model file
WHEN:
- Saving a new classifier fails part way through writing
THEN:
- The error is raised
- The partially written temporary file is removed
- The existing model file is left untouched
"""
model_file.write_bytes(b"existing model")
mocker.patch(
"documents.classifier.pickle.dump",
side_effect=RuntimeError("disk full"),
)
with pytest.raises(RuntimeError, match="disk full"):
classifier.save()
assert not model_file.with_name(f"{model_file.name}.part").exists()
assert model_file.read_bytes() == b"existing model"
class _StubProbaClassifier:
"""
A fake scikit-learn classifier exposing just enough of the API for
@@ -1017,3 +1105,135 @@ def test_preprocess_content_nltk_load_fail(mocker) -> None:
expected_preprocess_content = f.read().rstrip()
result = classifier.preprocess_content(content)
assert result == expected_preprocess_content
@pytest.mark.django_db
class TestClassifierTrainTagLabels:
@pytest.fixture(autouse=True)
def _simple_preprocess(self, mocker: MockerFixture) -> None:
mocker.patch.object(
DocumentClassifier,
"preprocess_content",
side_effect=dummy_preprocess,
)
@pytest.fixture
def auto_tags(self) -> list[Tag]:
return TagFactory.create_batch(2, matching_algorithm=MatchingModel.MATCH_AUTO)
def test_train_query_count_does_not_scale_with_documents(
self,
auto_tags: list[Tag],
) -> None:
"""
GIVEN:
- Documents with auto matching tags
WHEN:
- The classifier is trained, then more documents are added and it is
trained again
THEN:
- Both trainings run the same number of queries
"""
for doc in DocumentFactory.create_batch(2):
doc.tags.set(auto_tags)
with CaptureQueriesContext(connection) as few_documents:
DocumentClassifier().train()
for doc in DocumentFactory.create_batch(6):
doc.tags.set(auto_tags)
with CaptureQueriesContext(connection) as more_documents:
DocumentClassifier().train()
assert len(more_documents) == len(few_documents)
def test_train_uses_only_auto_tags_as_labels(
self,
auto_tags: list[Tag],
) -> None:
"""
GIVEN:
- Documents with both auto matching and non auto matching tags
WHEN:
- The classifier is trained
THEN:
- Only the auto matching tags are used as tag labels
"""
manual_tag = TagFactory(matching_algorithm=MatchingModel.MATCH_ANY)
first, second, third = DocumentFactory.create_batch(3)
first.tags.set([auto_tags[0], manual_tag])
second.tags.set([auto_tags[1], manual_tag])
third.tags.set([*auto_tags, manual_tag])
classifier = DocumentClassifier()
classifier.train()
assert list(classifier.tags_binarizer.classes_) == sorted(
tag.pk for tag in auto_tags
)
@pytest.mark.django_db
class TestClassifierTrainContent:
def test_train_content_follows_label_order_across_chunks(
self,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- More documents than fit in one content chunk
WHEN:
- The classifier is trained
THEN:
- Every document's content is preprocessed once, in document order
"""
mocker.patch("documents.classifier._CONTENT_CHUNK_SIZE", 2)
docs = DocumentFactory.create_batch(5)
preprocess = mocker.patch.object(
DocumentClassifier,
"preprocess_content",
side_effect=dummy_preprocess,
)
DocumentClassifier().train()
assert [call.args[0] for call in preprocess.call_args_list] == [
doc.content for doc in sorted(docs, key=lambda doc: doc.pk)
]
def test_train_document_deleted_while_training(
self,
mocker: MockerFixture,
) -> None:
"""
GIVEN:
- Two documents
WHEN:
- The second document is deleted after its labels were gathered, but
before its content is fetched
THEN:
- Training completes
- The deleted document is trained with empty content, keeping labels
and content aligned
"""
mocker.patch("documents.classifier._CONTENT_CHUNK_SIZE", 1)
first, second = DocumentFactory.create_batch(2)
def delete_second_then_preprocess(content: str, **kwargs) -> str:
if content == first.content:
second.delete()
return dummy_preprocess(content)
preprocess = mocker.patch.object(
DocumentClassifier,
"preprocess_content",
side_effect=delete_second_then_preprocess,
)
assert DocumentClassifier().train()
assert [call.args[0] for call in preprocess.call_args_list] == [
first.content,
"",
]
+60
View File
@@ -3769,6 +3769,66 @@ class TestWorkflows(
expected_str = "Error occurred sending email"
self.assertIn(expected_str, cm.output[0])
@override_settings(
EMAIL_ENABLED=True,
PAPERLESS_URL="http://localhost:8000",
)
@mock.patch("django.core.mail.message.EmailMessage.send")
def test_workflow_email_action_template_error(self, mock_email_send) -> None:
"""
GIVEN:
- Document added workflow with an email action whose body uses an
undefined template variable, followed by an assignment action
WHEN:
- Document consumption finishes
THEN:
- Error is logged, consumption is not aborted
- No email is sent
- Subsequent actions still run
"""
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
)
email_action = WorkflowActionEmail.objects.create(
subject="Test Notification: {{ doc_title }}",
body="Document Title: {{ title }}",
to="me@example.com",
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.EMAIL,
email=email_action,
order=0,
)
assignment_action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.ASSIGNMENT,
assign_correspondent=self.c2,
order=1,
)
w = Workflow.objects.create(
name="Workflow 1",
order=0,
)
w.triggers.add(trigger)
w.actions.add(action, assignment_action)
w.save()
doc = Document.objects.create(
title="sample test",
correspondent=self.c,
original_filename="sample.pdf",
)
with self.assertLogs("paperless.workflows", level="ERROR") as cm:
document_consumption_finished.send(
sender=self.__class__,
document=doc,
)
self.assertIn("'title' is undefined", cm.output[0])
mock_email_send.assert_not_called()
doc.refresh_from_db()
self.assertEqual(doc.correspondent, self.c2)
@override_settings(
PAPERLESS_EMAIL_HOST="localhost",
EMAIL_ENABLED=True,
+35 -35
View File
@@ -102,42 +102,42 @@ def execute_email_action(
)
return
subject = (
parse_w_workflow_placeholders(
action.email.subject,
context["correspondent"],
context["document_type"],
context["owner_username"],
context["added"],
context["filename"],
context["current_filename"],
context["created"],
context["title"],
context["doc_url"],
context["id"],
)
if action.email.subject
else ""
)
body = (
parse_w_workflow_placeholders(
action.email.body,
context["correspondent"],
context["document_type"],
context["owner_username"],
context["added"],
context["filename"],
context["current_filename"],
context["created"],
context["title"],
context["doc_url"],
context["id"],
)
if action.email.body
else ""
)
try:
subject = (
parse_w_workflow_placeholders(
action.email.subject,
context["correspondent"],
context["document_type"],
context["owner_username"],
context["added"],
context["filename"],
context["current_filename"],
context["created"],
context["title"],
context["doc_url"],
context["id"],
)
if action.email.subject
else ""
)
body = (
parse_w_workflow_placeholders(
action.email.body,
context["correspondent"],
context["document_type"],
context["owner_username"],
context["added"],
context["filename"],
context["current_filename"],
context["created"],
context["title"],
context["doc_url"],
context["id"],
)
if action.email.body
else ""
)
attachments: list[EmailAttachment] = []
if action.email.include_document:
attachment: EmailAttachment | None = None
+12 -12
View File
@@ -2,7 +2,7 @@ msgid ""
msgstr ""
"Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2026-09-15 14:27+0000\n"
"POT-Creation-Date: 2026-09-15 19:24+0000\n"
"PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n"
"Language-Team: English\n"
@@ -1631,49 +1631,49 @@ msgstr ""
msgid "workflow runs"
msgstr ""
#: documents/serialisers.py:525 documents/serialisers.py:882
#: documents/serialisers.py:2868 documents/views.py:343 documents/views.py:2719
#: documents/serialisers.py:514 documents/serialisers.py:871
#: documents/serialisers.py:2865 documents/views.py:343 documents/views.py:2719
#: paperless_mail/serialisers.py:156
msgid "Insufficient permissions."
msgstr ""
#: documents/serialisers.py:718
#: documents/serialisers.py:707
msgid "Invalid color."
msgstr ""
#: documents/serialisers.py:2341
#: documents/serialisers.py:2338
#, python-format
msgid "File type %(type)s not supported"
msgstr ""
#: documents/serialisers.py:2385
#: documents/serialisers.py:2382
#, python-format
msgid "Custom field id must be an integer: %(id)s"
msgstr ""
#: documents/serialisers.py:2392
#: documents/serialisers.py:2389
#, python-format
msgid "Custom field with id %(id)s does not exist"
msgstr ""
#: documents/serialisers.py:2409 documents/serialisers.py:2419
#: documents/serialisers.py:2406 documents/serialisers.py:2416
msgid ""
"Custom fields must be a list of integers or an object mapping ids to values."
msgstr ""
#: documents/serialisers.py:2414
#: documents/serialisers.py:2411
msgid "Some custom fields don't exist or were specified twice."
msgstr ""
#: documents/serialisers.py:2561
#: documents/serialisers.py:2558
msgid "Invalid variable detected."
msgstr ""
#: documents/serialisers.py:2924
#: documents/serialisers.py:2921
msgid "Duplicate document identifiers are not allowed."
msgstr ""
#: documents/serialisers.py:2954 documents/views.py:4742
#: documents/serialisers.py:2951 documents/views.py:4742
#, python-format
msgid "Documents not found: %(ids)s"
msgstr ""
+18
View File
@@ -235,6 +235,24 @@ class ApplicationConfigurationSerializer(
) -> list[str]:
return sorted(name for name in os.environ if name.startswith("PAPERLESS_"))
@staticmethod
def _require_json_object(field: str, value: object) -> None:
if value is not None and not isinstance(value, dict):
raise serializers.ValidationError(f"{field} must be a JSON object.")
def validate_user_args(self, value):
self._require_json_object("user_args", value)
return value
def validate_barcode_tag_mapping(self, value):
self._require_json_object("barcode_tag_mapping", value)
# Each value is the regex substitute applied to a matching barcode
if value is not None and not all(isinstance(v, str) for v in value.values()):
raise serializers.ValidationError(
"barcode_tag_mapping values must be strings.",
)
return value
def run_validation(self, data):
# Empty strings treated as None to avoid unexpected behavior
if "user_args" in data and data["user_args"] == "":