Compare commits

...
Author SHA1 Message Date
shamoonandGitHub 591e78cef4 Merge branch 'beta' into beta-sec-autocomplete-perms 2026-07-21 16:42:15 -07:00
stumpylog 708391976f To help readability, use NameTuple for the returns, so we know what list of pks is what 2026-07-21 16:16:15 -07:00
shamoon 7bfaa86c7c Fix: enforce current permissions in autocomplete 2026-07-21 16:03:22 -07:00
shamoonandGitHub 8aab34ea73 Security (beta): two small permission check fixes (#13186) 2026-07-21 16:50:28 +00:00
8404198ec8 Fix (beta): short-circuit ObjectOwnedOrGrantedPermissionsFilter for superusers (#13183)
Mirrors the existing short-circuit already present in its sibling class,
ObjectOwnedPermissionsFilter, which this one lacked.

Benchmarked as a superuser against a 100k-document corpus: no measurable
difference. django-guardian's own get_objects_for_user() (called via
super().filter_queryset()) already returns the queryset unrestricted for
superusers before this change, so the guardian permission tables were
never actually queried for superusers either way. This change only skips
building a redundant 3-way OR of conditions that are all subsets of the
already-unrestricted queryset -- worth keeping for clarity/consistency
with the sibling filter, not as a performance claim.

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-21 07:31:58 -07:00
d8d8872414 Fix (beta): compute num_notes via a subquery instead of Count()+distinct() (#13182)
DocumentViewSet.get_queryset() annotated num_notes via a LEFT JOIN to
documents_note plus Count(), which requires the database to aggregate
every matching document's note count before it can even sort or paginate
the result -- on every list request, not just filtered ones.

Switched to the same correlated-subquery pattern already used two lines
above for effective_content (Subquery + OuterRef), which Django compiles
to portable SQL across sqlite/postgresql/mariadb rather than a join-based
aggregate.

Benchmarked against a 100k synthetic document corpus (Postgres): the
plain document list request dropped from ~2.2-2.4s to ~0.9-1.0s. Note
that a separate, larger cost remains in the same queryset's plain
.distinct() call, which still forces a full sort over every matching
row regardless of this fix -- tracked separately, not addressed here.

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-20 13:59:44 -07:00
shamoonandGitHub ff609c2987 Change: update root modified timestamp on version changes (#13170) 2026-07-20 15:45:24 +00:00
Trenton HandGitHub 2b784e709b Fix: prevent tag assignment from reverting other pending workflow assignments (#13178)
apply_assignment_to_document() mutated tags directly on the shared document instance via add_nested_tags(), whose m2m_changed signal triggers update_filename_and_move_files() -> instance.refresh_from_db(), discarding any not-yet-saved fields (e.g. storage_path) staged by an earlier-ordered action in the same workflow. Apply tag changes to a freshly-fetched instance instead, matching the pattern already used in apply_removal_to_document().
2026-07-20 08:22:30 -07:00
Trenton HandGitHub c9716252f0 Performance: Add DB indexes for common query/sort patterns (#13167)
Adds indexes on Document.checksum, Document.page_count, and a composite
(owner, created) index on Document, plus composite (field, value_*)
indexes on CustomFieldInstance for each typed value column.

Benchmarked against a 100k-document / 200k-custom-field-instance SQLite
dataset: Document.checksum lookups ~69x faster, page_count sort ~40x,
owner+created list queries ~6x. CustomFieldInstance composite indexes
show large gains (10-24x) on realistic narrow-selectivity queries
matching the workflow scheduler and custom-field search code paths.
2026-07-20 13:12:54 +00:00
shamoon 5cf9152a40 Merge branch 'dev' into beta 2026-07-19 20:20:48 -07:00
4e52dd5710 Fix: cache per-request effective-document resolution for thumb/metadata/preview (#13166)
Django's condition() decorator invokes etag_func and last_modified_func
separately, and the view itself may resolve again -- each call to
resolve_effective_document_by_pk() was redoing the same root/version
DB lookups. Memoize the resolution on the request object so a single
thumb/metadata/preview request resolves the effective document once
instead of up to three times.

Related to paperless-ngx/paperless-ngx#13161.

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-19 18:04:16 -07:00
18 changed files with 512 additions and 71 deletions
+3 -3
View File
@@ -1,9 +1,9 @@
from datetime import UTC
from datetime import datetime
from typing import Any
from django.conf import settings
from django.core.cache import cache
from rest_framework.request import Request
from documents.caching import CACHE_5_MINUTES
from documents.caching import CACHE_50_MINUTES
@@ -117,7 +117,7 @@ def preview_last_modified(request, pk: int) -> datetime | None:
return doc.modified
def thumbnail_etag(request: Any, pk: int) -> str | None:
def thumbnail_etag(request: Request, pk: int) -> str | None:
"""
Thumbnails are version-dependent, so use the effective document checksum as
the ETag to invalidate cache when the latest version changes.
@@ -128,7 +128,7 @@ def thumbnail_etag(request: Any, pk: int) -> str | None:
return doc.checksum
def thumbnail_last_modified(request: Any, pk: int) -> datetime | None:
def thumbnail_last_modified(request: Request, pk: int) -> datetime | None:
"""
Returns the filesystem last modified either from cache or from filesystem.
Cache should be (slightly?) faster than filesystem
+5
View File
@@ -621,6 +621,11 @@ class ConsumerPlugin(
else:
original_document.save()
# Adding a version changes the effective document, so update root modified
Document.objects.filter(pk=root_doc.pk).update(
modified=timezone.now(),
)
# Create a log entry for the version addition, if enabled
if settings.AUDIT_LOG_ENABLED:
from auditlog.models import ( # type: ignore[import-untyped]
+2
View File
@@ -1024,6 +1024,8 @@ class ObjectOwnedOrGrantedPermissionsFilter(ObjectPermissionsFilter):
"""
def filter_queryset(self, request, queryset, view):
if request.user.is_superuser:
return queryset
objects_with_perms = super().filter_queryset(request, queryset, view)
objects_owned = queryset.filter(owner=request.user)
objects_unowned = queryset.filter(owner__isnull=True)
@@ -0,0 +1,73 @@
# Generated by Django 5.2.16 on 2026-07-19 20:33
import django.core.validators
from django.conf import settings
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "0021_widen_workflow_integer_fields"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AlterField(
model_name="document",
name="checksum",
field=models.CharField(
db_index=True,
editable=False,
help_text="The checksum of the original document.",
max_length=64,
verbose_name="checksum",
),
),
migrations.AlterField(
model_name="document",
name="page_count",
field=models.PositiveIntegerField(
db_index=True,
help_text="The number of pages of the document.",
null=True,
validators=[django.core.validators.MinValueValidator(1)],
verbose_name="page count",
),
),
migrations.AddIndex(
model_name="customfieldinstance",
index=models.Index(
fields=["field", "value_date"],
name="documents_c_field_i_30b51d_idx",
),
),
migrations.AddIndex(
model_name="customfieldinstance",
index=models.Index(
fields=["field", "value_int"],
name="documents_c_field_i_d25ab0_idx",
),
),
migrations.AddIndex(
model_name="customfieldinstance",
index=models.Index(
fields=["field", "value_float"],
name="documents_c_field_i_c35174_idx",
),
),
migrations.AddIndex(
model_name="customfieldinstance",
index=models.Index(
fields=["field", "value_monetary_amount"],
name="documents_c_field_i_53ce4e_idx",
),
),
migrations.AddIndex(
model_name="document",
index=models.Index(
fields=["owner", "created"],
name="documents_d_owner_i_ec6ab3_idx",
),
),
]
+11 -1
View File
@@ -217,6 +217,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
_("checksum"),
max_length=64,
editable=False,
db_index=True,
help_text=_("The checksum of the original document."),
)
@@ -234,7 +235,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
blank=False,
null=True,
unique=False,
db_index=False,
db_index=True,
validators=[MinValueValidator(1)],
help_text=_(
"The number of pages of the document.",
@@ -338,6 +339,9 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
ordering = ("-created",)
verbose_name = _("document")
verbose_name_plural = _("documents")
indexes = [
models.Index(fields=["owner", "created"]),
]
constraints = [
models.UniqueConstraint(
fields=["root_document", "version_index"],
@@ -1190,6 +1194,12 @@ class CustomFieldInstance(SoftDeleteModel):
ordering = ("created",)
verbose_name = _("custom field instance")
verbose_name_plural = _("custom field instances")
indexes = [
models.Index(fields=["field", "value_date"]),
models.Index(fields=["field", "value_int"]),
models.Index(fields=["field", "value_float"]),
models.Index(fields=["field", "value_monetary_amount"]),
]
constraints = [
models.UniqueConstraint(
fields=["document", "field"],
+88 -35
View File
@@ -11,6 +11,7 @@ from enum import StrEnum
from itertools import islice
from typing import TYPE_CHECKING
from typing import Final
from typing import NamedTuple
from typing import Self
from typing import TypedDict
from typing import TypeVar
@@ -21,6 +22,7 @@ import tantivy
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.utils.timezone import get_current_timezone
from guardian.shortcuts import get_groups_with_perms
from guardian.shortcuts import get_users_with_perms
from documents.search._query import build_permission_filter
@@ -45,6 +47,8 @@ if TYPE_CHECKING:
from pathlib import Path
from django.contrib.auth.models import AbstractUser
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.db.models import QuerySet
from documents.models import Document
@@ -59,6 +63,18 @@ _LOCK_BACKOFF_CAP: Final[float] = 10.0 # seconds
T = TypeVar("T")
class ViewerGrant(NamedTuple):
"""Direct user and group view grants for a single document.
Named fields (rather than a bare 2-tuple) so ``viewer_ids`` and
``viewer_group_ids`` can't be silently transposed at a call site — both
are ``list[int]``, so a positional swap would type-check cleanly.
"""
viewer_ids: list[int]
viewer_group_ids: list[int]
class SearchMode(StrEnum):
QUERY = "query"
TEXT = "text"
@@ -367,7 +383,7 @@ class TantivyBackend:
) -> tantivy.Query:
"""Wrap a query with a permission filter if the user is not a superuser."""
if user is not None:
permission_filter = build_permission_filter(self._schema, user)
permission_filter = self._build_permission_filter(user)
return tantivy.Query.boolean_query(
[
(tantivy.Occur.Must, query),
@@ -376,11 +392,21 @@ class TantivyBackend:
)
return query
def _build_permission_filter(self, user: AbstractUser) -> tantivy.Query:
"""Build a filter using the user's current group memberships."""
group_ids = user.groups.values_list("pk", flat=True)
return build_permission_filter(
self._schema,
user,
viewer_group_ids=group_ids,
)
def _build_tantivy_doc(
self,
document: Document,
effective_content: str | None = None,
viewer_ids: list[int] | None = None,
viewer_group_ids: list[int] | None = None,
) -> tantivy.Document:
"""Build a tantivy Document from a Django Document instance.
@@ -497,10 +523,26 @@ class TantivyBackend:
users_with_perms = get_users_with_perms(
document,
only_with_perms_in=["view_document"],
with_group_users=False,
)
viewer_ids = list(
cast("QuerySet[User]", users_with_perms).values_list("id", flat=True),
)
viewer_ids = [int(u.id) for u in users_with_perms]
for viewer_id in viewer_ids:
doc.add_unsigned("viewer_id", viewer_id)
if viewer_group_ids is None:
groups_with_perms = get_groups_with_perms(
document,
only_with_perms_in=["view_document"],
)
viewer_group_ids = list(
cast("QuerySet[Group]", groups_with_perms).values_list(
"id",
flat=True,
),
)
for viewer_group_id in viewer_group_ids:
doc.add_unsigned("viewer_group_id", viewer_group_id)
# Autocomplete words
text_sources = [document.title, content]
@@ -813,7 +855,7 @@ class TantivyBackend:
# Intersect with permission filter so autocomplete words from
# invisible documents don't leak to other users.
if user is not None and not user.is_superuser:
permission_query = build_permission_filter(self._schema, user)
permission_query = self._build_permission_filter(user)
matches = searcher.terms_with_prefix(
"autocomplete_word",
@@ -915,7 +957,7 @@ class TantivyBackend:
def rebuild(
self,
documents: QuerySet[Document],
iter_wrapper: IterWrapper[tuple[Document, list[int]]] = identity,
iter_wrapper: IterWrapper[tuple[Document, ViewerGrant]] = identity,
writer_heap_bytes: int = 512_000_000,
) -> None:
"""
@@ -928,8 +970,8 @@ class TantivyBackend:
documents: QuerySet of Document instances to index
iter_wrapper: Optional wrapper function for progress tracking
(e.g., progress bar). Wraps an iterable of
``(document, viewer_ids)`` pairs and should yield each pair
unchanged, advancing one step per document.
``(document, (viewer_ids, viewer_group_ids))`` pairs and should yield
each unchanged, advancing one step per document.
writer_heap_bytes: Tantivy writer memory budget (split across the
writer's threads). Larger values buffer more docs in RAM before
flushing a segment, deferring merge work; they do not avoid it.
@@ -953,11 +995,14 @@ class TantivyBackend:
documents_stream = _DocumentViewerStream(documents, chunk_size=1000)
try:
writer = new_index.writer(heap_size=writer_heap_bytes)
for document, viewer_ids in iter_wrapper(documents_stream):
for document, (viewer_ids, viewer_group_ids) in iter_wrapper(
documents_stream,
):
doc = self._build_tantivy_doc(
document,
document.get_effective_content(),
viewer_ids=viewer_ids,
viewer_group_ids=viewer_group_ids,
)
writer.add_document(doc)
writer.commit()
@@ -978,19 +1023,26 @@ def chunked(iterable, size):
yield chunk
class _DocumentViewerStream:
"""Yield (document, viewer_ids) pairs while batch-loading viewer ids.
_EMPTY_VIEWER_GRANT: Final[ViewerGrant] = ViewerGrant(
viewer_ids=[],
viewer_group_ids=[],
)
Viewer permissions are fetched one SQL query per chunk (see
``_bulk_get_viewer_ids``), but documents are yielded individually so a
class _DocumentViewerStream:
"""Yield document permission data while batch-loading grants.
Viewer permissions are fetched in batches (see
``_bulk_get_viewer_permissions``), but documents are yielded individually so a
progress bar wrapped around this stream advances per document rather than
jumping a whole chunk at a time. ``__len__`` lets the progress helper still
discover the total (it inspects ``QuerySet``/``Sized``).
The viewer ids travel with each document in the yielded pair rather than
through a separate mutable attribute, so the pairing survives regardless
of how ``iter_wrapper`` consumes the stream (buffering, batching, etc.) —
there is no reliance on the caller advancing this generator in lock-step.
The viewer and group ids travel with each document in the yielded pair
rather than through a separate mutable attribute, so the pairing survives
regardless of how ``iter_wrapper`` consumes the stream (buffering,
batching, etc.) — there is no reliance on the caller advancing this
generator in lock-step.
"""
def __init__(self, documents: QuerySet[Document], *, chunk_size: int) -> None:
@@ -1000,25 +1052,25 @@ class _DocumentViewerStream:
def __len__(self) -> int:
return self._documents.count()
def __iter__(self) -> Iterator[tuple[Document, list[int]]]:
def __iter__(self) -> Iterator[tuple[Document, ViewerGrant]]:
# iterator(chunk_size=…) streams from a server-side cursor instead of
# materialising the whole queryset in memory; since Django 4.1 it still
# honours prefetch_related, running the prefetches one batch at a time.
documents = self._documents.iterator(chunk_size=self._chunk_size)
for chunk in chunked(documents, self._chunk_size):
viewer_ids_by_pk = _bulk_get_viewer_ids([doc.pk for doc in chunk])
grants_by_pk = _bulk_get_viewer_permissions([doc.pk for doc in chunk])
for doc in chunk:
yield doc, viewer_ids_by_pk.get(doc.pk, [])
yield doc, grants_by_pk.get(doc.pk, _EMPTY_VIEWER_GRANT)
def _bulk_get_viewer_ids(doc_pks: Sequence[int]) -> dict[int, list[int]]:
"""Fetch all view_document permissions for a batch of documents in one query.
def _bulk_get_viewer_permissions(
doc_pks: Sequence[int],
) -> dict[int, ViewerGrant]:
"""Fetch direct user and group view grants for a batch of documents, keyed by pk.
Mirrors get_users_with_perms(doc, only_with_perms_in=["view_document"])
(with_group_users defaults to True there): a user counts as a viewer if
they hold the permission directly, OR via membership in a group that
holds the permission. Missing the group case would silently drop search
access for group-only viewers.
Group grants remain group IDs in the index so permission checks use the
requesting user's current memberships. Expanding groups to user IDs here
would leave stale access behind after a user is removed from a group.
"""
from collections import defaultdict
@@ -1032,6 +1084,7 @@ def _bulk_get_viewer_ids(doc_pks: Sequence[int]) -> dict[int, list[int]]:
str_pks = [str(pk) for pk in doc_pks]
viewer_map: dict[int, set[int]] = defaultdict(set)
viewer_group_map: dict[int, set[int]] = defaultdict(set)
# Fold the permission lookup into the query via a join on codename instead
# of a separate Permission.objects.get(), which would otherwise run once per
@@ -1045,22 +1098,22 @@ def _bulk_get_viewer_ids(doc_pks: Sequence[int]) -> dict[int, list[int]]:
for object_pk, user_id in user_qs:
viewer_map[int(object_pk)].add(user_id)
# User.groups has related_query_name="user", so group__user__id joins
# through the group membership m2m to the member users' ids in the same
# single-query fashion as user_qs above (values_list compiles to one SQL
# JOIN; no per-row Python-side lookups follow, so select_related /
# prefetch_related do not apply here).
group_qs = GroupObjectPermission.objects.filter(
content_type=ct,
permission__content_type=ct,
permission__codename="view_document",
object_pk__in=str_pks,
).values_list("object_pk", "group__user__id")
for object_pk, user_id in group_qs:
if user_id is not None:
viewer_map[int(object_pk)].add(user_id)
).values_list("object_pk", "group_id")
for object_pk, group_id in group_qs:
viewer_group_map[int(object_pk)].add(group_id)
return {object_pk: list(user_ids) for object_pk, user_ids in viewer_map.items()}
return {
object_pk: ViewerGrant(
viewer_ids=list(viewer_map.get(object_pk, ())),
viewer_group_ids=list(viewer_group_map.get(object_pk, ())),
)
for object_pk in viewer_map.keys() | viewer_group_map.keys()
}
# Module-level singleton with proper thread safety
+11 -1
View File
@@ -20,6 +20,7 @@ from documents.search._translate import SearchQueryError
from documents.search._translate import translate_query
if TYPE_CHECKING:
from collections.abc import Iterable
from datetime import tzinfo
from django.contrib.auth.base_user import AbstractBaseUser
@@ -114,6 +115,7 @@ def normalize_query(query: str) -> str:
def build_permission_filter(
schema: tantivy.Schema,
user: AbstractBaseUser,
viewer_group_ids: Iterable[int] = (),
) -> tantivy.Query:
"""
Build a query filter for user document permissions.
@@ -123,10 +125,12 @@ def build_permission_filter(
- Public documents (no owner) are visible to all users
- Private documents are visible to their owner
- Documents explicitly shared with the user are visible
- Documents shared with one of the user's current groups are visible
Args:
schema: Tantivy schema for field validation
user: User to check permissions for
viewer_group_ids: Current group memberships for the user
Returns:
Tantivy query that filters results to visible documents
@@ -140,7 +144,13 @@ def build_permission_filter(
)
owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
return tantivy.Query.disjunction_max_query([no_owner, owned, shared])
group_shared = [
tantivy.Query.term_query(schema, "viewer_group_id", group_id)
for group_id in viewer_group_ids
]
return tantivy.Query.disjunction_max_query(
[no_owner, owned, shared, *group_shared],
)
DEFAULT_SEARCH_FIELDS = [
+1
View File
@@ -102,6 +102,7 @@ def build_schema() -> tantivy.Schema:
"tag_id",
"owner_id",
"viewer_id",
"viewer_group_id",
):
sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
+92
View File
@@ -1419,6 +1419,30 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
@mock.patch("documents.views.bulk_edit.merge")
def test_merge_and_delete_requires_change_permission(self, m) -> None:
self.setup_mock(m, "merge")
user = User.objects.create_user(username="no-change")
user.user_permissions.add(
Permission.objects.get(codename="add_document"),
Permission.objects.get(codename="delete_document"),
)
self.client.force_authenticate(user=user)
response = self.client.post(
"/api/documents/merge/",
json.dumps(
{
"documents": [self.doc2.id, self.doc3.id],
"delete_originals": True,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
m.assert_not_called()
@mock.patch("documents.views.bulk_edit.merge")
def test_merge_invalid_parameters(self, m) -> None:
self.setup_mock(m, "merge")
@@ -1668,6 +1692,74 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_edit_pdf_update_requires_change_permission(self, m) -> None:
self.setup_mock(m, "edit_pdf")
user = User.objects.create_user(username="no-change")
self.client.force_authenticate(user=user)
response = self.client.post(
"/api/documents/edit_pdf/",
json.dumps(
{
"documents": [self.doc2.id],
"operations": [{"page": 1}],
"update_document": True,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
m.assert_not_called()
@mock.patch("documents.views.bulk_edit.remove_password")
@mock.patch("documents.views.bulk_edit.edit_pdf")
def test_delete_original_requires_delete_permission(
self,
edit_pdf_mock,
remove_password_mock,
) -> None:
self.setup_mock(edit_pdf_mock, "edit_pdf")
self.setup_mock(remove_password_mock, "remove_password")
user = User.objects.create_user(username="no-delete")
user.user_permissions.add(
Permission.objects.get(codename="add_document"),
Permission.objects.get(codename="change_document"),
)
self.client.force_authenticate(user=user)
cases = [
(
"/api/documents/edit_pdf/",
{
"documents": [self.doc2.id],
"operations": [{"page": 1}],
"delete_original": True,
},
edit_pdf_mock,
),
(
"/api/documents/remove_password/",
{
"documents": [self.doc2.id],
"password": "secret",
"delete_original": True,
},
remove_password_mock,
),
]
for endpoint, payload, operation_mock in cases:
with self.subTest(endpoint=endpoint):
response = self.client.post(
endpoint,
json.dumps(payload),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
operation_mock.assert_not_called()
@mock.patch("documents.views.bulk_edit.remove_password")
def test_remove_password(self, m) -> None:
self.setup_mock(m, "remove_password")
@@ -1,5 +1,6 @@
from __future__ import annotations
import datetime
from typing import TYPE_CHECKING
from unittest import TestCase
from unittest import mock
@@ -10,6 +11,7 @@ from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import FieldError
from django.core.files.uploadedfile import SimpleUploadedFile
from django.utils import timezone
from rest_framework import status
from rest_framework.test import APITestCase
@@ -137,6 +139,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
root_document=root,
content="v2-content",
)
original_modified = timezone.now() - datetime.timedelta(days=1)
Document.objects.filter(pk=root.pk).update(modified=original_modified)
with mock.patch("documents.search.get_backend"):
resp = self.client.delete(f"/api/documents/{root.id}/versions/{v2.id}/")
@@ -146,6 +150,7 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(resp.data["current_version_id"], v1.id)
root.refresh_from_db()
self.assertEqual(root.content, "root-content")
self.assertGreater(root.modified, original_modified)
with mock.patch("documents.search.get_backend"):
resp = self.client.delete(f"/api/documents/{root.id}/versions/{v1.id}/")
@@ -326,6 +331,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
root_document=root,
version_label="old",
)
original_modified = timezone.now() - datetime.timedelta(days=1)
Document.objects.filter(pk=root.pk).update(modified=original_modified)
resp = self.client.patch(
f"/api/documents/{root.id}/versions/{version.id}/",
@@ -339,6 +346,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(resp.data["version_label"], "Label 1")
self.assertEqual(resp.data["id"], version.id)
self.assertFalse(resp.data["is_root"])
root.refresh_from_db()
self.assertGreater(root.modified, original_modified)
def test_update_version_label_clears_on_blank(self) -> None:
root = Document.objects.create(
@@ -660,6 +669,26 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
def test_update_version_requires_global_change_permission(self) -> None:
user = User.objects.create_user(username="add-only")
user.user_permissions.add(Permission.objects.get(codename="add_document"))
root = Document.objects.create(
title="root",
checksum="root",
mime_type="application/pdf",
)
self.client.force_authenticate(user=user)
with mock.patch("documents.views.consume_file") as consume_mock:
resp = self.client.post(
f"/api/documents/{root.id}/update_version/",
{"document": self._make_pdf_upload()},
format="multipart",
)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
consume_mock.apply_async.assert_not_called()
def test_update_version_returns_404_for_missing_document(self) -> None:
resp = self.client.post(
"/api/documents/9999/update_version/",
@@ -131,6 +131,10 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
self.client.get("/api/saved_views/").status_code,
status.HTTP_403_FORBIDDEN,
)
self.assertEqual(
self.client.get("/api/search/autocomplete/?term=test").status_code,
status.HTTP_403_FORBIDDEN,
)
def test_api_sufficient_permissions(self) -> None:
user = User.objects.create_user(username="test")
+25
View File
@@ -832,6 +832,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
"""
u1 = User.objects.create_user("user1")
u2 = User.objects.create_user("user2")
u1.user_permissions.add(Permission.objects.get(codename="view_document"))
self.client.force_authenticate(user=u1)
@@ -878,6 +879,30 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.data, ["applebaum", "apples", "appletini"])
def test_search_autocomplete_group_revocation_is_immediate(self) -> None:
user = User.objects.create_user("group-user")
owner = User.objects.create_user("document-owner")
group = Group.objects.create(name="temporary-viewers")
user.user_permissions.add(Permission.objects.get(codename="view_document"))
user.groups.add(group)
document = Document.objects.create(
title="private",
content="canarysecretautocomplete",
checksum="group-revocation",
owner=owner,
)
assign_perm("view_document", group, document)
get_backend().add_or_update(document)
self.client.force_authenticate(user=user)
response = self.client.get("/api/search/autocomplete/?term=canarysecret")
self.assertEqual(response.data, ["canarysecretautocomplete"])
user.groups.remove(group)
response = self.client.get("/api/search/autocomplete/?term=canarysecret")
self.assertEqual(response.data, [])
def test_search_autocomplete_field_name_match(self) -> None:
"""
GIVEN:
+4
View File
@@ -767,6 +767,8 @@ class TestConsumer(
root_doc.archive_serial_number = 42
root_doc.save()
original_modified = timezone.now() - datetime.timedelta(days=1)
Document.objects.filter(pk=root_doc.pk).update(modified=original_modified)
actor = User.objects.create_user(
username="actor",
email="actor@example.com",
@@ -818,6 +820,8 @@ class TestConsumer(
self.assertIsNone(version.archive_serial_number)
self.assertEqual(version.original_filename, version_file.name)
self.assertTrue(bool(version.content))
root_doc.refresh_from_db()
self.assertGreater(root_doc.modified, original_modified)
@override_settings(AUDIT_LOG_ENABLED=True)
@mock.patch("documents.consumer.load_classifier")
@@ -1,8 +1,12 @@
from datetime import timedelta
from types import SimpleNamespace
from typing import TYPE_CHECKING
from typing import cast
from unittest import mock
from django.db import connection
from django.test import TestCase
from django.test.utils import CaptureQueriesContext
from django.utils import timezone
from documents.conditionals import metadata_etag
@@ -13,6 +17,9 @@ from documents.models import Document
from documents.tests.utils import DirectoriesMixin
from documents.versioning import resolve_effective_document_by_pk
if TYPE_CHECKING:
from rest_framework.request import Request
class TestConditionals(DirectoriesMixin, TestCase):
def test_metadata_etag_uses_latest_version_for_root_request(self) -> None:
@@ -29,14 +36,18 @@ class TestConditionals(DirectoriesMixin, TestCase):
mime_type="application/pdf",
root_document=root,
)
request = SimpleNamespace(query_params={})
request = cast("Request", SimpleNamespace(query_params={}))
self.assertEqual(
metadata_etag(request, root.id),
f"{latest.checksum}:{latest.modified.isoformat()}",
)
self.assertEqual(preview_etag(request, root.id), latest.archive_checksum)
self.assertEqual(thumbnail_etag(request, root.id), latest.checksum)
# preview_etag/thumbnail_etag resolve the same (pk, request) and should
# reuse metadata_etag's cached resolution instead of re-querying.
with CaptureQueriesContext(connection) as ctx:
self.assertEqual(preview_etag(request, root.id), latest.archive_checksum)
self.assertEqual(thumbnail_etag(request, root.id), latest.checksum)
self.assertEqual(len(ctx.captured_queries), 0)
def test_metadata_etag_changes_when_document_modified_changes(self) -> None:
doc = Document.objects.create(
@@ -44,15 +55,20 @@ class TestConditionals(DirectoriesMixin, TestCase):
checksum="same-checksum",
mime_type="application/pdf",
)
request = SimpleNamespace(query_params={})
original_etag = metadata_etag(request, doc.id)
# Each call simulates a separate incoming HTTP request, so it gets its
# own request object -- the per-request resolution cache must not leak
# a stale result across genuinely different requests.
original_etag = metadata_etag(
cast("Request", SimpleNamespace(query_params={})),
doc.id,
)
new_modified = timezone.now() + timedelta(seconds=5)
Document.objects.filter(id=doc.id).update(modified=new_modified)
self.assertNotEqual(metadata_etag(request, doc.id), original_etag)
second_request = cast("Request", SimpleNamespace(query_params={}))
self.assertNotEqual(metadata_etag(second_request, doc.id), original_etag)
self.assertEqual(
metadata_etag(request, doc.id),
metadata_etag(second_request, doc.id),
f"{doc.checksum}:{new_modified.isoformat()}",
)
@@ -76,9 +92,13 @@ class TestConditionals(DirectoriesMixin, TestCase):
root_document=other_root,
)
invalid_request = SimpleNamespace(query_params={"version": "not-a-number"})
unrelated_request = SimpleNamespace(
query_params={"version": str(other_version.id)},
invalid_request = cast(
"Request",
SimpleNamespace(query_params={"version": "not-a-number"}),
)
unrelated_request = cast(
"Request",
SimpleNamespace(query_params={"version": str(other_version.id)}),
)
self.assertIsNone(
@@ -105,7 +125,7 @@ class TestConditionals(DirectoriesMixin, TestCase):
latest.thumbnail_path.parent.mkdir(parents=True, exist_ok=True)
latest.thumbnail_path.write_bytes(b"thumb")
request = SimpleNamespace(query_params={})
request = cast("Request", SimpleNamespace(query_params={}))
with mock.patch(
"documents.conditionals.get_thumbnail_modified_key",
return_value="thumb-modified-key",
+63
View File
@@ -2938,6 +2938,69 @@ class TestWorkflows(
self.assertFalse(doc.tags.filter(pk=self.t1.pk).exists())
self.assertTrue(doc.tags.filter(pk=self.t2.pk).exists())
def test_document_updated_workflow_assignment_storage_path_persists_with_tag_assignment(
self,
) -> None:
"""
GIVEN:
- A document updated workflow filtered on a tag
- One assignment action assigns a storage path, a second (later-ordered)
assignment action adds a tag
WHEN:
- The document is updated and the workflow is triggered
THEN:
- Both the tag and the storage path are persisted
"""
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
trigger.filter_has_tags.add(self.t1)
assign_storage_path = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.ASSIGNMENT,
assign_storage_path=self.sp,
order=0,
)
assign_tag = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.ASSIGNMENT,
order=1,
)
assign_tag.assign_tags.add(self.t2)
assign_tag.save()
workflow = Workflow.objects.create(
name="Workflow assign storage path then tag",
order=0,
)
workflow.triggers.add(trigger)
workflow.actions.add(assign_storage_path, assign_tag)
workflow.save()
doc = Document.objects.create(
title="sample test",
mime_type="application/pdf",
checksum="assign-tag-and-storage-path",
original_filename="sample.pdf",
)
generated = generate_unique_filename(doc)
destination = (settings.ORIGINALS_DIR / generated).resolve()
create_source_path_directory(destination)
shutil.copy(self.SAMPLE_DIR / "simple.pdf", destination)
Document.objects.filter(pk=doc.pk).update(filename=generated.as_posix())
doc.refresh_from_db()
doc.tags.set([self.t1])
superuser = User.objects.create_superuser("superuser")
self.client.force_authenticate(user=superuser)
self.client.patch(
f"/api/documents/{doc.id}/",
{"title": "user update to trigger workflow"},
format="json",
)
doc.refresh_from_db()
self.assertEqual(doc.storage_path, self.sp)
self.assertTrue(doc.tags.filter(pk=self.t2.pk).exists())
def test_removal_action_document_updated_removeall(self) -> None:
"""
GIVEN:
+34 -11
View File
@@ -8,7 +8,7 @@ from typing import Any
from documents.models import Document
if TYPE_CHECKING:
from django.http import HttpRequest
from rest_framework.request import Request
class VersionResolutionError(StrEnum):
@@ -26,7 +26,7 @@ def _document_manager(*, include_deleted: bool) -> Any:
return Document.global_objects if include_deleted else Document.objects
def get_request_version_param(request: HttpRequest) -> str | None:
def get_request_version_param(request: Request) -> str | None:
if hasattr(request, "query_params"):
return request.query_params.get("version")
return None
@@ -57,7 +57,7 @@ def get_latest_version_for_root(
def resolve_requested_version_for_root(
root_doc: Document,
request: Any,
request: Request,
*,
include_deleted: bool = False,
) -> VersionResolution:
@@ -86,7 +86,7 @@ def resolve_requested_version_for_root(
def resolve_effective_document(
request_doc: Document,
request: Any,
request: Request,
*,
include_deleted: bool = False,
) -> VersionResolution:
@@ -107,18 +107,41 @@ def resolve_effective_document(
return VersionResolution(document=request_doc)
_EFFECTIVE_DOCUMENT_CACHE_ATTR = "_effective_document_resolution_cache"
def resolve_effective_document_by_pk(
pk: int,
request: Any,
request: Request,
*,
include_deleted: bool = False,
) -> VersionResolution:
# Django's `condition()` decorator (used for ETag/Last-Modified) invokes the
# etag_func and last_modified_func separately, and the view itself may resolve
# again -- all against the same request. Cache per-request so a single thumb/
# metadata/preview request doesn't redo this resolution multiple times.
cache = getattr(request, _EFFECTIVE_DOCUMENT_CACHE_ATTR, None)
if cache is None:
cache = {}
setattr(request, _EFFECTIVE_DOCUMENT_CACHE_ATTR, cache)
key = (pk, include_deleted)
if key in cache:
return cache[key]
manager = _document_manager(include_deleted=include_deleted)
request_doc = manager.only("id", "root_document_id").filter(pk=pk).first()
if request_doc is None:
return VersionResolution(document=None, error=VersionResolutionError.NOT_FOUND)
return resolve_effective_document(
request_doc,
request,
include_deleted=include_deleted,
)
resolution = VersionResolution(
document=None,
error=VersionResolutionError.NOT_FOUND,
)
else:
resolution = resolve_effective_document(
request_doc,
request,
include_deleted=include_deleted,
)
cache[key] = resolution
return resolution
+30 -7
View File
@@ -1040,12 +1040,23 @@ class DocumentViewSet(
.order_by("-id")
.values("content")[:1],
)
# A correlated subquery avoids the LEFT JOIN + Count() this used to
# be, which forced a GROUP BY aggregate over every matching document
# before the query could even be sorted or limited.
note_count = Subquery(
Note.objects.filter(document=OuterRef("pk"))
.order_by()
.values("document")
.annotate(count=Count("pk"))
.values("count"),
output_field=IntegerField(),
)
return (
Document.objects.filter(root_document__isnull=True)
.distinct()
.order_by("-created", "-id")
.annotate(effective_content=Coalesce(latest_version_content, F("content")))
.annotate(num_notes=Count("notes"))
.annotate(num_notes=Coalesce(note_count, 0))
.select_related("correspondent", "storage_path", "document_type", "owner")
.prefetch_related(
Prefetch(
@@ -1936,10 +1947,13 @@ class DocumentViewSet(
"root_document",
).get(pk=pk)
root_doc = get_root_document(request_doc)
if request.user is not None and not has_perms_owner_aware(
request.user,
"change_document",
root_doc,
if request.user is not None and (
not request.user.has_perm("documents.change_document")
or not has_perms_owner_aware(
request.user,
"change_document",
root_doc,
)
):
return HttpResponseForbidden("Insufficient permissions")
except Document.DoesNotExist:
@@ -2057,6 +2071,8 @@ class DocumentViewSet(
_backend.remove(version_doc.pk)
version_doc_id = version_doc.id
version_doc.delete()
root_doc.modified = timezone.now()
Document.objects.filter(pk=root_doc.pk).update(modified=root_doc.modified)
_backend.add_or_update(root_doc)
if settings.AUDIT_LOG_ENABLED:
actor = (
@@ -2136,6 +2152,8 @@ class DocumentViewSet(
old_label = version_doc.version_label
version_doc.version_label = serializer.validated_data["version_label"]
version_doc.save(update_fields=["version_label"])
root_doc.modified = timezone.now()
Document.objects.filter(pk=root_doc.pk).update(modified=root_doc.modified)
if settings.AUDIT_LOG_ENABLED and old_label != version_doc.version_label:
actor = (
@@ -2741,7 +2759,7 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
)
or (method == bulk_edit.edit_pdf and parameters.get("update_document"))
):
has_perms = user_is_owner_of_all_documents
has_perms = has_perms and user_is_owner_of_all_documents
# check global add permissions for methods that create documents
if (
@@ -2766,6 +2784,11 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
method in [bulk_edit.merge, bulk_edit.split]
and parameters.get("delete_originals")
)
or (
method in [bulk_edit.edit_pdf, bulk_edit.remove_password]
and parameters.get("delete_original")
and not parameters.get("update_document")
)
)
and not user.has_perm("documents.delete_document")
):
@@ -3366,7 +3389,7 @@ class SelectionDataView(GenericAPIView[Any]):
),
)
class SearchAutoCompleteView(GenericAPIView[Any]):
permission_classes = (IsAuthenticated,)
permission_classes = (IsAuthenticated, ViewDocumentsPermissions)
def get(self, request, format=None):
user = self.request.user if hasattr(self.request, "user") else None
+5 -1
View File
@@ -24,7 +24,11 @@ def apply_assignment_to_document(
action: WorkflowAction, annotated with 'has_assign_*' boolean fields
"""
if action.has_assign_tags:
document.add_nested_tags(action.assign_tags.all())
# Apply to a freshly-fetched instance rather than the shared `document`.
# Document.tags.add() fires an m2m_changed signal that ultimately calls
# instance.refresh_from_db(), which would discard any other unsaved
# assignment fields (e.g. storage_path) already staged on `document`.
Document.objects.get(pk=document.pk).add_nested_tags(action.assign_tags.all())
if action.assign_correspondent:
document.correspondent = action.assign_correspondent