mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-09 01:27:12 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3f258db0e2 | ||
|
|
642511c7b9 | ||
|
|
210d97a522 | ||
|
|
8a96190359 | ||
|
|
ece8769f59 |
No files matched your search
@@ -50,6 +50,7 @@ from documents.models import ShareLink
|
||||
from documents.models import ShareLinkBundle
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import annotate_authorizing_fields
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.versioning import annotate_effective_content
|
||||
@@ -1075,6 +1076,9 @@ class PermittedObjectsFilter(BaseFilterBackend):
|
||||
|
||||
include_granted: bool = True
|
||||
perm_codename: str | None = None
|
||||
# A self-referencing foreign key whose target authorizes a row, so a row is
|
||||
# judged by its parent's owner and grants (``Document.root_document``).
|
||||
parent_field: str | None = None
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
# Before the superuser and owner-only paths, neither of which consults
|
||||
@@ -1086,11 +1090,23 @@ class PermittedObjectsFilter(BaseFilterBackend):
|
||||
if request.user.is_superuser:
|
||||
return queryset
|
||||
if not self.include_granted:
|
||||
return queryset.filter(Q(owner=request.user) | Q(owner__isnull=True))
|
||||
owner_field = "owner"
|
||||
if self.parent_field is not None:
|
||||
queryset = annotate_authorizing_fields(queryset, self.parent_field)
|
||||
owner_field = "authorizing_owner"
|
||||
return queryset.filter(
|
||||
Q(**{owner_field: request.user.pk})
|
||||
| Q(**{f"{owner_field}__isnull": True}),
|
||||
)
|
||||
model = queryset.model
|
||||
perm = self.perm_codename or f"view_{model._meta.model_name}"
|
||||
return queryset.filter(
|
||||
id__in=permitted_object_ids(request.user, model, perm),
|
||||
id__in=permitted_object_ids(
|
||||
request.user,
|
||||
model,
|
||||
perm,
|
||||
parent_field=self.parent_field,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -67,18 +67,22 @@ class Command(PaperlessCommand):
|
||||
if options.get("recreate"):
|
||||
wipe_index(settings.INDEX_DIR)
|
||||
|
||||
documents = Document.objects.select_related(
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"storage_path",
|
||||
"owner",
|
||||
).prefetch_related(
|
||||
"tags",
|
||||
"notes__user",
|
||||
"custom_fields__field",
|
||||
"versions",
|
||||
"barcodes",
|
||||
"versions__barcodes",
|
||||
documents = (
|
||||
Document.objects.filter(root_document__isnull=True)
|
||||
.select_related(
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"storage_path",
|
||||
"owner",
|
||||
)
|
||||
.prefetch_related(
|
||||
"tags",
|
||||
"notes__user",
|
||||
"custom_fields__field",
|
||||
"versions",
|
||||
"barcodes",
|
||||
"versions__barcodes",
|
||||
)
|
||||
)
|
||||
total = documents.count()
|
||||
rebuild_kwargs = {}
|
||||
|
||||
@@ -6,9 +6,13 @@ from django.contrib.auth.models import Permission
|
||||
from django.contrib.auth.models import User
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.db.models import Case
|
||||
from django.db.models import CharField
|
||||
from django.db.models import Count
|
||||
from django.db.models import Exists
|
||||
from django.db.models import F
|
||||
from django.db.models import IntegerField
|
||||
from django.db.models import Model
|
||||
from django.db.models import OuterRef
|
||||
from django.db.models import Q
|
||||
from django.db.models import QuerySet
|
||||
from django.db.models import Value
|
||||
@@ -26,6 +30,7 @@ from rest_framework.permissions import BasePermission
|
||||
from rest_framework.permissions import DjangoObjectPermissions
|
||||
|
||||
from documents.models import Document
|
||||
from documents.versioning import get_root_document
|
||||
|
||||
|
||||
class PaperlessObjectPermissions(DjangoObjectPermissions):
|
||||
@@ -349,6 +354,7 @@ def permitted_object_ids(
|
||||
perm: str,
|
||||
*,
|
||||
include_deleted: bool = False,
|
||||
parent_field: str | None = None,
|
||||
) -> QuerySet[int]:
|
||||
"""
|
||||
Generic version of ``permitted_document_ids`` for any model with an
|
||||
@@ -357,6 +363,20 @@ def permitted_object_ids(
|
||||
soft-delete pattern (currently only ``Document``); for every other model
|
||||
it is accepted but has no effect, since those models have no soft-delete
|
||||
concept.
|
||||
|
||||
``parent_field`` names a self-referencing foreign key whose target
|
||||
authorizes the row (``Document.root_document``). A row with a parent is
|
||||
visible exactly when its parent is, judged by the parent's owner and
|
||||
grants, so the row's own owner and grants are ignored.
|
||||
|
||||
Guardian stores ``object_pk`` as a string, so each grant is an ``EXISTS``
|
||||
keyed on the row's id cast to a string, which can use guardian's
|
||||
(user, permission, object_pk) unique index. Casting every ``object_pk`` to
|
||||
an integer for an ``id IN (...)`` is not indexable, and MariaDB cannot
|
||||
materialize it inside the owner ``OR``, so it re-scans the user's grants
|
||||
for every row. The user's groups are matched with an ``IN`` subquery
|
||||
rather than a join through the membership table, which SQLite plans badly
|
||||
once the grant tables grow.
|
||||
"""
|
||||
has_soft_delete = hasattr(model, "global_objects")
|
||||
manager = (
|
||||
@@ -364,8 +384,14 @@ def permitted_object_ids(
|
||||
)
|
||||
base_qs = manager.all().only("id", "owner")
|
||||
|
||||
owner_field, key_field = "owner", "pk"
|
||||
if parent_field is not None:
|
||||
owner_field, key_field = "authorizing_owner", "authorizing_id"
|
||||
base_qs = annotate_authorizing_fields(base_qs, parent_field)
|
||||
unowned = Q(**{f"{owner_field}__isnull": True})
|
||||
|
||||
if user is None or not getattr(user, "is_authenticated", False):
|
||||
return base_qs.filter(owner__isnull=True).values_list("id", flat=True)
|
||||
return base_qs.filter(unowned).values_list("id", flat=True)
|
||||
|
||||
# Deactivated users get nothing, deactivated superusers included, so this
|
||||
# has to come before the superuser shortcut. guardian's
|
||||
@@ -389,26 +415,50 @@ def permitted_object_ids(
|
||||
"permission__content_type": content_type,
|
||||
}
|
||||
|
||||
user_perm_ids = (
|
||||
UserObjectPermission.objects.filter(user=user, **perm_filter)
|
||||
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
||||
.values_list("object_pk_int", flat=True)
|
||||
key_as_text = Cast(OuterRef(key_field), CharField(max_length=64))
|
||||
granted_to_user = Exists(
|
||||
UserObjectPermission.objects.filter(
|
||||
user=user,
|
||||
object_pk=key_as_text,
|
||||
**perm_filter,
|
||||
),
|
||||
)
|
||||
group_perm_ids = (
|
||||
GroupObjectPermission.objects.filter(group__user=user, **perm_filter)
|
||||
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
||||
.values_list("object_pk_int", flat=True)
|
||||
granted_to_group = Exists(
|
||||
GroupObjectPermission.objects.filter(
|
||||
group_id__in=user.groups.values("id"),
|
||||
object_pk=key_as_text,
|
||||
**perm_filter,
|
||||
),
|
||||
)
|
||||
permitted_ids = user_perm_ids.union(group_perm_ids)
|
||||
|
||||
return base_qs.filter(
|
||||
Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_ids),
|
||||
Q(**{owner_field: user.pk}) | unowned | granted_to_user | granted_to_group,
|
||||
).values_list("id", flat=True)
|
||||
|
||||
|
||||
ModelT = TypeVar("ModelT", bound=Model)
|
||||
|
||||
|
||||
def annotate_authorizing_fields(
|
||||
queryset: QuerySet[ModelT],
|
||||
parent_field: str,
|
||||
) -> QuerySet[ModelT]:
|
||||
"""
|
||||
Annotate each row with ``authorizing_id`` and ``authorizing_owner``: the id
|
||||
and owner of the row that authorizes it. A row with a parent (the
|
||||
self-referencing foreign key ``parent_field``) is authorized by its parent,
|
||||
any other row by itself.
|
||||
"""
|
||||
return queryset.annotate(
|
||||
authorizing_id=Coalesce(f"{parent_field}_id", "id"),
|
||||
authorizing_owner=Case(
|
||||
When(**{f"{parent_field}_id__isnull": True}, then=F("owner_id")),
|
||||
default=F(f"{parent_field}__owner_id"),
|
||||
output_field=IntegerField(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def user_is_unrestricted(user: User | None) -> bool:
|
||||
"""
|
||||
True when ``user`` means "no restriction at all" (an absent user, or an
|
||||
@@ -471,30 +521,16 @@ def permitted_document_ids(
|
||||
``include_deleted=True`` for callers that need to check permission on
|
||||
soft-deleted documents (e.g. trash restore). This intentionally avoids
|
||||
``get_objects_for_user`` to keep the subquery small and index-friendly.
|
||||
"""
|
||||
return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
|
||||
|
||||
|
||||
def documents_without_permitted_root(
|
||||
documents: QuerySet[Document],
|
||||
user: User | None,
|
||||
*,
|
||||
perm: str = "view_document",
|
||||
include_deleted: bool = False,
|
||||
) -> QuerySet[Document]:
|
||||
A version is authorized by its root document, so a version's own owner and
|
||||
grants never matter.
|
||||
"""
|
||||
The documents the user lacks ``perm`` on. Versions are authorized by their
|
||||
root document, so a version's own owner is ignored. A single query, without
|
||||
loading the documents or joining the root.
|
||||
"""
|
||||
return documents.annotate(
|
||||
root_id=Coalesce("root_document_id", "id"),
|
||||
).exclude(
|
||||
root_id__in=permitted_document_ids(
|
||||
user,
|
||||
perm=perm,
|
||||
include_deleted=include_deleted,
|
||||
),
|
||||
return permitted_object_ids(
|
||||
user,
|
||||
Document,
|
||||
perm,
|
||||
include_deleted=include_deleted,
|
||||
parent_field="root_document",
|
||||
)
|
||||
|
||||
|
||||
@@ -654,7 +690,14 @@ def has_perms_owner_aware(user, perms, obj):
|
||||
single-object check still has many production callers. Several callers
|
||||
remain across ``documents/``, ``paperless_mail/``, and ``paperless_ai/``
|
||||
-- grep for this function name before removing it.
|
||||
|
||||
A document version is authorized by its root document, like in
|
||||
``permitted_document_ids``, so a version's own owner and grants never
|
||||
matter. Fetch the root with ``select_related("root_document__owner")`` to
|
||||
avoid extra queries.
|
||||
"""
|
||||
if isinstance(obj, Document):
|
||||
obj = get_root_document(obj)
|
||||
checker = ObjectPermissionChecker(user)
|
||||
return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj)
|
||||
|
||||
|
||||
@@ -284,9 +284,14 @@ class WriteBatch:
|
||||
and adding the new version. This ensures stale document data (e.g., after
|
||||
permission changes) doesn't persist in the index.
|
||||
|
||||
Only root documents are indexed, with their effective content, so a
|
||||
version is indexed as its root document.
|
||||
|
||||
Args:
|
||||
document: Django Document instance to index
|
||||
"""
|
||||
if document.root_document_id is not None:
|
||||
document = document.root_document
|
||||
self.remove(document.pk)
|
||||
doc = self._backend._build_tantivy_doc(document)
|
||||
self._writer.add_document(doc)
|
||||
@@ -311,20 +316,22 @@ class WriteBatch:
|
||||
An id with no matching document (e.g. deleted between the caller
|
||||
collecting ids and the batch running) is silently skipped, matching
|
||||
``add_or_update()``'s existing single-document deferred-task behavior
|
||||
rather than erroring or leaving a stale index entry.
|
||||
rather than erroring or leaving a stale index entry. The id of a
|
||||
version stands for its root document.
|
||||
|
||||
Args:
|
||||
ids: Primary keys of Document instances to index
|
||||
"""
|
||||
from documents.models import Document
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.versioning import root_document_ids
|
||||
|
||||
ids = list(ids)
|
||||
if not ids:
|
||||
return
|
||||
|
||||
queryset = annotate_effective_content(
|
||||
Document.objects.filter(pk__in=ids)
|
||||
Document.objects.filter(pk__in=root_document_ids(ids))
|
||||
.select_related("correspondent", "document_type", "storage_path", "owner")
|
||||
.prefetch_related(
|
||||
"tags",
|
||||
|
||||
@@ -90,7 +90,6 @@ from documents.templating.utils import convert_format_str_to_template_format
|
||||
from documents.templating.workflows import validate_workflow_template
|
||||
from documents.validators import uri_validator
|
||||
from documents.validators import url_validator
|
||||
from documents.versioning import get_root_document
|
||||
from documents.versioning import has_prefetched_effective_content
|
||||
from documents.versioning import sort_versions_newest_first
|
||||
|
||||
@@ -2895,7 +2894,7 @@ class ShareLinkSerializer(OwnedObjectSerializer):
|
||||
and has_perms_owner_aware(
|
||||
self.user,
|
||||
"view_document",
|
||||
get_root_document(document),
|
||||
document,
|
||||
)
|
||||
):
|
||||
return document
|
||||
|
||||
@@ -490,23 +490,20 @@ def update_document_content_maybe_archive_file(
|
||||
shutil.move(thumbnail, document.thumbnail_path)
|
||||
|
||||
document.refresh_from_db()
|
||||
root_document = (
|
||||
document.root_document if document.root_document_id else document
|
||||
)
|
||||
logger.info(
|
||||
f"Updating index for document {root_document.pk} ({document.archive_checksum})",
|
||||
f"Updating index for document {document_id} ({document.archive_checksum})",
|
||||
)
|
||||
from documents.search import get_backend
|
||||
|
||||
get_backend().add_or_update(root_document)
|
||||
get_backend().add_or_update(document)
|
||||
|
||||
ai_config = AIConfig()
|
||||
if ai_config.llm_index_enabled:
|
||||
llm_index_add_or_update_document(root_document)
|
||||
llm_index_add_or_update_document(document)
|
||||
|
||||
clear_document_caches(document.pk)
|
||||
if root_document.pk != document.pk:
|
||||
clear_document_caches(root_document.pk)
|
||||
if document.root_document_id is not None:
|
||||
clear_document_caches(document.root_document_id)
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
|
||||
@@ -293,6 +293,80 @@ class TestAddOrUpdateIds:
|
||||
assert backend.search_ids("updated", user=None) == [doc.pk]
|
||||
|
||||
|
||||
class TestVersionsAreIndexedAsTheirRoot:
|
||||
"""Only root documents are indexed, with their effective content, so
|
||||
every write path that is handed a version indexes its root instead."""
|
||||
|
||||
@staticmethod
|
||||
def _root_with_version() -> tuple[Document, Document]:
|
||||
root = DocumentFactory(title="Statement", content="stale text")
|
||||
version = DocumentFactory(
|
||||
title="Statement",
|
||||
content="latest text",
|
||||
root_document=root,
|
||||
version_index=1,
|
||||
)
|
||||
return root, version
|
||||
|
||||
def test_add_or_update_indexes_the_root_of_a_version(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- The version is passed to add_or_update
|
||||
THEN:
|
||||
- The root is indexed with the version's text, and the version is not
|
||||
"""
|
||||
root, version = self._root_with_version()
|
||||
|
||||
backend.add_or_update(version)
|
||||
|
||||
assert backend.search_ids("latest", user=None) == [root.pk]
|
||||
assert backend.search_ids("stale", user=None) == []
|
||||
|
||||
def test_add_or_update_ids_indexes_each_root_once(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- Both ids are passed to add_or_update_ids
|
||||
THEN:
|
||||
- The root is indexed once and the version is not indexed
|
||||
"""
|
||||
root, version = self._root_with_version()
|
||||
|
||||
with backend.batch_update() as batch:
|
||||
batch.add_or_update_ids([version.pk, root.pk])
|
||||
|
||||
assert backend.search_ids("Statement", user=None) == [root.pk]
|
||||
assert backend.search_ids("latest", user=None) == [root.pk]
|
||||
|
||||
def test_add_or_update_ids_resolves_a_lone_version(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- Only the version's id is passed to add_or_update_ids
|
||||
THEN:
|
||||
- The root is indexed
|
||||
"""
|
||||
root, version = self._root_with_version()
|
||||
|
||||
with backend.batch_update() as batch:
|
||||
batch.add_or_update_ids([version.pk])
|
||||
|
||||
assert backend.search_ids("latest", user=None) == [root.pk]
|
||||
|
||||
|
||||
class TestSearch:
|
||||
"""Test search query parsing and matching via search_ids."""
|
||||
|
||||
|
||||
@@ -1196,6 +1196,59 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
self.assertIn(d3.id, result_ids)
|
||||
self.assertNotIn(d4.id, result_ids)
|
||||
|
||||
def test_search_more_like_version_uses_its_root(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document similar in content to a root document, and one that is not
|
||||
- A version of the root document, which is never indexed
|
||||
WHEN:
|
||||
- API request for more like the version
|
||||
THEN:
|
||||
- The documents similar to the root are returned, not the version
|
||||
"""
|
||||
indexed = {}
|
||||
for name, title, content, day in (
|
||||
("root", "bank statement 1", "things i paid for in august", (2019, 3, 4)),
|
||||
(
|
||||
"similar",
|
||||
"bank statement 3",
|
||||
"things i paid for in september",
|
||||
(2020, 7, 9),
|
||||
),
|
||||
(
|
||||
"other",
|
||||
"Quarterly Report",
|
||||
"quarterly revenue profit margin",
|
||||
(2021, 11, 30),
|
||||
),
|
||||
):
|
||||
with time_machine.travel(
|
||||
timezone.make_aware(datetime.datetime(*day)),
|
||||
tick=False,
|
||||
):
|
||||
indexed[name] = DocumentFactory(
|
||||
title=title,
|
||||
content=content,
|
||||
created=datetime.date(*day),
|
||||
added=timezone.make_aware(datetime.datetime(*day)),
|
||||
)
|
||||
version = DocumentFactory(
|
||||
root_document=indexed["root"],
|
||||
version_index=1,
|
||||
content="things i paid for in august",
|
||||
)
|
||||
backend = get_backend()
|
||||
for document in indexed.values():
|
||||
backend.add_or_update(document)
|
||||
|
||||
response = self.client.get(f"/api/documents/?more_like_id={version.id}")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
result_ids = [r["id"] for r in response.data["results"]]
|
||||
self.assertIn(indexed["similar"].id, result_ids)
|
||||
self.assertNotIn(indexed["other"].id, result_ids)
|
||||
self.assertNotIn(version.id, result_ids)
|
||||
|
||||
def test_more_like_requires_id_of_existing_document(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -22,6 +22,7 @@ from documents.models import Document
|
||||
from documents.tasks import update_document_content_maybe_archive_file
|
||||
from paperless_testing.assertions import FileSystemAssertsMixin
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
|
||||
sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
|
||||
|
||||
@@ -116,6 +117,27 @@ class TestMakeIndex:
|
||||
call_command("document_index", "reindex", skip_checks=True)
|
||||
mock_get_backend.return_value.rebuild.assert_called_once()
|
||||
|
||||
def test_reindex_skips_versions(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- The reindex command runs
|
||||
THEN:
|
||||
- Only the root document is handed to the rebuild, since a version
|
||||
is indexed as its root
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
DocumentFactory(root_document=root, version_index=1)
|
||||
mock_get_backend = mocker.patch(
|
||||
"documents.management.commands.document_index.get_backend",
|
||||
)
|
||||
|
||||
call_command("document_index", "reindex", skip_checks=True)
|
||||
|
||||
documents = mock_get_backend.return_value.rebuild.call_args.args[0]
|
||||
assert list(documents.values_list("pk", flat=True)) == [root.pk]
|
||||
|
||||
def test_optimize(self) -> None:
|
||||
"""Optimize command must execute without error (Tantivy handles optimization automatically)."""
|
||||
call_command("document_index", "optimize", skip_checks=True)
|
||||
|
||||
@@ -18,6 +18,7 @@ from documents.models import Correspondent
|
||||
from documents.models import DocumentType
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import has_perms_owner_aware
|
||||
from documents.permissions import permitted_document_ids
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.permissions import restrict_queryset_to_visible
|
||||
@@ -32,6 +33,8 @@ from paperless_testing.permissions import grant_global
|
||||
from paperless_testing.permissions import grant_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from paperless_testing.dirs import PaperlessDirs
|
||||
|
||||
|
||||
@@ -178,6 +181,382 @@ class TestPermittedDocumentIdsIncludeDeleted:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestPermittedDocumentIdsVersions:
|
||||
"""
|
||||
A version is authorized by its root document: the version's own owner and
|
||||
grants never matter.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("root_owner", "version_owner", "expected_visible"),
|
||||
[
|
||||
pytest.param(
|
||||
"other",
|
||||
"nobody",
|
||||
False,
|
||||
id="unowned-version-of-private-root",
|
||||
),
|
||||
pytest.param("other", "user", False, id="own-version-of-private-root"),
|
||||
pytest.param("user", "other", True, id="foreign-version-of-own-root"),
|
||||
pytest.param("user", "nobody", True, id="unowned-version-of-own-root"),
|
||||
pytest.param("nobody", "other", True, id="private-version-of-unowned-root"),
|
||||
],
|
||||
)
|
||||
def test_version_follows_root_owner(
|
||||
self,
|
||||
root_owner: str,
|
||||
version_owner: str,
|
||||
*,
|
||||
expected_visible: bool,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document and a version with differing owners
|
||||
WHEN:
|
||||
- The permitted document ids are resolved for the user
|
||||
THEN:
|
||||
- The version is visible exactly when its root is
|
||||
"""
|
||||
user = UserFactory()
|
||||
owners = {"user": user, "other": UserFactory(), "nobody": None}
|
||||
root = DocumentFactory(owner=owners[root_owner])
|
||||
version = DocumentFactory(root_document=root, owner=owners[version_owner])
|
||||
|
||||
visible = set(permitted_document_ids(user))
|
||||
|
||||
assert (version.pk in visible) is expected_visible
|
||||
assert (root.pk in visible) is expected_visible
|
||||
|
||||
@staticmethod
|
||||
def grantee(user: User, kind: str) -> User | Group:
|
||||
"""The user itself, or a new group the user belongs to."""
|
||||
if kind == "user":
|
||||
return user
|
||||
group = Group.objects.create(name="shared")
|
||||
user.groups.add(group)
|
||||
return group
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"grantee_kind",
|
||||
[pytest.param("user", id="user"), pytest.param("group", id="group")],
|
||||
)
|
||||
def test_grant_on_root_applies_to_version(self, grantee_kind: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A private root document shared with a user or one of their groups
|
||||
- A version of it owned by someone else
|
||||
WHEN:
|
||||
- The permitted document ids are resolved for the user
|
||||
THEN:
|
||||
- Both the root and the version are visible
|
||||
- A user without the grant sees neither
|
||||
"""
|
||||
user = UserFactory()
|
||||
stranger = UserFactory()
|
||||
root = DocumentFactory(owner=UserFactory())
|
||||
version = DocumentFactory(root_document=root, owner=UserFactory())
|
||||
grant_object(self.grantee(user, grantee_kind), root, "view_document")
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(user),
|
||||
expected_visible=[root.pk, version.pk],
|
||||
expected_hidden=[],
|
||||
)
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(stranger),
|
||||
expected_visible=[],
|
||||
expected_hidden=[root.pk, version.pk],
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"grantee_kind",
|
||||
[pytest.param("user", id="user"), pytest.param("group", id="group")],
|
||||
)
|
||||
def test_grant_on_version_is_ignored(self, grantee_kind: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A private root document
|
||||
- A version with an explicit grant for the user or one of their groups
|
||||
WHEN:
|
||||
- The permitted document ids are resolved for the user
|
||||
THEN:
|
||||
- Neither the root nor the version is visible
|
||||
"""
|
||||
user = UserFactory()
|
||||
root = DocumentFactory(owner=UserFactory())
|
||||
version = DocumentFactory(root_document=root, owner=UserFactory())
|
||||
grant_object(self.grantee(user, grantee_kind), version, "view_document")
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(user),
|
||||
expected_visible=[],
|
||||
expected_hidden=[root.pk, version.pk],
|
||||
)
|
||||
|
||||
def test_grant_on_one_root_does_not_reach_another_roots_version(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Two private roots, each with a version
|
||||
- The user may view only the first root
|
||||
WHEN:
|
||||
- The permitted document ids are resolved for the user
|
||||
THEN:
|
||||
- Only the first root and its version are visible
|
||||
"""
|
||||
user = UserFactory()
|
||||
first = DocumentFactory(owner=UserFactory())
|
||||
first_version = DocumentFactory(root_document=first, owner=UserFactory())
|
||||
second = DocumentFactory(owner=UserFactory())
|
||||
second_version = DocumentFactory(root_document=second, owner=user)
|
||||
grant_object(user, first, "view_document")
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(user),
|
||||
expected_visible=[first.pk, first_version.pk],
|
||||
expected_hidden=[second.pk, second_version.pk],
|
||||
)
|
||||
|
||||
def test_user_in_several_groups(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A user in two groups
|
||||
- Two private roots shared with one group each, and a third shared with nobody
|
||||
- A version of each root
|
||||
WHEN:
|
||||
- The permitted document ids are resolved for the user
|
||||
THEN:
|
||||
- The two shared roots and their versions are visible
|
||||
- The third root and its version are not
|
||||
"""
|
||||
user = UserFactory()
|
||||
groups = [Group.objects.create(name=f"group{i}") for i in range(2)]
|
||||
user.groups.add(*groups)
|
||||
shared = [DocumentFactory(owner=UserFactory()) for _ in groups]
|
||||
for root, group in zip(shared, groups, strict=True):
|
||||
grant_object(group, root, "view_document")
|
||||
unshared = DocumentFactory(owner=UserFactory())
|
||||
shared_versions = [
|
||||
DocumentFactory(root_document=root, owner=UserFactory()) for root in shared
|
||||
]
|
||||
unshared_version = DocumentFactory(root_document=unshared, owner=None)
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(user),
|
||||
expected_visible=[
|
||||
*(root.pk for root in shared),
|
||||
*(version.pk for version in shared_versions),
|
||||
],
|
||||
expected_hidden=[unshared.pk, unshared_version.pk],
|
||||
)
|
||||
|
||||
def test_permission_is_resolved_through_the_root(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A private root document where the user may view and change
|
||||
WHEN:
|
||||
- The permitted ids are resolved for view, change and delete
|
||||
THEN:
|
||||
- The version is visible for view and change only
|
||||
"""
|
||||
user = UserFactory()
|
||||
root = DocumentFactory(owner=UserFactory())
|
||||
version = DocumentFactory(root_document=root, owner=UserFactory())
|
||||
grant_object(user, root, "view_document", "change_document")
|
||||
|
||||
assert version.pk in set(permitted_document_ids(user))
|
||||
assert version.pk in set(permitted_document_ids(user, perm="change_document"))
|
||||
assert version.pk in set(
|
||||
permitted_document_ids(user, perm="documents.change_document"),
|
||||
)
|
||||
assert version.pk not in set(
|
||||
permitted_document_ids(user, perm="delete_document"),
|
||||
)
|
||||
|
||||
def test_anonymous_sees_versions_of_unowned_roots_only(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A version owned by nobody under a private root
|
||||
- A version owned by someone under an unowned root
|
||||
WHEN:
|
||||
- The permitted document ids are resolved for an anonymous user
|
||||
THEN:
|
||||
- Only the version of the unowned root is visible
|
||||
"""
|
||||
private_root = DocumentFactory(owner=UserFactory())
|
||||
private_version = DocumentFactory(root_document=private_root, owner=None)
|
||||
open_root = DocumentFactory(owner=None)
|
||||
open_version = DocumentFactory(root_document=open_root, owner=UserFactory())
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(AnonymousUser()),
|
||||
expected_visible=[open_root.pk, open_version.pk],
|
||||
expected_hidden=[private_root.pk, private_version.pk],
|
||||
)
|
||||
|
||||
def test_deleted_versions_follow_their_deleted_root(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A soft-deleted root document and its version, which deleting the
|
||||
root soft-deletes too; the version is owned by someone else
|
||||
WHEN:
|
||||
- The permitted document ids are resolved with and without deleted
|
||||
documents
|
||||
THEN:
|
||||
- Nothing is visible by default
|
||||
- With deleted documents included, the version is visible to the
|
||||
root's owner and not to the version's own owner
|
||||
"""
|
||||
owner = UserFactory()
|
||||
version_owner = UserFactory()
|
||||
root = DocumentFactory(owner=owner)
|
||||
version = DocumentFactory(root_document=root, owner=version_owner)
|
||||
root.delete()
|
||||
|
||||
assert not {root.pk, version.pk} & set(permitted_document_ids(owner))
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(owner, include_deleted=True),
|
||||
expected_visible=[root.pk, version.pk],
|
||||
expected_hidden=[],
|
||||
)
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(version_owner, include_deleted=True),
|
||||
expected_visible=[],
|
||||
expected_hidden=[root.pk, version.pk],
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"is_superuser",
|
||||
[
|
||||
pytest.param(False, id="regular-user"),
|
||||
pytest.param(True, id="superuser"),
|
||||
],
|
||||
)
|
||||
def test_inactive_user_sees_no_versions(self, *, is_superuser: bool) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An inactive user, possibly a superuser, who owns a root and its version
|
||||
WHEN:
|
||||
- The permitted document ids are resolved for them
|
||||
THEN:
|
||||
- Nothing is visible
|
||||
"""
|
||||
user = UserFactory(is_active=False, is_superuser=is_superuser)
|
||||
root = DocumentFactory(owner=user)
|
||||
version = DocumentFactory(root_document=root, owner=user)
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(user),
|
||||
expected_visible=[],
|
||||
expected_hidden=[root.pk, version.pk],
|
||||
)
|
||||
|
||||
def test_superuser_sees_all_versions(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A private root owned by someone else, with a version
|
||||
WHEN:
|
||||
- The permitted document ids are resolved for a superuser
|
||||
THEN:
|
||||
- Both the root and the version are visible
|
||||
"""
|
||||
superuser = UserFactory(superuser=True)
|
||||
root = DocumentFactory(owner=UserFactory())
|
||||
version = DocumentFactory(root_document=root, owner=UserFactory())
|
||||
|
||||
assert_visible_document_ids(
|
||||
permitted_document_ids(superuser),
|
||||
expected_visible=[root.pk, version.pk],
|
||||
expected_hidden=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestHasPermsOwnerAwareVersions:
|
||||
"""
|
||||
The single-object check agrees with permitted_document_ids: a version is
|
||||
authorized by its root document.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("root_owner", "version_owner", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"other",
|
||||
"nobody",
|
||||
False,
|
||||
id="unowned-version-of-private-root",
|
||||
),
|
||||
pytest.param("other", "user", False, id="own-version-of-private-root"),
|
||||
pytest.param("user", "other", True, id="foreign-version-of-own-root"),
|
||||
pytest.param("nobody", "other", True, id="private-version-of-unowned-root"),
|
||||
],
|
||||
)
|
||||
def test_version_follows_root_owner(
|
||||
self,
|
||||
root_owner: str,
|
||||
version_owner: str,
|
||||
*,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document and a version with differing owners
|
||||
WHEN:
|
||||
- The single-object check runs for the version
|
||||
THEN:
|
||||
- The version is allowed exactly when its root is
|
||||
"""
|
||||
user = UserFactory()
|
||||
owners = {"user": user, "other": UserFactory(), "nobody": None}
|
||||
root = DocumentFactory(owner=owners[root_owner])
|
||||
version = DocumentFactory(root_document=root, owner=owners[version_owner])
|
||||
|
||||
assert has_perms_owner_aware(user, "view_document", version) is expected
|
||||
assert has_perms_owner_aware(user, "view_document", root) is expected
|
||||
|
||||
def test_grant_on_root_applies_and_grant_on_version_does_not(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A private root with a version, and a second private root with a version
|
||||
- The user may change only the first root, and was granted the second
|
||||
root's version directly
|
||||
WHEN:
|
||||
- The single-object check runs for each version
|
||||
THEN:
|
||||
- Only the first root's version is allowed
|
||||
"""
|
||||
user = UserFactory()
|
||||
shared_root = DocumentFactory(owner=UserFactory())
|
||||
shared_version = DocumentFactory(root_document=shared_root, owner=UserFactory())
|
||||
private_root = DocumentFactory(owner=UserFactory())
|
||||
private_version = DocumentFactory(
|
||||
root_document=private_root,
|
||||
owner=UserFactory(),
|
||||
)
|
||||
grant_object(user, shared_root, "change_document")
|
||||
grant_object(user, private_version, "change_document")
|
||||
|
||||
assert has_perms_owner_aware(user, "change_document", shared_version)
|
||||
assert not has_perms_owner_aware(user, "change_document", private_version)
|
||||
|
||||
def test_other_models_use_their_own_owner(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A tag owned by someone else, and one owned by the user
|
||||
WHEN:
|
||||
- The single-object check runs for each
|
||||
THEN:
|
||||
- Only the user's own tag is allowed without a grant
|
||||
"""
|
||||
user = UserFactory()
|
||||
mine = TagFactory(owner=user)
|
||||
theirs = TagFactory(owner=UserFactory())
|
||||
|
||||
assert has_perms_owner_aware(user, "view_tag", mine)
|
||||
assert not has_perms_owner_aware(user, "view_tag", theirs)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestAiChatAllDocumentsPermissionBoundary:
|
||||
"""
|
||||
|
||||
@@ -310,7 +310,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
||||
|
||||
@mock.patch("documents.tasks.clear_document_caches")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
def test_update_content_version_indexes_root(
|
||||
def test_update_content_version_clears_caches_for_root(
|
||||
self,
|
||||
mock_get_backend: mock.Mock,
|
||||
mock_clear_caches: mock.Mock,
|
||||
@@ -321,8 +321,8 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
||||
WHEN:
|
||||
- Update content task is called for the version
|
||||
THEN:
|
||||
- The version's content is updated
|
||||
- The root document is indexed rather than the version
|
||||
- The version's content is updated, not the root's
|
||||
- The document is indexed
|
||||
- Caches are cleared for both
|
||||
"""
|
||||
root, version = self._create_root_with_version()
|
||||
@@ -334,8 +334,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
||||
"my document",
|
||||
)
|
||||
self.assertEqual(Document.objects.get(pk=root.pk).content, "root content")
|
||||
indexed = mock_get_backend.return_value.add_or_update.call_args.args[0]
|
||||
self.assertEqual(indexed.pk, root.pk)
|
||||
mock_get_backend.return_value.add_or_update.assert_called_once()
|
||||
mock_clear_caches.assert_has_calls(
|
||||
[mock.call(version.pk), mock.call(root.pk)],
|
||||
)
|
||||
@@ -343,7 +342,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
||||
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
|
||||
@mock.patch("documents.tasks.llm_index_add_or_update_document")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
def test_update_content_version_updates_llm_index_for_root(
|
||||
def test_update_content_version_updates_llm_index(
|
||||
self,
|
||||
mock_get_backend: mock.Mock,
|
||||
mock_llm_index: mock.Mock,
|
||||
@@ -355,14 +354,13 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
||||
WHEN:
|
||||
- Update content task is called for the version
|
||||
THEN:
|
||||
- The LLM index is updated for the root document, not the version
|
||||
- The LLM index is updated
|
||||
"""
|
||||
root, version = self._create_root_with_version()
|
||||
_, version = self._create_root_with_version()
|
||||
|
||||
tasks.update_document_content_maybe_archive_file(version.pk)
|
||||
|
||||
mock_llm_index.assert_called_once()
|
||||
self.assertEqual(mock_llm_index.call_args.args[0].pk, root.pk)
|
||||
|
||||
|
||||
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
||||
|
||||
@@ -17,9 +17,26 @@ from django.db.models.functions import RowNumber
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
from rest_framework.request import Request
|
||||
|
||||
|
||||
def root_document_ids(ids: Iterable[int]) -> QuerySet[int]:
|
||||
"""
|
||||
The ids of the root documents of the given documents: a root stands for
|
||||
itself and a version for its root. Only the indexes' bookkeeping needs
|
||||
this, since they hold root documents only.
|
||||
"""
|
||||
return (
|
||||
Document.objects.filter(pk__in=ids)
|
||||
.annotate(root_id=Coalesce("root_document_id", "id"))
|
||||
.order_by()
|
||||
.values_list("root_id", flat=True)
|
||||
.distinct()
|
||||
)
|
||||
|
||||
|
||||
def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
|
||||
"""
|
||||
Sorts versions so the newest one comes first using version_index and not on id,
|
||||
|
||||
+31
-54
@@ -174,7 +174,6 @@ from documents.permissions import TrashPermissions
|
||||
from documents.permissions import ViewDocumentsPermissions
|
||||
from documents.permissions import annotate_document_count_by_ids
|
||||
from documents.permissions import annotate_document_count_for_related_queryset
|
||||
from documents.permissions import documents_without_permitted_root
|
||||
from documents.permissions import get_document_count_filter_for_user
|
||||
from documents.permissions import get_objects_for_user_owner_aware
|
||||
from documents.permissions import has_global_statistics_permission
|
||||
@@ -330,9 +329,10 @@ def _get_tantivy_query_and_mode(params):
|
||||
def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
|
||||
try:
|
||||
more_like_doc_id = int(query_params["more_like_id"])
|
||||
more_like_doc = Document.objects.select_related("owner").get(
|
||||
pk=more_like_doc_id,
|
||||
)
|
||||
more_like_doc = Document.objects.select_related(
|
||||
"owner",
|
||||
"root_document__owner",
|
||||
).get(pk=more_like_doc_id)
|
||||
except (TypeError, ValueError, Document.DoesNotExist):
|
||||
raise PermissionDenied(_("Invalid more_like_id"))
|
||||
|
||||
@@ -343,7 +343,8 @@ def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
|
||||
):
|
||||
raise PermissionDenied(_("Insufficient permissions."))
|
||||
|
||||
return more_like_doc_id
|
||||
# Only root documents are indexed, a version stands for its root
|
||||
return more_like_doc.root_document_id or more_like_doc.pk
|
||||
|
||||
|
||||
class SearchParams(NamedTuple):
|
||||
@@ -1560,7 +1561,7 @@ class DocumentViewSet(
|
||||
if request.user is not None and not has_perms_owner_aware(
|
||||
request.user,
|
||||
"change_document",
|
||||
get_root_document(doc),
|
||||
doc,
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
@@ -1623,7 +1624,7 @@ class DocumentViewSet(
|
||||
if request.user is not None and not has_perms_owner_aware(
|
||||
request.user,
|
||||
"change_document",
|
||||
get_root_document(doc),
|
||||
doc,
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
@@ -1876,7 +1877,7 @@ class DocumentViewSet(
|
||||
if currentUser is not None and not has_perms_owner_aware(
|
||||
currentUser,
|
||||
"view_document",
|
||||
get_root_document(doc),
|
||||
doc,
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions to view notes")
|
||||
except Document.DoesNotExist:
|
||||
@@ -1898,7 +1899,7 @@ class DocumentViewSet(
|
||||
if currentUser is not None and not has_perms_owner_aware(
|
||||
currentUser,
|
||||
"change_document",
|
||||
get_root_document(doc),
|
||||
doc,
|
||||
):
|
||||
return HttpResponseForbidden(
|
||||
"Insufficient permissions to create notes",
|
||||
@@ -1941,7 +1942,7 @@ class DocumentViewSet(
|
||||
if currentUser is not None and not has_perms_owner_aware(
|
||||
currentUser,
|
||||
"change_document",
|
||||
get_root_document(doc),
|
||||
doc,
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions to delete notes")
|
||||
|
||||
@@ -1991,7 +1992,7 @@ class DocumentViewSet(
|
||||
if currentUser is not None and not has_perms_owner_aware(
|
||||
currentUser,
|
||||
"change_document",
|
||||
get_root_document(doc),
|
||||
doc,
|
||||
):
|
||||
return HttpResponseForbidden(
|
||||
"Insufficient permissions to add share link",
|
||||
@@ -2117,7 +2118,7 @@ class DocumentViewSet(
|
||||
documents = Document.objects.filter(pk__in=document_ids)
|
||||
if (
|
||||
request.user is not None
|
||||
and documents_without_permitted_root(documents, request.user).exists()
|
||||
and documents.exclude(id__in=permitted_document_ids(request.user)).exists()
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
@@ -2452,7 +2453,7 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
if not has_perms_owner_aware(
|
||||
request.user,
|
||||
"view_document",
|
||||
get_root_document(document),
|
||||
document,
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
@@ -3009,12 +3010,8 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
||||
user.has_perm(
|
||||
"documents.change_document",
|
||||
)
|
||||
and not Document.global_objects.filter(
|
||||
pk__in=[doc.pk for doc in root_docs],
|
||||
)
|
||||
.exclude(
|
||||
pk__in=permitted_document_ids(user, perm="change_document"),
|
||||
)
|
||||
and not Document.global_objects.filter(pk__in=documents)
|
||||
.exclude(pk__in=permitted_document_ids(user, perm="change_document"))
|
||||
.exists()
|
||||
)
|
||||
|
||||
@@ -3627,7 +3624,7 @@ class SelectionDataView(DocumentSelectionMixin, GenericAPIView[Any]):
|
||||
documents = Document.objects.filter(pk__in=ids)
|
||||
if (
|
||||
documents.count() != len(ids)
|
||||
or documents_without_permitted_root(documents, request.user).exists()
|
||||
or documents.exclude(id__in=permitted_document_ids(request.user)).exists()
|
||||
):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
@@ -4124,21 +4121,16 @@ class BulkDownloadView(DocumentSelectionMixin, GenericAPIView[Any]):
|
||||
validated_data=serializer.validated_data,
|
||||
)
|
||||
documents = Document.objects.filter(pk__in=ids)
|
||||
versioned_documents = []
|
||||
compression = serializer.validated_data.get("compression")
|
||||
content = serializer.validated_data.get("content")
|
||||
follow_filename_format = serializer.validated_data.get("follow_formatting")
|
||||
|
||||
permitted_ids = set(permitted_document_ids(request.user))
|
||||
for document in documents:
|
||||
root_doc = get_root_document(document)
|
||||
if root_doc.pk not in permitted_ids:
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
versioned_documents.append(
|
||||
get_latest_version_for_root(
|
||||
root_doc,
|
||||
),
|
||||
)
|
||||
if documents.exclude(id__in=permitted_document_ids(request.user)).exists():
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
versioned_documents = [
|
||||
get_latest_version_for_root(get_root_document(document))
|
||||
for document in documents
|
||||
]
|
||||
|
||||
if content == "both":
|
||||
strategy_class = OriginalAndArchiveStrategy
|
||||
@@ -4811,7 +4803,7 @@ class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
|
||||
)
|
||||
|
||||
denied_id = (
|
||||
documents_without_permitted_root(documents_qs, request.user)
|
||||
documents_qs.exclude(id__in=permitted_document_ids(request.user))
|
||||
.order_by("pk")
|
||||
.values_list("pk", flat=True)
|
||||
.first()
|
||||
@@ -5610,23 +5602,7 @@ class TrashView(ListModelMixin, PassUserMixin):
|
||||
|
||||
class _TrashPermittedObjectsFilter(PermittedObjectsFilter):
|
||||
include_granted = False
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
if request.user.is_superuser or not request.user.is_active:
|
||||
return super().filter_queryset(request, queryset, view)
|
||||
|
||||
# A version belongs to whoever owns its root
|
||||
def owned_or_unowned(prefix: str) -> Q:
|
||||
return Q(**{f"{prefix}owner": request.user}) | Q(
|
||||
**{f"{prefix}owner__isnull": True},
|
||||
)
|
||||
|
||||
return queryset.filter(
|
||||
(Q(root_document__isnull=True) & owned_or_unowned(""))
|
||||
| (
|
||||
Q(root_document__isnull=False) & owned_or_unowned("root_document__")
|
||||
),
|
||||
)
|
||||
parent_field = "root_document"
|
||||
|
||||
filter_backends = (_TrashPermittedObjectsFilter,)
|
||||
pagination_class = StandardPagination
|
||||
@@ -5657,11 +5633,12 @@ class TrashView(ListModelMixin, PassUserMixin):
|
||||
if doc_ids is not None
|
||||
else self.filter_queryset(self.get_queryset()).all()
|
||||
)
|
||||
if documents_without_permitted_root(
|
||||
docs,
|
||||
request.user,
|
||||
perm="delete_document",
|
||||
include_deleted=True,
|
||||
if docs.exclude(
|
||||
id__in=permitted_document_ids(
|
||||
request.user,
|
||||
perm="delete_document",
|
||||
include_deleted=True,
|
||||
),
|
||||
).exists():
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
action = serializer.validated_data.get("action")
|
||||
|
||||
@@ -7,6 +7,7 @@ from documents.models import Document
|
||||
from documents.permissions import permitted_object_ids
|
||||
from documents.permissions import restrict_queryset_to_visible
|
||||
from documents.permissions import user_is_unrestricted
|
||||
from documents.versioning import annotate_effective_content
|
||||
from paperless.config import AIConfig
|
||||
from paperless_ai.base_model import ClassificationSuggestions
|
||||
from paperless_ai.base_model import TaxonomyChoiceDict
|
||||
@@ -113,7 +114,7 @@ def build_prompt_without_rag(
|
||||
) -> str:
|
||||
filename = document.filename or ""
|
||||
content = truncate_content(
|
||||
document.content[:4000] or "",
|
||||
(document.get_effective_content() or "")[:4000],
|
||||
chunk_size=config.llm_embedding_chunk_size,
|
||||
context_size=config.llm_context_size,
|
||||
)
|
||||
@@ -227,7 +228,9 @@ def get_taxonomy_context(
|
||||
|
||||
# similar_documents is already ordered by descending weight; don't lose it.
|
||||
similar_document_ids = [s["document_id"] for s in similar_documents]
|
||||
similar_documents_by_id = Document.objects.in_bulk(similar_document_ids)
|
||||
similar_documents_by_id = annotate_effective_content(
|
||||
Document.objects.all(),
|
||||
).in_bulk(similar_document_ids)
|
||||
similar_docs = [
|
||||
similar_documents_by_id[document_id]
|
||||
for document_id in similar_document_ids
|
||||
@@ -235,7 +238,7 @@ def get_taxonomy_context(
|
||||
][:max_docs]
|
||||
context_blocks = []
|
||||
for similar in similar_docs:
|
||||
text = similar.content[:1000] or ""
|
||||
text = (similar.get_effective_content() or "")[:1000]
|
||||
title = similar.title or similar.filename or "Untitled"
|
||||
context_blocks.append(f"TITLE: {title}\n{text}")
|
||||
except Exception:
|
||||
|
||||
@@ -135,6 +135,6 @@ def build_llm_index_text(doc: Document) -> str:
|
||||
lines.append(f"Custom Field - {instance.field.name}: {instance}")
|
||||
|
||||
lines.append("\nContent:\n")
|
||||
lines.append(doc.content or "")
|
||||
lines.append(doc.get_effective_content() or "")
|
||||
|
||||
return _normalize_llm_index_text("\n".join(lines))
|
||||
@@ -17,6 +17,8 @@ from documents.models import PaperlessTask
|
||||
from documents.utils import IterWrapper
|
||||
from documents.utils import QuerySetStream
|
||||
from documents.utils import identity
|
||||
from documents.versioning import annotate_effective_content
|
||||
from documents.versioning import root_document_ids
|
||||
from paperless.config import AIConfig
|
||||
from paperless_ai.db import db_connection_released
|
||||
from paperless_ai.embedding import build_llm_index_text
|
||||
@@ -443,11 +445,11 @@ def update_llm_index(
|
||||
"Skipping LLM index update: migration check deferred; "
|
||||
"will retry next run."
|
||||
)
|
||||
documents = Document.objects.select_related(
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"storage_path",
|
||||
).prefetch_related("tags", "notes", "custom_fields__field")
|
||||
documents = annotate_effective_content(
|
||||
Document.objects.filter(root_document__isnull=True)
|
||||
.select_related("correspondent", "document_type", "storage_path")
|
||||
.prefetch_related("tags", "notes", "custom_fields__field"),
|
||||
)
|
||||
no_documents = not documents.exists()
|
||||
|
||||
# Fast exit before touching config: nothing to index and no existing index.
|
||||
@@ -483,7 +485,7 @@ def update_llm_index(
|
||||
msg = "LLM index rebuilt successfully."
|
||||
else:
|
||||
scoped_documents = (
|
||||
documents.filter(id__in=document_ids)
|
||||
documents.filter(id__in=root_document_ids(document_ids))
|
||||
if document_ids is not None
|
||||
else documents
|
||||
)
|
||||
@@ -510,7 +512,12 @@ def update_llm_index(
|
||||
|
||||
|
||||
def llm_index_add_or_update_document(document: Document):
|
||||
"""Add or atomically replace a document's chunks in the index."""
|
||||
"""
|
||||
Add or atomically replace a document's chunks in the index. Only root
|
||||
documents are indexed, so a version is indexed as its root document.
|
||||
"""
|
||||
if document.root_document_id is not None:
|
||||
document = document.root_document
|
||||
config = AIConfig()
|
||||
new_nodes = build_document_node(
|
||||
document,
|
||||
@@ -688,7 +695,7 @@ def retrieve_similar_nodes(
|
||||
)
|
||||
|
||||
query_text = truncate_embedding_query(
|
||||
(document.title or "") + "\n" + (document.content or ""),
|
||||
(document.title or "") + "\n" + (document.get_effective_content() or ""),
|
||||
chunk_size=config.llm_embedding_chunk_size,
|
||||
)
|
||||
# Hold the shared read lock for the whole retrieval so the connection is
|
||||
|
||||
@@ -55,6 +55,7 @@ def mock_document():
|
||||
doc.storage_path = None
|
||||
doc.archive_serial_number = "12345"
|
||||
doc.content = "This is the document content."
|
||||
doc.get_effective_content.return_value = "This is the document content."
|
||||
|
||||
cf1 = MagicMock(__str__=lambda x: "Value1")
|
||||
cf1.field = MagicMock()
|
||||
@@ -434,6 +435,55 @@ def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestClassifierEffectiveContent:
|
||||
"""A root document's text for the LLM is its newest version's content."""
|
||||
|
||||
@staticmethod
|
||||
def _root_with_version() -> Document:
|
||||
root = DocumentFactory(title="Statement", content="stale text")
|
||||
DocumentFactory(root_document=root, version_index=1, content="latest text")
|
||||
return root
|
||||
|
||||
def test_prompt_uses_the_newest_versions_content(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- The classification prompt is built for the root
|
||||
THEN:
|
||||
- It contains the newest version's content
|
||||
"""
|
||||
prompt = build_prompt_without_rag(self._root_with_version(), AIConfig())
|
||||
|
||||
assert "latest text" in prompt
|
||||
assert "stale text" not in prompt
|
||||
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_similar_document_context_uses_the_newest_versions_content(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A similar root document with a version
|
||||
WHEN:
|
||||
- The similar-document context is built
|
||||
THEN:
|
||||
- It contains the newest version's content
|
||||
"""
|
||||
similar = self._root_with_version()
|
||||
document = DocumentFactory(content="Some content")
|
||||
fake_nodes = [
|
||||
SimpleNamespace(metadata={"document_id": str(similar.pk)}, score=0.9),
|
||||
]
|
||||
|
||||
with patch(
|
||||
"paperless_ai.ai_classifier.retrieve_similar_nodes",
|
||||
return_value=fake_nodes,
|
||||
):
|
||||
_candidates, context = get_taxonomy_context(document, user=None)
|
||||
|
||||
assert context == "TITLE: Statement\nlatest text"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
|
||||
def test_get_taxonomy_context_no_similar_docs():
|
||||
|
||||
@@ -388,6 +388,108 @@ def test_update_llm_index_partial_update(
|
||||
assert after[str(doc2.pk)] == before[str(doc2.pk)]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestLlmIndexVersions:
|
||||
"""The LLM index holds root documents only: a version is indexed as its root."""
|
||||
|
||||
def test_add_or_update_document_indexes_a_version_as_its_root(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
mock_embed_model: FakeEmbedding,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- The version is passed to llm_index_add_or_update_document
|
||||
THEN:
|
||||
- Only the root document is in the index
|
||||
"""
|
||||
root = DocumentFactory(content="root content")
|
||||
version = DocumentFactory(root_document=root, version_index=1)
|
||||
|
||||
indexing.llm_index_add_or_update_document(version)
|
||||
|
||||
with indexing.get_vector_store() as store:
|
||||
indexed = store.get_modified_times()
|
||||
|
||||
assert set(indexed) == {str(root.pk)}
|
||||
|
||||
def test_rebuild_skips_versions(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
mock_embed_model: FakeEmbedding,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- The LLM index is rebuilt
|
||||
THEN:
|
||||
- Only the root document is in the index
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
DocumentFactory(root_document=root, version_index=1)
|
||||
|
||||
indexing.update_llm_index(rebuild=True)
|
||||
|
||||
with indexing.get_vector_store() as store:
|
||||
indexed = store.get_modified_times()
|
||||
|
||||
assert set(indexed) == {str(root.pk)}
|
||||
|
||||
def test_rebuild_indexes_the_newest_versions_content(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
mock_embed_model: FakeEmbedding,
|
||||
mocker: pytest_mock.MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with a version
|
||||
WHEN:
|
||||
- The LLM index is rebuilt
|
||||
THEN:
|
||||
- The root's text for the index is the version's content, answered
|
||||
from the query rather than a query per document
|
||||
"""
|
||||
root = DocumentFactory(content="stale text")
|
||||
DocumentFactory(root_document=root, version_index=1, content="latest text")
|
||||
spy = mocker.spy(indexing, "build_document_node")
|
||||
|
||||
indexing.update_llm_index(rebuild=True)
|
||||
|
||||
indexed = spy.call_args.args[0]
|
||||
assert indexed.pk == root.pk
|
||||
assert indexed.effective_content == "latest text"
|
||||
|
||||
def test_incremental_update_by_version_id_refreshes_the_root(
|
||||
self,
|
||||
temp_llm_index_dir: Path,
|
||||
mock_embed_model: FakeEmbedding,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An indexed root document with a version whose root was modified since
|
||||
WHEN:
|
||||
- An incremental update is scoped to the version's id
|
||||
THEN:
|
||||
- The root's entry is refreshed and no entry exists for the version
|
||||
"""
|
||||
root = DocumentFactory()
|
||||
version = DocumentFactory(root_document=root, version_index=1)
|
||||
indexing.update_llm_index(rebuild=True)
|
||||
Document.objects.filter(pk=root.pk).update(modified=timezone.now())
|
||||
root.refresh_from_db()
|
||||
|
||||
indexing.update_llm_index(document_ids=[version.pk])
|
||||
|
||||
with indexing.get_vector_store() as store:
|
||||
indexed = store.get_modified_times()
|
||||
|
||||
assert indexed == {str(root.pk): root.modified.isoformat()}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_or_update_document_updates_existing_entry(
|
||||
temp_llm_index_dir: Path,
|
||||
@@ -637,6 +739,7 @@ class TestLlmIndexAddOrUpdateDocumentEmptyContent:
|
||||
|
||||
doc = MagicMock(spec=Document)
|
||||
doc.id = 42
|
||||
doc.root_document_id = None
|
||||
# Must not raise
|
||||
indexing.llm_index_add_or_update_document(doc)
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from paperless_ai.embedding import _normalize_llm_index_text
|
||||
from paperless_ai.embedding import build_llm_index_text
|
||||
from paperless_ai.embedding import get_configured_model_name
|
||||
from paperless_ai.embedding import get_embedding_model
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -46,6 +47,7 @@ def mock_document():
|
||||
doc.correspondent.name = "Test Correspondent"
|
||||
doc.archive_serial_number = "12345"
|
||||
doc.content = "This is the document content."
|
||||
doc.get_effective_content.return_value = "This is the document content."
|
||||
|
||||
cf1 = MagicMock(__str__=lambda x: "Value1")
|
||||
cf1.field = MagicMock()
|
||||
@@ -280,7 +282,7 @@ def test_build_llm_index_text(mock_document):
|
||||
|
||||
|
||||
def test_build_llm_index_text_normalizes_ocr_punctuation_runs(mock_document):
|
||||
mock_document.content = (
|
||||
mock_document.get_effective_content.return_value = (
|
||||
"Introduction ................................................ 7\n"
|
||||
"Hardware Limitation ________________________________________ 9\n"
|
||||
"Keep short punctuation like INV-100 and ellipses..."
|
||||
@@ -294,6 +296,43 @@ def test_build_llm_index_text_normalizes_ocr_punctuation_runs(mock_document):
|
||||
assert "ellipses..." in result
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestBuildLlmIndexTextVersions:
|
||||
"""A root document is indexed with its effective content, like in the search index."""
|
||||
|
||||
def test_root_uses_the_newest_versions_content(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document with two versions
|
||||
WHEN:
|
||||
- The LLM index text is built for the root
|
||||
THEN:
|
||||
- It contains the newest version's content and not the others'
|
||||
"""
|
||||
root = DocumentFactory(content="stale text")
|
||||
DocumentFactory(root_document=root, version_index=1, content="older text")
|
||||
DocumentFactory(root_document=root, version_index=2, content="latest text")
|
||||
|
||||
text = build_llm_index_text(root)
|
||||
|
||||
assert "latest text" in text
|
||||
assert "stale text" not in text
|
||||
assert "older text" not in text
|
||||
|
||||
def test_root_without_versions_uses_its_own_content(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A root document without versions
|
||||
WHEN:
|
||||
- The LLM index text is built for it
|
||||
THEN:
|
||||
- It contains the document's own content
|
||||
"""
|
||||
root = DocumentFactory(content="own text")
|
||||
|
||||
assert "own text" in build_llm_index_text(root)
|
||||
|
||||
|
||||
def test_normalize_llm_index_text_collapses_ocr_leaders_without_joining_lines():
|
||||
assert _normalize_llm_index_text("A........B\nC____D----E") == "A B\nC D E"
|
||||
|
||||
|
||||
Reference in new issue
Block a user