mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-09 09:37:12 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
210d97a522 | ||
|
|
8a96190359 | ||
|
|
ece8769f59 |
No files matched your search
@@ -67,18 +67,22 @@ class Command(PaperlessCommand):
|
|||||||
if options.get("recreate"):
|
if options.get("recreate"):
|
||||||
wipe_index(settings.INDEX_DIR)
|
wipe_index(settings.INDEX_DIR)
|
||||||
|
|
||||||
documents = Document.objects.select_related(
|
documents = (
|
||||||
"correspondent",
|
Document.objects.filter(root_document__isnull=True)
|
||||||
"document_type",
|
.select_related(
|
||||||
"storage_path",
|
"correspondent",
|
||||||
"owner",
|
"document_type",
|
||||||
).prefetch_related(
|
"storage_path",
|
||||||
"tags",
|
"owner",
|
||||||
"notes__user",
|
)
|
||||||
"custom_fields__field",
|
.prefetch_related(
|
||||||
"versions",
|
"tags",
|
||||||
"barcodes",
|
"notes__user",
|
||||||
"versions__barcodes",
|
"custom_fields__field",
|
||||||
|
"versions",
|
||||||
|
"barcodes",
|
||||||
|
"versions__barcodes",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
total = documents.count()
|
total = documents.count()
|
||||||
rebuild_kwargs = {}
|
rebuild_kwargs = {}
|
||||||
|
|||||||
@@ -6,9 +6,13 @@ from django.contrib.auth.models import Permission
|
|||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
from django.contrib.contenttypes.models import ContentType
|
from django.contrib.contenttypes.models import ContentType
|
||||||
from django.db.models import Case
|
from django.db.models import Case
|
||||||
|
from django.db.models import CharField
|
||||||
from django.db.models import Count
|
from django.db.models import Count
|
||||||
|
from django.db.models import Exists
|
||||||
|
from django.db.models import F
|
||||||
from django.db.models import IntegerField
|
from django.db.models import IntegerField
|
||||||
from django.db.models import Model
|
from django.db.models import Model
|
||||||
|
from django.db.models import OuterRef
|
||||||
from django.db.models import Q
|
from django.db.models import Q
|
||||||
from django.db.models import QuerySet
|
from django.db.models import QuerySet
|
||||||
from django.db.models import Value
|
from django.db.models import Value
|
||||||
@@ -26,6 +30,7 @@ from rest_framework.permissions import BasePermission
|
|||||||
from rest_framework.permissions import DjangoObjectPermissions
|
from rest_framework.permissions import DjangoObjectPermissions
|
||||||
|
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
|
from documents.versioning import get_root_document
|
||||||
|
|
||||||
|
|
||||||
class PaperlessObjectPermissions(DjangoObjectPermissions):
|
class PaperlessObjectPermissions(DjangoObjectPermissions):
|
||||||
@@ -349,6 +354,7 @@ def permitted_object_ids(
|
|||||||
perm: str,
|
perm: str,
|
||||||
*,
|
*,
|
||||||
include_deleted: bool = False,
|
include_deleted: bool = False,
|
||||||
|
parent_field: str | None = None,
|
||||||
) -> QuerySet[int]:
|
) -> QuerySet[int]:
|
||||||
"""
|
"""
|
||||||
Generic version of ``permitted_document_ids`` for any model with an
|
Generic version of ``permitted_document_ids`` for any model with an
|
||||||
@@ -357,6 +363,20 @@ def permitted_object_ids(
|
|||||||
soft-delete pattern (currently only ``Document``); for every other model
|
soft-delete pattern (currently only ``Document``); for every other model
|
||||||
it is accepted but has no effect, since those models have no soft-delete
|
it is accepted but has no effect, since those models have no soft-delete
|
||||||
concept.
|
concept.
|
||||||
|
|
||||||
|
``parent_field`` names a self-referencing foreign key whose target
|
||||||
|
authorizes the row (``Document.root_document``). A row with a parent is
|
||||||
|
visible exactly when its parent is, judged by the parent's owner and
|
||||||
|
grants, so the row's own owner and grants are ignored.
|
||||||
|
|
||||||
|
Guardian stores ``object_pk`` as a string, so each grant is an ``EXISTS``
|
||||||
|
keyed on the row's id cast to a string, which can use guardian's
|
||||||
|
(user, permission, object_pk) unique index. Casting every ``object_pk`` to
|
||||||
|
an integer for an ``id IN (...)`` is not indexable, and MariaDB cannot
|
||||||
|
materialize it inside the owner ``OR``, so it re-scans the user's grants
|
||||||
|
for every row. The user's groups are matched with an ``IN`` subquery
|
||||||
|
rather than a join through the membership table, which SQLite plans badly
|
||||||
|
once the grant tables grow.
|
||||||
"""
|
"""
|
||||||
has_soft_delete = hasattr(model, "global_objects")
|
has_soft_delete = hasattr(model, "global_objects")
|
||||||
manager = (
|
manager = (
|
||||||
@@ -364,8 +384,21 @@ def permitted_object_ids(
|
|||||||
)
|
)
|
||||||
base_qs = manager.all().only("id", "owner")
|
base_qs = manager.all().only("id", "owner")
|
||||||
|
|
||||||
|
owner_field, key_field = "owner", "pk"
|
||||||
|
if parent_field is not None:
|
||||||
|
owner_field, key_field = "authorizing_owner", "authorizing_id"
|
||||||
|
base_qs = base_qs.annotate(
|
||||||
|
authorizing_id=Coalesce(f"{parent_field}_id", "id"),
|
||||||
|
authorizing_owner=Case(
|
||||||
|
When(**{f"{parent_field}_id__isnull": True}, then=F("owner_id")),
|
||||||
|
default=F(f"{parent_field}__owner_id"),
|
||||||
|
output_field=IntegerField(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
unowned = Q(**{f"{owner_field}__isnull": True})
|
||||||
|
|
||||||
if user is None or not getattr(user, "is_authenticated", False):
|
if user is None or not getattr(user, "is_authenticated", False):
|
||||||
return base_qs.filter(owner__isnull=True).values_list("id", flat=True)
|
return base_qs.filter(unowned).values_list("id", flat=True)
|
||||||
|
|
||||||
# Deactivated users get nothing, deactivated superusers included, so this
|
# Deactivated users get nothing, deactivated superusers included, so this
|
||||||
# has to come before the superuser shortcut. guardian's
|
# has to come before the superuser shortcut. guardian's
|
||||||
@@ -389,20 +422,24 @@ def permitted_object_ids(
|
|||||||
"permission__content_type": content_type,
|
"permission__content_type": content_type,
|
||||||
}
|
}
|
||||||
|
|
||||||
user_perm_ids = (
|
key_as_text = Cast(OuterRef(key_field), CharField(max_length=64))
|
||||||
UserObjectPermission.objects.filter(user=user, **perm_filter)
|
granted_to_user = Exists(
|
||||||
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
UserObjectPermission.objects.filter(
|
||||||
.values_list("object_pk_int", flat=True)
|
user=user,
|
||||||
|
object_pk=key_as_text,
|
||||||
|
**perm_filter,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
group_perm_ids = (
|
granted_to_group = Exists(
|
||||||
GroupObjectPermission.objects.filter(group__user=user, **perm_filter)
|
GroupObjectPermission.objects.filter(
|
||||||
.annotate(object_pk_int=Cast("object_pk", IntegerField()))
|
group_id__in=user.groups.values("id"),
|
||||||
.values_list("object_pk_int", flat=True)
|
object_pk=key_as_text,
|
||||||
|
**perm_filter,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
permitted_ids = user_perm_ids.union(group_perm_ids)
|
|
||||||
|
|
||||||
return base_qs.filter(
|
return base_qs.filter(
|
||||||
Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_ids),
|
Q(**{owner_field: user.pk}) | unowned | granted_to_user | granted_to_group,
|
||||||
).values_list("id", flat=True)
|
).values_list("id", flat=True)
|
||||||
|
|
||||||
|
|
||||||
@@ -471,30 +508,16 @@ def permitted_document_ids(
|
|||||||
``include_deleted=True`` for callers that need to check permission on
|
``include_deleted=True`` for callers that need to check permission on
|
||||||
soft-deleted documents (e.g. trash restore). This intentionally avoids
|
soft-deleted documents (e.g. trash restore). This intentionally avoids
|
||||||
``get_objects_for_user`` to keep the subquery small and index-friendly.
|
``get_objects_for_user`` to keep the subquery small and index-friendly.
|
||||||
"""
|
|
||||||
return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
|
|
||||||
|
|
||||||
|
A version is authorized by its root document, so a version's own owner and
|
||||||
def documents_without_permitted_root(
|
grants never matter.
|
||||||
documents: QuerySet[Document],
|
|
||||||
user: User | None,
|
|
||||||
*,
|
|
||||||
perm: str = "view_document",
|
|
||||||
include_deleted: bool = False,
|
|
||||||
) -> QuerySet[Document]:
|
|
||||||
"""
|
"""
|
||||||
The documents the user lacks ``perm`` on. Versions are authorized by their
|
return permitted_object_ids(
|
||||||
root document, so a version's own owner is ignored. A single query, without
|
user,
|
||||||
loading the documents or joining the root.
|
Document,
|
||||||
"""
|
perm,
|
||||||
return documents.annotate(
|
include_deleted=include_deleted,
|
||||||
root_id=Coalesce("root_document_id", "id"),
|
parent_field="root_document",
|
||||||
).exclude(
|
|
||||||
root_id__in=permitted_document_ids(
|
|
||||||
user,
|
|
||||||
perm=perm,
|
|
||||||
include_deleted=include_deleted,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -654,7 +677,14 @@ def has_perms_owner_aware(user, perms, obj):
|
|||||||
single-object check still has many production callers. Several callers
|
single-object check still has many production callers. Several callers
|
||||||
remain across ``documents/``, ``paperless_mail/``, and ``paperless_ai/``
|
remain across ``documents/``, ``paperless_mail/``, and ``paperless_ai/``
|
||||||
-- grep for this function name before removing it.
|
-- grep for this function name before removing it.
|
||||||
|
|
||||||
|
A document version is authorized by its root document, like in
|
||||||
|
``permitted_document_ids``, so a version's own owner and grants never
|
||||||
|
matter. Fetch the root with ``select_related("root_document__owner")`` to
|
||||||
|
avoid extra queries.
|
||||||
"""
|
"""
|
||||||
|
if isinstance(obj, Document):
|
||||||
|
obj = get_root_document(obj)
|
||||||
checker = ObjectPermissionChecker(user)
|
checker = ObjectPermissionChecker(user)
|
||||||
return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj)
|
return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj)
|
||||||
|
|
||||||
|
|||||||
@@ -284,9 +284,14 @@ class WriteBatch:
|
|||||||
and adding the new version. This ensures stale document data (e.g., after
|
and adding the new version. This ensures stale document data (e.g., after
|
||||||
permission changes) doesn't persist in the index.
|
permission changes) doesn't persist in the index.
|
||||||
|
|
||||||
|
Only root documents are indexed, with their effective content, so a
|
||||||
|
version is indexed as its root document.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
document: Django Document instance to index
|
document: Django Document instance to index
|
||||||
"""
|
"""
|
||||||
|
if document.root_document_id is not None:
|
||||||
|
document = document.root_document
|
||||||
self.remove(document.pk)
|
self.remove(document.pk)
|
||||||
doc = self._backend._build_tantivy_doc(document)
|
doc = self._backend._build_tantivy_doc(document)
|
||||||
self._writer.add_document(doc)
|
self._writer.add_document(doc)
|
||||||
@@ -311,20 +316,22 @@ class WriteBatch:
|
|||||||
An id with no matching document (e.g. deleted between the caller
|
An id with no matching document (e.g. deleted between the caller
|
||||||
collecting ids and the batch running) is silently skipped, matching
|
collecting ids and the batch running) is silently skipped, matching
|
||||||
``add_or_update()``'s existing single-document deferred-task behavior
|
``add_or_update()``'s existing single-document deferred-task behavior
|
||||||
rather than erroring or leaving a stale index entry.
|
rather than erroring or leaving a stale index entry. The id of a
|
||||||
|
version stands for its root document.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
ids: Primary keys of Document instances to index
|
ids: Primary keys of Document instances to index
|
||||||
"""
|
"""
|
||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
from documents.versioning import annotate_effective_content
|
from documents.versioning import annotate_effective_content
|
||||||
|
from documents.versioning import root_document_ids
|
||||||
|
|
||||||
ids = list(ids)
|
ids = list(ids)
|
||||||
if not ids:
|
if not ids:
|
||||||
return
|
return
|
||||||
|
|
||||||
queryset = annotate_effective_content(
|
queryset = annotate_effective_content(
|
||||||
Document.objects.filter(pk__in=ids)
|
Document.objects.filter(pk__in=root_document_ids(ids))
|
||||||
.select_related("correspondent", "document_type", "storage_path", "owner")
|
.select_related("correspondent", "document_type", "storage_path", "owner")
|
||||||
.prefetch_related(
|
.prefetch_related(
|
||||||
"tags",
|
"tags",
|
||||||
|
|||||||
@@ -90,7 +90,6 @@ from documents.templating.utils import convert_format_str_to_template_format
|
|||||||
from documents.templating.workflows import validate_workflow_template
|
from documents.templating.workflows import validate_workflow_template
|
||||||
from documents.validators import uri_validator
|
from documents.validators import uri_validator
|
||||||
from documents.validators import url_validator
|
from documents.validators import url_validator
|
||||||
from documents.versioning import get_root_document
|
|
||||||
from documents.versioning import has_prefetched_effective_content
|
from documents.versioning import has_prefetched_effective_content
|
||||||
from documents.versioning import sort_versions_newest_first
|
from documents.versioning import sort_versions_newest_first
|
||||||
|
|
||||||
@@ -2895,7 +2894,7 @@ class ShareLinkSerializer(OwnedObjectSerializer):
|
|||||||
and has_perms_owner_aware(
|
and has_perms_owner_aware(
|
||||||
self.user,
|
self.user,
|
||||||
"view_document",
|
"view_document",
|
||||||
get_root_document(document),
|
document,
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
return document
|
return document
|
||||||
|
|||||||
@@ -490,23 +490,20 @@ def update_document_content_maybe_archive_file(
|
|||||||
shutil.move(thumbnail, document.thumbnail_path)
|
shutil.move(thumbnail, document.thumbnail_path)
|
||||||
|
|
||||||
document.refresh_from_db()
|
document.refresh_from_db()
|
||||||
root_document = (
|
|
||||||
document.root_document if document.root_document_id else document
|
|
||||||
)
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Updating index for document {root_document.pk} ({document.archive_checksum})",
|
f"Updating index for document {document_id} ({document.archive_checksum})",
|
||||||
)
|
)
|
||||||
from documents.search import get_backend
|
from documents.search import get_backend
|
||||||
|
|
||||||
get_backend().add_or_update(root_document)
|
get_backend().add_or_update(document)
|
||||||
|
|
||||||
ai_config = AIConfig()
|
ai_config = AIConfig()
|
||||||
if ai_config.llm_index_enabled:
|
if ai_config.llm_index_enabled:
|
||||||
llm_index_add_or_update_document(root_document)
|
llm_index_add_or_update_document(document)
|
||||||
|
|
||||||
clear_document_caches(document.pk)
|
clear_document_caches(document.pk)
|
||||||
if root_document.pk != document.pk:
|
if document.root_document_id is not None:
|
||||||
clear_document_caches(root_document.pk)
|
clear_document_caches(document.root_document_id)
|
||||||
|
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception(
|
logger.exception(
|
||||||
|
|||||||
@@ -293,6 +293,80 @@ class TestAddOrUpdateIds:
|
|||||||
assert backend.search_ids("updated", user=None) == [doc.pk]
|
assert backend.search_ids("updated", user=None) == [doc.pk]
|
||||||
|
|
||||||
|
|
||||||
|
class TestVersionsAreIndexedAsTheirRoot:
|
||||||
|
"""Only root documents are indexed, with their effective content, so
|
||||||
|
every write path that is handed a version indexes its root instead."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _root_with_version() -> tuple[Document, Document]:
|
||||||
|
root = DocumentFactory(title="Statement", content="stale text")
|
||||||
|
version = DocumentFactory(
|
||||||
|
title="Statement",
|
||||||
|
content="latest text",
|
||||||
|
root_document=root,
|
||||||
|
version_index=1,
|
||||||
|
)
|
||||||
|
return root, version
|
||||||
|
|
||||||
|
def test_add_or_update_indexes_the_root_of_a_version(
|
||||||
|
self,
|
||||||
|
backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A root document with a version
|
||||||
|
WHEN:
|
||||||
|
- The version is passed to add_or_update
|
||||||
|
THEN:
|
||||||
|
- The root is indexed with the version's text, and the version is not
|
||||||
|
"""
|
||||||
|
root, version = self._root_with_version()
|
||||||
|
|
||||||
|
backend.add_or_update(version)
|
||||||
|
|
||||||
|
assert backend.search_ids("latest", user=None) == [root.pk]
|
||||||
|
assert backend.search_ids("stale", user=None) == []
|
||||||
|
|
||||||
|
def test_add_or_update_ids_indexes_each_root_once(
|
||||||
|
self,
|
||||||
|
backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A root document with a version
|
||||||
|
WHEN:
|
||||||
|
- Both ids are passed to add_or_update_ids
|
||||||
|
THEN:
|
||||||
|
- The root is indexed once and the version is not indexed
|
||||||
|
"""
|
||||||
|
root, version = self._root_with_version()
|
||||||
|
|
||||||
|
with backend.batch_update() as batch:
|
||||||
|
batch.add_or_update_ids([version.pk, root.pk])
|
||||||
|
|
||||||
|
assert backend.search_ids("Statement", user=None) == [root.pk]
|
||||||
|
assert backend.search_ids("latest", user=None) == [root.pk]
|
||||||
|
|
||||||
|
def test_add_or_update_ids_resolves_a_lone_version(
|
||||||
|
self,
|
||||||
|
backend: TantivyBackend,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A root document with a version
|
||||||
|
WHEN:
|
||||||
|
- Only the version's id is passed to add_or_update_ids
|
||||||
|
THEN:
|
||||||
|
- The root is indexed
|
||||||
|
"""
|
||||||
|
root, version = self._root_with_version()
|
||||||
|
|
||||||
|
with backend.batch_update() as batch:
|
||||||
|
batch.add_or_update_ids([version.pk])
|
||||||
|
|
||||||
|
assert backend.search_ids("latest", user=None) == [root.pk]
|
||||||
|
|
||||||
|
|
||||||
class TestSearch:
|
class TestSearch:
|
||||||
"""Test search query parsing and matching via search_ids."""
|
"""Test search query parsing and matching via search_ids."""
|
||||||
|
|
||||||
|
|||||||
@@ -1196,6 +1196,59 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
|||||||
self.assertIn(d3.id, result_ids)
|
self.assertIn(d3.id, result_ids)
|
||||||
self.assertNotIn(d4.id, result_ids)
|
self.assertNotIn(d4.id, result_ids)
|
||||||
|
|
||||||
|
def test_search_more_like_version_uses_its_root(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A document similar in content to a root document, and one that is not
|
||||||
|
- A version of the root document, which is never indexed
|
||||||
|
WHEN:
|
||||||
|
- API request for more like the version
|
||||||
|
THEN:
|
||||||
|
- The documents similar to the root are returned, not the version
|
||||||
|
"""
|
||||||
|
indexed = {}
|
||||||
|
for name, title, content, day in (
|
||||||
|
("root", "bank statement 1", "things i paid for in august", (2019, 3, 4)),
|
||||||
|
(
|
||||||
|
"similar",
|
||||||
|
"bank statement 3",
|
||||||
|
"things i paid for in september",
|
||||||
|
(2020, 7, 9),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"other",
|
||||||
|
"Quarterly Report",
|
||||||
|
"quarterly revenue profit margin",
|
||||||
|
(2021, 11, 30),
|
||||||
|
),
|
||||||
|
):
|
||||||
|
with time_machine.travel(
|
||||||
|
timezone.make_aware(datetime.datetime(*day)),
|
||||||
|
tick=False,
|
||||||
|
):
|
||||||
|
indexed[name] = DocumentFactory(
|
||||||
|
title=title,
|
||||||
|
content=content,
|
||||||
|
created=datetime.date(*day),
|
||||||
|
added=timezone.make_aware(datetime.datetime(*day)),
|
||||||
|
)
|
||||||
|
version = DocumentFactory(
|
||||||
|
root_document=indexed["root"],
|
||||||
|
version_index=1,
|
||||||
|
content="things i paid for in august",
|
||||||
|
)
|
||||||
|
backend = get_backend()
|
||||||
|
for document in indexed.values():
|
||||||
|
backend.add_or_update(document)
|
||||||
|
|
||||||
|
response = self.client.get(f"/api/documents/?more_like_id={version.id}")
|
||||||
|
|
||||||
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
|
result_ids = [r["id"] for r in response.data["results"]]
|
||||||
|
self.assertIn(indexed["similar"].id, result_ids)
|
||||||
|
self.assertNotIn(indexed["other"].id, result_ids)
|
||||||
|
self.assertNotIn(version.id, result_ids)
|
||||||
|
|
||||||
def test_more_like_requires_id_of_existing_document(self) -> None:
|
def test_more_like_requires_id_of_existing_document(self) -> None:
|
||||||
"""
|
"""
|
||||||
GIVEN:
|
GIVEN:
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from documents.models import Document
|
|||||||
from documents.tasks import update_document_content_maybe_archive_file
|
from documents.tasks import update_document_content_maybe_archive_file
|
||||||
from paperless_testing.assertions import FileSystemAssertsMixin
|
from paperless_testing.assertions import FileSystemAssertsMixin
|
||||||
from paperless_testing.dirs import DirectoriesMixin
|
from paperless_testing.dirs import DirectoriesMixin
|
||||||
|
from paperless_testing.factories import DocumentFactory
|
||||||
|
|
||||||
sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
|
sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
|
||||||
|
|
||||||
@@ -116,6 +117,27 @@ class TestMakeIndex:
|
|||||||
call_command("document_index", "reindex", skip_checks=True)
|
call_command("document_index", "reindex", skip_checks=True)
|
||||||
mock_get_backend.return_value.rebuild.assert_called_once()
|
mock_get_backend.return_value.rebuild.assert_called_once()
|
||||||
|
|
||||||
|
def test_reindex_skips_versions(self, mocker: MockerFixture) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A root document with a version
|
||||||
|
WHEN:
|
||||||
|
- The reindex command runs
|
||||||
|
THEN:
|
||||||
|
- Only the root document is handed to the rebuild, since a version
|
||||||
|
is indexed as its root
|
||||||
|
"""
|
||||||
|
root = DocumentFactory()
|
||||||
|
DocumentFactory(root_document=root, version_index=1)
|
||||||
|
mock_get_backend = mocker.patch(
|
||||||
|
"documents.management.commands.document_index.get_backend",
|
||||||
|
)
|
||||||
|
|
||||||
|
call_command("document_index", "reindex", skip_checks=True)
|
||||||
|
|
||||||
|
documents = mock_get_backend.return_value.rebuild.call_args.args[0]
|
||||||
|
assert list(documents.values_list("pk", flat=True)) == [root.pk]
|
||||||
|
|
||||||
def test_optimize(self) -> None:
|
def test_optimize(self) -> None:
|
||||||
"""Optimize command must execute without error (Tantivy handles optimization automatically)."""
|
"""Optimize command must execute without error (Tantivy handles optimization automatically)."""
|
||||||
call_command("document_index", "optimize", skip_checks=True)
|
call_command("document_index", "optimize", skip_checks=True)
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from documents.models import Correspondent
|
|||||||
from documents.models import DocumentType
|
from documents.models import DocumentType
|
||||||
from documents.models import StoragePath
|
from documents.models import StoragePath
|
||||||
from documents.models import Tag
|
from documents.models import Tag
|
||||||
|
from documents.permissions import has_perms_owner_aware
|
||||||
from documents.permissions import permitted_document_ids
|
from documents.permissions import permitted_document_ids
|
||||||
from documents.permissions import permitted_object_ids
|
from documents.permissions import permitted_object_ids
|
||||||
from documents.permissions import restrict_queryset_to_visible
|
from documents.permissions import restrict_queryset_to_visible
|
||||||
@@ -32,6 +33,8 @@ from paperless_testing.permissions import grant_global
|
|||||||
from paperless_testing.permissions import grant_object
|
from paperless_testing.permissions import grant_object
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from django.contrib.auth.models import User
|
||||||
|
|
||||||
from paperless_testing.dirs import PaperlessDirs
|
from paperless_testing.dirs import PaperlessDirs
|
||||||
|
|
||||||
|
|
||||||
@@ -178,6 +181,382 @@ class TestPermittedDocumentIdsIncludeDeleted:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
class TestPermittedDocumentIdsVersions:
|
||||||
|
"""
|
||||||
|
A version is authorized by its root document: the version's own owner and
|
||||||
|
grants never matter.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("root_owner", "version_owner", "expected_visible"),
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
"other",
|
||||||
|
"nobody",
|
||||||
|
False,
|
||||||
|
id="unowned-version-of-private-root",
|
||||||
|
),
|
||||||
|
pytest.param("other", "user", False, id="own-version-of-private-root"),
|
||||||
|
pytest.param("user", "other", True, id="foreign-version-of-own-root"),
|
||||||
|
pytest.param("user", "nobody", True, id="unowned-version-of-own-root"),
|
||||||
|
pytest.param("nobody", "other", True, id="private-version-of-unowned-root"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_version_follows_root_owner(
|
||||||
|
self,
|
||||||
|
root_owner: str,
|
||||||
|
version_owner: str,
|
||||||
|
*,
|
||||||
|
expected_visible: bool,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A root document and a version with differing owners
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved for the user
|
||||||
|
THEN:
|
||||||
|
- The version is visible exactly when its root is
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
owners = {"user": user, "other": UserFactory(), "nobody": None}
|
||||||
|
root = DocumentFactory(owner=owners[root_owner])
|
||||||
|
version = DocumentFactory(root_document=root, owner=owners[version_owner])
|
||||||
|
|
||||||
|
visible = set(permitted_document_ids(user))
|
||||||
|
|
||||||
|
assert (version.pk in visible) is expected_visible
|
||||||
|
assert (root.pk in visible) is expected_visible
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def grantee(user: User, kind: str) -> User | Group:
|
||||||
|
"""The user itself, or a new group the user belongs to."""
|
||||||
|
if kind == "user":
|
||||||
|
return user
|
||||||
|
group = Group.objects.create(name="shared")
|
||||||
|
user.groups.add(group)
|
||||||
|
return group
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"grantee_kind",
|
||||||
|
[pytest.param("user", id="user"), pytest.param("group", id="group")],
|
||||||
|
)
|
||||||
|
def test_grant_on_root_applies_to_version(self, grantee_kind: str) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A private root document shared with a user or one of their groups
|
||||||
|
- A version of it owned by someone else
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved for the user
|
||||||
|
THEN:
|
||||||
|
- Both the root and the version are visible
|
||||||
|
- A user without the grant sees neither
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
stranger = UserFactory()
|
||||||
|
root = DocumentFactory(owner=UserFactory())
|
||||||
|
version = DocumentFactory(root_document=root, owner=UserFactory())
|
||||||
|
grant_object(self.grantee(user, grantee_kind), root, "view_document")
|
||||||
|
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(user),
|
||||||
|
expected_visible=[root.pk, version.pk],
|
||||||
|
expected_hidden=[],
|
||||||
|
)
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(stranger),
|
||||||
|
expected_visible=[],
|
||||||
|
expected_hidden=[root.pk, version.pk],
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"grantee_kind",
|
||||||
|
[pytest.param("user", id="user"), pytest.param("group", id="group")],
|
||||||
|
)
|
||||||
|
def test_grant_on_version_is_ignored(self, grantee_kind: str) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A private root document
|
||||||
|
- A version with an explicit grant for the user or one of their groups
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved for the user
|
||||||
|
THEN:
|
||||||
|
- Neither the root nor the version is visible
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
root = DocumentFactory(owner=UserFactory())
|
||||||
|
version = DocumentFactory(root_document=root, owner=UserFactory())
|
||||||
|
grant_object(self.grantee(user, grantee_kind), version, "view_document")
|
||||||
|
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(user),
|
||||||
|
expected_visible=[],
|
||||||
|
expected_hidden=[root.pk, version.pk],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_grant_on_one_root_does_not_reach_another_roots_version(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- Two private roots, each with a version
|
||||||
|
- The user may view only the first root
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved for the user
|
||||||
|
THEN:
|
||||||
|
- Only the first root and its version are visible
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
first = DocumentFactory(owner=UserFactory())
|
||||||
|
first_version = DocumentFactory(root_document=first, owner=UserFactory())
|
||||||
|
second = DocumentFactory(owner=UserFactory())
|
||||||
|
second_version = DocumentFactory(root_document=second, owner=user)
|
||||||
|
grant_object(user, first, "view_document")
|
||||||
|
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(user),
|
||||||
|
expected_visible=[first.pk, first_version.pk],
|
||||||
|
expected_hidden=[second.pk, second_version.pk],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_user_in_several_groups(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A user in two groups
|
||||||
|
- Two private roots shared with one group each, and a third shared with nobody
|
||||||
|
- A version of each root
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved for the user
|
||||||
|
THEN:
|
||||||
|
- The two shared roots and their versions are visible
|
||||||
|
- The third root and its version are not
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
groups = [Group.objects.create(name=f"group{i}") for i in range(2)]
|
||||||
|
user.groups.add(*groups)
|
||||||
|
shared = [DocumentFactory(owner=UserFactory()) for _ in groups]
|
||||||
|
for root, group in zip(shared, groups, strict=True):
|
||||||
|
grant_object(group, root, "view_document")
|
||||||
|
unshared = DocumentFactory(owner=UserFactory())
|
||||||
|
shared_versions = [
|
||||||
|
DocumentFactory(root_document=root, owner=UserFactory()) for root in shared
|
||||||
|
]
|
||||||
|
unshared_version = DocumentFactory(root_document=unshared, owner=None)
|
||||||
|
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(user),
|
||||||
|
expected_visible=[
|
||||||
|
*(root.pk for root in shared),
|
||||||
|
*(version.pk for version in shared_versions),
|
||||||
|
],
|
||||||
|
expected_hidden=[unshared.pk, unshared_version.pk],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_permission_is_resolved_through_the_root(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A private root document where the user may view and change
|
||||||
|
WHEN:
|
||||||
|
- The permitted ids are resolved for view, change and delete
|
||||||
|
THEN:
|
||||||
|
- The version is visible for view and change only
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
root = DocumentFactory(owner=UserFactory())
|
||||||
|
version = DocumentFactory(root_document=root, owner=UserFactory())
|
||||||
|
grant_object(user, root, "view_document", "change_document")
|
||||||
|
|
||||||
|
assert version.pk in set(permitted_document_ids(user))
|
||||||
|
assert version.pk in set(permitted_document_ids(user, perm="change_document"))
|
||||||
|
assert version.pk in set(
|
||||||
|
permitted_document_ids(user, perm="documents.change_document"),
|
||||||
|
)
|
||||||
|
assert version.pk not in set(
|
||||||
|
permitted_document_ids(user, perm="delete_document"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_anonymous_sees_versions_of_unowned_roots_only(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A version owned by nobody under a private root
|
||||||
|
- A version owned by someone under an unowned root
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved for an anonymous user
|
||||||
|
THEN:
|
||||||
|
- Only the version of the unowned root is visible
|
||||||
|
"""
|
||||||
|
private_root = DocumentFactory(owner=UserFactory())
|
||||||
|
private_version = DocumentFactory(root_document=private_root, owner=None)
|
||||||
|
open_root = DocumentFactory(owner=None)
|
||||||
|
open_version = DocumentFactory(root_document=open_root, owner=UserFactory())
|
||||||
|
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(AnonymousUser()),
|
||||||
|
expected_visible=[open_root.pk, open_version.pk],
|
||||||
|
expected_hidden=[private_root.pk, private_version.pk],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_deleted_versions_follow_their_deleted_root(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A soft-deleted root document and its version, which deleting the
|
||||||
|
root soft-deletes too; the version is owned by someone else
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved with and without deleted
|
||||||
|
documents
|
||||||
|
THEN:
|
||||||
|
- Nothing is visible by default
|
||||||
|
- With deleted documents included, the version is visible to the
|
||||||
|
root's owner and not to the version's own owner
|
||||||
|
"""
|
||||||
|
owner = UserFactory()
|
||||||
|
version_owner = UserFactory()
|
||||||
|
root = DocumentFactory(owner=owner)
|
||||||
|
version = DocumentFactory(root_document=root, owner=version_owner)
|
||||||
|
root.delete()
|
||||||
|
|
||||||
|
assert not {root.pk, version.pk} & set(permitted_document_ids(owner))
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(owner, include_deleted=True),
|
||||||
|
expected_visible=[root.pk, version.pk],
|
||||||
|
expected_hidden=[],
|
||||||
|
)
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(version_owner, include_deleted=True),
|
||||||
|
expected_visible=[],
|
||||||
|
expected_hidden=[root.pk, version.pk],
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"is_superuser",
|
||||||
|
[
|
||||||
|
pytest.param(False, id="regular-user"),
|
||||||
|
pytest.param(True, id="superuser"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_inactive_user_sees_no_versions(self, *, is_superuser: bool) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- An inactive user, possibly a superuser, who owns a root and its version
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved for them
|
||||||
|
THEN:
|
||||||
|
- Nothing is visible
|
||||||
|
"""
|
||||||
|
user = UserFactory(is_active=False, is_superuser=is_superuser)
|
||||||
|
root = DocumentFactory(owner=user)
|
||||||
|
version = DocumentFactory(root_document=root, owner=user)
|
||||||
|
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(user),
|
||||||
|
expected_visible=[],
|
||||||
|
expected_hidden=[root.pk, version.pk],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_superuser_sees_all_versions(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A private root owned by someone else, with a version
|
||||||
|
WHEN:
|
||||||
|
- The permitted document ids are resolved for a superuser
|
||||||
|
THEN:
|
||||||
|
- Both the root and the version are visible
|
||||||
|
"""
|
||||||
|
superuser = UserFactory(superuser=True)
|
||||||
|
root = DocumentFactory(owner=UserFactory())
|
||||||
|
version = DocumentFactory(root_document=root, owner=UserFactory())
|
||||||
|
|
||||||
|
assert_visible_document_ids(
|
||||||
|
permitted_document_ids(superuser),
|
||||||
|
expected_visible=[root.pk, version.pk],
|
||||||
|
expected_hidden=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
class TestHasPermsOwnerAwareVersions:
|
||||||
|
"""
|
||||||
|
The single-object check agrees with permitted_document_ids: a version is
|
||||||
|
authorized by its root document.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("root_owner", "version_owner", "expected"),
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
"other",
|
||||||
|
"nobody",
|
||||||
|
False,
|
||||||
|
id="unowned-version-of-private-root",
|
||||||
|
),
|
||||||
|
pytest.param("other", "user", False, id="own-version-of-private-root"),
|
||||||
|
pytest.param("user", "other", True, id="foreign-version-of-own-root"),
|
||||||
|
pytest.param("nobody", "other", True, id="private-version-of-unowned-root"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_version_follows_root_owner(
|
||||||
|
self,
|
||||||
|
root_owner: str,
|
||||||
|
version_owner: str,
|
||||||
|
*,
|
||||||
|
expected: bool,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A root document and a version with differing owners
|
||||||
|
WHEN:
|
||||||
|
- The single-object check runs for the version
|
||||||
|
THEN:
|
||||||
|
- The version is allowed exactly when its root is
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
owners = {"user": user, "other": UserFactory(), "nobody": None}
|
||||||
|
root = DocumentFactory(owner=owners[root_owner])
|
||||||
|
version = DocumentFactory(root_document=root, owner=owners[version_owner])
|
||||||
|
|
||||||
|
assert has_perms_owner_aware(user, "view_document", version) is expected
|
||||||
|
assert has_perms_owner_aware(user, "view_document", root) is expected
|
||||||
|
|
||||||
|
def test_grant_on_root_applies_and_grant_on_version_does_not(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A private root with a version, and a second private root with a version
|
||||||
|
- The user may change only the first root, and was granted the second
|
||||||
|
root's version directly
|
||||||
|
WHEN:
|
||||||
|
- The single-object check runs for each version
|
||||||
|
THEN:
|
||||||
|
- Only the first root's version is allowed
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
shared_root = DocumentFactory(owner=UserFactory())
|
||||||
|
shared_version = DocumentFactory(root_document=shared_root, owner=UserFactory())
|
||||||
|
private_root = DocumentFactory(owner=UserFactory())
|
||||||
|
private_version = DocumentFactory(
|
||||||
|
root_document=private_root,
|
||||||
|
owner=UserFactory(),
|
||||||
|
)
|
||||||
|
grant_object(user, shared_root, "change_document")
|
||||||
|
grant_object(user, private_version, "change_document")
|
||||||
|
|
||||||
|
assert has_perms_owner_aware(user, "change_document", shared_version)
|
||||||
|
assert not has_perms_owner_aware(user, "change_document", private_version)
|
||||||
|
|
||||||
|
def test_other_models_use_their_own_owner(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A tag owned by someone else, and one owned by the user
|
||||||
|
WHEN:
|
||||||
|
- The single-object check runs for each
|
||||||
|
THEN:
|
||||||
|
- Only the user's own tag is allowed without a grant
|
||||||
|
"""
|
||||||
|
user = UserFactory()
|
||||||
|
mine = TagFactory(owner=user)
|
||||||
|
theirs = TagFactory(owner=UserFactory())
|
||||||
|
|
||||||
|
assert has_perms_owner_aware(user, "view_tag", mine)
|
||||||
|
assert not has_perms_owner_aware(user, "view_tag", theirs)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
class TestAiChatAllDocumentsPermissionBoundary:
|
class TestAiChatAllDocumentsPermissionBoundary:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -310,7 +310,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
|||||||
|
|
||||||
@mock.patch("documents.tasks.clear_document_caches")
|
@mock.patch("documents.tasks.clear_document_caches")
|
||||||
@mock.patch("documents.search.get_backend")
|
@mock.patch("documents.search.get_backend")
|
||||||
def test_update_content_version_indexes_root(
|
def test_update_content_version_clears_caches_for_root(
|
||||||
self,
|
self,
|
||||||
mock_get_backend: mock.Mock,
|
mock_get_backend: mock.Mock,
|
||||||
mock_clear_caches: mock.Mock,
|
mock_clear_caches: mock.Mock,
|
||||||
@@ -321,8 +321,8 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
|||||||
WHEN:
|
WHEN:
|
||||||
- Update content task is called for the version
|
- Update content task is called for the version
|
||||||
THEN:
|
THEN:
|
||||||
- The version's content is updated
|
- The version's content is updated, not the root's
|
||||||
- The root document is indexed rather than the version
|
- The document is indexed
|
||||||
- Caches are cleared for both
|
- Caches are cleared for both
|
||||||
"""
|
"""
|
||||||
root, version = self._create_root_with_version()
|
root, version = self._create_root_with_version()
|
||||||
@@ -334,8 +334,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
|||||||
"my document",
|
"my document",
|
||||||
)
|
)
|
||||||
self.assertEqual(Document.objects.get(pk=root.pk).content, "root content")
|
self.assertEqual(Document.objects.get(pk=root.pk).content, "root content")
|
||||||
indexed = mock_get_backend.return_value.add_or_update.call_args.args[0]
|
mock_get_backend.return_value.add_or_update.assert_called_once()
|
||||||
self.assertEqual(indexed.pk, root.pk)
|
|
||||||
mock_clear_caches.assert_has_calls(
|
mock_clear_caches.assert_has_calls(
|
||||||
[mock.call(version.pk), mock.call(root.pk)],
|
[mock.call(version.pk), mock.call(root.pk)],
|
||||||
)
|
)
|
||||||
@@ -343,7 +342,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
|||||||
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
|
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
|
||||||
@mock.patch("documents.tasks.llm_index_add_or_update_document")
|
@mock.patch("documents.tasks.llm_index_add_or_update_document")
|
||||||
@mock.patch("documents.search.get_backend")
|
@mock.patch("documents.search.get_backend")
|
||||||
def test_update_content_version_updates_llm_index_for_root(
|
def test_update_content_version_updates_llm_index(
|
||||||
self,
|
self,
|
||||||
mock_get_backend: mock.Mock,
|
mock_get_backend: mock.Mock,
|
||||||
mock_llm_index: mock.Mock,
|
mock_llm_index: mock.Mock,
|
||||||
@@ -355,14 +354,13 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
|||||||
WHEN:
|
WHEN:
|
||||||
- Update content task is called for the version
|
- Update content task is called for the version
|
||||||
THEN:
|
THEN:
|
||||||
- The LLM index is updated for the root document, not the version
|
- The LLM index is updated
|
||||||
"""
|
"""
|
||||||
root, version = self._create_root_with_version()
|
_, version = self._create_root_with_version()
|
||||||
|
|
||||||
tasks.update_document_content_maybe_archive_file(version.pk)
|
tasks.update_document_content_maybe_archive_file(version.pk)
|
||||||
|
|
||||||
mock_llm_index.assert_called_once()
|
mock_llm_index.assert_called_once()
|
||||||
self.assertEqual(mock_llm_index.call_args.args[0].pk, root.pk)
|
|
||||||
|
|
||||||
|
|
||||||
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
||||||
|
|||||||
@@ -17,9 +17,26 @@ from django.db.models.functions import RowNumber
|
|||||||
from documents.models import Document
|
from documents.models import Document
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
from rest_framework.request import Request
|
from rest_framework.request import Request
|
||||||
|
|
||||||
|
|
||||||
|
def root_document_ids(ids: Iterable[int]) -> QuerySet[int]:
|
||||||
|
"""
|
||||||
|
The ids of the root documents of the given documents: a root stands for
|
||||||
|
itself and a version for its root. Only the indexes' bookkeeping needs
|
||||||
|
this, since they hold root documents only.
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
Document.objects.filter(pk__in=ids)
|
||||||
|
.annotate(root_id=Coalesce("root_document_id", "id"))
|
||||||
|
.order_by()
|
||||||
|
.values_list("root_id", flat=True)
|
||||||
|
.distinct()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
|
def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
|
||||||
"""
|
"""
|
||||||
Sorts versions so the newest one comes first using version_index and not on id,
|
Sorts versions so the newest one comes first using version_index and not on id,
|
||||||
|
|||||||
+30
-37
@@ -174,7 +174,6 @@ from documents.permissions import TrashPermissions
|
|||||||
from documents.permissions import ViewDocumentsPermissions
|
from documents.permissions import ViewDocumentsPermissions
|
||||||
from documents.permissions import annotate_document_count_by_ids
|
from documents.permissions import annotate_document_count_by_ids
|
||||||
from documents.permissions import annotate_document_count_for_related_queryset
|
from documents.permissions import annotate_document_count_for_related_queryset
|
||||||
from documents.permissions import documents_without_permitted_root
|
|
||||||
from documents.permissions import get_document_count_filter_for_user
|
from documents.permissions import get_document_count_filter_for_user
|
||||||
from documents.permissions import get_objects_for_user_owner_aware
|
from documents.permissions import get_objects_for_user_owner_aware
|
||||||
from documents.permissions import has_global_statistics_permission
|
from documents.permissions import has_global_statistics_permission
|
||||||
@@ -330,9 +329,10 @@ def _get_tantivy_query_and_mode(params):
|
|||||||
def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
|
def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
|
||||||
try:
|
try:
|
||||||
more_like_doc_id = int(query_params["more_like_id"])
|
more_like_doc_id = int(query_params["more_like_id"])
|
||||||
more_like_doc = Document.objects.select_related("owner").get(
|
more_like_doc = Document.objects.select_related(
|
||||||
pk=more_like_doc_id,
|
"owner",
|
||||||
)
|
"root_document__owner",
|
||||||
|
).get(pk=more_like_doc_id)
|
||||||
except (TypeError, ValueError, Document.DoesNotExist):
|
except (TypeError, ValueError, Document.DoesNotExist):
|
||||||
raise PermissionDenied(_("Invalid more_like_id"))
|
raise PermissionDenied(_("Invalid more_like_id"))
|
||||||
|
|
||||||
@@ -343,7 +343,8 @@ def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
|
|||||||
):
|
):
|
||||||
raise PermissionDenied(_("Insufficient permissions."))
|
raise PermissionDenied(_("Insufficient permissions."))
|
||||||
|
|
||||||
return more_like_doc_id
|
# Only root documents are indexed, a version stands for its root
|
||||||
|
return more_like_doc.root_document_id or more_like_doc.pk
|
||||||
|
|
||||||
|
|
||||||
class SearchParams(NamedTuple):
|
class SearchParams(NamedTuple):
|
||||||
@@ -1560,7 +1561,7 @@ class DocumentViewSet(
|
|||||||
if request.user is not None and not has_perms_owner_aware(
|
if request.user is not None and not has_perms_owner_aware(
|
||||||
request.user,
|
request.user,
|
||||||
"change_document",
|
"change_document",
|
||||||
get_root_document(doc),
|
doc,
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
|
|
||||||
@@ -1623,7 +1624,7 @@ class DocumentViewSet(
|
|||||||
if request.user is not None and not has_perms_owner_aware(
|
if request.user is not None and not has_perms_owner_aware(
|
||||||
request.user,
|
request.user,
|
||||||
"change_document",
|
"change_document",
|
||||||
get_root_document(doc),
|
doc,
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
|
|
||||||
@@ -1876,7 +1877,7 @@ class DocumentViewSet(
|
|||||||
if currentUser is not None and not has_perms_owner_aware(
|
if currentUser is not None and not has_perms_owner_aware(
|
||||||
currentUser,
|
currentUser,
|
||||||
"view_document",
|
"view_document",
|
||||||
get_root_document(doc),
|
doc,
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions to view notes")
|
return HttpResponseForbidden("Insufficient permissions to view notes")
|
||||||
except Document.DoesNotExist:
|
except Document.DoesNotExist:
|
||||||
@@ -1898,7 +1899,7 @@ class DocumentViewSet(
|
|||||||
if currentUser is not None and not has_perms_owner_aware(
|
if currentUser is not None and not has_perms_owner_aware(
|
||||||
currentUser,
|
currentUser,
|
||||||
"change_document",
|
"change_document",
|
||||||
get_root_document(doc),
|
doc,
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden(
|
return HttpResponseForbidden(
|
||||||
"Insufficient permissions to create notes",
|
"Insufficient permissions to create notes",
|
||||||
@@ -1941,7 +1942,7 @@ class DocumentViewSet(
|
|||||||
if currentUser is not None and not has_perms_owner_aware(
|
if currentUser is not None and not has_perms_owner_aware(
|
||||||
currentUser,
|
currentUser,
|
||||||
"change_document",
|
"change_document",
|
||||||
get_root_document(doc),
|
doc,
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions to delete notes")
|
return HttpResponseForbidden("Insufficient permissions to delete notes")
|
||||||
|
|
||||||
@@ -1991,7 +1992,7 @@ class DocumentViewSet(
|
|||||||
if currentUser is not None and not has_perms_owner_aware(
|
if currentUser is not None and not has_perms_owner_aware(
|
||||||
currentUser,
|
currentUser,
|
||||||
"change_document",
|
"change_document",
|
||||||
get_root_document(doc),
|
doc,
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden(
|
return HttpResponseForbidden(
|
||||||
"Insufficient permissions to add share link",
|
"Insufficient permissions to add share link",
|
||||||
@@ -2117,7 +2118,7 @@ class DocumentViewSet(
|
|||||||
documents = Document.objects.filter(pk__in=document_ids)
|
documents = Document.objects.filter(pk__in=document_ids)
|
||||||
if (
|
if (
|
||||||
request.user is not None
|
request.user is not None
|
||||||
and documents_without_permitted_root(documents, request.user).exists()
|
and documents.exclude(id__in=permitted_document_ids(request.user)).exists()
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
|
|
||||||
@@ -2452,7 +2453,7 @@ class ChatStreamingView(GenericAPIView[Any]):
|
|||||||
if not has_perms_owner_aware(
|
if not has_perms_owner_aware(
|
||||||
request.user,
|
request.user,
|
||||||
"view_document",
|
"view_document",
|
||||||
get_root_document(document),
|
document,
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
|
|
||||||
@@ -3009,12 +3010,8 @@ class DocumentOperationPermissionMixin(PassUserMixin, DocumentSelectionMixin):
|
|||||||
user.has_perm(
|
user.has_perm(
|
||||||
"documents.change_document",
|
"documents.change_document",
|
||||||
)
|
)
|
||||||
and not Document.global_objects.filter(
|
and not Document.global_objects.filter(pk__in=documents)
|
||||||
pk__in=[doc.pk for doc in root_docs],
|
.exclude(pk__in=permitted_document_ids(user, perm="change_document"))
|
||||||
)
|
|
||||||
.exclude(
|
|
||||||
pk__in=permitted_document_ids(user, perm="change_document"),
|
|
||||||
)
|
|
||||||
.exists()
|
.exists()
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -3627,7 +3624,7 @@ class SelectionDataView(DocumentSelectionMixin, GenericAPIView[Any]):
|
|||||||
documents = Document.objects.filter(pk__in=ids)
|
documents = Document.objects.filter(pk__in=ids)
|
||||||
if (
|
if (
|
||||||
documents.count() != len(ids)
|
documents.count() != len(ids)
|
||||||
or documents_without_permitted_root(documents, request.user).exists()
|
or documents.exclude(id__in=permitted_document_ids(request.user)).exists()
|
||||||
):
|
):
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
|
|
||||||
@@ -4124,21 +4121,16 @@ class BulkDownloadView(DocumentSelectionMixin, GenericAPIView[Any]):
|
|||||||
validated_data=serializer.validated_data,
|
validated_data=serializer.validated_data,
|
||||||
)
|
)
|
||||||
documents = Document.objects.filter(pk__in=ids)
|
documents = Document.objects.filter(pk__in=ids)
|
||||||
versioned_documents = []
|
|
||||||
compression = serializer.validated_data.get("compression")
|
compression = serializer.validated_data.get("compression")
|
||||||
content = serializer.validated_data.get("content")
|
content = serializer.validated_data.get("content")
|
||||||
follow_filename_format = serializer.validated_data.get("follow_formatting")
|
follow_filename_format = serializer.validated_data.get("follow_formatting")
|
||||||
|
|
||||||
permitted_ids = set(permitted_document_ids(request.user))
|
if documents.exclude(id__in=permitted_document_ids(request.user)).exists():
|
||||||
for document in documents:
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
root_doc = get_root_document(document)
|
versioned_documents = [
|
||||||
if root_doc.pk not in permitted_ids:
|
get_latest_version_for_root(get_root_document(document))
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
for document in documents
|
||||||
versioned_documents.append(
|
]
|
||||||
get_latest_version_for_root(
|
|
||||||
root_doc,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
if content == "both":
|
if content == "both":
|
||||||
strategy_class = OriginalAndArchiveStrategy
|
strategy_class = OriginalAndArchiveStrategy
|
||||||
@@ -4811,7 +4803,7 @@ class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
|
|||||||
)
|
)
|
||||||
|
|
||||||
denied_id = (
|
denied_id = (
|
||||||
documents_without_permitted_root(documents_qs, request.user)
|
documents_qs.exclude(id__in=permitted_document_ids(request.user))
|
||||||
.order_by("pk")
|
.order_by("pk")
|
||||||
.values_list("pk", flat=True)
|
.values_list("pk", flat=True)
|
||||||
.first()
|
.first()
|
||||||
@@ -5657,11 +5649,12 @@ class TrashView(ListModelMixin, PassUserMixin):
|
|||||||
if doc_ids is not None
|
if doc_ids is not None
|
||||||
else self.filter_queryset(self.get_queryset()).all()
|
else self.filter_queryset(self.get_queryset()).all()
|
||||||
)
|
)
|
||||||
if documents_without_permitted_root(
|
if docs.exclude(
|
||||||
docs,
|
id__in=permitted_document_ids(
|
||||||
request.user,
|
request.user,
|
||||||
perm="delete_document",
|
perm="delete_document",
|
||||||
include_deleted=True,
|
include_deleted=True,
|
||||||
|
),
|
||||||
).exists():
|
).exists():
|
||||||
return HttpResponseForbidden("Insufficient permissions")
|
return HttpResponseForbidden("Insufficient permissions")
|
||||||
action = serializer.validated_data.get("action")
|
action = serializer.validated_data.get("action")
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from documents.models import PaperlessTask
|
|||||||
from documents.utils import IterWrapper
|
from documents.utils import IterWrapper
|
||||||
from documents.utils import QuerySetStream
|
from documents.utils import QuerySetStream
|
||||||
from documents.utils import identity
|
from documents.utils import identity
|
||||||
|
from documents.versioning import root_document_ids
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
from paperless_ai.db import db_connection_released
|
from paperless_ai.db import db_connection_released
|
||||||
from paperless_ai.embedding import build_llm_index_text
|
from paperless_ai.embedding import build_llm_index_text
|
||||||
@@ -443,11 +444,11 @@ def update_llm_index(
|
|||||||
"Skipping LLM index update: migration check deferred; "
|
"Skipping LLM index update: migration check deferred; "
|
||||||
"will retry next run."
|
"will retry next run."
|
||||||
)
|
)
|
||||||
documents = Document.objects.select_related(
|
documents = (
|
||||||
"correspondent",
|
Document.objects.filter(root_document__isnull=True)
|
||||||
"document_type",
|
.select_related("correspondent", "document_type", "storage_path")
|
||||||
"storage_path",
|
.prefetch_related("tags", "notes", "custom_fields__field")
|
||||||
).prefetch_related("tags", "notes", "custom_fields__field")
|
)
|
||||||
no_documents = not documents.exists()
|
no_documents = not documents.exists()
|
||||||
|
|
||||||
# Fast exit before touching config: nothing to index and no existing index.
|
# Fast exit before touching config: nothing to index and no existing index.
|
||||||
@@ -483,7 +484,7 @@ def update_llm_index(
|
|||||||
msg = "LLM index rebuilt successfully."
|
msg = "LLM index rebuilt successfully."
|
||||||
else:
|
else:
|
||||||
scoped_documents = (
|
scoped_documents = (
|
||||||
documents.filter(id__in=document_ids)
|
documents.filter(id__in=root_document_ids(document_ids))
|
||||||
if document_ids is not None
|
if document_ids is not None
|
||||||
else documents
|
else documents
|
||||||
)
|
)
|
||||||
@@ -510,7 +511,12 @@ def update_llm_index(
|
|||||||
|
|
||||||
|
|
||||||
def llm_index_add_or_update_document(document: Document):
|
def llm_index_add_or_update_document(document: Document):
|
||||||
"""Add or atomically replace a document's chunks in the index."""
|
"""
|
||||||
|
Add or atomically replace a document's chunks in the index. Only root
|
||||||
|
documents are indexed, so a version is indexed as its root document.
|
||||||
|
"""
|
||||||
|
if document.root_document_id is not None:
|
||||||
|
document = document.root_document
|
||||||
config = AIConfig()
|
config = AIConfig()
|
||||||
new_nodes = build_document_node(
|
new_nodes = build_document_node(
|
||||||
document,
|
document,
|
||||||
|
|||||||
@@ -388,6 +388,83 @@ def test_update_llm_index_partial_update(
|
|||||||
assert after[str(doc2.pk)] == before[str(doc2.pk)]
|
assert after[str(doc2.pk)] == before[str(doc2.pk)]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
class TestLlmIndexVersions:
|
||||||
|
"""The LLM index holds root documents only: a version is indexed as its root."""
|
||||||
|
|
||||||
|
def test_add_or_update_document_indexes_a_version_as_its_root(
|
||||||
|
self,
|
||||||
|
temp_llm_index_dir: Path,
|
||||||
|
mock_embed_model: FakeEmbedding,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A root document with a version
|
||||||
|
WHEN:
|
||||||
|
- The version is passed to llm_index_add_or_update_document
|
||||||
|
THEN:
|
||||||
|
- Only the root document is in the index
|
||||||
|
"""
|
||||||
|
root = DocumentFactory(content="root content")
|
||||||
|
version = DocumentFactory(root_document=root, version_index=1)
|
||||||
|
|
||||||
|
indexing.llm_index_add_or_update_document(version)
|
||||||
|
|
||||||
|
with indexing.get_vector_store() as store:
|
||||||
|
indexed = store.get_modified_times()
|
||||||
|
|
||||||
|
assert set(indexed) == {str(root.pk)}
|
||||||
|
|
||||||
|
def test_rebuild_skips_versions(
|
||||||
|
self,
|
||||||
|
temp_llm_index_dir: Path,
|
||||||
|
mock_embed_model: FakeEmbedding,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- A root document with a version
|
||||||
|
WHEN:
|
||||||
|
- The LLM index is rebuilt
|
||||||
|
THEN:
|
||||||
|
- Only the root document is in the index
|
||||||
|
"""
|
||||||
|
root = DocumentFactory()
|
||||||
|
DocumentFactory(root_document=root, version_index=1)
|
||||||
|
|
||||||
|
indexing.update_llm_index(rebuild=True)
|
||||||
|
|
||||||
|
with indexing.get_vector_store() as store:
|
||||||
|
indexed = store.get_modified_times()
|
||||||
|
|
||||||
|
assert set(indexed) == {str(root.pk)}
|
||||||
|
|
||||||
|
def test_incremental_update_by_version_id_refreshes_the_root(
|
||||||
|
self,
|
||||||
|
temp_llm_index_dir: Path,
|
||||||
|
mock_embed_model: FakeEmbedding,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN:
|
||||||
|
- An indexed root document with a version whose root was modified since
|
||||||
|
WHEN:
|
||||||
|
- An incremental update is scoped to the version's id
|
||||||
|
THEN:
|
||||||
|
- The root's entry is refreshed and no entry exists for the version
|
||||||
|
"""
|
||||||
|
root = DocumentFactory()
|
||||||
|
version = DocumentFactory(root_document=root, version_index=1)
|
||||||
|
indexing.update_llm_index(rebuild=True)
|
||||||
|
Document.objects.filter(pk=root.pk).update(modified=timezone.now())
|
||||||
|
root.refresh_from_db()
|
||||||
|
|
||||||
|
indexing.update_llm_index(document_ids=[version.pk])
|
||||||
|
|
||||||
|
with indexing.get_vector_store() as store:
|
||||||
|
indexed = store.get_modified_times()
|
||||||
|
|
||||||
|
assert indexed == {str(root.pk): root.modified.isoformat()}
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
def test_add_or_update_document_updates_existing_entry(
|
def test_add_or_update_document_updates_existing_entry(
|
||||||
temp_llm_index_dir: Path,
|
temp_llm_index_dir: Path,
|
||||||
@@ -637,6 +714,7 @@ class TestLlmIndexAddOrUpdateDocumentEmptyContent:
|
|||||||
|
|
||||||
doc = MagicMock(spec=Document)
|
doc = MagicMock(spec=Document)
|
||||||
doc.id = 42
|
doc.id = 42
|
||||||
|
doc.root_document_id = None
|
||||||
# Must not raise
|
# Must not raise
|
||||||
indexing.llm_index_add_or_update_document(doc)
|
indexing.llm_index_add_or_update_document(doc)
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user